mobridge-mcp 0.0.6 → 0.0.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/main.obf.cjs +1 -1
- package/dist/protocol/schemas/tools.schema.json +6 -1
- package/integrity.json +2 -2
- package/loader.cjs +1 -1
- package/package.json +1 -1
- package/resources/android/app-developer-debug.apk +0 -0
- package/resources/ios/MobileSemantic/MobileSemantic.xcodeproj/project.pbxproj +85 -213
- package/resources/ios/MobileSemantic/MobileSemantic.xcodeproj/xcshareddata/xcschemes/MobileSemanticUITests.xcscheme +2 -31
- package/resources/ios/MobileSemantic/UITests/BridgeServer.swift +77 -6
- package/resources/ios/MobileSemantic/UITests/MSHID.h +25 -0
- package/resources/ios/MobileSemantic/UITests/MSHID.m +155 -0
- package/resources/ios/MobileSemantic/UITests/MobileSemanticUITests-Bridging.h +2 -0
- package/resources/ios/MobileSemantic/UITests/XcuiSnapshot.swift +19 -4
- package/resources/ios/MobileSemantic/gen_project.rb +15 -22
- package/resources/ios/MobileSemantic/Fixture/MobileSemanticFixtureApp.swift +0 -50
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import Foundation
|
|
2
2
|
import XCTest
|
|
3
3
|
import Darwin
|
|
4
|
+
import ObjectiveC
|
|
4
5
|
|
|
5
6
|
// The XCTest Runner (WDA-style): a long-running UI test that binds a TCP socket on 127.0.0.1:8500 and
|
|
6
7
|
// services length-prefixed JSON-RPC requests by doing REAL XCUIApplication/XCUIElement work. TS stays
|
|
@@ -14,15 +15,33 @@ final class BridgeServer: XCTestCase {
|
|
|
14
15
|
private let runnerInstanceId = UUID().uuidString
|
|
15
16
|
private var sessionToken = ""
|
|
16
17
|
private var app: XCUIApplication!
|
|
17
|
-
// The bundle id of the app currently under automation.
|
|
18
|
-
// app.open with a `bundleId` switches
|
|
19
|
-
// so CMS can read/act on App
|
|
18
|
+
// The bundle id of the app currently under automation. Default = SpringBoard (the home screen), since
|
|
19
|
+
// this is a standalone UI-test Runner with NO host app of its own. app.open with a `bundleId` switches
|
|
20
|
+
// this to any already-installed app (XCUIApplication(bundleIdentifier:)), so CMS can read/act on App
|
|
21
|
+
// Store / system apps without re-signing them; only this Runner is signed.
|
|
22
|
+
private static let springboard = "com.apple.springboard"
|
|
20
23
|
private var targetBundleId: String? = nil
|
|
21
24
|
|
|
25
|
+
// PERF (the single biggest iOS speedup): XCUITest blocks EVERY query/gesture until the app reaches a
|
|
26
|
+
// quiescent state (animations + main-runloop idle). A media feed (TikTok, a game, any animation) never
|
|
27
|
+
// idles, so each call waits the full ~animation timeout and appears to hang / times out. Neutralize the
|
|
28
|
+
// private `waitForQuiescenceIncludingAnimationsIdle:` to a no-op — the same technique WebDriverAgent /
|
|
29
|
+
// Appium use. Guarded: if Apple renames it, we simply keep the default (slow but correct) behavior.
|
|
30
|
+
private static func disableQuiescenceWaiting() {
|
|
31
|
+
guard let cls = NSClassFromString("XCUIApplicationProcess") else { return }
|
|
32
|
+
let sel = NSSelectorFromString("waitForQuiescenceIncludingAnimationsIdle:")
|
|
33
|
+
guard let method = class_getInstanceMethod(cls, sel) else { return }
|
|
34
|
+
let noop: @convention(block) (AnyObject, Bool) -> Void = { _, _ in }
|
|
35
|
+
method_setImplementation(method, imp_implementationWithBlock(noop))
|
|
36
|
+
FileHandle.standardError.write("[runner] quiescence wait disabled (fast mode)\n".data(using: .utf8)!)
|
|
37
|
+
}
|
|
38
|
+
|
|
22
39
|
func testServe() throws {
|
|
23
40
|
continueAfterFailure = true
|
|
24
|
-
|
|
25
|
-
app
|
|
41
|
+
BridgeServer.disableQuiescenceWaiting()
|
|
42
|
+
// Standalone Runner (no fixture host app): default to SpringBoard, which is always running, so a
|
|
43
|
+
// screen.get before any app.open reads the home screen instead of launching a nonexistent host.
|
|
44
|
+
app = XCUIApplication(bundleIdentifier: BridgeServer.springboard)
|
|
26
45
|
|
|
27
46
|
let listenFd = try openListener(port: BridgeServer.port)
|
|
28
47
|
defer { close(listenFd) }
|
|
@@ -73,6 +92,8 @@ final class BridgeServer: XCTestCase {
|
|
|
73
92
|
return ok(snapshot())
|
|
74
93
|
case "screen.act":
|
|
75
94
|
return screenAct(params, ok: ok, err: err)
|
|
95
|
+
case "gesture.perform":
|
|
96
|
+
return gesturePerform(params, ok: ok, err: err)
|
|
76
97
|
case "system.action":
|
|
77
98
|
let a = params["action"] as? String ?? ""
|
|
78
99
|
// rc.2: DISPATCH + evidence only. HOME is business-level with no intrinsic predicate; the
|
|
@@ -88,6 +109,56 @@ final class BridgeServer: XCTestCase {
|
|
|
88
109
|
}
|
|
89
110
|
}
|
|
90
111
|
|
|
112
|
+
// Coordinate gesture (mobile.gesture.perform): TAP / LONG_PRESS / SWIPE / DRAG by absolute screen
|
|
113
|
+
// points, for surfaces with no semantic node to act on (media feeds, canvas, maps). Points map onto
|
|
114
|
+
// the app window via a (0,0)-normalized coordinate + a point offset. PINCH is not expressible via a
|
|
115
|
+
// single XCUICoordinate, so it is honestly rejected.
|
|
116
|
+
private func gesturePerform(_ p: [String: Any], ok: ([String: Any]) -> ([String: Any], Bool), err: (String, String, [String]) -> ([String: Any], Bool)) -> ([String: Any], Bool) {
|
|
117
|
+
func num(_ v: Any?) -> Double? { if let d = v as? Double { return d }; if let i = v as? Int { return Double(i) }; if let s = v as? String { return Double(s) }; return nil }
|
|
118
|
+
let kind = (p["kind"] as? String ?? "TAP").uppercased()
|
|
119
|
+
guard let x = num(p["x"]), let y = num(p["y"]) else {
|
|
120
|
+
return err("ACTION_REJECTED", "gesture.perform needs numeric x,y.", ["Pass x,y (and x2,y2 for SWIPE/DRAG) in screen points."])
|
|
121
|
+
}
|
|
122
|
+
// Blind coordinate gesture: NO before/after snapshot (a heavy media feed's tree is slow to capture
|
|
123
|
+
// and needs no semantic verification here — the caller does an explicit screen.get if it wants one).
|
|
124
|
+
// PRIMARY path is MSHID: synthesized HID touch events that bypass the app process entirely, so a
|
|
125
|
+
// swipe on TikTok is injected in ~ms instead of ~1.2s via XCUICoordinate (which waits on the app).
|
|
126
|
+
// XCUICoordinate stays as a fallback only if the private HID API is unavailable on this OS.
|
|
127
|
+
let start = CGPoint(x: x, y: y)
|
|
128
|
+
var endPoint = start
|
|
129
|
+
if kind == "SWIPE" || kind == "DRAG" {
|
|
130
|
+
guard let x2 = num(p["x2"]), let y2 = num(p["y2"]) else {
|
|
131
|
+
return err("ACTION_REJECTED", "\(kind) needs x2,y2 (end point).", ["Pass x2,y2 in screen points."])
|
|
132
|
+
}
|
|
133
|
+
endPoint = CGPoint(x: x2, y: y2)
|
|
134
|
+
}
|
|
135
|
+
guard ["TAP", "LONG_PRESS", "SWIPE", "DRAG"].contains(kind) else {
|
|
136
|
+
return err("ACTION_UNSUPPORTED", "gesture kind '\(kind)' is not supported on iOS.", ["Use TAP, LONG_PRESS, SWIPE, or DRAG."])
|
|
137
|
+
}
|
|
138
|
+
// PRIMARY: HID injection (MODERN_HID iOS15+ → LEGACY_HID iOS14), which bypasses the app process so
|
|
139
|
+
// a swipe on a heavy/animated app lands in ~ms. MSHID returns the injector name, or nil → fall back
|
|
140
|
+
// to XCUICoordinate (slow, but always available) so the gesture is never lost.
|
|
141
|
+
var method: String?
|
|
142
|
+
switch kind {
|
|
143
|
+
case "TAP": method = MSHID.tap(at: start)
|
|
144
|
+
case "LONG_PRESS": method = MSHID.longPress(at: start, duration: 0.7)
|
|
145
|
+
case "SWIPE": method = MSHID.swipe(from: start, to: endPoint, duration: 0.12)
|
|
146
|
+
case "DRAG": method = MSHID.drag(from: start, to: endPoint, duration: 0.6)
|
|
147
|
+
default: break
|
|
148
|
+
}
|
|
149
|
+
if method == nil {
|
|
150
|
+
let origin = app.coordinate(withNormalizedOffset: CGVector(dx: 0, dy: 0))
|
|
151
|
+
let s = origin.withOffset(CGVector(dx: x, dy: y))
|
|
152
|
+
switch kind {
|
|
153
|
+
case "TAP": s.tap()
|
|
154
|
+
case "LONG_PRESS": s.press(forDuration: 0.7)
|
|
155
|
+
default: s.press(forDuration: kind == "DRAG" ? 0.6 : 0.08, thenDragTo: origin.withOffset(CGVector(dx: endPoint.x, dy: endPoint.y)))
|
|
156
|
+
}
|
|
157
|
+
method = "COORDINATE"
|
|
158
|
+
}
|
|
159
|
+
return ok(["dispatch": ["status": "DISPATCHED", "method": method!, "fallbackType": "GESTURE"], "screenChanged": true])
|
|
160
|
+
}
|
|
161
|
+
|
|
91
162
|
private func isValidBundleId(_ b: String) -> Bool {
|
|
92
163
|
// reverse-DNS-ish: segments of [A-Za-z0-9-] joined by dots; guards against argv/format injection.
|
|
93
164
|
guard b.count <= 255 else { return false }
|
|
@@ -95,7 +166,7 @@ final class BridgeServer: XCTestCase {
|
|
|
95
166
|
}
|
|
96
167
|
|
|
97
168
|
private func snapshot() -> [String: Any] {
|
|
98
|
-
let bundle = targetBundleId ??
|
|
169
|
+
let bundle = targetBundleId ?? BridgeServer.springboard
|
|
99
170
|
if let snap = try? app.snapshot() {
|
|
100
171
|
return Xcui.toScreenSnapshot(snap, bundleId: bundle, deviceId: "ios-sim")
|
|
101
172
|
}
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
// MSHID — HID-level gesture injection facade (Appium/WebDriverAgent technique). Synthesizes touch
|
|
2
|
+
// events WITHOUT querying the app's accessibility tree and WITHOUT waiting on the app process, so a
|
|
3
|
+
// TAP/SWIPE/DRAG/LONG_PRESS lands in ~ms regardless of how heavy/animated the foreground app is.
|
|
4
|
+
//
|
|
5
|
+
// Version-durable: a GestureInjector chain tries the MODERN path first
|
|
6
|
+
// (XCSynthesizedEventRecord + XCPointerEventPath via XCUIDevice.eventSynthesizer, iOS 15+/current),
|
|
7
|
+
// then the LEGACY path (XCEventGenerator, iOS 14). Each method returns the injector name that
|
|
8
|
+
// succeeded ("MODERN_HID" / "LEGACY_HID") or nil, so the Swift caller can fall back to XCUICoordinate
|
|
9
|
+
// and report which path ran. All coordinates are ABSOLUTE screen points.
|
|
10
|
+
#import <Foundation/Foundation.h>
|
|
11
|
+
#import <CoreGraphics/CoreGraphics.h>
|
|
12
|
+
|
|
13
|
+
NS_ASSUME_NONNULL_BEGIN
|
|
14
|
+
|
|
15
|
+
@interface MSHID : NSObject
|
|
16
|
+
/// True when any HID injector is usable on this OS (else the caller uses XCUICoordinate).
|
|
17
|
+
+ (BOOL)available;
|
|
18
|
+
/// Each returns the method name that injected the gesture ("MODERN_HID"/"LEGACY_HID"), or nil on failure.
|
|
19
|
+
+ (nullable NSString *)tapAt:(CGPoint)point;
|
|
20
|
+
+ (nullable NSString *)longPressAt:(CGPoint)point duration:(double)duration;
|
|
21
|
+
+ (nullable NSString *)swipeFrom:(CGPoint)from to:(CGPoint)to duration:(double)duration;
|
|
22
|
+
+ (nullable NSString *)dragFrom:(CGPoint)from to:(CGPoint)to duration:(double)duration;
|
|
23
|
+
@end
|
|
24
|
+
|
|
25
|
+
NS_ASSUME_NONNULL_END
|
|
@@ -0,0 +1,155 @@
|
|
|
1
|
+
#import "MSHID.h"
|
|
2
|
+
#import <XCTest/XCTest.h>
|
|
3
|
+
|
|
4
|
+
// ---- Private XCTest event APIs (signatures from appium/WebDriverAgent PrivateHeaders) --------------
|
|
5
|
+
|
|
6
|
+
// MODERN (iOS 15+/current): build a pointer path, wrap it in a synthesized event, dispatch via the
|
|
7
|
+
// device's event synthesizer. This is what WDA switched to when Apple reworked XCTest internals.
|
|
8
|
+
@interface XCPointerEventPath : NSObject
|
|
9
|
+
- (instancetype)initForTouchAtPoint:(CGPoint)point offset:(double)offset;
|
|
10
|
+
- (void)moveToPoint:(CGPoint)point atOffset:(double)offset;
|
|
11
|
+
- (void)liftUpAtOffset:(double)offset;
|
|
12
|
+
@end
|
|
13
|
+
|
|
14
|
+
@interface XCSynthesizedEventRecord : NSObject
|
|
15
|
+
- (instancetype)initWithName:(NSString *)name interfaceOrientation:(long long)orientation;
|
|
16
|
+
- (void)addPointerEventPath:(XCPointerEventPath *)path;
|
|
17
|
+
@end
|
|
18
|
+
|
|
19
|
+
@interface NSObject (MSEventSynthesizer)
|
|
20
|
+
- (id)eventSynthesizer; // on XCUIDevice
|
|
21
|
+
- (void)synthesizeEvent:(id)event completion:(void (^)(BOOL result, NSError *error))completion; // on the synthesizer
|
|
22
|
+
@end
|
|
23
|
+
|
|
24
|
+
// LEGACY (iOS 14): the older singleton generator.
|
|
25
|
+
@interface XCEventGenerator : NSObject
|
|
26
|
+
+ (instancetype)sharedGenerator;
|
|
27
|
+
- (double)pressAtPoint:(CGPoint)point forDuration:(double)duration liftAtPoint:(CGPoint)liftPoint velocity:(double)velocity orientation:(long long)orientation name:(NSString *)name handler:(void (^)(id record, NSError *error))handler;
|
|
28
|
+
- (double)pressAtPoint:(CGPoint)point forDuration:(double)duration orientation:(long long)orientation handler:(void (^)(id record, NSError *error))handler;
|
|
29
|
+
@end
|
|
30
|
+
|
|
31
|
+
// Portrait; the CMS coordinates are already in portrait screen points.
|
|
32
|
+
static const long long kPortrait = 1;
|
|
33
|
+
|
|
34
|
+
// Run an async injector and wait (bounded) for its completion. The Runner's socket loop is NOT the main
|
|
35
|
+
// thread, so a plain semaphore wait here does not deadlock a completion delivered on the main queue.
|
|
36
|
+
static BOOL MSAwait(double timeout, void (^inject)(dispatch_semaphore_t sem)) {
|
|
37
|
+
dispatch_semaphore_t sem = dispatch_semaphore_create(0);
|
|
38
|
+
@try { inject(sem); }
|
|
39
|
+
@catch (NSException *e) { return NO; }
|
|
40
|
+
return dispatch_semaphore_wait(sem, dispatch_time(DISPATCH_TIME_NOW, (int64_t)(timeout * NSEC_PER_SEC))) == 0;
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
// ---- Injector protocol + two implementations ------------------------------------------------------
|
|
44
|
+
|
|
45
|
+
@protocol MSGestureInjector <NSObject>
|
|
46
|
+
- (BOOL)available;
|
|
47
|
+
- (NSString *)name;
|
|
48
|
+
- (BOOL)tapAt:(CGPoint)p;
|
|
49
|
+
- (BOOL)longPressAt:(CGPoint)p duration:(double)dur;
|
|
50
|
+
- (BOOL)swipeFrom:(CGPoint)from to:(CGPoint)to duration:(double)dur;
|
|
51
|
+
- (BOOL)dragFrom:(CGPoint)from to:(CGPoint)to duration:(double)dur;
|
|
52
|
+
@end
|
|
53
|
+
|
|
54
|
+
// MODERN — XCSynthesizedEventRecord + XCPointerEventPath via XCUIDevice.eventSynthesizer.
|
|
55
|
+
@interface MSModernInjector : NSObject <MSGestureInjector>
|
|
56
|
+
@end
|
|
57
|
+
@implementation MSModernInjector
|
|
58
|
+
- (NSString *)name { return @"MODERN_HID"; }
|
|
59
|
+
- (id)synthesizer {
|
|
60
|
+
XCUIDevice *dev = [XCUIDevice sharedDevice];
|
|
61
|
+
if (![dev respondsToSelector:@selector(eventSynthesizer)]) return nil;
|
|
62
|
+
return [dev eventSynthesizer];
|
|
63
|
+
}
|
|
64
|
+
- (BOOL)available {
|
|
65
|
+
return NSClassFromString(@"XCSynthesizedEventRecord") && NSClassFromString(@"XCPointerEventPath") && [self synthesizer];
|
|
66
|
+
}
|
|
67
|
+
// Build a single-finger path: touch-down at `from`, optional hold, move to `to`, lift. A tap is from==to.
|
|
68
|
+
- (BOOL)injectPathFrom:(CGPoint)from to:(CGPoint)to hold:(double)hold move:(double)move name:(NSString *)name {
|
|
69
|
+
id synth = [self synthesizer]; if (!synth) return NO;
|
|
70
|
+
Class pathC = NSClassFromString(@"XCPointerEventPath");
|
|
71
|
+
Class recC = NSClassFromString(@"XCSynthesizedEventRecord");
|
|
72
|
+
if (!pathC || !recC) return NO;
|
|
73
|
+
XCPointerEventPath *path = [[pathC alloc] initForTouchAtPoint:from offset:0.0];
|
|
74
|
+
double t = 0.0;
|
|
75
|
+
if (hold > 0) { [path moveToPoint:from atOffset:(t += hold)]; }
|
|
76
|
+
if (!CGPointEqualToPoint(from, to)) { [path moveToPoint:to atOffset:(t += MAX(move, 0.01))]; }
|
|
77
|
+
else { t += 0.05; }
|
|
78
|
+
[path liftUpAtOffset:t];
|
|
79
|
+
XCSynthesizedEventRecord *rec = [[recC alloc] initWithName:name interfaceOrientation:kPortrait];
|
|
80
|
+
[rec addPointerEventPath:path];
|
|
81
|
+
__block BOOL ok = NO;
|
|
82
|
+
BOOL fired = MSAwait(t + 5.0, ^(dispatch_semaphore_t sem) {
|
|
83
|
+
[synth synthesizeEvent:rec completion:^(BOOL result, NSError *error) { ok = (error == nil); dispatch_semaphore_signal(sem); }];
|
|
84
|
+
});
|
|
85
|
+
return fired && ok;
|
|
86
|
+
}
|
|
87
|
+
- (BOOL)tapAt:(CGPoint)p { return [self injectPathFrom:p to:p hold:0 move:0 name:@"tap"]; }
|
|
88
|
+
- (BOOL)longPressAt:(CGPoint)p duration:(double)dur { return [self injectPathFrom:p to:p hold:dur move:0 name:@"longpress"]; }
|
|
89
|
+
- (BOOL)swipeFrom:(CGPoint)f to:(CGPoint)t duration:(double)dur { return [self injectPathFrom:f to:t hold:0 move:dur name:@"swipe"]; }
|
|
90
|
+
- (BOOL)dragFrom:(CGPoint)f to:(CGPoint)t duration:(double)dur { return [self injectPathFrom:f to:t hold:0.5 move:dur name:@"drag"]; }
|
|
91
|
+
@end
|
|
92
|
+
|
|
93
|
+
// LEGACY — XCEventGenerator (iOS 14).
|
|
94
|
+
@interface MSLegacyInjector : NSObject <MSGestureInjector>
|
|
95
|
+
@end
|
|
96
|
+
@implementation MSLegacyInjector
|
|
97
|
+
- (NSString *)name { return @"LEGACY_HID"; }
|
|
98
|
+
- (nullable XCEventGenerator *)gen {
|
|
99
|
+
Class cls = NSClassFromString(@"XCEventGenerator");
|
|
100
|
+
if (!cls) return nil;
|
|
101
|
+
@try { return [cls sharedGenerator]; } @catch (NSException *e) { return nil; }
|
|
102
|
+
}
|
|
103
|
+
- (BOOL)available { return [self gen] != nil; }
|
|
104
|
+
- (BOOL)tapAt:(CGPoint)p {
|
|
105
|
+
XCEventGenerator *g = [self gen]; if (!g) return NO;
|
|
106
|
+
return MSAwait(5.0, ^(dispatch_semaphore_t sem) {
|
|
107
|
+
[g pressAtPoint:p forDuration:0.0 liftAtPoint:p velocity:0 orientation:kPortrait name:@"tap" handler:^(id r, NSError *e) { dispatch_semaphore_signal(sem); }];
|
|
108
|
+
});
|
|
109
|
+
}
|
|
110
|
+
- (BOOL)longPressAt:(CGPoint)p duration:(double)dur {
|
|
111
|
+
XCEventGenerator *g = [self gen]; if (!g) return NO;
|
|
112
|
+
return MSAwait(dur + 5.0, ^(dispatch_semaphore_t sem) {
|
|
113
|
+
[g pressAtPoint:p forDuration:dur orientation:kPortrait handler:^(id r, NSError *e) { dispatch_semaphore_signal(sem); }];
|
|
114
|
+
});
|
|
115
|
+
}
|
|
116
|
+
- (BOOL)swipeFrom:(CGPoint)f to:(CGPoint)t duration:(double)dur { return [self dragFrom:f to:t duration:dur press:0.0]; }
|
|
117
|
+
- (BOOL)dragFrom:(CGPoint)f to:(CGPoint)t duration:(double)dur { return [self dragFrom:f to:t duration:dur press:0.5]; }
|
|
118
|
+
- (BOOL)dragFrom:(CGPoint)f to:(CGPoint)t duration:(double)dur press:(double)press {
|
|
119
|
+
XCEventGenerator *g = [self gen]; if (!g) return NO;
|
|
120
|
+
double vel = hypot(t.x - f.x, t.y - f.y) / MAX(dur, 0.05);
|
|
121
|
+
return MSAwait(dur + press + 5.0, ^(dispatch_semaphore_t sem) {
|
|
122
|
+
[g pressAtPoint:f forDuration:press liftAtPoint:t velocity:vel orientation:kPortrait name:@"swipe" handler:^(id r, NSError *e) { dispatch_semaphore_signal(sem); }];
|
|
123
|
+
});
|
|
124
|
+
}
|
|
125
|
+
@end
|
|
126
|
+
|
|
127
|
+
// ---- Facade: try MODERN then LEGACY; return the name that ran (or nil → caller uses XCUICoordinate) --
|
|
128
|
+
|
|
129
|
+
@implementation MSHID
|
|
130
|
+
|
|
131
|
+
+ (NSArray<id<MSGestureInjector>> *)chain {
|
|
132
|
+
static NSArray *chain; static dispatch_once_t once;
|
|
133
|
+
dispatch_once(&once, ^{ chain = @[[MSModernInjector new], [MSLegacyInjector new]]; });
|
|
134
|
+
return chain;
|
|
135
|
+
}
|
|
136
|
+
|
|
137
|
+
+ (BOOL)available {
|
|
138
|
+
for (id<MSGestureInjector> inj in [self chain]) { if ([inj available]) return YES; }
|
|
139
|
+
return NO;
|
|
140
|
+
}
|
|
141
|
+
|
|
142
|
+
+ (nullable NSString *)run:(BOOL (^)(id<MSGestureInjector> inj))op {
|
|
143
|
+
for (id<MSGestureInjector> inj in [self chain]) {
|
|
144
|
+
if (![inj available]) continue;
|
|
145
|
+
if (op(inj)) return [inj name];
|
|
146
|
+
}
|
|
147
|
+
return nil;
|
|
148
|
+
}
|
|
149
|
+
|
|
150
|
+
+ (nullable NSString *)tapAt:(CGPoint)p { return [self run:^BOOL(id<MSGestureInjector> i) { return [i tapAt:p]; }]; }
|
|
151
|
+
+ (nullable NSString *)longPressAt:(CGPoint)p duration:(double)d { return [self run:^BOOL(id<MSGestureInjector> i) { return [i longPressAt:p duration:d]; }]; }
|
|
152
|
+
+ (nullable NSString *)swipeFrom:(CGPoint)f to:(CGPoint)t duration:(double)d { return [self run:^BOOL(id<MSGestureInjector> i) { return [i swipeFrom:f to:t duration:d]; }]; }
|
|
153
|
+
+ (nullable NSString *)dragFrom:(CGPoint)f to:(CGPoint)t duration:(double)d { return [self run:^BOOL(id<MSGestureInjector> i) { return [i dragFrom:f to:t duration:d]; }]; }
|
|
154
|
+
|
|
155
|
+
@end
|
|
@@ -41,9 +41,23 @@ enum Xcui {
|
|
|
41
41
|
static func toScreenSnapshot(_ root: XCUIElementSnapshot, bundleId: String, deviceId: String) -> [String: Any] {
|
|
42
42
|
var nodes: [[String: Any]] = []
|
|
43
43
|
var counter = 0
|
|
44
|
-
|
|
45
|
-
|
|
44
|
+
var truncated = false
|
|
45
|
+
// PERF (C): bound the tree on heavy apps (a TikTok/feed hierarchy is huge and slow to serialize on
|
|
46
|
+
// a 2GB device). Cap depth + total nodes; env-overridable. Truncation is reported, never silent.
|
|
47
|
+
let env = ProcessInfo.processInfo.environment
|
|
48
|
+
let maxDepth = Int(env["MSM_IOS_MAX_DEPTH"] ?? "") ?? 60
|
|
49
|
+
let maxNodes = Int(env["MSM_IOS_MAX_NODES"] ?? "") ?? 3000
|
|
50
|
+
func walk(_ el: XCUIElementSnapshot, parentId: String?, depth: Int) {
|
|
51
|
+
if nodes.count >= maxNodes || depth > maxDepth { truncated = true; return }
|
|
46
52
|
let f = el.frame
|
|
53
|
+
// PERF: drop zero-area layout artifacts (they cannot be seen or tapped), but keep descending so
|
|
54
|
+
// any visible children still surface — this trims the tree on dense apps without losing a
|
|
55
|
+
// real target. The child keeps the current parentId so the hierarchy stays connected.
|
|
56
|
+
if f.width <= 0 || f.height <= 0 {
|
|
57
|
+
for c in el.children { walk(c, parentId: parentId, depth: depth + 1) }
|
|
58
|
+
return
|
|
59
|
+
}
|
|
60
|
+
let id = "n\(counter)"; counter += 1
|
|
47
61
|
var node: [String: Any] = [
|
|
48
62
|
"nodeId": id,
|
|
49
63
|
"role": role(for: el.elementType),
|
|
@@ -58,9 +72,9 @@ enum Xcui {
|
|
|
58
72
|
let isSecure = el.elementType == .secureTextField
|
|
59
73
|
node["state"] = ["enabled": el.isEnabled, "selected": el.isSelected, "password": isSecure]
|
|
60
74
|
nodes.append(node)
|
|
61
|
-
for c in el.children { walk(c, parentId: id) }
|
|
75
|
+
for c in el.children { walk(c, parentId: id, depth: depth + 1) }
|
|
62
76
|
}
|
|
63
|
-
walk(root, parentId: nil)
|
|
77
|
+
walk(root, parentId: nil, depth: 0)
|
|
64
78
|
let th = treeHash(nodes)
|
|
65
79
|
return [
|
|
66
80
|
"schemaVersion": 1, "platform": "IOS", "deviceId": deviceId,
|
|
@@ -68,6 +82,7 @@ enum Xcui {
|
|
|
68
82
|
"timestamp": ISO8601DateFormatter().string(from: Date()),
|
|
69
83
|
"packageName": bundleId, "appId": bundleId, "windowId": 0, "orientation": "portrait",
|
|
70
84
|
"screenStateId": "ios:\(th.prefix(4))", "compact": true, "nodes": nodes, "treeHash": th,
|
|
85
|
+
"truncated": truncated,
|
|
71
86
|
]
|
|
72
87
|
}
|
|
73
88
|
}
|
|
@@ -1,6 +1,8 @@
|
|
|
1
1
|
#!/usr/bin/env ruby
|
|
2
|
-
# Generate MobileSemantic.xcodeproj
|
|
3
|
-
#
|
|
2
|
+
# Generate MobileSemantic.xcodeproj — ONE target: the standalone MobileSemanticUITests UI-test bundle
|
|
3
|
+
# (WebDriverAgent style: no host application). The bundle's BridgeServer IS the device driver; it drives
|
|
4
|
+
# arbitrary apps via XCUIApplication(bundleIdentifier:), so it needs no fixture/host app of its own.
|
|
5
|
+
# No Xcode GUI, no xcodegen. Run: `ruby gen_project.rb`.
|
|
4
6
|
require 'xcodeproj'
|
|
5
7
|
|
|
6
8
|
here = File.dirname(File.expand_path(__FILE__))
|
|
@@ -12,22 +14,17 @@ DEPLOY = '14.0'
|
|
|
12
14
|
# with this development team. Override with MSM_DEV_TEAM to use a different Apple account.
|
|
13
15
|
DEV_TEAM = ENV['MSM_DEV_TEAM'] || 'A2KNU2X4F7'
|
|
14
16
|
|
|
15
|
-
# ---
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
ref = app_group.new_file(f)
|
|
20
|
-
app.source_build_phase.add_file_reference(ref)
|
|
21
|
-
end
|
|
22
|
-
|
|
23
|
-
# --- UI-test bundle target ---------------------------------------------------
|
|
17
|
+
# --- UI-test bundle target (the ONLY target: the driver) ---------------------
|
|
18
|
+
# A target-less ui_test_bundle: xcodebuild auto-generates the MobileSemanticUITests-Runner host app,
|
|
19
|
+
# exactly like WebDriverAgent. No TEST_TARGET_NAME / no app dependency, so there is no second app to
|
|
20
|
+
# install or confuse with the driver.
|
|
24
21
|
uitest = proj.new_target(:ui_test_bundle, 'MobileSemanticUITests', :ios, DEPLOY)
|
|
25
22
|
ui_group = proj.new_group('UITests', 'UITests')
|
|
26
|
-
|
|
23
|
+
# Swift sources + the ObjC HID shim (MSHID.m). A bridging header (set below) exposes MSHID to Swift.
|
|
24
|
+
Dir.glob(File.join(here, 'UITests', '*.{swift,m}')).sort.each do |f|
|
|
27
25
|
ref = ui_group.new_file(f)
|
|
28
26
|
uitest.source_build_phase.add_file_reference(ref)
|
|
29
27
|
end
|
|
30
|
-
uitest.add_dependency(app)
|
|
31
28
|
|
|
32
29
|
# --- build settings ----------------------------------------------------------
|
|
33
30
|
common = {
|
|
@@ -43,16 +40,13 @@ common = {
|
|
|
43
40
|
'MARKETING_VERSION' => '1.0',
|
|
44
41
|
'ALWAYS_SEARCH_USER_PATHS' => 'NO',
|
|
45
42
|
}
|
|
46
|
-
app.build_configurations.each do |c|
|
|
47
|
-
c.build_settings.merge!(common)
|
|
48
|
-
c.build_settings['PRODUCT_BUNDLE_IDENTIFIER'] = 'com.msm.MobileSemanticFixture'
|
|
49
|
-
c.build_settings['INFOPLIST_KEY_UILaunchScreen_Generation'] = 'YES'
|
|
50
|
-
c.build_settings['INFOPLIST_KEY_UIApplicationSupportsIndirectInputEvents'] = 'YES'
|
|
51
|
-
end
|
|
52
43
|
uitest.build_configurations.each do |c|
|
|
53
44
|
c.build_settings.merge!(common)
|
|
54
45
|
c.build_settings['PRODUCT_BUNDLE_IDENTIFIER'] = 'com.msm.MobileSemanticUITests'
|
|
55
|
-
|
|
46
|
+
# Bridging header so Swift can call the ObjC HID shim (MSHID); enable ObjC modules.
|
|
47
|
+
c.build_settings['SWIFT_OBJC_BRIDGING_HEADER'] = 'UITests/MobileSemanticUITests-Bridging.h'
|
|
48
|
+
c.build_settings['CLANG_ENABLE_MODULES'] = 'YES'
|
|
49
|
+
c.build_settings['SWIFT_INSTALL_OBJC_HEADER'] = 'NO'
|
|
56
50
|
end
|
|
57
51
|
|
|
58
52
|
# --- shared scheme so `xcodebuild -scheme MobileSemanticUITests test` works ---
|
|
@@ -61,7 +55,6 @@ proj.save
|
|
|
61
55
|
scheme = Xcodeproj::XCScheme.new
|
|
62
56
|
scheme.add_build_target(uitest)
|
|
63
57
|
scheme.add_test_target(uitest)
|
|
64
|
-
scheme.set_launch_target(app)
|
|
65
58
|
scheme.save_as(proj_path, 'MobileSemanticUITests', true)
|
|
66
59
|
|
|
67
|
-
puts "generated #{proj_path}"
|
|
60
|
+
puts "generated #{proj_path} (standalone UITests-Runner, no fixture app)"
|
|
@@ -1,50 +0,0 @@
|
|
|
1
|
-
import SwiftUI
|
|
2
|
-
|
|
3
|
-
// Deterministic fixture app under test. Deployment target 14.0 so the same binary runs on the
|
|
4
|
-
// physical iPhone 7 Plus (iOS 14.2) later. Elements carry stable accessibilityIdentifiers so the
|
|
5
|
-
// TS resolver can locate by automationId; two same-labelled "Continue" buttons give a controlled
|
|
6
|
-
// AMBIGUOUS_MATCH case; signing in flips to a "welcome" screen for wait/diff.
|
|
7
|
-
@main
|
|
8
|
-
struct MobileSemanticFixtureApp: App {
|
|
9
|
-
var body: some Scene {
|
|
10
|
-
WindowGroup { ContentView() }
|
|
11
|
-
}
|
|
12
|
-
}
|
|
13
|
-
|
|
14
|
-
struct ContentView: View {
|
|
15
|
-
@State private var username = ""
|
|
16
|
-
@State private var signedIn = false
|
|
17
|
-
@State private var playing = false
|
|
18
|
-
|
|
19
|
-
var body: some View {
|
|
20
|
-
if signedIn {
|
|
21
|
-
VStack(spacing: 16) {
|
|
22
|
-
Text("Welcome")
|
|
23
|
-
.accessibilityIdentifier("welcome")
|
|
24
|
-
// A plain Button whose OWN title flips on tap — a reliable XCUI hit target (unlike a
|
|
25
|
-
// SwiftUI switch), giving a deterministic NODE_STATE_CHANGED postcondition.
|
|
26
|
-
Button(playing ? "Pause" : "Play") { playing.toggle() }
|
|
27
|
-
.accessibilityIdentifier("playPause")
|
|
28
|
-
Button("Sign Out") { signedIn = false }
|
|
29
|
-
.accessibilityIdentifier("signOut")
|
|
30
|
-
}
|
|
31
|
-
.padding()
|
|
32
|
-
} else {
|
|
33
|
-
VStack(spacing: 16) {
|
|
34
|
-
Text("Sign In").font(.title)
|
|
35
|
-
TextField("Username", text: $username)
|
|
36
|
-
.accessibilityIdentifier("username")
|
|
37
|
-
.textFieldStyle(.roundedBorder)
|
|
38
|
-
Button("Sign In") { signedIn = true }
|
|
39
|
-
.accessibilityIdentifier("signIn")
|
|
40
|
-
// Two buttons with the SAME visible label "Continue" -> ambiguity when a NodeRef
|
|
41
|
-
// relies on ROLE_LABEL alone; distinct identifiers let a stronger locator disambiguate.
|
|
42
|
-
HStack(spacing: 24) {
|
|
43
|
-
Button("Continue") {}.accessibilityIdentifier("continuePrimary")
|
|
44
|
-
Button("Continue") {}.accessibilityIdentifier("continueSecondary")
|
|
45
|
-
}
|
|
46
|
-
}
|
|
47
|
-
.padding()
|
|
48
|
-
}
|
|
49
|
-
}
|
|
50
|
-
}
|