mcp-baepsae 5.1.0 → 6.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README-KR.md +98 -33
- package/README.md +101 -35
- package/bundled/baepsae-native +0 -0
- package/dist/backend.d.ts +26 -0
- package/dist/backend.d.ts.map +1 -0
- package/dist/backend.js +79 -0
- package/dist/backend.js.map +1 -0
- package/dist/index.js +3 -1
- package/dist/index.js.map +1 -1
- package/dist/tool-manifest.d.ts +12 -0
- package/dist/tool-manifest.d.ts.map +1 -0
- package/dist/tool-manifest.js +79 -0
- package/dist/tool-manifest.js.map +1 -0
- package/dist/tools/info.d.ts.map +1 -1
- package/dist/tools/info.js +104 -5
- package/dist/tools/info.js.map +1 -1
- package/dist/tools/input.js +7 -6
- package/dist/tools/input.js.map +1 -1
- package/dist/tools/media.d.ts.map +1 -1
- package/dist/tools/media.js +137 -11
- package/dist/tools/media.js.map +1 -1
- package/dist/tools/simulator.js +7 -7
- package/dist/tools/simulator.js.map +1 -1
- package/dist/tools/system.d.ts.map +1 -1
- package/dist/tools/system.js +2 -2
- package/dist/tools/system.js.map +1 -1
- package/dist/tools/ui.d.ts.map +1 -1
- package/dist/tools/ui.js +126 -8
- package/dist/tools/ui.js.map +1 -1
- package/dist/tools/workflow.d.ts +3 -0
- package/dist/tools/workflow.d.ts.map +1 -0
- package/dist/tools/workflow.js +434 -0
- package/dist/tools/workflow.js.map +1 -0
- package/dist/types.d.ts +15 -0
- package/dist/types.d.ts.map +1 -1
- package/dist/utils.d.ts +19 -3
- package/dist/utils.d.ts.map +1 -1
- package/dist/utils.js +110 -5
- package/dist/utils.js.map +1 -1
- package/dist/version.d.ts +1 -1
- package/dist/version.js +1 -1
- package/native/Sources/Commands/InputCommands.swift +53 -33
- package/native/Sources/Commands/SystemCommands.swift +86 -0
- package/native/Sources/Commands/UICommands.swift +254 -35
- package/native/Sources/Commands/WindowCommands.swift +11 -4
- package/native/Sources/IndigoHID/IndigoHIDClient.swift +222 -0
- package/native/Sources/IndigoHID/IndigoHIDCoordinates.swift +74 -0
- package/native/Sources/IndigoHID/IndigoHIDEvents.swift +63 -0
- package/native/Sources/IndigoHID/IndigoHIDLoader.swift +102 -0
- package/native/Sources/IndigoHID/IndigoHIDTypes.swift +41 -0
- package/native/Sources/Types.swift +26 -0
- package/native/Sources/Utils.swift +653 -13
- package/native/Sources/Version.swift +1 -1
- package/native/Sources/main.swift +55 -8
- package/native/Tests/BaepsaeNativeTests/BinaryInvocationTests.swift +54 -6
- package/package.json +12 -3
- package/scripts/dump-tabbar-actions.mjs +312 -0
- package/scripts/generate-tool-manifest.mjs +75 -0
- package/scripts/research-coordinate-calibration.mjs +276 -0
- package/scripts/research-input-channels.mjs +327 -0
- package/scripts/research-tap-tab-grid.mjs +271 -0
- package/scripts/verify-media-capture.mjs +99 -0
|
@@ -9,13 +9,15 @@ func handleDescribeUI(_ parsed: ParsedOptions) throws -> Int32 {
|
|
|
9
9
|
|
|
10
10
|
let appRoot = try accessibilityRootElement(for: target)
|
|
11
11
|
var targetRoot = appRoot
|
|
12
|
+
var usedSimulatorContentScope = false
|
|
12
13
|
|
|
13
14
|
switch target {
|
|
14
|
-
case .simulator:
|
|
15
|
+
case .simulator(let udid):
|
|
15
16
|
// Default behavior: Focus on in-app content group unless --all is specified
|
|
16
17
|
if !parsed.flags.contains("--all") {
|
|
17
|
-
if let contentGroup = simulatorContentRootElement(from: appRoot) {
|
|
18
|
+
if let contentGroup = simulatorContentRootElement(from: appRoot, udid: udid) {
|
|
18
19
|
targetRoot = contentGroup
|
|
20
|
+
usedSimulatorContentScope = true
|
|
19
21
|
}
|
|
20
22
|
}
|
|
21
23
|
case .macApp:
|
|
@@ -32,6 +34,7 @@ func handleDescribeUI(_ parsed: ParsedOptions) throws -> Int32 {
|
|
|
32
34
|
if let focusId = parsed.options["--focus-id"] {
|
|
33
35
|
if let found = findAccessibilityElement(in: appRoot, identifier: focusId, label: nil) {
|
|
34
36
|
targetRoot = found
|
|
37
|
+
usedSimulatorContentScope = false
|
|
35
38
|
} else {
|
|
36
39
|
throw NativeError.commandFailed("Could not find element with id: \(focusId)")
|
|
37
40
|
}
|
|
@@ -41,6 +44,7 @@ func handleDescribeUI(_ parsed: ParsedOptions) throws -> Int32 {
|
|
|
41
44
|
if let rootElementId = parsed.options["--root-element-id"] {
|
|
42
45
|
if let found = findAccessibilityElement(in: appRoot, identifier: rootElementId, label: nil) {
|
|
43
46
|
targetRoot = found
|
|
47
|
+
usedSimulatorContentScope = false
|
|
44
48
|
} else {
|
|
45
49
|
throw NativeError.commandFailed("Could not find element with id: \(rootElementId)")
|
|
46
50
|
}
|
|
@@ -68,10 +72,17 @@ func handleDescribeUI(_ parsed: ParsedOptions) throws -> Int32 {
|
|
|
68
72
|
}
|
|
69
73
|
}
|
|
70
74
|
|
|
71
|
-
|
|
75
|
+
var lines = describeAccessibilityTree(from: targetRoot, options: descOpts)
|
|
72
76
|
if lines.isEmpty {
|
|
73
77
|
throw NativeError.commandFailed("No accessibility elements found.")
|
|
74
78
|
}
|
|
79
|
+
if usedSimulatorContentScope {
|
|
80
|
+
let targetUdid = simulatorUdid(from: target)
|
|
81
|
+
let auxiliaryLabels = simulatorAuxiliaryContainerLabels(from: appRoot, excluding: targetRoot, udid: targetUdid)
|
|
82
|
+
if let hint = formatSimulatorAuxiliaryContainerHint(auxiliaryLabels) {
|
|
83
|
+
lines.append(hint)
|
|
84
|
+
}
|
|
85
|
+
}
|
|
75
86
|
let report = lines.joined(separator: "\n")
|
|
76
87
|
|
|
77
88
|
if let output = parsed.options["--output"] {
|
|
@@ -95,8 +106,10 @@ func handleSearchUI(_ parsed: ParsedOptions) throws -> Int32 {
|
|
|
95
106
|
|
|
96
107
|
let appRoot = try accessibilityRootElement(for: target)
|
|
97
108
|
var searchRoot = appRoot
|
|
98
|
-
|
|
99
|
-
|
|
109
|
+
var simulatorContentRoot: UIElement? = nil
|
|
110
|
+
if case .simulator(let udid) = target {
|
|
111
|
+
if let contentGroup = simulatorContentRootElement(from: appRoot, udid: udid) {
|
|
112
|
+
simulatorContentRoot = contentGroup
|
|
100
113
|
searchRoot = contentGroup
|
|
101
114
|
}
|
|
102
115
|
}
|
|
@@ -117,7 +130,24 @@ func handleSearchUI(_ parsed: ParsedOptions) throws -> Int32 {
|
|
|
117
130
|
}
|
|
118
131
|
|
|
119
132
|
let results = searchAccessibilityElements(in: searchRoot, query: query, options: searchOpts)
|
|
120
|
-
if results.isEmpty {
|
|
133
|
+
if results.isEmpty, case .simulator = target, let simulatorContentRoot {
|
|
134
|
+
let targetUdid = simulatorUdid(from: target)
|
|
135
|
+
let auxiliaryCandidates = simulatorAuxiliaryContainerCandidates(from: appRoot, excluding: simulatorContentRoot, udid: targetUdid)
|
|
136
|
+
let auxiliaryResults = searchAccessibilityElements(
|
|
137
|
+
in: auxiliaryCandidates.map(\.element),
|
|
138
|
+
query: query,
|
|
139
|
+
options: searchOpts
|
|
140
|
+
)
|
|
141
|
+
if auxiliaryResults.isEmpty {
|
|
142
|
+
print("No elements found matching query: \(query) in simulator app content or auxiliary containers.")
|
|
143
|
+
if let hint = formatSimulatorAuxiliaryContainerHint(auxiliaryCandidates.map(\.label)) {
|
|
144
|
+
print(hint)
|
|
145
|
+
}
|
|
146
|
+
} else {
|
|
147
|
+
print("[Fallback] No matches in simulator content scope; searched auxiliary containers outside iOSContentGroup.")
|
|
148
|
+
print(auxiliaryResults.joined(separator: "\n"))
|
|
149
|
+
}
|
|
150
|
+
} else if results.isEmpty {
|
|
121
151
|
print("No elements found matching query: \(query)")
|
|
122
152
|
} else {
|
|
123
153
|
print(results.joined(separator: "\n"))
|
|
@@ -142,15 +172,22 @@ func handleTap(_ parsed: ParsedOptions) throws -> Int32 {
|
|
|
142
172
|
}
|
|
143
173
|
|
|
144
174
|
let root = try accessibilityRootElement(for: target)
|
|
145
|
-
let
|
|
175
|
+
let targetUdid = simulatorUdid(from: target)
|
|
176
|
+
let simulatorContentRoot = simulatorContentRootElement(from: root, udid: targetUdid)
|
|
177
|
+
let searchRoots: [UIElement]
|
|
146
178
|
if case .simulator = target, !includeAll {
|
|
147
|
-
|
|
179
|
+
if let contentRoot = simulatorContentRoot {
|
|
180
|
+
let auxiliaryRoots = simulatorAuxiliaryContainerCandidates(from: root, excluding: contentRoot, udid: targetUdid).map(\.element)
|
|
181
|
+
searchRoots = [contentRoot] + auxiliaryRoots
|
|
182
|
+
} else {
|
|
183
|
+
searchRoots = [root]
|
|
184
|
+
}
|
|
148
185
|
} else {
|
|
149
|
-
|
|
186
|
+
searchRoots = [root]
|
|
150
187
|
}
|
|
151
188
|
|
|
152
189
|
guard let matchedElement = findAccessibilityElement(
|
|
153
|
-
in:
|
|
190
|
+
in: searchRoots,
|
|
154
191
|
identifier: accessibilityId,
|
|
155
192
|
label: accessibilityLabel
|
|
156
193
|
) else {
|
|
@@ -163,7 +200,10 @@ func handleTap(_ parsed: ParsedOptions) throws -> Int32 {
|
|
|
163
200
|
}
|
|
164
201
|
let selectorText = selectors.joined(separator: " and ")
|
|
165
202
|
if case .simulator = target, !includeAll {
|
|
166
|
-
|
|
203
|
+
let auxiliaryLabels = simulatorContentRoot.map {
|
|
204
|
+
simulatorAuxiliaryContainerLabels(from: root, excluding: $0, udid: targetUdid)
|
|
205
|
+
} ?? []
|
|
206
|
+
throw NativeError.commandFailed(simulatorSelectorNotFoundMessage(selectorText: selectorText, auxiliaryLabels: auxiliaryLabels))
|
|
167
207
|
}
|
|
168
208
|
throw NativeError.commandFailed("No accessibility element matched \(selectorText).")
|
|
169
209
|
}
|
|
@@ -174,18 +214,7 @@ func handleTap(_ parsed: ParsedOptions) throws -> Int32 {
|
|
|
174
214
|
}
|
|
175
215
|
sendDoubleClick(at: CGPoint(x: frame.midX, y: frame.midY))
|
|
176
216
|
} else {
|
|
177
|
-
|
|
178
|
-
if actions.contains(kAXPressAction as String) {
|
|
179
|
-
let status = AXUIElementPerformAction(matchedElement, kAXPressAction as CFString)
|
|
180
|
-
if status != .success {
|
|
181
|
-
throw NativeError.commandFailed("Matched accessibility element but AXPress failed with status \(status.rawValue).")
|
|
182
|
-
}
|
|
183
|
-
} else if let frame = FrameAttribute(matchedElement) {
|
|
184
|
-
let point = CGPoint(x: frame.midX, y: frame.midY)
|
|
185
|
-
sendClick(at: point)
|
|
186
|
-
} else {
|
|
187
|
-
throw NativeError.commandFailed("Matched accessibility element has no AXPress action or frame for fallback click.")
|
|
188
|
-
}
|
|
217
|
+
try performPrimaryAction(on: matchedElement)
|
|
189
218
|
}
|
|
190
219
|
|
|
191
220
|
if postDelay > 0 {
|
|
@@ -206,12 +235,114 @@ func handleTap(_ parsed: ParsedOptions) throws -> Int32 {
|
|
|
206
235
|
if preDelay > 0 {
|
|
207
236
|
Thread.sleep(forTimeInterval: preDelay)
|
|
208
237
|
}
|
|
209
|
-
let
|
|
210
|
-
|
|
211
|
-
|
|
238
|
+
let backend = resolveInputBackend(for: target)
|
|
239
|
+
switch backend {
|
|
240
|
+
case .indigoHID(let client):
|
|
241
|
+
if isDouble {
|
|
242
|
+
_ = client.tap(x: x, y: y)
|
|
243
|
+
Thread.sleep(forTimeInterval: 0.1)
|
|
244
|
+
_ = client.tap(x: x, y: y)
|
|
245
|
+
} else {
|
|
246
|
+
_ = client.tap(x: x, y: y)
|
|
247
|
+
}
|
|
248
|
+
case .cgevent:
|
|
249
|
+
let point = try pointInWindow(x: x, y: y, for: target)
|
|
250
|
+
if isDouble {
|
|
251
|
+
sendDoubleClick(at: point)
|
|
252
|
+
} else {
|
|
253
|
+
sendClick(at: point)
|
|
254
|
+
}
|
|
255
|
+
}
|
|
256
|
+
if postDelay > 0 {
|
|
257
|
+
Thread.sleep(forTimeInterval: postDelay)
|
|
258
|
+
}
|
|
259
|
+
return 0
|
|
260
|
+
}
|
|
261
|
+
|
|
262
|
+
func handleTapTab(_ parsed: ParsedOptions) throws -> Int32 {
|
|
263
|
+
let target = try resolveTarget(from: parsed)
|
|
264
|
+
guard let indexStr = parsed.options["--index"], let index = Int(indexStr) else {
|
|
265
|
+
throw NativeError.invalidArguments("tap-tab requires --index <N>.")
|
|
266
|
+
}
|
|
267
|
+
|
|
268
|
+
let preDelay = try optionalDoubleOption("--pre-delay", from: parsed) ?? 0
|
|
269
|
+
let postDelay = try optionalDoubleOption("--post-delay", from: parsed) ?? 0
|
|
270
|
+
|
|
271
|
+
try ensureAccessibilityTrusted()
|
|
272
|
+
try activateTarget(target)
|
|
273
|
+
|
|
274
|
+
let appRoot = try accessibilityRootElement(for: target)
|
|
275
|
+
let targetUdid = simulatorUdid(from: target)
|
|
276
|
+
let searchRoot: UIElement = appRoot
|
|
277
|
+
|
|
278
|
+
guard let tabBar = findTabBarElement(in: searchRoot, simulatorUdid: targetUdid) else {
|
|
279
|
+
throw NativeError.commandFailed("No tab bar found in the application UI. Ensure the app has a visible tab bar.")
|
|
280
|
+
}
|
|
281
|
+
|
|
282
|
+
guard let frame = FrameAttribute(tabBar) else {
|
|
283
|
+
throw NativeError.commandFailed("Tab bar element has no frame attribute.")
|
|
284
|
+
}
|
|
285
|
+
|
|
286
|
+
let tabCount: Int
|
|
287
|
+
if let tabCountStr = parsed.options["--tab-count"], let tc = Int(tabCountStr) {
|
|
288
|
+
tabCount = tc
|
|
212
289
|
} else {
|
|
213
|
-
|
|
290
|
+
let children = Children(tabBar)
|
|
291
|
+
tabCount = children.count
|
|
292
|
+
}
|
|
293
|
+
|
|
294
|
+
guard tabCount > 0 else {
|
|
295
|
+
throw NativeError.commandFailed("Tab bar has no children and --tab-count was not specified.")
|
|
296
|
+
}
|
|
297
|
+
|
|
298
|
+
guard index >= 0 && index < tabCount else {
|
|
299
|
+
throw NativeError.invalidArguments("Tab index \(index) is out of range. Valid range: 0..\(tabCount - 1)")
|
|
300
|
+
}
|
|
301
|
+
|
|
302
|
+
if preDelay > 0 {
|
|
303
|
+
Thread.sleep(forTimeInterval: preDelay)
|
|
214
304
|
}
|
|
305
|
+
|
|
306
|
+
let actionableItems = actionableTabBarItems(in: tabBar)
|
|
307
|
+
if actionableItems.count == tabCount {
|
|
308
|
+
try performPrimaryAction(on: actionableItems[index])
|
|
309
|
+
if postDelay > 0 {
|
|
310
|
+
Thread.sleep(forTimeInterval: postDelay)
|
|
311
|
+
}
|
|
312
|
+
return 0
|
|
313
|
+
}
|
|
314
|
+
|
|
315
|
+
if case .simulator = target,
|
|
316
|
+
let contentRoot = simulatorContentRootElement(from: appRoot, udid: targetUdid) {
|
|
317
|
+
let proxyItems = semanticProxyTabButtons(in: contentRoot, excluding: tabBar, expectedCount: tabCount)
|
|
318
|
+
if proxyItems.count == tabCount {
|
|
319
|
+
try performPrimaryAction(on: proxyItems[index])
|
|
320
|
+
if postDelay > 0 {
|
|
321
|
+
Thread.sleep(forTimeInterval: postDelay)
|
|
322
|
+
}
|
|
323
|
+
return 0
|
|
324
|
+
}
|
|
325
|
+
}
|
|
326
|
+
|
|
327
|
+
let tabWidth = frame.width / CGFloat(tabCount)
|
|
328
|
+
let tapX = frame.origin.x + tabWidth * CGFloat(index) + tabWidth / 2.0
|
|
329
|
+
let tapY = frame.origin.y + frame.height / 2.0
|
|
330
|
+
|
|
331
|
+
let backend = resolveInputBackend(for: target)
|
|
332
|
+
switch backend {
|
|
333
|
+
case .indigoHID(let client):
|
|
334
|
+
guard let contentBounds = simulatorContentBounds(udid: targetUdid) else {
|
|
335
|
+
throw NativeError.commandFailed("Cannot determine simulator content bounds for IndigoHID coordinate conversion. Ensure Simulator is running.")
|
|
336
|
+
}
|
|
337
|
+
let relX = Double(tapX) - Double(contentBounds.origin.x)
|
|
338
|
+
let relY = Double(tapY) - Double(contentBounds.origin.y)
|
|
339
|
+
if !client.tap(x: relX, y: relY) {
|
|
340
|
+
throw NativeError.commandFailed("IndigoHID tap failed. The simulator may not be responding.")
|
|
341
|
+
}
|
|
342
|
+
case .cgevent:
|
|
343
|
+
sendClick(at: CGPoint(x: tapX, y: tapY))
|
|
344
|
+
}
|
|
345
|
+
|
|
215
346
|
if postDelay > 0 {
|
|
216
347
|
Thread.sleep(forTimeInterval: postDelay)
|
|
217
348
|
}
|
|
@@ -233,10 +364,69 @@ func handleType(_ parsed: ParsedOptions) throws -> Int32 {
|
|
|
233
364
|
if text.isEmpty {
|
|
234
365
|
throw NativeError.invalidArguments("type requires text input.")
|
|
235
366
|
}
|
|
236
|
-
|
|
367
|
+
|
|
368
|
+
let methodStr = parsed.options["--method"] ?? "auto"
|
|
369
|
+
let usePaste: Bool
|
|
370
|
+
switch methodStr {
|
|
371
|
+
case "paste":
|
|
372
|
+
usePaste = true
|
|
373
|
+
case "keyboard":
|
|
374
|
+
usePaste = false
|
|
375
|
+
default: // auto
|
|
376
|
+
if case .simulator = target {
|
|
377
|
+
usePaste = true
|
|
378
|
+
} else {
|
|
379
|
+
usePaste = false
|
|
380
|
+
}
|
|
381
|
+
}
|
|
382
|
+
|
|
383
|
+
if usePaste {
|
|
384
|
+
try pasteText(text, target: target)
|
|
385
|
+
} else {
|
|
386
|
+
let backend = resolveInputBackend(for: target)
|
|
387
|
+
switch backend {
|
|
388
|
+
case .indigoHID(let client):
|
|
389
|
+
_ = client.typeText(text)
|
|
390
|
+
case .cgevent:
|
|
391
|
+
sendText(text)
|
|
392
|
+
}
|
|
393
|
+
}
|
|
237
394
|
return 0
|
|
238
395
|
}
|
|
239
396
|
|
|
397
|
+
func pasteText(_ text: String, target: TargetApp) throws {
|
|
398
|
+
switch target {
|
|
399
|
+
case .simulator(let udid):
|
|
400
|
+
try setSimulatorPasteboard(text, udid: udid)
|
|
401
|
+
// Cmd+V: Command keycode=55, V keycode=9
|
|
402
|
+
sendKeyCombo(modifiers: [55], key: 9)
|
|
403
|
+
Thread.sleep(forTimeInterval: 0.2)
|
|
404
|
+
case .macApp:
|
|
405
|
+
let pasteboard = NSPasteboard.general
|
|
406
|
+
let original = pasteboard.string(forType: .string)
|
|
407
|
+
|
|
408
|
+
pasteboard.clearContents()
|
|
409
|
+
pasteboard.setString(text, forType: .string)
|
|
410
|
+
|
|
411
|
+
// Cmd+V: Command keycode=55, V keycode=9
|
|
412
|
+
sendKeyCombo(modifiers: [55], key: 9)
|
|
413
|
+
Thread.sleep(forTimeInterval: 0.15)
|
|
414
|
+
|
|
415
|
+
// Restore original clipboard content
|
|
416
|
+
pasteboard.clearContents()
|
|
417
|
+
if let original = original {
|
|
418
|
+
pasteboard.setString(original, forType: .string)
|
|
419
|
+
}
|
|
420
|
+
}
|
|
421
|
+
}
|
|
422
|
+
|
|
423
|
+
func setSimulatorPasteboard(_ text: String, udid: String) throws {
|
|
424
|
+
let status = try runProcess("/usr/bin/xcrun", ["simctl", "pbcopy", udid], stdinText: text)
|
|
425
|
+
if status != 0 {
|
|
426
|
+
throw NativeError.commandFailed("Failed to set simulator pasteboard (exit code \(status)).")
|
|
427
|
+
}
|
|
428
|
+
}
|
|
429
|
+
|
|
240
430
|
func handleSwipe(_ parsed: ParsedOptions) throws -> Int32 {
|
|
241
431
|
let target = try resolveTarget(from: parsed)
|
|
242
432
|
let startX = try requiredOption("--start-x", from: parsed)
|
|
@@ -257,9 +447,15 @@ func handleSwipe(_ parsed: ParsedOptions) throws -> Int32 {
|
|
|
257
447
|
if preDelay > 0 {
|
|
258
448
|
Thread.sleep(forTimeInterval: preDelay)
|
|
259
449
|
}
|
|
260
|
-
let
|
|
261
|
-
|
|
262
|
-
|
|
450
|
+
let backend = resolveInputBackend(for: target)
|
|
451
|
+
switch backend {
|
|
452
|
+
case .indigoHID(let client):
|
|
453
|
+
_ = client.swipe(fromX: startXValue, fromY: startYValue, toX: endXValue, toY: endYValue, duration: duration)
|
|
454
|
+
case .cgevent:
|
|
455
|
+
let start = try pointForInput(x: startXValue, y: startYValue, for: target)
|
|
456
|
+
let end = try pointForInput(x: endXValue, y: endYValue, for: target)
|
|
457
|
+
sendSwipe(from: start, to: end, duration: duration)
|
|
458
|
+
}
|
|
263
459
|
if postDelay > 0 {
|
|
264
460
|
Thread.sleep(forTimeInterval: postDelay)
|
|
265
461
|
}
|
|
@@ -275,12 +471,35 @@ func handleScroll(_ parsed: ParsedOptions) throws -> Int32 {
|
|
|
275
471
|
if deltaX == 0 && deltaY == 0 {
|
|
276
472
|
throw NativeError.invalidArguments("scroll requires --delta-x and/or --delta-y.")
|
|
277
473
|
}
|
|
278
|
-
|
|
474
|
+
|
|
279
475
|
let xRaw = parsed.options["-x"]
|
|
280
476
|
let yRaw = parsed.options["-y"]
|
|
281
|
-
|
|
282
|
-
|
|
477
|
+
let xValue = xRaw.flatMap(Double.init)
|
|
478
|
+
let yValue = yRaw.flatMap(Double.init)
|
|
479
|
+
|
|
480
|
+
switch target {
|
|
481
|
+
case .simulator:
|
|
482
|
+
let scrollPoint = try simulatorScrollAnchorPoint(x: xValue, y: yValue)
|
|
483
|
+
let scrollDistance = simulatorScrollDistance(deltaX: deltaX, deltaY: deltaY)
|
|
484
|
+
let start = CGPoint(x: scrollPoint.x - scrollDistance.width / 2, y: scrollPoint.y - scrollDistance.height / 2)
|
|
485
|
+
let end = CGPoint(x: scrollPoint.x + scrollDistance.width / 2, y: scrollPoint.y + scrollDistance.height / 2)
|
|
486
|
+
|
|
487
|
+
let backend = resolveInputBackend(for: target)
|
|
488
|
+
switch backend {
|
|
489
|
+
case .indigoHID(let client):
|
|
490
|
+
_ = client.swipe(fromX: Double(start.x), fromY: Double(start.y), toX: Double(end.x), toY: Double(end.y), duration: 0.45, steps: 18)
|
|
491
|
+
case .cgevent:
|
|
492
|
+
let targetUdid = simulatorUdid(from: target)
|
|
493
|
+
let startPoint = try pointInSimulatorContent(x: Double(start.x), y: Double(start.y), udid: targetUdid)
|
|
494
|
+
let endPoint = try pointInSimulatorContent(x: Double(end.x), y: Double(end.y), udid: targetUdid)
|
|
495
|
+
sendSwipe(from: startPoint, to: endPoint, duration: 0.45)
|
|
496
|
+
}
|
|
497
|
+
case .macApp:
|
|
498
|
+
var scrollPoint: CGPoint? = nil
|
|
499
|
+
if let xValue, let yValue {
|
|
500
|
+
scrollPoint = try pointForInput(x: xValue, y: yValue, for: target)
|
|
501
|
+
}
|
|
502
|
+
sendScrollWheel(at: scrollPoint, deltaX: Int32(deltaX), deltaY: Int32(deltaY))
|
|
283
503
|
}
|
|
284
|
-
sendScrollWheel(at: scrollPoint, deltaX: Int32(deltaX), deltaY: Int32(deltaY))
|
|
285
504
|
return 0
|
|
286
505
|
}
|
|
@@ -18,7 +18,7 @@ func handleRightClick(_ parsed: ParsedOptions) throws -> Int32 {
|
|
|
18
18
|
let root = try accessibilityRootElement(for: target)
|
|
19
19
|
let searchRoot: UIElement
|
|
20
20
|
if case .simulator = target, !includeAll {
|
|
21
|
-
searchRoot = simulatorContentRootElement(from: root) ?? root
|
|
21
|
+
searchRoot = simulatorContentRootElement(from: root, udid: simulatorUdid(from: target)) ?? root
|
|
22
22
|
} else {
|
|
23
23
|
searchRoot = root
|
|
24
24
|
}
|
|
@@ -71,14 +71,21 @@ func handleDragDrop(_ parsed: ParsedOptions) throws -> Int32 {
|
|
|
71
71
|
throw NativeError.invalidArguments("drag-drop requires numeric start/end coordinates.")
|
|
72
72
|
}
|
|
73
73
|
let duration = try optionalDoubleOption("--duration", from: parsed) ?? 0.5
|
|
74
|
+
let holdDuration = try optionalDoubleOption("--hold-duration", from: parsed) ?? 0.7
|
|
74
75
|
let preDelay = try optionalDoubleOption("--pre-delay", from: parsed) ?? 0
|
|
75
76
|
let postDelay = try optionalDoubleOption("--post-delay", from: parsed) ?? 0
|
|
76
77
|
if preDelay > 0 {
|
|
77
78
|
Thread.sleep(forTimeInterval: preDelay)
|
|
78
79
|
}
|
|
79
|
-
let
|
|
80
|
-
|
|
81
|
-
|
|
80
|
+
let backend = resolveInputBackend(for: target)
|
|
81
|
+
switch backend {
|
|
82
|
+
case .indigoHID(let client):
|
|
83
|
+
_ = client.drag(fromX: startXVal, fromY: startYVal, toX: endXVal, toY: endYVal, holdDuration: holdDuration, moveDuration: duration)
|
|
84
|
+
case .cgevent:
|
|
85
|
+
let start = try pointForInput(x: startXVal, y: startYVal, for: target)
|
|
86
|
+
let end = try pointForInput(x: endXVal, y: endYVal, for: target)
|
|
87
|
+
sendDrag(from: start, to: end, holdDuration: holdDuration, moveDuration: duration)
|
|
88
|
+
}
|
|
82
89
|
if postDelay > 0 {
|
|
83
90
|
Thread.sleep(forTimeInterval: postDelay)
|
|
84
91
|
}
|
|
@@ -0,0 +1,222 @@
|
|
|
1
|
+
import Foundation
|
|
2
|
+
|
|
3
|
+
// MARK: - IndigoHID Client
|
|
4
|
+
|
|
5
|
+
/// High-level API for sending HID events to an iOS Simulator via IndigoHID.
|
|
6
|
+
final class IndigoHIDClient {
|
|
7
|
+
let udid: String
|
|
8
|
+
let port: mach_port_t
|
|
9
|
+
let loader: IndigoHIDLoader
|
|
10
|
+
let coords: IndigoHIDCoordinates
|
|
11
|
+
|
|
12
|
+
/// Initialize with a simulator UDID.
|
|
13
|
+
/// Returns nil if IndigoHID is not available or the Mach port cannot be resolved.
|
|
14
|
+
init?(udid: String) {
|
|
15
|
+
let loader = IndigoHIDLoader.shared
|
|
16
|
+
guard loader.isAvailable else { return nil }
|
|
17
|
+
guard let port = resolveIndigoHIDPort(udid: udid) else { return nil }
|
|
18
|
+
|
|
19
|
+
self.udid = udid
|
|
20
|
+
self.port = port
|
|
21
|
+
self.loader = loader
|
|
22
|
+
|
|
23
|
+
let screenSize = resolveSimulatorScreenSize(udid: udid)
|
|
24
|
+
self.coords = IndigoHIDCoordinates(screenWidth: screenSize.width, screenHeight: screenSize.height)
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
// MARK: - Touch
|
|
28
|
+
|
|
29
|
+
/// Send a single tap at the given pixel coordinates.
|
|
30
|
+
func tap(x: Double, y: Double) -> Bool {
|
|
31
|
+
let (nx, ny) = coords.normalize(x: x, y: y)
|
|
32
|
+
|
|
33
|
+
// Touch began
|
|
34
|
+
guard let beginMsg = loader.createTouchMessage(phase: .began, x: nx, y: ny) else { return false }
|
|
35
|
+
guard sendIndigoHIDMessage(beginMsg, to: port) else { return false }
|
|
36
|
+
|
|
37
|
+
Thread.sleep(forTimeInterval: 0.05)
|
|
38
|
+
|
|
39
|
+
// Touch ended
|
|
40
|
+
guard let endMsg = loader.createTouchMessage(phase: .ended, x: nx, y: ny) else { return false }
|
|
41
|
+
return sendIndigoHIDMessage(endMsg, to: port)
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
/// Send a swipe gesture from start to end coordinates (in pixels).
|
|
45
|
+
func swipe(fromX: Double, fromY: Double, toX: Double, toY: Double, duration: Double? = nil, steps: Int = 10) -> Bool {
|
|
46
|
+
let (sx, sy) = coords.normalize(x: fromX, y: fromY)
|
|
47
|
+
let (ex, ey) = coords.normalize(x: toX, y: toY)
|
|
48
|
+
|
|
49
|
+
// Touch began
|
|
50
|
+
guard let beginMsg = loader.createTouchMessage(phase: .began, x: sx, y: sy) else { return false }
|
|
51
|
+
guard sendIndigoHIDMessage(beginMsg, to: port) else { return false }
|
|
52
|
+
|
|
53
|
+
let stepDuration = (duration ?? 0.3) / Double(steps)
|
|
54
|
+
|
|
55
|
+
// Touch moved sequence
|
|
56
|
+
for step in 1...steps {
|
|
57
|
+
let progress = Double(step) / Double(steps)
|
|
58
|
+
let cx = sx + (ex - sx) * progress
|
|
59
|
+
let cy = sy + (ey - sy) * progress
|
|
60
|
+
guard let moveMsg = loader.createTouchMessage(phase: .moved, x: cx, y: cy) else { return false }
|
|
61
|
+
guard sendIndigoHIDMessage(moveMsg, to: port) else { return false }
|
|
62
|
+
Thread.sleep(forTimeInterval: stepDuration)
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
// Touch ended
|
|
66
|
+
guard let endMsg = loader.createTouchMessage(phase: .ended, x: ex, y: ey) else { return false }
|
|
67
|
+
return sendIndigoHIDMessage(endMsg, to: port)
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
/// Send a drag gesture (long press + move + release).
|
|
71
|
+
func drag(fromX: Double, fromY: Double, toX: Double, toY: Double, holdDuration: Double = 0.5, moveDuration: Double? = nil) -> Bool {
|
|
72
|
+
let (sx, sy) = coords.normalize(x: fromX, y: fromY)
|
|
73
|
+
let (ex, ey) = coords.normalize(x: toX, y: toY)
|
|
74
|
+
|
|
75
|
+
// Touch began (long press)
|
|
76
|
+
guard let beginMsg = loader.createTouchMessage(phase: .began, x: sx, y: sy) else { return false }
|
|
77
|
+
guard sendIndigoHIDMessage(beginMsg, to: port) else { return false }
|
|
78
|
+
|
|
79
|
+
if holdDuration > 0 {
|
|
80
|
+
Thread.sleep(forTimeInterval: holdDuration)
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
// A small warmup move helps SwiftUI DragGesture transition into
|
|
84
|
+
// an active drag before the long move sequence begins.
|
|
85
|
+
let warmupProgress = 0.02
|
|
86
|
+
let warmupX = sx + (ex - sx) * warmupProgress
|
|
87
|
+
let warmupY = sy + (ey - sy) * warmupProgress
|
|
88
|
+
guard let warmupMsg = loader.createTouchMessage(phase: .moved, x: warmupX, y: warmupY) else { return false }
|
|
89
|
+
guard sendIndigoHIDMessage(warmupMsg, to: port) else { return false }
|
|
90
|
+
Thread.sleep(forTimeInterval: 0.05)
|
|
91
|
+
|
|
92
|
+
// Move sequence
|
|
93
|
+
let steps = 18
|
|
94
|
+
let stepDuration = max((moveDuration ?? 0.6) / Double(steps), 0.02)
|
|
95
|
+
for step in 1...steps {
|
|
96
|
+
let progress = Double(step) / Double(steps)
|
|
97
|
+
let cx = sx + (ex - sx) * progress
|
|
98
|
+
let cy = sy + (ey - sy) * progress
|
|
99
|
+
guard let moveMsg = loader.createTouchMessage(phase: .moved, x: cx, y: cy) else { return false }
|
|
100
|
+
guard sendIndigoHIDMessage(moveMsg, to: port) else { return false }
|
|
101
|
+
Thread.sleep(forTimeInterval: stepDuration)
|
|
102
|
+
}
|
|
103
|
+
|
|
104
|
+
Thread.sleep(forTimeInterval: 0.08)
|
|
105
|
+
|
|
106
|
+
// Touch ended
|
|
107
|
+
guard let endMsg = loader.createTouchMessage(phase: .ended, x: ex, y: ey) else { return false }
|
|
108
|
+
return sendIndigoHIDMessage(endMsg, to: port)
|
|
109
|
+
}
|
|
110
|
+
|
|
111
|
+
/// Type text using HID keyboard events.
|
|
112
|
+
/// This bypasses iOS autocomplete by sending raw HID keycodes.
|
|
113
|
+
func typeText(_ text: String) -> Bool {
|
|
114
|
+
let usagePage: UInt32 = 0x07 // Keyboard/Keypad page
|
|
115
|
+
|
|
116
|
+
for char in text {
|
|
117
|
+
guard let usage = hidUsageForCharacter(char) else { continue }
|
|
118
|
+
|
|
119
|
+
let needsShift = characterNeedsShift(char)
|
|
120
|
+
|
|
121
|
+
if needsShift {
|
|
122
|
+
// Shift down (usage 0xE1 = Left Shift)
|
|
123
|
+
if let shiftDown = loader.createKeyboardMessage(usagePage: usagePage, usage: 0xE1, operation: .keyDown) {
|
|
124
|
+
_ = sendIndigoHIDMessage(shiftDown, to: port)
|
|
125
|
+
}
|
|
126
|
+
}
|
|
127
|
+
|
|
128
|
+
// Key down
|
|
129
|
+
if let keyDown = loader.createKeyboardMessage(usagePage: usagePage, usage: usage, operation: .keyDown) {
|
|
130
|
+
_ = sendIndigoHIDMessage(keyDown, to: port)
|
|
131
|
+
}
|
|
132
|
+
Thread.sleep(forTimeInterval: 0.02)
|
|
133
|
+
|
|
134
|
+
// Key up
|
|
135
|
+
if let keyUp = loader.createKeyboardMessage(usagePage: usagePage, usage: usage, operation: .keyUp) {
|
|
136
|
+
_ = sendIndigoHIDMessage(keyUp, to: port)
|
|
137
|
+
}
|
|
138
|
+
|
|
139
|
+
if needsShift {
|
|
140
|
+
// Shift up
|
|
141
|
+
if let shiftUp = loader.createKeyboardMessage(usagePage: usagePage, usage: 0xE1, operation: .keyUp) {
|
|
142
|
+
_ = sendIndigoHIDMessage(shiftUp, to: port)
|
|
143
|
+
}
|
|
144
|
+
}
|
|
145
|
+
|
|
146
|
+
Thread.sleep(forTimeInterval: 0.02)
|
|
147
|
+
}
|
|
148
|
+
|
|
149
|
+
return true
|
|
150
|
+
}
|
|
151
|
+
}
|
|
152
|
+
|
|
153
|
+
// MARK: - HID Usage Mapping
|
|
154
|
+
|
|
155
|
+
/// Map a character to its USB HID Usage ID (usage page 0x07).
|
|
156
|
+
private func hidUsageForCharacter(_ char: Character) -> UInt32? {
|
|
157
|
+
let lower = char.lowercased().first ?? char
|
|
158
|
+
switch lower {
|
|
159
|
+
case "a": return 0x04
|
|
160
|
+
case "b": return 0x05
|
|
161
|
+
case "c": return 0x06
|
|
162
|
+
case "d": return 0x07
|
|
163
|
+
case "e": return 0x08
|
|
164
|
+
case "f": return 0x09
|
|
165
|
+
case "g": return 0x0A
|
|
166
|
+
case "h": return 0x0B
|
|
167
|
+
case "i": return 0x0C
|
|
168
|
+
case "j": return 0x0D
|
|
169
|
+
case "k": return 0x0E
|
|
170
|
+
case "l": return 0x0F
|
|
171
|
+
case "m": return 0x10
|
|
172
|
+
case "n": return 0x11
|
|
173
|
+
case "o": return 0x12
|
|
174
|
+
case "p": return 0x13
|
|
175
|
+
case "q": return 0x14
|
|
176
|
+
case "r": return 0x15
|
|
177
|
+
case "s": return 0x16
|
|
178
|
+
case "t": return 0x17
|
|
179
|
+
case "u": return 0x18
|
|
180
|
+
case "v": return 0x19
|
|
181
|
+
case "w": return 0x1A
|
|
182
|
+
case "x": return 0x1B
|
|
183
|
+
case "y": return 0x1C
|
|
184
|
+
case "z": return 0x1D
|
|
185
|
+
case "1", "!": return 0x1E
|
|
186
|
+
case "2", "@": return 0x1F
|
|
187
|
+
case "3", "#": return 0x20
|
|
188
|
+
case "4", "$": return 0x21
|
|
189
|
+
case "5", "%": return 0x22
|
|
190
|
+
case "6", "^": return 0x23
|
|
191
|
+
case "7", "&": return 0x24
|
|
192
|
+
case "8", "*": return 0x25
|
|
193
|
+
case "9", "(": return 0x26
|
|
194
|
+
case "0", ")": return 0x27
|
|
195
|
+
case "\n": return 0x28 // Return
|
|
196
|
+
case "\t": return 0x2B // Tab
|
|
197
|
+
case " ": return 0x2C // Space
|
|
198
|
+
case "-", "_": return 0x2D
|
|
199
|
+
case "=", "+": return 0x2E
|
|
200
|
+
case "[", "{": return 0x2F
|
|
201
|
+
case "]", "}": return 0x30
|
|
202
|
+
case "\\", "|": return 0x31
|
|
203
|
+
case ";", ":": return 0x33
|
|
204
|
+
case "'", "\"": return 0x34
|
|
205
|
+
case "`", "~": return 0x35
|
|
206
|
+
case ",", "<": return 0x36
|
|
207
|
+
case ".", ">": return 0x37
|
|
208
|
+
case "/", "?": return 0x38
|
|
209
|
+
default: return nil
|
|
210
|
+
}
|
|
211
|
+
}
|
|
212
|
+
|
|
213
|
+
/// Check if a character requires the Shift modifier.
|
|
214
|
+
private func characterNeedsShift(_ char: Character) -> Bool {
|
|
215
|
+
let shiftChars: Set<Character> = [
|
|
216
|
+
"A", "B", "C", "D", "E", "F", "G", "H", "I", "J", "K", "L", "M",
|
|
217
|
+
"N", "O", "P", "Q", "R", "S", "T", "U", "V", "W", "X", "Y", "Z",
|
|
218
|
+
"!", "@", "#", "$", "%", "^", "&", "*", "(", ")",
|
|
219
|
+
"_", "+", "{", "}", "|", ":", "\"", "~", "<", ">", "?"
|
|
220
|
+
]
|
|
221
|
+
return shiftChars.contains(char)
|
|
222
|
+
}
|