wrangle 0.1.0 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +24 -0
- data/README.md +94 -3
- data/exe/wrangle +556 -33
- data/lib/wrangle/decision_provider.rb +128 -0
- data/lib/wrangle/desktop_autonomy.rb +61 -0
- data/lib/wrangle/desktop_decider.rb +263 -0
- data/lib/wrangle/desktop_dispatch.rb +67 -0
- data/lib/wrangle/desktop_effect.rb +198 -0
- data/lib/wrangle/desktop_observation.rb +252 -0
- data/lib/wrangle/desktop_policy.rb +47 -0
- data/lib/wrangle/desktop_progressive_observation.rb +34 -0
- data/lib/wrangle/desktop_proposal.rb +75 -0
- data/lib/wrangle/desktop_session_autonomy.rb +344 -0
- data/lib/wrangle/desktop_session_server.rb +374 -0
- data/lib/wrangle/desktop_task.rb +67 -0
- data/lib/wrangle/errors.rb +36 -0
- data/lib/wrangle/event_log.rb +51 -0
- data/lib/wrangle/jev.rb +4 -2
- data/lib/wrangle/macos/helper.swift +593 -0
- data/lib/wrangle/macos_driver.rb +200 -0
- data/lib/wrangle/macos_helper.rb +194 -0
- data/lib/wrangle/observation.rb +1 -1
- data/lib/wrangle/provider_conformance.jsonl +8 -0
- data/lib/wrangle/provider_factory.rb +106 -0
- data/lib/wrangle/provider_qualification.rb +73 -0
- data/lib/wrangle/run_loop.rb +4 -2
- data/lib/wrangle/safari.rb +7 -4
- data/lib/wrangle/scope_registry.rb +152 -0
- data/lib/wrangle/session_server.rb +15 -9
- data/lib/wrangle/tart_guest_driver.rb +251 -0
- data/lib/wrangle/timing.rb +12 -0
- data/lib/wrangle/version.rb +1 -1
- data/lib/wrangle.rb +20 -1
- data/skills/wrangle/SKILL.md +37 -4
- metadata +29 -7
|
@@ -0,0 +1,593 @@
|
|
|
1
|
+
import Cocoa
|
|
2
|
+
import ApplicationServices
|
|
3
|
+
import CoreGraphics
|
|
4
|
+
import Darwin
|
|
5
|
+
|
|
6
|
+
let protocolVersion = "1.2"
|
|
7
|
+
let maxSnapshotNodes = 1_500
|
|
8
|
+
let maxChildrenPerNode = 120
|
|
9
|
+
let maxTextCharacters = 500
|
|
10
|
+
|
|
11
|
+
enum HelperFailure: Error {
|
|
12
|
+
case refused(String, String, String?, [String: Any]?)
|
|
13
|
+
}
|
|
14
|
+
|
|
15
|
+
// LaunchServices leaves launchDate unset for directly executed app bundles such as `tart run`.
|
|
16
|
+
// Kernel process birth time remains available there and still detects PID reuse across app restarts.
|
|
17
|
+
func processInstance(_ app: NSRunningApplication) throws -> String {
|
|
18
|
+
let pid = app.processIdentifier
|
|
19
|
+
var info = proc_bsdinfo()
|
|
20
|
+
let expected = MemoryLayout<proc_bsdinfo>.size
|
|
21
|
+
let actual = proc_pidinfo(pid, PROC_PIDTBSDINFO, 0, &info, Int32(expected))
|
|
22
|
+
guard actual == expected, info.pbi_start_tvsec > 0, info.pbi_start_tvusec < 1_000_000 else {
|
|
23
|
+
throw HelperFailure.refused("scope_changed", "The application has no stable launch identity", nil, nil)
|
|
24
|
+
}
|
|
25
|
+
return "macos-proc-v2:\(pid):\(info.pbi_start_tvsec):\(info.pbi_start_tvusec)"
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
func application(_ pid: pid_t, expected: String?) throws -> NSRunningApplication {
|
|
29
|
+
guard let app = NSRunningApplication(processIdentifier: pid), !app.isTerminated else {
|
|
30
|
+
throw HelperFailure.refused("scope_changed", "The scoped application is no longer running", nil, nil)
|
|
31
|
+
}
|
|
32
|
+
let actual = try processInstance(app)
|
|
33
|
+
if let expected, expected != actual {
|
|
34
|
+
throw HelperFailure.refused("scope_changed", "The scoped application process was replaced", nil,
|
|
35
|
+
["expected": expected, "actual": actual])
|
|
36
|
+
}
|
|
37
|
+
return app
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
func runningApplications(named wanted: String) -> [NSRunningApplication] {
|
|
41
|
+
let needle = wanted.lowercased()
|
|
42
|
+
return NSWorkspace.shared.runningApplications.filter { app in
|
|
43
|
+
guard !app.isTerminated else { return false }
|
|
44
|
+
return app.bundleIdentifier?.lowercased() == needle || app.localizedName?.lowercased() == needle
|
|
45
|
+
}
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
func rect(_ dictionary: [String: Any]) -> [String: Double]? {
|
|
49
|
+
guard let bounds = CGRect(dictionaryRepresentation: dictionary as CFDictionary) else { return nil }
|
|
50
|
+
return ["x": bounds.origin.x, "y": bounds.origin.y,
|
|
51
|
+
"width": bounds.size.width, "height": bounds.size.height]
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
func windows(appName: String, titles: Bool) throws -> [[String: Any]] {
|
|
55
|
+
let apps = runningApplications(named: appName)
|
|
56
|
+
guard !apps.isEmpty else {
|
|
57
|
+
throw HelperFailure.refused("app_not_found", "Application is not running", "Open \(appName) and try again", nil)
|
|
58
|
+
}
|
|
59
|
+
let byPID = Dictionary(uniqueKeysWithValues: try apps.map {
|
|
60
|
+
(Int($0.processIdentifier), (try processInstance($0), $0.bundleIdentifier ?? ""))
|
|
61
|
+
})
|
|
62
|
+
let frontPID = NSWorkspace.shared.frontmostApplication?.processIdentifier
|
|
63
|
+
let records = CGWindowListCopyWindowInfo([.optionOnScreenOnly, .excludeDesktopElements], kCGNullWindowID)
|
|
64
|
+
as? [[String: Any]] ?? []
|
|
65
|
+
var firstForPID = Set<Int>()
|
|
66
|
+
return records.compactMap { record in
|
|
67
|
+
guard let pid = record[kCGWindowOwnerPID as String] as? Int,
|
|
68
|
+
let identity = byPID[pid],
|
|
69
|
+
(record[kCGWindowLayer as String] as? Int) == 0,
|
|
70
|
+
let number = record[kCGWindowNumber as String] as? Int,
|
|
71
|
+
let rawBounds = record[kCGWindowBounds as String] as? [String: Any],
|
|
72
|
+
let bounds = rect(rawBounds), bounds["width", default: 0] > 1, bounds["height", default: 0] > 1
|
|
73
|
+
else { return nil }
|
|
74
|
+
let focused = pid_t(pid) == frontPID && !firstForPID.contains(pid)
|
|
75
|
+
firstForPID.insert(pid)
|
|
76
|
+
var result: [String: Any] = [
|
|
77
|
+
"id": "w-\(number)", "number": number, "pid": pid, "process_instance": identity.0,
|
|
78
|
+
"bundle_id": identity.1, "app_name": record[kCGWindowOwnerName as String] as? String ?? appName,
|
|
79
|
+
"bounds": bounds, "focused": focused,
|
|
80
|
+
"visible": (record[kCGWindowIsOnscreen as String] as? Bool) ?? true
|
|
81
|
+
]
|
|
82
|
+
if titles, let title = record[kCGWindowName as String] as? String { result["title"] = title }
|
|
83
|
+
return result
|
|
84
|
+
}
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
func displays() -> [[String: Any]] {
|
|
88
|
+
guard let primary = NSScreen.screens.first else { return [] }
|
|
89
|
+
let top = primary.frame.maxY
|
|
90
|
+
return NSScreen.screens.enumerated().map { index, screen in
|
|
91
|
+
let frame = screen.frame
|
|
92
|
+
let visible = screen.visibleFrame
|
|
93
|
+
let number = (screen.deviceDescription[NSDeviceDescriptionKey("NSScreenNumber")] as? NSNumber)?.uint32Value ?? 0
|
|
94
|
+
return [
|
|
95
|
+
"id": "display-\(number)", "index": index, "primary": index == 0,
|
|
96
|
+
"scale": screen.backingScaleFactor,
|
|
97
|
+
"bounds": ["x": frame.minX, "y": top - frame.maxY,
|
|
98
|
+
"width": frame.width, "height": frame.height],
|
|
99
|
+
"visible_bounds": ["x": visible.minX, "y": top - visible.maxY,
|
|
100
|
+
"width": visible.width, "height": visible.height]
|
|
101
|
+
]
|
|
102
|
+
}
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
func attribute(_ element: AXUIElement, _ name: String) -> CFTypeRef? {
|
|
106
|
+
var value: CFTypeRef?
|
|
107
|
+
guard AXUIElementCopyAttributeValue(element, name as CFString, &value) == .success else { return nil }
|
|
108
|
+
return value
|
|
109
|
+
}
|
|
110
|
+
|
|
111
|
+
func boolean(_ element: AXUIElement, _ name: String) -> Bool? {
|
|
112
|
+
attribute(element, name) as? Bool
|
|
113
|
+
}
|
|
114
|
+
|
|
115
|
+
func string(_ element: AXUIElement, _ name: String) -> String? {
|
|
116
|
+
attribute(element, name) as? String
|
|
117
|
+
}
|
|
118
|
+
|
|
119
|
+
func point(_ value: CFTypeRef?) -> CGPoint? {
|
|
120
|
+
guard let value, CFGetTypeID(value) == AXValueGetTypeID() else { return nil }
|
|
121
|
+
var result = CGPoint.zero
|
|
122
|
+
return AXValueGetValue(value as! AXValue, .cgPoint, &result) ? result : nil
|
|
123
|
+
}
|
|
124
|
+
|
|
125
|
+
func size(_ value: CFTypeRef?) -> CGSize? {
|
|
126
|
+
guard let value, CFGetTypeID(value) == AXValueGetTypeID() else { return nil }
|
|
127
|
+
var result = CGSize.zero
|
|
128
|
+
return AXValueGetValue(value as! AXValue, .cgSize, &result) ? result : nil
|
|
129
|
+
}
|
|
130
|
+
|
|
131
|
+
func elementBounds(_ element: AXUIElement) -> CGRect? {
|
|
132
|
+
guard let origin = point(attribute(element, kAXPositionAttribute)),
|
|
133
|
+
let dimensions = size(attribute(element, kAXSizeAttribute)),
|
|
134
|
+
dimensions.width >= 0, dimensions.height >= 0 else { return nil }
|
|
135
|
+
return CGRect(origin: origin, size: dimensions)
|
|
136
|
+
}
|
|
137
|
+
|
|
138
|
+
func boundsDictionary(_ bounds: CGRect) -> [String: Double] {
|
|
139
|
+
["x": bounds.minX, "y": bounds.minY, "width": bounds.width, "height": bounds.height]
|
|
140
|
+
}
|
|
141
|
+
|
|
142
|
+
func inputBounds(_ value: Any?) -> CGRect? {
|
|
143
|
+
guard let dictionary = value as? [String: Any],
|
|
144
|
+
let x = dictionary["x"] as? NSNumber, let y = dictionary["y"] as? NSNumber,
|
|
145
|
+
let width = dictionary["width"] as? NSNumber, let height = dictionary["height"] as? NSNumber
|
|
146
|
+
else { return nil }
|
|
147
|
+
return CGRect(x: x.doubleValue, y: y.doubleValue, width: width.doubleValue, height: height.doubleValue)
|
|
148
|
+
}
|
|
149
|
+
|
|
150
|
+
func nearlyEqual(_ first: CGRect, _ second: CGRect, tolerance: Double = 2) -> Bool {
|
|
151
|
+
abs(first.minX - second.minX) <= tolerance && abs(first.minY - second.minY) <= tolerance &&
|
|
152
|
+
abs(first.width - second.width) <= tolerance && abs(first.height - second.height) <= tolerance
|
|
153
|
+
}
|
|
154
|
+
|
|
155
|
+
func overlapScore(_ first: CGRect, _ second: CGRect) -> Double {
|
|
156
|
+
let overlap = first.intersection(second)
|
|
157
|
+
guard !overlap.isNull else { return 0 }
|
|
158
|
+
let smaller = min(first.width * first.height, second.width * second.height)
|
|
159
|
+
return smaller > 0 ? (overlap.width * overlap.height) / smaller : 0
|
|
160
|
+
}
|
|
161
|
+
|
|
162
|
+
func windowNumber(_ identifier: String) -> CGWindowID? {
|
|
163
|
+
guard identifier.hasPrefix("w-"), let number = UInt32(identifier.dropFirst(2)), number > 0 else { return nil }
|
|
164
|
+
return CGWindowID(number)
|
|
165
|
+
}
|
|
166
|
+
|
|
167
|
+
func verifyCGWindow(pid: pid_t, identifier: String, expectedBounds: CGRect) throws -> CGRect {
|
|
168
|
+
guard let number = windowNumber(identifier) else {
|
|
169
|
+
throw HelperFailure.refused("bad_request", "Invalid scoped window identifier", nil, nil)
|
|
170
|
+
}
|
|
171
|
+
let records = CGWindowListCopyWindowInfo(.optionIncludingWindow, number) as? [[String: Any]] ?? []
|
|
172
|
+
guard let record = records.first(where: {
|
|
173
|
+
($0[kCGWindowNumber as String] as? NSNumber)?.uint32Value == number
|
|
174
|
+
}), (record[kCGWindowOwnerPID as String] as? NSNumber)?.int32Value == pid,
|
|
175
|
+
let rawBounds = record[kCGWindowBounds as String] as? [String: Any],
|
|
176
|
+
let currentBounds = rect(rawBounds).flatMap({ inputBounds($0) }),
|
|
177
|
+
nearlyEqual(currentBounds, expectedBounds)
|
|
178
|
+
else {
|
|
179
|
+
throw HelperFailure.refused("scope_changed", "The exact CoreGraphics window changed before AX access", nil,
|
|
180
|
+
["window_id": identifier])
|
|
181
|
+
}
|
|
182
|
+
return currentBounds
|
|
183
|
+
}
|
|
184
|
+
|
|
185
|
+
func axChildren(_ element: AXUIElement) -> [AXUIElement] {
|
|
186
|
+
attribute(element, kAXChildrenAttribute) as? [AXUIElement] ?? []
|
|
187
|
+
}
|
|
188
|
+
|
|
189
|
+
func sessionLocked() -> Bool {
|
|
190
|
+
let session = CGSessionCopyCurrentDictionary() as? [String: Any]
|
|
191
|
+
return (session?["CGSSessionScreenIsLocked"] as? Bool) == true
|
|
192
|
+
}
|
|
193
|
+
|
|
194
|
+
func exactAXWindow(pid: pid_t, identifier: String, expectedBounds: CGRect) throws -> AXUIElement {
|
|
195
|
+
if sessionLocked() {
|
|
196
|
+
throw HelperFailure.refused("session_locked", "The macOS session is locked", "Unlock it and observe again", nil)
|
|
197
|
+
}
|
|
198
|
+
let currentBounds = try verifyCGWindow(pid: pid, identifier: identifier, expectedBounds: expectedBounds)
|
|
199
|
+
let applicationElement = AXUIElementCreateApplication(pid)
|
|
200
|
+
let candidates = axChildren(applicationElement).filter {
|
|
201
|
+
string($0, kAXRoleAttribute) == kAXWindowRole && elementBounds($0).map {
|
|
202
|
+
overlapScore($0, currentBounds) >= 0.90 &&
|
|
203
|
+
abs($0.midX - currentBounds.midX) <= 32 && abs($0.midY - currentBounds.midY) <= 32
|
|
204
|
+
} == true
|
|
205
|
+
}
|
|
206
|
+
guard candidates.count == 1, let match = candidates.first else {
|
|
207
|
+
throw HelperFailure.refused(
|
|
208
|
+
candidates.isEmpty ? "action_not_supported" : "ambiguous_target",
|
|
209
|
+
candidates.isEmpty ? "No AX window matched the exact CoreGraphics root" :
|
|
210
|
+
"More than one AX window matched the exact CoreGraphics root",
|
|
211
|
+
nil, ["window_id": identifier, "candidate_count": candidates.count]
|
|
212
|
+
)
|
|
213
|
+
}
|
|
214
|
+
return match
|
|
215
|
+
}
|
|
216
|
+
|
|
217
|
+
func normalizedRole(_ raw: String?) -> String {
|
|
218
|
+
guard var role = raw, !role.isEmpty else { return "unknown" }
|
|
219
|
+
if role.hasPrefix("AX") { role.removeFirst(2) }
|
|
220
|
+
return role.lowercased().replacingOccurrences(of: "_", with: "")
|
|
221
|
+
}
|
|
222
|
+
|
|
223
|
+
func normalizedSubrole(_ raw: String?) -> String? {
|
|
224
|
+
guard var role = raw, !role.isEmpty else { return nil }
|
|
225
|
+
if role.hasPrefix("AX") { role.removeFirst(2) }
|
|
226
|
+
return role.lowercased().replacingOccurrences(of: "_", with: "")
|
|
227
|
+
}
|
|
228
|
+
|
|
229
|
+
func directElementName(_ element: AXUIElement, role: String) -> String {
|
|
230
|
+
for name in [kAXTitleAttribute, kAXDescriptionAttribute, kAXHelpAttribute, "AXLabel"] {
|
|
231
|
+
if let value = string(element, name), !value.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty {
|
|
232
|
+
return String(value.prefix(maxTextCharacters))
|
|
233
|
+
}
|
|
234
|
+
}
|
|
235
|
+
if role == "statictext", let value = string(element, kAXValueAttribute) {
|
|
236
|
+
return String(value.prefix(maxTextCharacters))
|
|
237
|
+
}
|
|
238
|
+
return ""
|
|
239
|
+
}
|
|
240
|
+
|
|
241
|
+
func elementName(_ element: AXUIElement, role: String) -> String {
|
|
242
|
+
let direct = directElementName(element, role: role)
|
|
243
|
+
if !direct.isEmpty || !["row", "treeitem", "cell"].contains(role) { return direct }
|
|
244
|
+
|
|
245
|
+
var queue = axChildren(element).map { ($0, 1) }
|
|
246
|
+
var visited = 0
|
|
247
|
+
while !queue.isEmpty && visited < 24 {
|
|
248
|
+
let (child, depth) = queue.removeFirst()
|
|
249
|
+
visited += 1
|
|
250
|
+
let childRole = normalizedRole(string(child, kAXRoleAttribute))
|
|
251
|
+
let name = directElementName(child, role: childRole)
|
|
252
|
+
if !name.isEmpty { return name }
|
|
253
|
+
if depth < 2 { queue.append(contentsOf: axChildren(child).map { ($0, depth + 1) }) }
|
|
254
|
+
}
|
|
255
|
+
return ""
|
|
256
|
+
}
|
|
257
|
+
|
|
258
|
+
func scalarValue(_ element: AXUIElement, secure: Bool) -> Any? {
|
|
259
|
+
guard !secure, let value = attribute(element, kAXValueAttribute) else { return nil }
|
|
260
|
+
if let text = value as? String { return String(text.prefix(maxTextCharacters)) }
|
|
261
|
+
if let flag = value as? Bool { return flag }
|
|
262
|
+
if let number = value as? NSNumber { return number }
|
|
263
|
+
return nil
|
|
264
|
+
}
|
|
265
|
+
|
|
266
|
+
func isSettable(_ element: AXUIElement, _ name: String) -> Bool {
|
|
267
|
+
var settable = DarwinBoolean(false)
|
|
268
|
+
return AXUIElementIsAttributeSettable(element, name as CFString, &settable) == .success && settable.boolValue
|
|
269
|
+
}
|
|
270
|
+
|
|
271
|
+
func actionNames(_ element: AXUIElement) -> [String] {
|
|
272
|
+
var names: CFArray?
|
|
273
|
+
guard AXUIElementCopyActionNames(element, &names) == .success else { return [] }
|
|
274
|
+
return names as? [String] ?? []
|
|
275
|
+
}
|
|
276
|
+
|
|
277
|
+
func operationMap(_ element: AXUIElement, role: String, subrole: String?) -> [String: String] {
|
|
278
|
+
if boolean(element, kAXEnabledAttribute) == false ||
|
|
279
|
+
["closebutton", "minimizebutton", "zoombutton", "fullscreenbutton"].contains(subrole ?? "") {
|
|
280
|
+
return [:]
|
|
281
|
+
}
|
|
282
|
+
let names = actionNames(element)
|
|
283
|
+
var operations: [String: String] = [:]
|
|
284
|
+
let textRole = ["textfield", "textarea", "searchfield", "combobox"].contains(role)
|
|
285
|
+
if textRole && isSettable(element, kAXValueAttribute) {
|
|
286
|
+
operations["SET_TEXT"] = "AXSetValue"
|
|
287
|
+
operations["CLEAR"] = "AXSetValue"
|
|
288
|
+
}
|
|
289
|
+
if ["checkbox", "switch", "radiobutton"].contains(role), names.contains(kAXPressAction) {
|
|
290
|
+
operations["TOGGLE"] = kAXPressAction
|
|
291
|
+
} else if ["row", "treeitem", "cell"].contains(role),
|
|
292
|
+
isSettable(element, kAXSelectedAttribute), boolean(element, kAXSelectedAttribute) != true {
|
|
293
|
+
operations["PRESS"] = "AXSetSelected"
|
|
294
|
+
} else if let action = [kAXPressAction, "AXOpen", "AXConfirm"].first(where: names.contains) {
|
|
295
|
+
operations["PRESS"] = action
|
|
296
|
+
}
|
|
297
|
+
if isSettable(element, kAXExpandedAttribute), let expanded = boolean(element, kAXExpandedAttribute) {
|
|
298
|
+
operations[expanded ? "COLLAPSE" : "EXPAND"] = "AXSetExpanded"
|
|
299
|
+
}
|
|
300
|
+
return operations
|
|
301
|
+
}
|
|
302
|
+
|
|
303
|
+
func elementStates(_ element: AXUIElement) -> [String] {
|
|
304
|
+
var states: [String] = []
|
|
305
|
+
if boolean(element, kAXEnabledAttribute) == true { states.append("enabled") }
|
|
306
|
+
if boolean(element, kAXFocusedAttribute) == true { states.append("focused") }
|
|
307
|
+
if boolean(element, kAXSelectedAttribute) == true { states.append("selected") }
|
|
308
|
+
if boolean(element, kAXExpandedAttribute) == true { states.append("expanded") }
|
|
309
|
+
if let value = boolean(element, kAXValueAttribute), value { states.append("checked") }
|
|
310
|
+
return states
|
|
311
|
+
}
|
|
312
|
+
|
|
313
|
+
func targetSignature(_ element: AXUIElement, path: [Int], role: String, subrole: String?, name: String,
|
|
314
|
+
operations: [String: String]) -> [String: Any] {
|
|
315
|
+
var signature: [String: Any] = ["path": path, "role": role, "name": name, "operations": operations]
|
|
316
|
+
if let subrole { signature["subrole"] = subrole }
|
|
317
|
+
if let identifier = string(element, kAXIdentifierAttribute), !identifier.isEmpty {
|
|
318
|
+
signature["identifier"] = identifier
|
|
319
|
+
}
|
|
320
|
+
if let bounds = elementBounds(element) { signature["bounds"] = boundsDictionary(bounds) }
|
|
321
|
+
return signature
|
|
322
|
+
}
|
|
323
|
+
|
|
324
|
+
func targetKey(path: [Int], role: String) -> String {
|
|
325
|
+
"native:\(path.map(String.init).joined(separator: ".")):\(role)"
|
|
326
|
+
}
|
|
327
|
+
|
|
328
|
+
func resolvePath(root: AXUIElement, path: [Int]) throws -> AXUIElement {
|
|
329
|
+
var current = root
|
|
330
|
+
for index in path {
|
|
331
|
+
let children = axChildren(current)
|
|
332
|
+
guard index >= 0, index < children.count else {
|
|
333
|
+
throw HelperFailure.refused("stale_ref", "The native AX target path no longer exists", nil, nil)
|
|
334
|
+
}
|
|
335
|
+
current = children[index]
|
|
336
|
+
}
|
|
337
|
+
return current
|
|
338
|
+
}
|
|
339
|
+
|
|
340
|
+
func validateTarget(_ element: AXUIElement, target: [String: Any]) throws {
|
|
341
|
+
let role = normalizedRole(string(element, kAXRoleAttribute))
|
|
342
|
+
let subrole = normalizedSubrole(string(element, kAXSubroleAttribute))
|
|
343
|
+
let name = elementName(element, role: role)
|
|
344
|
+
guard role == target["role"] as? String, name == target["name"] as? String,
|
|
345
|
+
subrole == target["subrole"] as? String else {
|
|
346
|
+
throw HelperFailure.refused("stale_ref", "The native AX target identity changed", nil, nil)
|
|
347
|
+
}
|
|
348
|
+
if let expected = target["identifier"] as? String,
|
|
349
|
+
string(element, kAXIdentifierAttribute) != expected {
|
|
350
|
+
throw HelperFailure.refused("stale_ref", "The native AX target identifier changed", nil, nil)
|
|
351
|
+
}
|
|
352
|
+
if let expected = inputBounds(target["bounds"]),
|
|
353
|
+
elementBounds(element).map({ nearlyEqual($0, expected) }) != true {
|
|
354
|
+
throw HelperFailure.refused("stale_ref", "The native AX target bounds changed", nil, nil)
|
|
355
|
+
}
|
|
356
|
+
}
|
|
357
|
+
|
|
358
|
+
func snapshot(pid: pid_t, processInstance expected: String, windowID: String, bounds: CGRect,
|
|
359
|
+
rootTarget: [String: Any]?, maxDepth: Int) throws -> [String: Any] {
|
|
360
|
+
_ = try application(pid, expected: expected)
|
|
361
|
+
let window = try exactAXWindow(pid: pid, identifier: windowID, expectedBounds: bounds)
|
|
362
|
+
let root: AXUIElement
|
|
363
|
+
let rootPath: [Int]
|
|
364
|
+
if let rootTarget {
|
|
365
|
+
rootPath = rootTarget["path"] as? [Int] ?? []
|
|
366
|
+
root = try resolvePath(root: window, path: rootPath)
|
|
367
|
+
try validateTarget(root, target: rootTarget)
|
|
368
|
+
} else {
|
|
369
|
+
root = window
|
|
370
|
+
rootPath = []
|
|
371
|
+
}
|
|
372
|
+
|
|
373
|
+
let snapshotID = UUID().uuidString.replacingOccurrences(of: "-", with: "").prefix(16).lowercased()
|
|
374
|
+
var targets: [String: Any] = [:]
|
|
375
|
+
var sequence = 0
|
|
376
|
+
var nodeCount = 0
|
|
377
|
+
var complete = true
|
|
378
|
+
|
|
379
|
+
func visit(_ element: AXUIElement, path: [Int], depth: Int, ancestors: [AXUIElement]) -> [String: Any] {
|
|
380
|
+
nodeCount += 1
|
|
381
|
+
let role = normalizedRole(string(element, kAXRoleAttribute))
|
|
382
|
+
let subrole = normalizedSubrole(string(element, kAXSubroleAttribute))
|
|
383
|
+
let name = elementName(element, role: role)
|
|
384
|
+
let secure = subrole == "securetextfield" ||
|
|
385
|
+
name.range(of: "password|passcode|security code", options: [.regularExpression, .caseInsensitive]) != nil
|
|
386
|
+
let operations = operationMap(element, role: role, subrole: subrole)
|
|
387
|
+
var node: [String: Any] = ["role": role, "name": name, "states": elementStates(element)]
|
|
388
|
+
if let subrole { node["subrole"] = subrole }
|
|
389
|
+
if let value = scalarValue(element, secure: secure) { node["value"] = value }
|
|
390
|
+
|
|
391
|
+
let children = axChildren(element)
|
|
392
|
+
var included: [[String: Any]] = []
|
|
393
|
+
var omitted = 0
|
|
394
|
+
if depth >= maxDepth {
|
|
395
|
+
omitted = children.count
|
|
396
|
+
} else {
|
|
397
|
+
for (index, child) in children.enumerated() {
|
|
398
|
+
if index >= maxChildrenPerNode || nodeCount >= maxSnapshotNodes {
|
|
399
|
+
omitted = children.count - index
|
|
400
|
+
break
|
|
401
|
+
}
|
|
402
|
+
if ancestors.contains(where: { CFEqual($0, child) }) {
|
|
403
|
+
omitted += 1
|
|
404
|
+
continue
|
|
405
|
+
}
|
|
406
|
+
included.append(visit(child, path: path + [index], depth: depth + 1,
|
|
407
|
+
ancestors: ancestors + [element]))
|
|
408
|
+
}
|
|
409
|
+
}
|
|
410
|
+
if omitted > 0 {
|
|
411
|
+
complete = false
|
|
412
|
+
node["children_count"] = omitted
|
|
413
|
+
}
|
|
414
|
+
if !included.isEmpty { node["children"] = included }
|
|
415
|
+
if !operations.isEmpty || omitted > 0 {
|
|
416
|
+
sequence += 1
|
|
417
|
+
let reference = "@n\(snapshotID):e\(sequence)"
|
|
418
|
+
node["ref_id"] = reference
|
|
419
|
+
node["operations"] = Array(operations.keys).sorted()
|
|
420
|
+
node["target_key"] = targetKey(path: path, role: role)
|
|
421
|
+
targets[reference] = targetSignature(element, path: path, role: role, subrole: subrole,
|
|
422
|
+
name: name, operations: operations)
|
|
423
|
+
}
|
|
424
|
+
return node
|
|
425
|
+
}
|
|
426
|
+
|
|
427
|
+
let tree = visit(root, path: rootPath, depth: 0, ancestors: [])
|
|
428
|
+
return ["snapshot_id": String(snapshotID), "complete": complete,
|
|
429
|
+
"window": ["id": windowID], "tree": tree, "targets": targets,
|
|
430
|
+
"provenance": ["ax", "macos_helper_native"]]
|
|
431
|
+
}
|
|
432
|
+
|
|
433
|
+
func expandedAccessibility(_ app: AXUIElement) -> Bool {
|
|
434
|
+
var queue = axChildren(app)
|
|
435
|
+
var visited = 0
|
|
436
|
+
while !queue.isEmpty && visited < 600 {
|
|
437
|
+
let element = queue.removeFirst()
|
|
438
|
+
visited += 1
|
|
439
|
+
if string(element, kAXRoleAttribute) == "AXWebArea" { return true }
|
|
440
|
+
queue.append(contentsOf: axChildren(element).prefix(80))
|
|
441
|
+
}
|
|
442
|
+
return false
|
|
443
|
+
}
|
|
444
|
+
|
|
445
|
+
func enableAccessibility(pid: pid_t, expected: String?) throws -> [String: Any] {
|
|
446
|
+
if sessionLocked() {
|
|
447
|
+
throw HelperFailure.refused("session_locked", "The macOS session is locked", "Unlock it and try again", nil)
|
|
448
|
+
}
|
|
449
|
+
let app = try application(pid, expected: expected)
|
|
450
|
+
let element = AXUIElementCreateApplication(pid)
|
|
451
|
+
let attributeName = "AXManualAccessibility"
|
|
452
|
+
let before = boolean(element, attributeName)
|
|
453
|
+
let result = AXUIElementSetAttributeValue(element, attributeName as CFString, kCFBooleanTrue)
|
|
454
|
+
let after = boolean(element, attributeName)
|
|
455
|
+
let expanded = expandedAccessibility(element)
|
|
456
|
+
guard (result == .success && after == true) || expanded else {
|
|
457
|
+
throw HelperFailure.refused("action_not_supported", "Application did not expose expanded accessibility", nil,
|
|
458
|
+
["ax_error": result.rawValue, "attribute_verified": after == true,
|
|
459
|
+
"expansion_verified": expanded])
|
|
460
|
+
}
|
|
461
|
+
return ["pid": pid, "process_instance": try processInstance(app),
|
|
462
|
+
"changed": before != true, "verified": true,
|
|
463
|
+
"attribute_verified": after == true, "expansion_verified": expanded]
|
|
464
|
+
}
|
|
465
|
+
|
|
466
|
+
func executeNative(pid: pid_t, processInstance expected: String, windowID: String, bounds: CGRect,
|
|
467
|
+
target: [String: Any], operation: String, text: String?) throws -> [String: Any] {
|
|
468
|
+
_ = try application(pid, expected: expected)
|
|
469
|
+
let window = try exactAXWindow(pid: pid, identifier: windowID, expectedBounds: bounds)
|
|
470
|
+
guard let path = target["path"] as? [Int] else {
|
|
471
|
+
throw HelperFailure.refused("bad_request", "Native action target has no path", nil, nil)
|
|
472
|
+
}
|
|
473
|
+
let element = try resolvePath(root: window, path: path)
|
|
474
|
+
try validateTarget(element, target: target)
|
|
475
|
+
let role = normalizedRole(string(element, kAXRoleAttribute))
|
|
476
|
+
let subrole = normalizedSubrole(string(element, kAXSubroleAttribute))
|
|
477
|
+
let currentOperations = operationMap(element, role: role, subrole: subrole)
|
|
478
|
+
guard let operations = target["operations"] as? [String: String], let native = operations[operation],
|
|
479
|
+
currentOperations[operation] == native else {
|
|
480
|
+
throw HelperFailure.refused("stale_ref", "Operation is no longer observed on the native AX target", nil, nil)
|
|
481
|
+
}
|
|
482
|
+
|
|
483
|
+
let result: AXError
|
|
484
|
+
if native == "AXSetValue" {
|
|
485
|
+
let value = operation == "CLEAR" ? "" : text
|
|
486
|
+
guard let value else {
|
|
487
|
+
throw HelperFailure.refused("bad_request", "SET_TEXT requires exact text", nil, nil)
|
|
488
|
+
}
|
|
489
|
+
result = AXUIElementSetAttributeValue(element, kAXValueAttribute as CFString, value as CFTypeRef)
|
|
490
|
+
} else if native == "AXSetExpanded" {
|
|
491
|
+
result = AXUIElementSetAttributeValue(element, kAXExpandedAttribute as CFString,
|
|
492
|
+
(operation == "EXPAND" ? kCFBooleanTrue : kCFBooleanFalse))
|
|
493
|
+
} else if native == "AXSetSelected" {
|
|
494
|
+
result = AXUIElementSetAttributeValue(element, kAXSelectedAttribute as CFString, kCFBooleanTrue)
|
|
495
|
+
} else {
|
|
496
|
+
result = AXUIElementPerformAction(element, native as CFString)
|
|
497
|
+
}
|
|
498
|
+
if result == .success {
|
|
499
|
+
return ["dispatch": "delivered", "effect": "pending_verification", "driver": "macos_helper_native"]
|
|
500
|
+
}
|
|
501
|
+
if result == .actionUnsupported || result == .attributeUnsupported || result == .illegalArgument {
|
|
502
|
+
return ["dispatch": "not_delivered", "code": "AX_\(result.rawValue)",
|
|
503
|
+
"driver": "macos_helper_native"]
|
|
504
|
+
}
|
|
505
|
+
return ["dispatch": "delivery_unknown", "code": "AX_\(result.rawValue)",
|
|
506
|
+
"driver": "macos_helper_native"]
|
|
507
|
+
}
|
|
508
|
+
|
|
509
|
+
func requiredScope(_ input: [String: Any]) throws -> (pid_t, String, String, CGRect) {
|
|
510
|
+
guard let pid = input["pid"] as? Int, pid > 0,
|
|
511
|
+
let instance = input["process_instance"] as? String, !instance.isEmpty,
|
|
512
|
+
let windowID = input["window_id"] as? String, !windowID.isEmpty,
|
|
513
|
+
let bounds = inputBounds(input["bounds"]) else {
|
|
514
|
+
throw HelperFailure.refused("bad_request", "Native AX operation requires exact scope evidence", nil, nil)
|
|
515
|
+
}
|
|
516
|
+
return (pid_t(pid), instance, windowID, bounds)
|
|
517
|
+
}
|
|
518
|
+
|
|
519
|
+
func request() throws -> [String: Any] {
|
|
520
|
+
guard CommandLine.arguments.count == 2,
|
|
521
|
+
let data = CommandLine.arguments[1].data(using: .utf8),
|
|
522
|
+
let input = try JSONSerialization.jsonObject(with: data) as? [String: Any],
|
|
523
|
+
let operation = input["op"] as? String
|
|
524
|
+
else { throw HelperFailure.refused("bad_request", "Expected one JSON request argument", nil, nil) }
|
|
525
|
+
|
|
526
|
+
switch operation {
|
|
527
|
+
case "ping":
|
|
528
|
+
return ["version": protocolVersion, "platform": "macos", "accessibility": AXIsProcessTrusted(),
|
|
529
|
+
"session_locked": sessionLocked()]
|
|
530
|
+
case "displays":
|
|
531
|
+
let found = displays()
|
|
532
|
+
guard !found.isEmpty else {
|
|
533
|
+
throw HelperFailure.refused("incomplete_observation", "macOS returned no attached displays", nil, nil)
|
|
534
|
+
}
|
|
535
|
+
return ["displays": found]
|
|
536
|
+
case "windows":
|
|
537
|
+
guard let app = input["app"] as? String, !app.isEmpty else {
|
|
538
|
+
throw HelperFailure.refused("bad_request", "Window discovery requires an application", nil, nil)
|
|
539
|
+
}
|
|
540
|
+
return ["windows": try windows(appName: app, titles: input["titles"] as? Bool ?? false)]
|
|
541
|
+
case "frontmost":
|
|
542
|
+
guard let app = NSWorkspace.shared.frontmostApplication,
|
|
543
|
+
let name = app.bundleIdentifier ?? app.localizedName else {
|
|
544
|
+
throw HelperFailure.refused("scope_changed", "There is no frontmost application", nil, nil)
|
|
545
|
+
}
|
|
546
|
+
let found = try windows(appName: name, titles: input["titles"] as? Bool ?? false)
|
|
547
|
+
guard let first = found.first(where: { ($0["focused"] as? Bool) == true }) ?? found.first else {
|
|
548
|
+
throw HelperFailure.refused("scope_changed", "The frontmost application has no visible window", nil, nil)
|
|
549
|
+
}
|
|
550
|
+
return ["window": first]
|
|
551
|
+
case "enable_accessibility":
|
|
552
|
+
guard let pid = input["pid"] as? Int else {
|
|
553
|
+
throw HelperFailure.refused("bad_request", "Accessibility setup requires a PID", nil, nil)
|
|
554
|
+
}
|
|
555
|
+
return try enableAccessibility(pid: pid_t(pid), expected: input["process_instance"] as? String)
|
|
556
|
+
case "snapshot":
|
|
557
|
+
let (pid, instance, windowID, bounds) = try requiredScope(input)
|
|
558
|
+
let depth = min(max(input["max_depth"] as? Int ?? 18, 1), 24)
|
|
559
|
+
return try snapshot(pid: pid, processInstance: instance, windowID: windowID, bounds: bounds,
|
|
560
|
+
rootTarget: input["root_target"] as? [String: Any], maxDepth: depth)
|
|
561
|
+
case "execute":
|
|
562
|
+
let (pid, instance, windowID, bounds) = try requiredScope(input)
|
|
563
|
+
guard let target = input["target"] as? [String: Any],
|
|
564
|
+
let action = input["operation"] as? String else {
|
|
565
|
+
throw HelperFailure.refused("bad_request", "Native AX execution requires a target and operation", nil, nil)
|
|
566
|
+
}
|
|
567
|
+
return try executeNative(pid: pid, processInstance: instance, windowID: windowID, bounds: bounds,
|
|
568
|
+
target: target, operation: action, text: input["text"] as? String)
|
|
569
|
+
default:
|
|
570
|
+
throw HelperFailure.refused("bad_request", "Unknown operation \(operation)", nil, nil)
|
|
571
|
+
}
|
|
572
|
+
}
|
|
573
|
+
|
|
574
|
+
func emit(_ object: [String: Any]) {
|
|
575
|
+
let data = try! JSONSerialization.data(withJSONObject: object, options: [.sortedKeys])
|
|
576
|
+
print(String(data: data, encoding: .utf8)!)
|
|
577
|
+
}
|
|
578
|
+
|
|
579
|
+
do {
|
|
580
|
+
emit(["version": protocolVersion, "ok": true, "value": try request()])
|
|
581
|
+
} catch HelperFailure.refused(let code, let message, let suggestion, let details) {
|
|
582
|
+
var error: [String: Any] = ["code": code, "message": message,
|
|
583
|
+
"disposition": ["delivery": "not_delivered", "retry": "unsafe"]]
|
|
584
|
+
if let suggestion { error["suggestion"] = suggestion }
|
|
585
|
+
if let details { error["details"] = details }
|
|
586
|
+
emit(["version": protocolVersion, "ok": false, "error": error])
|
|
587
|
+
exit(1)
|
|
588
|
+
} catch {
|
|
589
|
+
emit(["version": protocolVersion, "ok": false,
|
|
590
|
+
"error": ["code": "bridge_error", "message": "macOS helper failed",
|
|
591
|
+
"disposition": ["delivery": "not_delivered", "retry": "unsafe"]]])
|
|
592
|
+
exit(1)
|
|
593
|
+
}
|