@qilin-agent/experimental-cua-macos 0.0.0-stage → 3.1.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.i18n.yaml +52 -0
- package/README.md +225 -2
- package/README.zh.md +226 -0
- package/lib/index.js +573 -0
- package/lib/types/guidance.d.ts +4 -0
- package/lib/types/index.d.ts +19 -0
- package/lib/types/native-build.d.ts +47 -0
- package/lib/types/shared-native.d.ts +13 -0
- package/native/LICENSE.cua-driver +21 -0
- package/native/Package.swift +13 -0
- package/native/README.md +130 -0
- package/native/README.zh.md +131 -0
- package/native/Sources/CuaNative/AXAccess.swift +525 -0
- package/native/Sources/CuaNative/AXHover.swift +37 -0
- package/native/Sources/CuaNative/AXTextFormatter.swift +120 -0
- package/native/Sources/CuaNative/AXWorker.swift +489 -0
- package/native/Sources/CuaNative/ApplicationAuthorization.swift +69 -0
- package/native/Sources/CuaNative/ApplicationDiscovery.swift +92 -0
- package/native/Sources/CuaNative/ClipboardPaste.swift +138 -0
- package/native/Sources/CuaNative/KeyboardInput.swift +260 -0
- package/native/Sources/CuaNative/Main.swift +37 -0
- package/native/Sources/CuaNative/NativePlatform.swift +118 -0
- package/native/Sources/CuaNative/NativeProtocol.generated.swift +33 -0
- package/native/Sources/CuaNative/PIDEventPoster.swift +85 -0
- package/native/Sources/CuaNative/Protocol.swift +149 -0
- package/native/Sources/CuaNative/RequestAdmission.swift +58 -0
- package/native/Sources/CuaNative/RequestCoordinator.swift +251 -0
- package/native/Sources/CuaNative/RequestScheduler.swift +107 -0
- package/native/Sources/CuaNative/Runtime.swift +170 -0
- package/native/Sources/CuaNative/ScreenshotTrace.swift +46 -0
- package/native/Sources/CuaNative/SharedHelperProxy.swift +124 -0
- package/native/Sources/CuaNative/SharedHelperRuntime.swift +289 -0
- package/native/Sources/CuaNative/SharedHelperServer.swift +188 -0
- package/native/Sources/CuaNative/TextSelection.swift +38 -0
- package/native/Sources/CuaNative/WindowCapture.swift +165 -0
- package/native/Sources/CuaNative/WindowGeometry.swift +54 -0
- package/native/Sources/CuaNative/WindowPointerInput.swift +300 -0
- package/native/Sources/CuaNative/WindowSpaces.swift +133 -0
- package/native/protocol.schema.json +1030 -0
- package/native/tests/swift/AXFailureTests.swift +166 -0
- package/native/tests/swift/AXTextFormatterTests.swift +128 -0
- package/native/tests/swift/ApplicationDiscoveryTests.swift +59 -0
- package/native/tests/swift/ClipboardPasteTests.swift +136 -0
- package/native/tests/swift/ForegroundTests.swift +42 -0
- package/native/tests/swift/InputMappingTests.swift +34 -0
- package/native/tests/swift/PermissionIdentityTests.swift +24 -0
- package/native/tests/swift/PointerTimingTests.swift +10 -0
- package/native/tests/swift/ProtocolTests.swift +118 -0
- package/native/tests/swift/ScreenshotTraceTests.swift +23 -0
- package/native/tests/swift/SharedHelperTests.swift +90 -0
- package/native/tests/swift/SharedSocketTests.swift +57 -0
- package/native/tests/swift/SystemHoverTests.swift +27 -0
- package/native/tests/swift/TextSelectionTests.swift +47 -0
- package/native/tests/swift/WindowCaptureTests.swift +80 -0
- package/native/tests/swift/WindowGeometryTests.swift +105 -0
- package/native/tests/swift/WindowSpacesTests.swift +26 -0
- package/native/tests/test_swift.py +27 -0
- package/native/upstream.json +67 -0
- package/package.json +88 -3
|
@@ -0,0 +1,525 @@
|
|
|
1
|
+
// Checked AX access and tree projection. Behavioral reference: native/upstream.json (MIT Cua Driver).
|
|
2
|
+
import Foundation
|
|
3
|
+
import ApplicationServices
|
|
4
|
+
import Darwin
|
|
5
|
+
|
|
6
|
+
private typealias AXWindowFunction = @convention(c) (AXUIElement, UnsafeMutablePointer<CGWindowID>) -> AXError
|
|
7
|
+
|
|
8
|
+
struct AXProjection {
|
|
9
|
+
let tree: String
|
|
10
|
+
let elements: [JSONValue]
|
|
11
|
+
let indexed: [Int: AXUIElement]
|
|
12
|
+
let focusedWindowID: UInt32?
|
|
13
|
+
let truncated: Bool
|
|
14
|
+
}
|
|
15
|
+
|
|
16
|
+
struct AXNodeObservation {
|
|
17
|
+
let role: String
|
|
18
|
+
let attribute: (String) throws -> CFTypeRef?
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
final class AXAccess {
|
|
22
|
+
let timeout: Float
|
|
23
|
+
let cancellation: CancellationLatch
|
|
24
|
+
private let framework: UnsafeMutableRawPointer?
|
|
25
|
+
private let windowFunction: AXWindowFunction?
|
|
26
|
+
private var enabledProcesses: Set<ProcessIdentityKey> = []
|
|
27
|
+
|
|
28
|
+
init(timeout: Float, cancellation: CancellationLatch) {
|
|
29
|
+
self.timeout = timeout
|
|
30
|
+
self.cancellation = cancellation
|
|
31
|
+
framework = dlopen("/System/Library/Frameworks/ApplicationServices.framework/ApplicationServices", RTLD_LAZY | RTLD_LOCAL)
|
|
32
|
+
if let framework, let symbol = dlsym(framework, "_AXUIElementGetWindow") {
|
|
33
|
+
windowFunction = unsafeBitCast(symbol, to: AXWindowFunction.self)
|
|
34
|
+
} else { windowFunction = nil }
|
|
35
|
+
}
|
|
36
|
+
deinit { if let framework { dlclose(framework) } }
|
|
37
|
+
|
|
38
|
+
private func prepare(_ element: AXUIElement) throws {
|
|
39
|
+
try cancellation.check()
|
|
40
|
+
let error = AXUIElementSetMessagingTimeout(element, timeout)
|
|
41
|
+
guard error == .success else { throw failure(error, "set AX messaging timeout") }
|
|
42
|
+
}
|
|
43
|
+
func failure(_ error: AXError, _ operation: String) -> NativeFailure {
|
|
44
|
+
NativeFailure(code: error == .apiDisabled ? "PERMISSION_DENIED" : "NATIVE_ERROR",
|
|
45
|
+
message: "Accessibility \(operation) failed with error \(error.rawValue)", accessibilityError: error.rawValue)
|
|
46
|
+
}
|
|
47
|
+
private func absent(_ error: AXError) -> Bool { error == .attributeUnsupported || error == .noValue || error == .notImplemented }
|
|
48
|
+
static func optionalAttributeUnavailable(_ error: AXError, attribute: String) -> Bool {
|
|
49
|
+
// File-panel controls can lack descriptive metadata while their role
|
|
50
|
+
// and accessibility children remain readable.
|
|
51
|
+
guard error == .failure else { return false }
|
|
52
|
+
let optional: Set<String> = ["AXValue", "AXTitle", "AXDescription", "AXIdentifier", "AXPlaceholderValue",
|
|
53
|
+
"AXSubrole", "AXHelp", "AXURL", "AXFocused", "AXExpanded", "AXEnabled", "AXSelected", "AXSelectedText", "AXSelectedTextRange", "AXPosition", "AXSize", "AXMain", "AXMinimized"]
|
|
54
|
+
return optional.contains(attribute)
|
|
55
|
+
}
|
|
56
|
+
static func actionsUnavailable(_ error: AXError) -> Bool {
|
|
57
|
+
error == .failure || error == .actionUnsupported
|
|
58
|
+
}
|
|
59
|
+
func attribute(_ element: AXUIElement, _ name: String) throws -> CFTypeRef? {
|
|
60
|
+
try prepare(element)
|
|
61
|
+
var value: CFTypeRef?
|
|
62
|
+
let error = AXUIElementCopyAttributeValue(element, name as CFString, &value)
|
|
63
|
+
if absent(error) || Self.optionalAttributeUnavailable(error, attribute: name) { return nil }
|
|
64
|
+
guard error == .success else { throw failure(error, "read \(name)") }
|
|
65
|
+
guard let value else { throw NativeFailure(code: "NATIVE_ERROR", message: "Accessibility \(name) returned no value after success") }
|
|
66
|
+
return value
|
|
67
|
+
}
|
|
68
|
+
func string(_ element: AXUIElement, _ name: String) throws -> String? {
|
|
69
|
+
try stringValue(try attribute(element, name), attribute: name)
|
|
70
|
+
}
|
|
71
|
+
private func stringValue(_ raw: CFTypeRef?, attribute name: String) throws -> String? {
|
|
72
|
+
guard let value = raw else { return nil }
|
|
73
|
+
guard CFGetTypeID(value) == CFStringGetTypeID() else { throw NativeFailure(code: "NATIVE_ERROR", message: "Accessibility \(name) is not a string") }
|
|
74
|
+
return value as? String
|
|
75
|
+
}
|
|
76
|
+
func boolean(_ element: AXUIElement, _ name: String) throws -> Bool? {
|
|
77
|
+
try booleanValue(try attribute(element, name), attribute: name)
|
|
78
|
+
}
|
|
79
|
+
private func booleanValue(_ raw: CFTypeRef?, attribute name: String) throws -> Bool? {
|
|
80
|
+
guard let value = raw else { return nil }
|
|
81
|
+
guard CFGetTypeID(value) == CFBooleanGetTypeID() else { throw NativeFailure(code: "NATIVE_ERROR", message: "Accessibility \(name) is not a boolean") }
|
|
82
|
+
return (value as? NSNumber)?.boolValue
|
|
83
|
+
}
|
|
84
|
+
func elementValue(_ value: CFTypeRef, _ name: String) throws -> AXUIElement {
|
|
85
|
+
guard CFGetTypeID(value) == AXUIElementGetTypeID() else { throw NativeFailure(code: "NATIVE_ERROR", message: "Accessibility \(name) is not an element") }
|
|
86
|
+
return unsafeDowncast(value, to: AXUIElement.self)
|
|
87
|
+
}
|
|
88
|
+
func element(_ element: AXUIElement, _ name: String) throws -> AXUIElement? {
|
|
89
|
+
guard let value = try attribute(element, name) else { return nil }
|
|
90
|
+
return try elementValue(value, name)
|
|
91
|
+
}
|
|
92
|
+
func element(at point: CGPoint, owner: ProcessIdentity) throws -> AXUIElement? {
|
|
93
|
+
let app = AXUIElementCreateApplication(owner.pid)
|
|
94
|
+
try prepare(app)
|
|
95
|
+
var hit: AXUIElement?
|
|
96
|
+
let error = AXUIElementCopyElementAtPosition(app, Float(point.x), Float(point.y), &hit)
|
|
97
|
+
if absent(error) { return nil }
|
|
98
|
+
guard error == .success else { throw failure(error, "find element at hover point") }
|
|
99
|
+
return hit
|
|
100
|
+
}
|
|
101
|
+
func children(_ element: AXUIElement, _ name: String = "AXChildren") throws -> [AXUIElement] {
|
|
102
|
+
try childValues(try attribute(element, name), attribute: name)
|
|
103
|
+
}
|
|
104
|
+
private func childValues(_ raw: CFTypeRef?, attribute name: String) throws -> [AXUIElement] {
|
|
105
|
+
guard let value = raw else { return [] }
|
|
106
|
+
guard CFGetTypeID(value) == CFArrayGetTypeID(), let array = value as? [AnyObject] else {
|
|
107
|
+
throw NativeFailure(code: "NATIVE_ERROR", message: "Accessibility \(name) is not an array")
|
|
108
|
+
}
|
|
109
|
+
return try array.map { try elementValue($0, name) }
|
|
110
|
+
}
|
|
111
|
+
func windowID(_ element: AXUIElement) throws -> UInt32? {
|
|
112
|
+
try prepare(element)
|
|
113
|
+
guard let windowFunction else { return nil }
|
|
114
|
+
var id: UInt32 = 0
|
|
115
|
+
let error = windowFunction(element, &id)
|
|
116
|
+
if absent(error) { return nil }
|
|
117
|
+
guard error == .success else { throw failure(error, "resolve window ID") }
|
|
118
|
+
return id == 0 ? nil : id
|
|
119
|
+
}
|
|
120
|
+
func focusedWindowID(_ pid: Int32) throws -> UInt32? {
|
|
121
|
+
guard let focused = try element(AXUIElementCreateApplication(pid), "AXFocusedWindow") else { return nil }
|
|
122
|
+
return try windowID(focused)
|
|
123
|
+
}
|
|
124
|
+
/// Restore a selected or unambiguous window without raising it or requesting application activation.
|
|
125
|
+
func restoreWindow(_ requested: UInt32?, owner: ProcessIdentity) throws -> UInt32? {
|
|
126
|
+
let app = AXUIElementCreateApplication(owner.pid)
|
|
127
|
+
try clearBooleanAttribute(app, "AXHidden")
|
|
128
|
+
let windows = try children(app, "AXWindows")
|
|
129
|
+
let selected: AXUIElement?
|
|
130
|
+
if let requested {
|
|
131
|
+
_ = try NativePlatform.ownedWindow(owner.pid, requested)
|
|
132
|
+
selected = try windows.first { try windowID($0) == requested }
|
|
133
|
+
guard selected != nil else { throw NativeFailure(code: "WINDOW_UNAVAILABLE", message: "Window is not exposed for restoration; read fresh state before deciding whether to activate") }
|
|
134
|
+
} else {
|
|
135
|
+
selected = try element(app, "AXFocusedWindow") ?? element(app, "AXMainWindow") ?? (windows.count == 1 ? windows[0] : nil)
|
|
136
|
+
}
|
|
137
|
+
guard let selected else { return nil }
|
|
138
|
+
try clearBooleanAttribute(selected, "AXMinimized")
|
|
139
|
+
return try windowID(selected)
|
|
140
|
+
}
|
|
141
|
+
private func clearBooleanAttribute(_ element: AXUIElement, _ name: String) throws {
|
|
142
|
+
guard try boolean(element, name) == true else { return }
|
|
143
|
+
guard try writable(element, name) == true else {
|
|
144
|
+
throw NativeFailure(code: "UNSUPPORTED", message: "Cannot restore \(name) without activation; read fresh state before deciding whether to activate")
|
|
145
|
+
}
|
|
146
|
+
try prepare(element)
|
|
147
|
+
let error = AXUIElementSetAttributeValue(element, name as CFString, kCFBooleanFalse)
|
|
148
|
+
guard error == .success else { throw failure(error, "clear \(name)") }
|
|
149
|
+
}
|
|
150
|
+
func actions(_ element: AXUIElement) throws -> [String] {
|
|
151
|
+
try prepare(element)
|
|
152
|
+
var names: CFArray?
|
|
153
|
+
let error = AXUIElementCopyActionNames(element, &names)
|
|
154
|
+
if absent(error) || Self.actionsUnavailable(error) { return [] }
|
|
155
|
+
guard error == .success else { throw failure(error, "read actions") }
|
|
156
|
+
guard let names else { return [] }
|
|
157
|
+
guard let result = names as? [String] else { throw NativeFailure(code: "NATIVE_ERROR", message: "Accessibility actions are not strings") }
|
|
158
|
+
return result
|
|
159
|
+
}
|
|
160
|
+
func writable(_ element: AXUIElement, _ name: String) throws -> Bool? {
|
|
161
|
+
try prepare(element)
|
|
162
|
+
var result: DarwinBoolean = false
|
|
163
|
+
let error = AXUIElementIsAttributeSettable(element, name as CFString, &result)
|
|
164
|
+
if absent(error) { return nil }
|
|
165
|
+
guard error == .success else { throw failure(error, "read attribute writability") }
|
|
166
|
+
return result.boolValue
|
|
167
|
+
}
|
|
168
|
+
func requireEnabled(_ element: AXUIElement) throws {
|
|
169
|
+
if try boolean(element, "AXEnabled") == false { throw NativeFailure(code: "NATIVE_ERROR", message: "Accessibility element is disabled") }
|
|
170
|
+
}
|
|
171
|
+
func perform(_ element: AXUIElement, _ action: String) throws {
|
|
172
|
+
try requireEnabled(element)
|
|
173
|
+
try prepare(element)
|
|
174
|
+
let error = AXUIElementPerformAction(element, action as CFString)
|
|
175
|
+
guard error == .success else { throw failure(error, "perform \(action)") }
|
|
176
|
+
}
|
|
177
|
+
func setValue(_ element: AXUIElement, _ value: String) throws {
|
|
178
|
+
try requireEnabled(element)
|
|
179
|
+
try prepare(element)
|
|
180
|
+
let error = AXUIElementSetAttributeValue(element, "AXValue" as CFString, value as CFString)
|
|
181
|
+
guard error == .success else { throw failure(error, "set AXValue") }
|
|
182
|
+
}
|
|
183
|
+
func pasteCommand(_ pid: Int32, maxElements: Int, maxDepth: Int) throws -> AXUIElement {
|
|
184
|
+
guard let menu = try element(AXUIElementCreateApplication(pid), "AXMenuBar") else {
|
|
185
|
+
throw NativeFailure(code: "UNSUPPORTED", message: "The application does not expose a Paste menu command")
|
|
186
|
+
}
|
|
187
|
+
var pending: [(AXUIElement, Int)] = [(menu, 0)]
|
|
188
|
+
var found: AXUIElement?
|
|
189
|
+
var seen: [CFHashCode: [AXUIElement]] = [:]
|
|
190
|
+
var count = 0
|
|
191
|
+
while let (node, depth) = pending.popLast() {
|
|
192
|
+
let hash = CFHash(node)
|
|
193
|
+
if seen[hash, default: []].contains(where: { CFEqual($0, node) }) { continue }
|
|
194
|
+
seen[hash, default: []].append(node)
|
|
195
|
+
guard count < maxElements, depth <= maxDepth else {
|
|
196
|
+
throw NativeFailure(code: "UNSUPPORTED", message: "Paste menu discovery exceeded its observation limits")
|
|
197
|
+
}
|
|
198
|
+
count += 1
|
|
199
|
+
if try string(node, "AXRole") == "AXMenuItem", try string(node, "AXMenuItemCmdChar")?.lowercased() == "v" {
|
|
200
|
+
let raw = try attribute(node, "AXMenuItemCmdModifiers")
|
|
201
|
+
if let raw, CFGetTypeID(raw) == CFNumberGetTypeID(), let number = raw as? NSNumber, number.intValue == 0,
|
|
202
|
+
try actions(node).contains("AXPress") {
|
|
203
|
+
guard found == nil else { throw NativeFailure(code: "UNSUPPORTED", message: "The application exposes ambiguous Paste commands") }
|
|
204
|
+
found = node
|
|
205
|
+
}
|
|
206
|
+
}
|
|
207
|
+
for child in try children(node) { pending.append((child, depth + 1)) }
|
|
208
|
+
}
|
|
209
|
+
guard let found else { throw NativeFailure(code: "UNSUPPORTED", message: "No standard Paste command is available") }
|
|
210
|
+
return found
|
|
211
|
+
}
|
|
212
|
+
func selectText(_ element: AXUIElement, text: String, prefix: String?, suffix: String?, mode: String) throws {
|
|
213
|
+
try requireEnabled(element)
|
|
214
|
+
guard try writable(element, "AXSelectedTextRange") == true, let original = try string(element, "AXValue") else {
|
|
215
|
+
throw NativeFailure(code: "UNSUPPORTED", message: "This control does not expose writable text selection")
|
|
216
|
+
}
|
|
217
|
+
var range = try TextSelection.range(in: original, text: text, prefix: prefix, suffix: suffix, mode: mode)
|
|
218
|
+
guard let value = AXValueCreate(.cfRange, &range) else {
|
|
219
|
+
throw NativeFailure(code: "NATIVE_ERROR", message: "Could not create the text selection range")
|
|
220
|
+
}
|
|
221
|
+
guard try string(element, "AXValue") == original else {
|
|
222
|
+
throw NativeFailure(code: "STALE_SNAPSHOT", message: "Text changed before selection; read fresh state")
|
|
223
|
+
}
|
|
224
|
+
try prepare(element)
|
|
225
|
+
let error = AXUIElementSetAttributeValue(element, "AXSelectedTextRange" as CFString, value)
|
|
226
|
+
guard error == .success else { throw failure(error, "set text selection") }
|
|
227
|
+
let actual = try textRange(attribute(element, "AXSelectedTextRange"))
|
|
228
|
+
guard actual?.location == range.location, actual?.length == range.length, try string(element, "AXValue") == original else {
|
|
229
|
+
throw NativeFailure(code: "SELECTION_CHANGED", message: "The requested selection was not retained; read fresh state before input")
|
|
230
|
+
}
|
|
231
|
+
}
|
|
232
|
+
private func textRange(_ raw: CFTypeRef?) throws -> CFRange? {
|
|
233
|
+
guard let raw else { return nil }
|
|
234
|
+
guard CFGetTypeID(raw) == AXValueGetTypeID() else {
|
|
235
|
+
throw NativeFailure(code: "NATIVE_ERROR", message: "AXSelectedTextRange is not an accessibility value")
|
|
236
|
+
}
|
|
237
|
+
let value = unsafeDowncast(raw, to: AXValue.self)
|
|
238
|
+
var range = CFRange()
|
|
239
|
+
guard AXValueGetType(value) == .cfRange, AXValueGetValue(value, .cfRange, &range), range.location >= 0, range.length >= 0 else {
|
|
240
|
+
throw NativeFailure(code: "NATIVE_ERROR", message: "AXSelectedTextRange is not a valid range")
|
|
241
|
+
}
|
|
242
|
+
return range
|
|
243
|
+
}
|
|
244
|
+
func bounds(_ element: AXUIElement) throws -> WindowBounds? {
|
|
245
|
+
guard let position = try attribute(element, "AXPosition") else { return nil }
|
|
246
|
+
return try bounds(position: position, size: attribute(element, "AXSize"))
|
|
247
|
+
}
|
|
248
|
+
/// Resolve an exact AX window or sheet without enabling accessibility or traversing its controls.
|
|
249
|
+
func windowBounds(_ id: UInt32, owner: ProcessIdentity) throws -> WindowBounds? {
|
|
250
|
+
let app = AXUIElementCreateApplication(owner.pid)
|
|
251
|
+
var windows = try children(app, "AXWindows")
|
|
252
|
+
if let focused = try element(app, "AXFocusedWindow"), !windows.contains(where: { CFEqual($0, focused) }) {
|
|
253
|
+
windows.append(focused)
|
|
254
|
+
}
|
|
255
|
+
for window in windows {
|
|
256
|
+
let role = try string(window, "AXRole")
|
|
257
|
+
guard role == "AXWindow" || role == "AXSheet" else { continue }
|
|
258
|
+
if try windowID(window) == id { return try bounds(window) }
|
|
259
|
+
for sheet in try children(window) where try string(sheet, "AXRole") == "AXSheet" {
|
|
260
|
+
if try windowID(sheet) == id { return try bounds(sheet) }
|
|
261
|
+
}
|
|
262
|
+
}
|
|
263
|
+
return nil
|
|
264
|
+
}
|
|
265
|
+
private func bounds(position: CFTypeRef?, size: CFTypeRef?) throws -> WindowBounds? {
|
|
266
|
+
guard let position, let size else { return nil }
|
|
267
|
+
guard CFGetTypeID(position) == AXValueGetTypeID(), CFGetTypeID(size) == AXValueGetTypeID() else { return nil }
|
|
268
|
+
let positionValue = unsafeDowncast(position, to: AXValue.self)
|
|
269
|
+
let sizeValue = unsafeDowncast(size, to: AXValue.self)
|
|
270
|
+
var point = CGPoint.zero
|
|
271
|
+
var dimensions = CGSize.zero
|
|
272
|
+
guard AXValueGetValue(positionValue, .cgPoint, &point), AXValueGetValue(sizeValue, .cgSize, &dimensions),
|
|
273
|
+
point.x.isFinite, point.y.isFinite, dimensions.width.isFinite, dimensions.height.isFinite else { return nil }
|
|
274
|
+
return WindowBounds(x: point.x, y: point.y, width: dimensions.width, height: dimensions.height)
|
|
275
|
+
}
|
|
276
|
+
private func controlValue(_ raw: CFTypeRef?) -> String? {
|
|
277
|
+
guard let value = raw else { return nil }
|
|
278
|
+
if CFGetTypeID(value) == CFStringGetTypeID() { return value as? String }
|
|
279
|
+
if CFGetTypeID(value) == CFBooleanGetTypeID() { return ((value as? NSNumber)?.boolValue == true) ? "1" : "0" }
|
|
280
|
+
if CFGetTypeID(value) == CFNumberGetTypeID() { return (value as? NSNumber)?.stringValue }
|
|
281
|
+
return nil
|
|
282
|
+
}
|
|
283
|
+
// A node's values are retained only for this observation. Input always reads
|
|
284
|
+
// live attributes, and unsupported batch APIs retain the individual read path.
|
|
285
|
+
private func metadata(_ element: AXUIElement) throws -> [String: CFTypeRef]? {
|
|
286
|
+
try prepare(element)
|
|
287
|
+
let names = ["AXRole", "AXTitle", "AXValue", "AXPlaceholderValue", "AXDescription", "AXEnabled",
|
|
288
|
+
"AXIdentifier", "AXPosition", "AXSize", "AXChildren", "AXSelected", "AXSelectedText", "AXSelectedTextRange",
|
|
289
|
+
"AXSubrole", "AXHelp", "AXURL", "AXFocused", "AXExpanded"]
|
|
290
|
+
var output: CFArray?
|
|
291
|
+
let error = AXUIElementCopyMultipleAttributeValues(element, names as CFArray, [], &output)
|
|
292
|
+
if error == .notImplemented || error == .attributeUnsupported { return nil }
|
|
293
|
+
guard error == .success else { throw failure(error, "read node attributes") }
|
|
294
|
+
guard let output, CFArrayGetCount(output) == names.count else {
|
|
295
|
+
throw NativeFailure(code: "NATIVE_ERROR", message: "Accessibility batch returned mismatched attribute values")
|
|
296
|
+
}
|
|
297
|
+
let values = output as [AnyObject]
|
|
298
|
+
return Dictionary(uniqueKeysWithValues: zip(names, values))
|
|
299
|
+
}
|
|
300
|
+
static func batchAttribute(_ value: CFTypeRef, name: String) throws -> CFTypeRef? {
|
|
301
|
+
if CFGetTypeID(value) == CFNullGetTypeID() { return nil }
|
|
302
|
+
if CFGetTypeID(value) == AXValueGetTypeID() {
|
|
303
|
+
let wrapped = unsafeDowncast(value, to: AXValue.self)
|
|
304
|
+
if AXValueGetType(wrapped) == .axError {
|
|
305
|
+
var error = AXError.success
|
|
306
|
+
guard AXValueGetValue(wrapped, .axError, &error), error != .success else {
|
|
307
|
+
throw NativeFailure(code: "NATIVE_ERROR", message: "Accessibility batch returned an invalid attribute error")
|
|
308
|
+
}
|
|
309
|
+
if error == .attributeUnsupported || error == .noValue || error == .notImplemented
|
|
310
|
+
|| optionalAttributeUnavailable(error, attribute: name) { return nil }
|
|
311
|
+
throw NativeFailure(code: error == .apiDisabled ? "PERMISSION_DENIED" : "NATIVE_ERROR",
|
|
312
|
+
message: "Accessibility read \(name) failed with error \(error.rawValue)", accessibilityError: error.rawValue)
|
|
313
|
+
}
|
|
314
|
+
}
|
|
315
|
+
return value
|
|
316
|
+
}
|
|
317
|
+
|
|
318
|
+
// Decode metadata lazily while retaining errors for every consumed attribute.
|
|
319
|
+
func nodeObservation(attributes: [String: CFTypeRef]?, fallback: @escaping (String) throws -> CFTypeRef?) throws -> AXNodeObservation {
|
|
320
|
+
func observed(_ name: String) throws -> CFTypeRef? {
|
|
321
|
+
guard let attributes else { return try fallback(name) }
|
|
322
|
+
guard let value = attributes[name] else { throw NativeFailure(code: "NATIVE_ERROR", message: "Missing requested observation attribute") }
|
|
323
|
+
return try Self.batchAttribute(value, name: name)
|
|
324
|
+
}
|
|
325
|
+
let role = try stringValue(observed("AXRole"), attribute: "AXRole") ?? "AXUnknown"
|
|
326
|
+
return AXNodeObservation(role: role, attribute: observed)
|
|
327
|
+
}
|
|
328
|
+
|
|
329
|
+
private func enableAX(_ app: AXUIElement, identity: ProcessIdentity) throws {
|
|
330
|
+
let key = ProcessIdentityKey(identity)
|
|
331
|
+
if enabledProcesses.contains(key) { return }
|
|
332
|
+
try prepare(app)
|
|
333
|
+
var error = AXUIElementSetAttributeValue(app, "AXManualAccessibility" as CFString, kCFBooleanTrue)
|
|
334
|
+
if error == .attributeUnsupported { error = AXUIElementSetAttributeValue(app, "AXEnhancedUserInterface" as CFString, kCFBooleanTrue) }
|
|
335
|
+
if error == .apiDisabled { throw failure(error, "enable application accessibility") }
|
|
336
|
+
if error == .success { enabledProcesses.insert(key) }
|
|
337
|
+
}
|
|
338
|
+
|
|
339
|
+
func project(identity: ProcessIdentity, windowID requested: UInt32?, maxElements: Int, maxDepth: Int, includeMenus: Bool = false, details: Bool = false) throws -> AXProjection {
|
|
340
|
+
let app = AXUIElementCreateApplication(identity.pid)
|
|
341
|
+
try enableAX(app, identity: identity)
|
|
342
|
+
var roots = try children(app)
|
|
343
|
+
func appendUnique(_ element: AXUIElement) {
|
|
344
|
+
if !roots.contains(where: { CFEqual($0, element) }) { roots.append(element) }
|
|
345
|
+
}
|
|
346
|
+
for root in try children(app, "AXWindows") { appendUnique(root) }
|
|
347
|
+
let focused = try element(app, "AXFocusedWindow")
|
|
348
|
+
if let focused { appendUnique(focused) }
|
|
349
|
+
var cursor = 0
|
|
350
|
+
while cursor < roots.count && cursor < maxElements {
|
|
351
|
+
let root = roots[cursor]
|
|
352
|
+
cursor += 1
|
|
353
|
+
let role = try string(root, "AXRole")
|
|
354
|
+
guard role == "AXWindow" || role == "AXSheet" else { continue }
|
|
355
|
+
for child in try children(root) where roots.count < maxElements {
|
|
356
|
+
if try string(child, "AXRole") == "AXSheet" { appendUnique(child) }
|
|
357
|
+
}
|
|
358
|
+
}
|
|
359
|
+
let focusedID = try focused.flatMap { try windowID($0) }
|
|
360
|
+
if let requested {
|
|
361
|
+
var matches: [AXUIElement] = []
|
|
362
|
+
var nonWindows: [AXUIElement] = []
|
|
363
|
+
var anonymousSheets: [AXUIElement] = []
|
|
364
|
+
for root in roots {
|
|
365
|
+
let role = try string(root, "AXRole") ?? ""
|
|
366
|
+
if role == "AXWindow" || role == "AXSheet" {
|
|
367
|
+
let id = try windowID(root)
|
|
368
|
+
if id == requested { matches.append(root) }
|
|
369
|
+
else if role == "AXSheet" && id == nil { anonymousSheets.append(root) }
|
|
370
|
+
} else { nonWindows.append(root) }
|
|
371
|
+
}
|
|
372
|
+
if matches.isEmpty {
|
|
373
|
+
_ = try NativePlatform.ownedWindow(identity.pid, requested)
|
|
374
|
+
return AXProjection(tree: "", elements: [], indexed: [:], focusedWindowID: focusedID, truncated: false)
|
|
375
|
+
}
|
|
376
|
+
let dialog = try matches.contains { root in
|
|
377
|
+
let role = try string(root, "AXRole")
|
|
378
|
+
let subrole = try string(root, "AXSubrole")
|
|
379
|
+
let identifier = try string(root, "AXIdentifier")
|
|
380
|
+
return role == "AXSheet" || ["AXDialog", "AXSystemDialog", "AXSheet"].contains(subrole ?? "") || ["open-panel", "save-panel"].contains(identifier ?? "")
|
|
381
|
+
}
|
|
382
|
+
roots = matches
|
|
383
|
+
for root in nonWindows {
|
|
384
|
+
let role = try string(root, "AXRole")
|
|
385
|
+
if !dialog || role != "AXMenuBar" { roots.append(root) }
|
|
386
|
+
}
|
|
387
|
+
if !dialog { roots.append(contentsOf: anonymousSheets) }
|
|
388
|
+
}
|
|
389
|
+
var visited: [CFHashCode: [AXUIElement]] = [:]
|
|
390
|
+
var count = 0
|
|
391
|
+
var elementLimitReached = false
|
|
392
|
+
var depthLimitReached = false
|
|
393
|
+
var menusCollapsed = false
|
|
394
|
+
var lines = ["AX scope=\(requested == nil ? "application" : "window") offsets=UTF-16\(details ? " bounds=global-points" : "")"]
|
|
395
|
+
var elements: [JSONValue] = []
|
|
396
|
+
var indexed: [Int: AXUIElement] = [:]
|
|
397
|
+
func walk(_ element: AXUIElement, level: Int, depth: Int) throws {
|
|
398
|
+
try cancellation.check()
|
|
399
|
+
let hash = CFHash(element)
|
|
400
|
+
if visited[hash, default: []].contains(where: { CFEqual($0, element) }) { return }
|
|
401
|
+
guard count < maxElements else { elementLimitReached = true; return }
|
|
402
|
+
guard depth <= maxDepth else { depthLimitReached = true; return }
|
|
403
|
+
visited[hash, default: []].append(element)
|
|
404
|
+
count += 1
|
|
405
|
+
let observation = try nodeObservation(attributes: metadata(element), fallback: { try self.attribute(element, $0) })
|
|
406
|
+
let role = observation.role
|
|
407
|
+
let observed = observation.attribute
|
|
408
|
+
let title = try stringValue(observed("AXTitle"), attribute: "AXTitle")
|
|
409
|
+
let value = try controlValue(observed("AXValue"))
|
|
410
|
+
let placeholder = try stringValue(observed("AXPlaceholderValue"), attribute: "AXPlaceholderValue")
|
|
411
|
+
let description = try stringValue(observed("AXDescription"), attribute: "AXDescription")
|
|
412
|
+
let actionNames = try self.actions(element)
|
|
413
|
+
let valueWritable = AXTextNode.valueRoles.contains(role) ? try writable(element, "AXValue") : nil
|
|
414
|
+
let enabled = try booleanValue(observed("AXEnabled"), attribute: "AXEnabled")
|
|
415
|
+
let selected = try booleanValue(observed("AXSelected"), attribute: "AXSelected")
|
|
416
|
+
let selectionWritable = AXTextNode.textRoles.contains(role) ? try writable(element, "AXSelectedTextRange") : nil
|
|
417
|
+
let selectedRange = AXTextNode.textRoles.contains(role) ? try textRange(observed("AXSelectedTextRange")) : nil
|
|
418
|
+
let selectedText = AXTextNode.textRoles.contains(role) ? try stringValue(observed("AXSelectedText"), attribute: "AXSelectedText") : nil
|
|
419
|
+
let identifier = try stringValue(observed("AXIdentifier"), attribute: "AXIdentifier")
|
|
420
|
+
let subrole = try stringValue(observed("AXSubrole"), attribute: "AXSubrole")
|
|
421
|
+
let help = try stringValue(observed("AXHelp"), attribute: "AXHelp")
|
|
422
|
+
let rawURL = try observed("AXURL")
|
|
423
|
+
let url: String?
|
|
424
|
+
if let rawURL, CFGetTypeID(rawURL) == CFURLGetTypeID() { url = (rawURL as? URL)?.absoluteString }
|
|
425
|
+
else { url = try stringValue(rawURL, attribute: "AXURL") }
|
|
426
|
+
let focused = try booleanValue(observed("AXFocused"), attribute: "AXFocused")
|
|
427
|
+
let expanded = try booleanValue(observed("AXExpanded"), attribute: "AXExpanded")
|
|
428
|
+
let addressable = enabled != false && (!actionNames.isEmpty || valueWritable == true || selectionWritable == true)
|
|
429
|
+
var node = AXTextNode(index: nil, role: role, title: title, value: value, description: description,
|
|
430
|
+
actions: actionNames, enabled: enabled, valueWritable: valueWritable,
|
|
431
|
+
selectionWritable: selectionWritable, selected: selected, selectedRange: selectedRange,
|
|
432
|
+
selectedText: selectedText, subrole: subrole, placeholder: placeholder, help: help,
|
|
433
|
+
identifier: identifier, url: url, focused: focused, expanded: expanded)
|
|
434
|
+
node.validateSelection()
|
|
435
|
+
if node.transparent || (!addressable && !node.hasMeaning && role != "AXWindow" && role != "AXSheet") {
|
|
436
|
+
for child in try childValues(observed("AXChildren"), attribute: "AXChildren") { try walk(child, level: level, depth: depth + 1) }
|
|
437
|
+
return
|
|
438
|
+
}
|
|
439
|
+
let index: Int? = addressable ? indexed.count : nil
|
|
440
|
+
if let index { indexed[index] = element }
|
|
441
|
+
node.index = index
|
|
442
|
+
let frame: WindowBounds?
|
|
443
|
+
if let position = try observed("AXPosition") { frame = try bounds(position: position, size: observed("AXSize")) }
|
|
444
|
+
else { frame = nil }
|
|
445
|
+
node.bounds = frame
|
|
446
|
+
lines.append(AXTextFormatter.line(node, level: level, details: details))
|
|
447
|
+
if let index {
|
|
448
|
+
let frameJSON: JSONValue = frame.map { .array([.number($0.x), .number($0.y), .number($0.width), .number($0.height)]) } ?? .null
|
|
449
|
+
elements.append(.object([
|
|
450
|
+
"index": .number(Double(index)), "role": .string(role), "title": title.map(JSONValue.string) ?? .null,
|
|
451
|
+
"value": value.map(JSONValue.string) ?? .null, "description": description.map(JSONValue.string) ?? .null,
|
|
452
|
+
"identifier": identifier.map(JSONValue.string) ?? .null,
|
|
453
|
+
"bounds": frameJSON, "actions": .array(actionNames.map(JSONValue.string)), "enabled": enabled.map(JSONValue.bool) ?? .null,
|
|
454
|
+
"valueWritable": valueWritable.map(JSONValue.bool) ?? .null, "selectionWritable": selectionWritable.map(JSONValue.bool) ?? .null,
|
|
455
|
+
"selected": selected.map(JSONValue.bool) ?? .null,
|
|
456
|
+
"selectedText": node.selectedText.map(JSONValue.string) ?? .null,
|
|
457
|
+
"selectedRange": node.selectedRange.map { .array([.number(Double($0.location)), .number(Double($0.length))]) } ?? .null,
|
|
458
|
+
]))
|
|
459
|
+
}
|
|
460
|
+
if role == "AXMenuBarItem" && !includeMenus { menusCollapsed = true; return }
|
|
461
|
+
for child in try childValues(observed("AXChildren"), attribute: "AXChildren") { try walk(child, level: level + 1, depth: depth + 1) }
|
|
462
|
+
}
|
|
463
|
+
for root in roots { try walk(root, level: 0, depth: 0) }
|
|
464
|
+
if menusCollapsed { lines.append("Menu contents omitted; request includeMenus=true to expand.") }
|
|
465
|
+
let truncated = elementLimitReached || depthLimitReached
|
|
466
|
+
if truncated { lines.append(AXTextFormatter.truncation(maxElements: elementLimitReached ? maxElements : nil, maxDepth: depthLimitReached ? maxDepth : nil)) }
|
|
467
|
+
return AXProjection(tree: lines.joined(separator: "\n"), elements: elements, indexed: indexed, focusedWindowID: focusedID, truncated: truncated)
|
|
468
|
+
}
|
|
469
|
+
|
|
470
|
+
func enclosingWindow(_ element: AXUIElement, owner: ProcessIdentity) throws -> WindowInfo {
|
|
471
|
+
var current: AXUIElement? = element
|
|
472
|
+
var visited: [AXUIElement] = []
|
|
473
|
+
for _ in 0..<100 {
|
|
474
|
+
guard let node = current else { break }
|
|
475
|
+
if visited.contains(where: { CFEqual($0, node) }) { break }
|
|
476
|
+
visited.append(node)
|
|
477
|
+
let role = try string(node, "AXRole")
|
|
478
|
+
if role == "AXWindow" || role == "AXSheet", let id = try windowID(node) {
|
|
479
|
+
return try WindowGeometry.resolve(owner: owner, windowID: id) { try bounds(node) }
|
|
480
|
+
}
|
|
481
|
+
current = try self.element(node, "AXParent")
|
|
482
|
+
}
|
|
483
|
+
throw NativeFailure(code: "WINDOW_UNAVAILABLE", message: "Element has no app-owned window or sheet; refresh getAXState")
|
|
484
|
+
}
|
|
485
|
+
func keyboardRecipient(_ owner: ProcessIdentity, catalog: ApplicationCatalogSnapshot) throws -> ProcessIdentity {
|
|
486
|
+
var current = try element(AXUIElementCreateApplication(owner.pid), "AXFocusedWindow")
|
|
487
|
+
var visited: [AXUIElement] = []
|
|
488
|
+
for _ in 0..<100 {
|
|
489
|
+
guard let node = current else { return owner }
|
|
490
|
+
if visited.contains(where: { CFEqual($0, node) }) { break }
|
|
491
|
+
visited.append(node)
|
|
492
|
+
if ["open-panel", "save-panel"].contains(try string(node, "AXIdentifier") ?? "") {
|
|
493
|
+
guard let hostID = try windowID(node) else { throw NativeFailure(code: "PANEL_NOT_READY", message: "File panel has no window ID") }
|
|
494
|
+
_ = try NativePlatform.ownedWindow(owner.pid, hostID)
|
|
495
|
+
var recipient: (ProcessIdentity, UInt32)?
|
|
496
|
+
for child in try children(node) {
|
|
497
|
+
guard let id = try windowID(child), let window = NativePlatform.windows(allLayers: true).first(where: { $0.windowID == id }), window.pid != owner.pid else { continue }
|
|
498
|
+
guard catalog.application(window.pid)?.bundleID == "com.apple.appkit.xpc.openAndSavePanelService" else {
|
|
499
|
+
throw NativeFailure(code: "PANEL_NOT_READY", message: "File panel refers to an unsupported remote process")
|
|
500
|
+
}
|
|
501
|
+
let process = try NativePlatform.identity(window.pid)
|
|
502
|
+
if let old = recipient, old.0 != process || old.1 != id { throw NativeFailure(code: "PANEL_NOT_READY", message: "File panel has ambiguous remote recipients") }
|
|
503
|
+
recipient = (process, id)
|
|
504
|
+
}
|
|
505
|
+
guard let (process, id) = recipient else { throw NativeFailure(code: "PANEL_NOT_READY", message: "System file panel has no ready input recipient") }
|
|
506
|
+
var focus = try element(AXUIElementCreateApplication(process.pid), "AXFocusedWindow")
|
|
507
|
+
for _ in 0..<100 {
|
|
508
|
+
guard let focused = focus else { break }
|
|
509
|
+
if try windowID(focused) == id { try NativePlatform.verify(process); return process }
|
|
510
|
+
focus = try element(focused, "AXParent")
|
|
511
|
+
}
|
|
512
|
+
throw NativeFailure(code: "PANEL_NOT_READY", message: "System file panel focus changed; observe fresh state")
|
|
513
|
+
}
|
|
514
|
+
current = try element(node, "AXParent")
|
|
515
|
+
}
|
|
516
|
+
throw NativeFailure(code: "PANEL_NOT_READY", message: "Focused accessibility ancestry exceeds the traversal limit")
|
|
517
|
+
}
|
|
518
|
+
}
|
|
519
|
+
|
|
520
|
+
private struct ProcessIdentityKey: Hashable {
|
|
521
|
+
let pid: Int32
|
|
522
|
+
let seconds: UInt64
|
|
523
|
+
let microseconds: UInt64
|
|
524
|
+
init(_ value: ProcessIdentity) { pid = value.pid; seconds = value.startedSeconds; microseconds = value.startedMicroseconds }
|
|
525
|
+
}
|
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
// Alternate accessibility UI can remain revealed without moving the system pointer.
|
|
2
|
+
import ApplicationServices
|
|
3
|
+
|
|
4
|
+
final class AXHover {
|
|
5
|
+
private var revealed: (element: AXUIElement, windowID: UInt32)?
|
|
6
|
+
|
|
7
|
+
/// Entering revealed descendants preserves their parent UI; leaving restores it once.
|
|
8
|
+
func move(to target: AXUIElement?, in route: WindowRoute, access: AXAccess) throws -> Bool {
|
|
9
|
+
var current = target
|
|
10
|
+
var visited: [AXUIElement] = []
|
|
11
|
+
var alternate: AXUIElement?
|
|
12
|
+
while let node = current {
|
|
13
|
+
guard !visited.contains(where: { CFEqual($0, node) }) else {
|
|
14
|
+
throw NativeFailure(code: "NATIVE_ERROR", message: "Accessibility hover ancestry contains a cycle; read fresh state")
|
|
15
|
+
}
|
|
16
|
+
visited.append(node)
|
|
17
|
+
if let revealed, revealed.windowID == route.windowID, CFEqual(revealed.element, node) { return true }
|
|
18
|
+
let actions = try access.actions(node)
|
|
19
|
+
if alternate == nil, actions.contains("AXShowAlternateUI"), actions.contains("AXShowDefaultUI"),
|
|
20
|
+
try access.boolean(node, "AXEnabled") != false { alternate = node }
|
|
21
|
+
let role = try access.string(node, "AXRole")
|
|
22
|
+
if role == "AXWindow" || role == "AXSheet" || role == "AXApplication" { break }
|
|
23
|
+
current = try access.element(node, "AXParent")
|
|
24
|
+
}
|
|
25
|
+
let leaving = revealed != nil
|
|
26
|
+
if let revealed {
|
|
27
|
+
try access.perform(revealed.element, "AXShowDefaultUI")
|
|
28
|
+
self.revealed = nil
|
|
29
|
+
}
|
|
30
|
+
if let alternate {
|
|
31
|
+
try access.perform(alternate, "AXShowAlternateUI")
|
|
32
|
+
revealed = (alternate, route.windowID)
|
|
33
|
+
return true
|
|
34
|
+
}
|
|
35
|
+
return leaving
|
|
36
|
+
}
|
|
37
|
+
}
|