@anionex/dsh-computer-use 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.i18n.yaml +6 -0
- package/README.md +307 -0
- package/README.zh.md +307 -0
- package/assets/computer-use-fixture.png +0 -0
- package/assets/cursor.png +0 -0
- package/cordis.patch.yml +13 -0
- package/docs/interaction-policy.i18n.yaml +4 -0
- package/docs/interaction-policy.md +140 -0
- package/docs/interaction-policy.zh.md +140 -0
- package/lib/approval-policy.js +12 -0
- package/lib/approval-policy.js.map +1 -0
- package/lib/artifacts.js +79 -0
- package/lib/artifacts.js.map +1 -0
- package/lib/backend.js +3 -0
- package/lib/backend.js.map +1 -0
- package/lib/client.js +349 -0
- package/lib/client.js.map +1 -0
- package/lib/config.js +120 -0
- package/lib/config.js.map +1 -0
- package/lib/confirmations.js +99 -0
- package/lib/confirmations.js.map +1 -0
- package/lib/diff.js +58 -0
- package/lib/diff.js.map +1 -0
- package/lib/errors.js +22 -0
- package/lib/errors.js.map +1 -0
- package/lib/exposure.js +231 -0
- package/lib/exposure.js.map +1 -0
- package/lib/index.js +56 -0
- package/lib/index.js.map +1 -0
- package/lib/leases.js +259 -0
- package/lib/leases.js.map +1 -0
- package/lib/providers/macos.js +148 -0
- package/lib/providers/macos.js.map +1 -0
- package/lib/providers/native-helper.js +356 -0
- package/lib/providers/native-helper.js.map +1 -0
- package/lib/service.js +480 -0
- package/lib/service.js.map +1 -0
- package/lib/skill.js +119 -0
- package/lib/skill.js.map +1 -0
- package/lib/target-resolver.js +180 -0
- package/lib/target-resolver.js.map +1 -0
- package/lib/tools.js +493 -0
- package/lib/tools.js.map +1 -0
- package/lib/types/approval-policy.d.ts +13 -0
- package/lib/types/approval-policy.d.ts.map +1 -0
- package/lib/types/artifacts.d.ts +10 -0
- package/lib/types/artifacts.d.ts.map +1 -0
- package/lib/types/backend.d.ts +94 -0
- package/lib/types/backend.d.ts.map +1 -0
- package/lib/types/client/index.d.ts +78 -0
- package/lib/types/client/index.d.ts.map +1 -0
- package/lib/types/config.d.ts +75 -0
- package/lib/types/config.d.ts.map +1 -0
- package/lib/types/confirmations.d.ts +23 -0
- package/lib/types/confirmations.d.ts.map +1 -0
- package/lib/types/diff.d.ts +5 -0
- package/lib/types/diff.d.ts.map +1 -0
- package/lib/types/errors.d.ts +15 -0
- package/lib/types/errors.d.ts.map +1 -0
- package/lib/types/exposure.d.ts +30 -0
- package/lib/types/exposure.d.ts.map +1 -0
- package/lib/types/index.d.ts +21 -0
- package/lib/types/index.d.ts.map +1 -0
- package/lib/types/leases.d.ts +80 -0
- package/lib/types/leases.d.ts.map +1 -0
- package/lib/types/providers/macos.d.ts +15 -0
- package/lib/types/providers/macos.d.ts.map +1 -0
- package/lib/types/providers/native-helper.d.ts +36 -0
- package/lib/types/providers/native-helper.d.ts.map +1 -0
- package/lib/types/service.d.ts +52 -0
- package/lib/types/service.d.ts.map +1 -0
- package/lib/types/skill.d.ts +9 -0
- package/lib/types/skill.d.ts.map +1 -0
- package/lib/types/target-resolver.d.ts +38 -0
- package/lib/types/target-resolver.d.ts.map +1 -0
- package/lib/types/tools.d.ts +6 -0
- package/lib/types/tools.d.ts.map +1 -0
- package/lib/types/types.d.ts +246 -0
- package/lib/types/types.d.ts.map +1 -0
- package/lib/types/web.d.ts +32 -0
- package/lib/types/web.d.ts.map +1 -0
- package/lib/types.js +9 -0
- package/lib/types.js.map +1 -0
- package/lib/web.js +163 -0
- package/lib/web.js.map +1 -0
- package/native/macos/Sources/Fixture/main.swift +366 -0
- package/native/macos/Sources/Helper/CursorImage.swift +11 -0
- package/native/macos/Sources/Helper/CursorOverlay.swift +488 -0
- package/native/macos/Sources/Helper/TargetedPointer.swift +314 -0
- package/native/macos/Sources/Helper/main.swift +1140 -0
- package/native/macos/Sources/Monitor/main.swift +139 -0
- package/native/macos/bin/dsh-computer-use-helper +0 -0
- package/native/macos/manifest.json +14 -0
- package/package.json +128 -0
- package/scripts/build-client.mjs +24 -0
- package/scripts/build-native.mjs +158 -0
- package/scripts/check-native.mjs +92 -0
- package/scripts/model-e2e.mjs +291 -0
- package/scripts/session-transcript.mjs +117 -0
- package/scripts/validate.mjs +256 -0
- package/src/approval-policy.ts +16 -0
- package/src/artifacts.ts +93 -0
- package/src/backend.ts +103 -0
- package/src/client/index.tsx +539 -0
- package/src/config.ts +196 -0
- package/src/confirmations.ts +128 -0
- package/src/diff.ts +52 -0
- package/src/errors.ts +39 -0
- package/src/exposure.ts +236 -0
- package/src/index.ts +66 -0
- package/src/leases.ts +337 -0
- package/src/providers/macos.ts +185 -0
- package/src/providers/native-helper.ts +393 -0
- package/src/service.ts +583 -0
- package/src/skill.ts +123 -0
- package/src/target-resolver.ts +237 -0
- package/src/tools.ts +528 -0
- package/src/types.ts +299 -0
- package/src/web.ts +180 -0
|
@@ -0,0 +1,1140 @@
|
|
|
1
|
+
import AppKit
|
|
2
|
+
import ApplicationServices
|
|
3
|
+
import CoreGraphics
|
|
4
|
+
import CryptoKit
|
|
5
|
+
import Darwin
|
|
6
|
+
import Foundation
|
|
7
|
+
import ScreenCaptureKit
|
|
8
|
+
|
|
9
|
+
private let helperVersion = "0.1.0"
|
|
10
|
+
|
|
11
|
+
private struct HelperError: Error {
|
|
12
|
+
let code: String
|
|
13
|
+
let message: String
|
|
14
|
+
}
|
|
15
|
+
|
|
16
|
+
private struct ElementRecord {
|
|
17
|
+
let index: Int
|
|
18
|
+
let locator: [Int]
|
|
19
|
+
let element: AXUIElement
|
|
20
|
+
let json: [String: Any]
|
|
21
|
+
let hashLine: String
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
private struct ObservationSnapshot {
|
|
25
|
+
let app: NSRunningApplication
|
|
26
|
+
let appJSON: [String: Any]
|
|
27
|
+
let frontmost: Bool
|
|
28
|
+
let window: AXUIElement?
|
|
29
|
+
let windowJSON: [String: Any]?
|
|
30
|
+
let stateHash: String
|
|
31
|
+
let treeText: String
|
|
32
|
+
let truncated: Bool
|
|
33
|
+
let elements: [ElementRecord]
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
private func fail(_ code: String, _ message: String) -> HelperError {
|
|
37
|
+
HelperError(code: code, message: message)
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
private func string(_ object: Any?, _ field: String) throws -> String {
|
|
41
|
+
guard let value = object as? String, !value.isEmpty else {
|
|
42
|
+
throw fail("COMPUTER_PROVIDER_FAILURE", "missing or invalid \(field)")
|
|
43
|
+
}
|
|
44
|
+
return value
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
private func stringAllowingEmpty(_ object: Any?, _ field: String) throws -> String {
|
|
48
|
+
guard let value = object as? String else {
|
|
49
|
+
throw fail("COMPUTER_PROVIDER_FAILURE", "missing or invalid \(field)")
|
|
50
|
+
}
|
|
51
|
+
return value
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
private func int(_ object: Any?, _ field: String) throws -> Int {
|
|
55
|
+
guard let number = object as? NSNumber else {
|
|
56
|
+
throw fail("COMPUTER_PROVIDER_FAILURE", "missing or invalid \(field)")
|
|
57
|
+
}
|
|
58
|
+
return number.intValue
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
private func double(_ object: Any?, _ field: String) throws -> Double {
|
|
62
|
+
guard let number = object as? NSNumber else {
|
|
63
|
+
throw fail("COMPUTER_PROVIDER_FAILURE", "missing or invalid \(field)")
|
|
64
|
+
}
|
|
65
|
+
return number.doubleValue
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
private func bool(_ object: Any?, default fallback: Bool = false) -> Bool {
|
|
69
|
+
(object as? NSNumber)?.boolValue ?? fallback
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
private func dictionary(_ object: Any?, _ field: String) throws -> [String: Any] {
|
|
73
|
+
guard let value = object as? [String: Any] else {
|
|
74
|
+
throw fail("COMPUTER_PROVIDER_FAILURE", "missing or invalid \(field)")
|
|
75
|
+
}
|
|
76
|
+
return value
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
private func dictionaries(_ object: Any?) -> [[String: Any]] {
|
|
80
|
+
object as? [[String: Any]] ?? []
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
private enum TransportEndpoint: Hashable {
|
|
84
|
+
case pipe(handle: UInt64, peer: UInt64)
|
|
85
|
+
case unixSocket(handle: UInt64, peer: UInt64)
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
private func processDescriptors(_ pid: Int32) -> [proc_fdinfo] {
|
|
89
|
+
let capacity = Int(max(proc_pidinfo(pid, PROC_PIDLISTFDS, 0, nil, 0), 0))
|
|
90
|
+
guard capacity >= MemoryLayout<proc_fdinfo>.size else { return [] }
|
|
91
|
+
var descriptors = Array(repeating: proc_fdinfo(), count: capacity / MemoryLayout<proc_fdinfo>.size)
|
|
92
|
+
let bytes = descriptors.withUnsafeMutableBytes { buffer in
|
|
93
|
+
proc_pidinfo(pid, PROC_PIDLISTFDS, 0, buffer.baseAddress, Int32(buffer.count))
|
|
94
|
+
}
|
|
95
|
+
guard bytes > 0 else { return [] }
|
|
96
|
+
return Array(descriptors.prefix(Int(bytes) / MemoryLayout<proc_fdinfo>.size))
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
private func transportEndpoint(_ pid: Int32, _ fd: Int32) -> TransportEndpoint? {
|
|
100
|
+
var pipe = pipe_fdinfo()
|
|
101
|
+
let pipeSize = MemoryLayout<pipe_fdinfo>.size
|
|
102
|
+
if proc_pidfdinfo(pid, fd, PROC_PIDFDPIPEINFO, &pipe, Int32(pipeSize)) == pipeSize {
|
|
103
|
+
return .pipe(handle: pipe.pipeinfo.pipe_handle, peer: pipe.pipeinfo.pipe_peerhandle)
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
var socket = socket_fdinfo()
|
|
107
|
+
let socketSize = MemoryLayout<socket_fdinfo>.size
|
|
108
|
+
guard proc_pidfdinfo(pid, fd, PROC_PIDFDSOCKETINFO, &socket, Int32(socketSize)) == socketSize,
|
|
109
|
+
socket.psi.soi_family == AF_UNIX,
|
|
110
|
+
socket.psi.soi_type == SOCK_STREAM,
|
|
111
|
+
socket.psi.soi_kind == SOCKINFO_UN else { return nil }
|
|
112
|
+
return .unixSocket(
|
|
113
|
+
handle: socket.psi.soi_so,
|
|
114
|
+
peer: socket.psi.soi_proto.pri_un.unsi_conn_so
|
|
115
|
+
)
|
|
116
|
+
}
|
|
117
|
+
|
|
118
|
+
private func transportEndpointsMatch(_ child: TransportEndpoint, _ parent: TransportEndpoint) -> Bool {
|
|
119
|
+
switch (child, parent) {
|
|
120
|
+
case let (.pipe(childHandle, childPeer), .pipe(parentHandle, parentPeer)),
|
|
121
|
+
let (.unixSocket(childHandle, childPeer), .unixSocket(parentHandle, parentPeer)):
|
|
122
|
+
return childPeer != 0
|
|
123
|
+
&& childHandle == parentPeer
|
|
124
|
+
&& childPeer == parentHandle
|
|
125
|
+
default:
|
|
126
|
+
return false
|
|
127
|
+
}
|
|
128
|
+
}
|
|
129
|
+
|
|
130
|
+
private func disconnectedUnixSocket(_ endpoint: TransportEndpoint) -> Bool {
|
|
131
|
+
if case let .unixSocket(_, peer) = endpoint { return peer == 0 }
|
|
132
|
+
return false
|
|
133
|
+
}
|
|
134
|
+
|
|
135
|
+
private func parentOwnsStandardTransport() -> Bool {
|
|
136
|
+
let childEndpoints = [STDIN_FILENO, STDOUT_FILENO, STDERR_FILENO].compactMap { fd in
|
|
137
|
+
transportEndpoint(getpid(), fd).map { (fd, $0) }
|
|
138
|
+
}
|
|
139
|
+
guard childEndpoints.count == 3,
|
|
140
|
+
Set(childEndpoints.map(\.1)).count == 3 else { return false }
|
|
141
|
+
|
|
142
|
+
let parentEndpoints = processDescriptors(getppid()).compactMap { descriptor in
|
|
143
|
+
transportEndpoint(getppid(), descriptor.proc_fd)
|
|
144
|
+
}
|
|
145
|
+
return childEndpoints.allSatisfy { fd, child in
|
|
146
|
+
let matches = parentEndpoints.filter { transportEndpointsMatch(child, $0) }
|
|
147
|
+
if matches.count == 1 { return true }
|
|
148
|
+
// The batch stdin writer may close its parent endpoint before the helper
|
|
149
|
+
// reaches main. Node represents that standard stream as a disconnected
|
|
150
|
+
// Unix socket, while stdout and stderr remain provably parent-owned.
|
|
151
|
+
return fd == STDIN_FILENO && matches.isEmpty && disconnectedUnixSocket(child)
|
|
152
|
+
}
|
|
153
|
+
}
|
|
154
|
+
|
|
155
|
+
private func requireHostTransport() throws {
|
|
156
|
+
guard getpgrp() == getpid(),
|
|
157
|
+
parentOwnsStandardTransport() else {
|
|
158
|
+
throw fail("COMPUTER_ACTION_BLOCKED", "native helper requires managed parent transport; use the registered Computer Use Tools")
|
|
159
|
+
}
|
|
160
|
+
}
|
|
161
|
+
|
|
162
|
+
private func permissionAccessibility() -> String {
|
|
163
|
+
AXIsProcessTrusted() ? "granted" : "denied"
|
|
164
|
+
}
|
|
165
|
+
|
|
166
|
+
private func permissionScreenRecording() -> String {
|
|
167
|
+
CGPreflightScreenCaptureAccess() ? "granted" : "denied"
|
|
168
|
+
}
|
|
169
|
+
|
|
170
|
+
private func runningApps() -> [NSRunningApplication] {
|
|
171
|
+
NSWorkspace.shared.runningApplications
|
|
172
|
+
.filter { !$0.isTerminated && $0.activationPolicy == .regular && $0.bundleIdentifier != nil }
|
|
173
|
+
.sorted {
|
|
174
|
+
let lhs = $0.localizedName ?? $0.bundleIdentifier ?? ""
|
|
175
|
+
let rhs = $1.localizedName ?? $1.bundleIdentifier ?? ""
|
|
176
|
+
return lhs.localizedCaseInsensitiveCompare(rhs) == .orderedAscending
|
|
177
|
+
}
|
|
178
|
+
}
|
|
179
|
+
|
|
180
|
+
private func appJSON(_ app: NSRunningApplication) throws -> [String: Any] {
|
|
181
|
+
guard let bundleId = app.bundleIdentifier else {
|
|
182
|
+
throw fail("COMPUTER_APP_NOT_FOUND", "running application has no bundle identifier")
|
|
183
|
+
}
|
|
184
|
+
return [
|
|
185
|
+
"bundleId": bundleId,
|
|
186
|
+
"pid": Int(app.processIdentifier),
|
|
187
|
+
"name": app.localizedName ?? bundleId,
|
|
188
|
+
]
|
|
189
|
+
}
|
|
190
|
+
|
|
191
|
+
private func resolveApp(_ selector: [String: Any]) throws -> NSRunningApplication {
|
|
192
|
+
let bundleId = selector["bundleId"] as? String
|
|
193
|
+
let pid = (selector["pid"] as? NSNumber)?.int32Value
|
|
194
|
+
let name = selector["name"] as? String
|
|
195
|
+
if bundleId == nil && pid == nil && name == nil {
|
|
196
|
+
throw fail("COMPUTER_APP_NOT_FOUND", "app selector needs bundleId, pid, or unique name")
|
|
197
|
+
}
|
|
198
|
+
let matches = runningApps().filter { app in
|
|
199
|
+
if let bundleId, app.bundleIdentifier != bundleId { return false }
|
|
200
|
+
if let pid, app.processIdentifier != pid { return false }
|
|
201
|
+
if let name {
|
|
202
|
+
let display = app.localizedName ?? ""
|
|
203
|
+
if display.compare(name, options: [.caseInsensitive, .diacriticInsensitive]) != .orderedSame { return false }
|
|
204
|
+
}
|
|
205
|
+
return true
|
|
206
|
+
}
|
|
207
|
+
guard matches.count == 1, let app = matches.first else {
|
|
208
|
+
if matches.isEmpty { throw fail("COMPUTER_APP_NOT_FOUND", "no running application uniquely matches the selector") }
|
|
209
|
+
if bundleId != nil {
|
|
210
|
+
throw fail("COMPUTER_APP_NOT_FOUND", "multiple running processes match the bundleId; add the exact pid from computer_list_apps")
|
|
211
|
+
}
|
|
212
|
+
throw fail("COMPUTER_APP_NOT_FOUND", "application name is ambiguous; use bundleId and, if necessary, pid")
|
|
213
|
+
}
|
|
214
|
+
return app
|
|
215
|
+
}
|
|
216
|
+
|
|
217
|
+
private func axCopy(_ element: AXUIElement, _ attribute: CFString) -> Any? {
|
|
218
|
+
var value: CFTypeRef?
|
|
219
|
+
let result = AXUIElementCopyAttributeValue(element, attribute, &value)
|
|
220
|
+
return result == .success ? value : nil
|
|
221
|
+
}
|
|
222
|
+
|
|
223
|
+
private func axString(_ element: AXUIElement, _ attribute: CFString) -> String? {
|
|
224
|
+
if let value = axCopy(element, attribute) as? String, !value.isEmpty { return value }
|
|
225
|
+
if let value = axCopy(element, attribute) as? NSAttributedString, !value.string.isEmpty { return value.string }
|
|
226
|
+
return nil
|
|
227
|
+
}
|
|
228
|
+
|
|
229
|
+
private func axBool(_ element: AXUIElement, _ attribute: CFString) -> Bool? {
|
|
230
|
+
(axCopy(element, attribute) as? NSNumber)?.boolValue
|
|
231
|
+
}
|
|
232
|
+
|
|
233
|
+
private func axPoint(_ element: AXUIElement, _ attribute: CFString) -> CGPoint? {
|
|
234
|
+
guard let value = axCopy(element, attribute), CFGetTypeID(value as CFTypeRef) == AXValueGetTypeID() else { return nil }
|
|
235
|
+
let axValue = value as! AXValue
|
|
236
|
+
guard AXValueGetType(axValue) == .cgPoint else { return nil }
|
|
237
|
+
var point = CGPoint.zero
|
|
238
|
+
return AXValueGetValue(axValue, .cgPoint, &point) ? point : nil
|
|
239
|
+
}
|
|
240
|
+
|
|
241
|
+
private func axSize(_ element: AXUIElement, _ attribute: CFString) -> CGSize? {
|
|
242
|
+
guard let value = axCopy(element, attribute), CFGetTypeID(value as CFTypeRef) == AXValueGetTypeID() else { return nil }
|
|
243
|
+
let axValue = value as! AXValue
|
|
244
|
+
guard AXValueGetType(axValue) == .cgSize else { return nil }
|
|
245
|
+
var size = CGSize.zero
|
|
246
|
+
return AXValueGetValue(axValue, .cgSize, &size) ? size : nil
|
|
247
|
+
}
|
|
248
|
+
|
|
249
|
+
private func axFrame(_ element: AXUIElement) -> CGRect? {
|
|
250
|
+
guard let position = axPoint(element, kAXPositionAttribute as CFString),
|
|
251
|
+
let size = axSize(element, kAXSizeAttribute as CFString),
|
|
252
|
+
size.width >= 0, size.height >= 0 else { return nil }
|
|
253
|
+
return CGRect(origin: position, size: size)
|
|
254
|
+
}
|
|
255
|
+
|
|
256
|
+
private func rectJSON(_ rect: CGRect) -> [String: Any] {
|
|
257
|
+
["x": rect.origin.x, "y": rect.origin.y, "width": rect.size.width, "height": rect.size.height]
|
|
258
|
+
}
|
|
259
|
+
|
|
260
|
+
private func axActions(_ element: AXUIElement) -> [String] {
|
|
261
|
+
var names: CFArray?
|
|
262
|
+
guard AXUIElementCopyActionNames(element, &names) == .success, let names else { return [] }
|
|
263
|
+
return (names as NSArray).compactMap { $0 as? String }.sorted()
|
|
264
|
+
}
|
|
265
|
+
|
|
266
|
+
private func axChildren(_ element: AXUIElement) -> [AXUIElement] {
|
|
267
|
+
guard let values = axCopy(element, kAXChildrenAttribute as CFString) as? [AnyObject] else { return [] }
|
|
268
|
+
return values.compactMap { value in
|
|
269
|
+
guard CFGetTypeID(value) == AXUIElementGetTypeID() else { return nil }
|
|
270
|
+
return (value as! AXUIElement)
|
|
271
|
+
}
|
|
272
|
+
}
|
|
273
|
+
|
|
274
|
+
private func pressWithDescendantFallback(_ element: AXUIElement) -> Bool {
|
|
275
|
+
// Only descend when the selected element advertised AXPress; otherwise the
|
|
276
|
+
// element/coordinate fallback below owns the click decision.
|
|
277
|
+
guard axActions(element).contains(kAXPressAction as String) else { return false }
|
|
278
|
+
var visited = Set<CFHashCode>()
|
|
279
|
+
return pressRecursive(element, remainingDepth: 4, visited: &visited)
|
|
280
|
+
}
|
|
281
|
+
|
|
282
|
+
private func pressRecursive(_ element: AXUIElement, remainingDepth: Int, visited: inout Set<CFHashCode>) -> Bool {
|
|
283
|
+
guard remainingDepth >= 0 else { return false }
|
|
284
|
+
let identity = CFHash(element)
|
|
285
|
+
guard visited.insert(identity).inserted else { return false }
|
|
286
|
+
if axActions(element).contains(kAXPressAction as String),
|
|
287
|
+
AXUIElementPerformAction(element, kAXPressAction as CFString) == .success {
|
|
288
|
+
return true
|
|
289
|
+
}
|
|
290
|
+
for child in axChildren(element) {
|
|
291
|
+
if pressRecursive(child, remainingDepth: remainingDepth - 1, visited: &visited) { return true }
|
|
292
|
+
}
|
|
293
|
+
return false
|
|
294
|
+
}
|
|
295
|
+
|
|
296
|
+
private func sanitize(_ value: String, max: Int = 240) -> String {
|
|
297
|
+
let collapsed = value.replacingOccurrences(of: "\\s+", with: " ", options: .regularExpression).trimmingCharacters(in: .whitespacesAndNewlines)
|
|
298
|
+
guard collapsed.count > max else { return collapsed }
|
|
299
|
+
return String(collapsed.prefix(max)) + "..."
|
|
300
|
+
}
|
|
301
|
+
|
|
302
|
+
private func jsonString(_ value: String) -> String {
|
|
303
|
+
let data = try? JSONSerialization.data(withJSONObject: [value])
|
|
304
|
+
let encoded = data.flatMap { String(data: $0, encoding: .utf8) } ?? "[\"\"]"
|
|
305
|
+
return String(encoded.dropFirst().dropLast())
|
|
306
|
+
}
|
|
307
|
+
|
|
308
|
+
private func chosenWindow(_ appElement: AXUIElement) -> AXUIElement? {
|
|
309
|
+
if let focused = axCopy(appElement, kAXFocusedWindowAttribute as CFString) as AnyObject?,
|
|
310
|
+
CFGetTypeID(focused) == AXUIElementGetTypeID() {
|
|
311
|
+
return (focused as! AXUIElement)
|
|
312
|
+
}
|
|
313
|
+
if let main = axCopy(appElement, kAXMainWindowAttribute as CFString) as AnyObject?,
|
|
314
|
+
CFGetTypeID(main) == AXUIElementGetTypeID() {
|
|
315
|
+
return (main as! AXUIElement)
|
|
316
|
+
}
|
|
317
|
+
return axChildren(appElement).first
|
|
318
|
+
}
|
|
319
|
+
|
|
320
|
+
private func windowNumber(_ element: AXUIElement) -> Int? {
|
|
321
|
+
(axCopy(element, "AXWindowNumber" as CFString) as? NSNumber)?.intValue
|
|
322
|
+
}
|
|
323
|
+
|
|
324
|
+
private func windowNumber(
|
|
325
|
+
app: NSRunningApplication,
|
|
326
|
+
frame: CGRect?,
|
|
327
|
+
title: String?
|
|
328
|
+
) -> Int? {
|
|
329
|
+
guard let windows = CGWindowListCopyWindowInfo(
|
|
330
|
+
[.optionOnScreenOnly, .excludeDesktopElements],
|
|
331
|
+
kCGNullWindowID
|
|
332
|
+
) as? [[String: Any]] else { return nil }
|
|
333
|
+
let candidates = windows.filter { window in
|
|
334
|
+
guard (window[kCGWindowOwnerPID as String] as? NSNumber)?.int32Value == app.processIdentifier,
|
|
335
|
+
(window[kCGWindowLayer as String] as? NSNumber)?.intValue == 0 else { return false }
|
|
336
|
+
if let frame {
|
|
337
|
+
guard let bounds = window[kCGWindowBounds as String] as? [String: Any],
|
|
338
|
+
let candidate = CGRect(dictionaryRepresentation: bounds as CFDictionary) else { return false }
|
|
339
|
+
let tolerance: CGFloat = 2
|
|
340
|
+
guard abs(candidate.minX - frame.minX) <= tolerance,
|
|
341
|
+
abs(candidate.minY - frame.minY) <= tolerance,
|
|
342
|
+
abs(candidate.width - frame.width) <= tolerance,
|
|
343
|
+
abs(candidate.height - frame.height) <= tolerance else { return false }
|
|
344
|
+
}
|
|
345
|
+
if let title, !title.isEmpty {
|
|
346
|
+
guard let candidateTitle = window[kCGWindowName as String] as? String,
|
|
347
|
+
candidateTitle == title else { return false }
|
|
348
|
+
}
|
|
349
|
+
return true
|
|
350
|
+
}
|
|
351
|
+
guard candidates.count == 1 else { return nil }
|
|
352
|
+
return (candidates[0][kCGWindowNumber as String] as? NSNumber)?.intValue
|
|
353
|
+
}
|
|
354
|
+
|
|
355
|
+
private func sha256(_ value: String) -> String {
|
|
356
|
+
SHA256.hash(data: Data(value.utf8)).map { String(format: "%02x", $0) }.joined()
|
|
357
|
+
}
|
|
358
|
+
|
|
359
|
+
private func observeSnapshot(app: NSRunningApplication, limits: [String: Any]) throws -> ObservationSnapshot {
|
|
360
|
+
guard AXIsProcessTrusted() else {
|
|
361
|
+
throw fail("COMPUTER_PERMISSION_REQUIRED", "macOS Accessibility permission is required for the DSH helper")
|
|
362
|
+
}
|
|
363
|
+
let maxNodes = try int(limits["maxNodes"], "limits.maxNodes")
|
|
364
|
+
let maxDepth = try int(limits["maxDepth"], "limits.maxDepth")
|
|
365
|
+
let maxTextBytes = try int(limits["maxTextBytes"], "limits.maxTextBytes")
|
|
366
|
+
guard maxNodes > 0, maxDepth > 0, maxTextBytes > 0 else {
|
|
367
|
+
throw fail("COMPUTER_PROVIDER_FAILURE", "observation limits must be positive")
|
|
368
|
+
}
|
|
369
|
+
let appElement = AXUIElementCreateApplication(app.processIdentifier)
|
|
370
|
+
let rootWindow = chosenWindow(appElement)
|
|
371
|
+
let root = rootWindow ?? appElement
|
|
372
|
+
let frame = rootWindow.flatMap(axFrame)
|
|
373
|
+
let title = rootWindow.flatMap { axString($0, kAXTitleAttribute as CFString) }
|
|
374
|
+
let windowId = rootWindow.flatMap(windowNumber)
|
|
375
|
+
?? windowNumber(app: app, frame: frame, title: title)
|
|
376
|
+
var windowJSON: [String: Any]?
|
|
377
|
+
if let frame {
|
|
378
|
+
var json: [String: Any] = ["frame": rectJSON(frame)]
|
|
379
|
+
if let title { json["title"] = title }
|
|
380
|
+
if let windowId { json["id"] = windowId }
|
|
381
|
+
windowJSON = json
|
|
382
|
+
}
|
|
383
|
+
|
|
384
|
+
var queue: [(AXUIElement, [Int], Int)] = [(root, [], 0)]
|
|
385
|
+
var cursor = 0
|
|
386
|
+
var seen = Set<CFHashCode>()
|
|
387
|
+
var records: [ElementRecord] = []
|
|
388
|
+
var lines: [String] = []
|
|
389
|
+
var textBytes = 0
|
|
390
|
+
var truncated = false
|
|
391
|
+
|
|
392
|
+
while cursor < queue.count {
|
|
393
|
+
if records.count >= maxNodes { truncated = true; break }
|
|
394
|
+
let (element, locator, depth) = queue[cursor]
|
|
395
|
+
cursor += 1
|
|
396
|
+
let identity = CFHash(element)
|
|
397
|
+
if seen.contains(identity) { continue }
|
|
398
|
+
seen.insert(identity)
|
|
399
|
+
let role = axString(element, kAXRoleAttribute as CFString) ?? "AXUnknown"
|
|
400
|
+
let subrole = axString(element, kAXSubroleAttribute as CFString)
|
|
401
|
+
let nativeIdentifier = axString(element, kAXIdentifierAttribute as CFString)
|
|
402
|
+
let titleValue = axString(element, kAXTitleAttribute as CFString)
|
|
403
|
+
let descriptionValue = axString(element, kAXDescriptionAttribute as CFString)
|
|
404
|
+
let labelValue = titleValue ?? descriptionValue
|
|
405
|
+
let secure = role == "AXSecureTextField" || subrole == "AXSecureTextField"
|
|
406
|
+
let valueValue: String? = secure ? "[secure]" : {
|
|
407
|
+
guard let raw = axCopy(element, kAXValueAttribute as CFString) else { return nil }
|
|
408
|
+
if let string = raw as? String { return sanitize(string) }
|
|
409
|
+
if let number = raw as? NSNumber { return number.stringValue }
|
|
410
|
+
return nil
|
|
411
|
+
}()
|
|
412
|
+
let enabled = axBool(element, kAXEnabledAttribute as CFString)
|
|
413
|
+
let focused = axBool(element, kAXFocusedAttribute as CFString)
|
|
414
|
+
let selected = axBool(element, kAXSelectedAttribute as CFString)
|
|
415
|
+
let elementFrame = axFrame(element)
|
|
416
|
+
let actions = axActions(element)
|
|
417
|
+
let index = records.count
|
|
418
|
+
var json: [String: Any] = [
|
|
419
|
+
"index": index,
|
|
420
|
+
"role": role,
|
|
421
|
+
"actions": actions,
|
|
422
|
+
"locator": locator,
|
|
423
|
+
]
|
|
424
|
+
if let subrole { json["subrole"] = subrole }
|
|
425
|
+
if let nativeIdentifier { json["nativeIdentifier"] = sanitize(nativeIdentifier) }
|
|
426
|
+
if let titleValue { json["title"] = sanitize(titleValue) }
|
|
427
|
+
if let labelValue { json["label"] = sanitize(labelValue) }
|
|
428
|
+
if let valueValue { json["value"] = valueValue }
|
|
429
|
+
if let enabled { json["enabled"] = enabled }
|
|
430
|
+
if let focused { json["focused"] = focused }
|
|
431
|
+
if let selected { json["selected"] = selected }
|
|
432
|
+
if let elementFrame { json["frame"] = rectJSON(elementFrame) }
|
|
433
|
+
let frameKey = elementFrame.map { "\(Int($0.origin.x)),\(Int($0.origin.y)),\(Int($0.width)),\(Int($0.height))" } ?? ""
|
|
434
|
+
let hashLine = [locator.map(String.init).joined(separator: "."), nativeIdentifier ?? "", role, subrole ?? "", titleValue ?? "", labelValue ?? "", valueValue ?? "", String(enabled ?? true), String(focused ?? false), String(selected ?? false), frameKey, actions.joined(separator: ",")].joined(separator: "|")
|
|
435
|
+
var line = String(repeating: " ", count: min(depth, 20)) + "[\(index)] \(role)"
|
|
436
|
+
if let titleValue { line += " \(jsonString(sanitize(titleValue)))" }
|
|
437
|
+
else if let labelValue { line += " \(jsonString(sanitize(labelValue)))" }
|
|
438
|
+
if let valueValue { line += " value=\(jsonString(valueValue))" }
|
|
439
|
+
if enabled == false { line += " disabled" }
|
|
440
|
+
if focused == true { line += " focused" }
|
|
441
|
+
if selected == true { line += " selected" }
|
|
442
|
+
let lineBytes = line.lengthOfBytes(using: .utf8) + (lines.isEmpty ? 0 : 1)
|
|
443
|
+
if textBytes + lineBytes <= maxTextBytes {
|
|
444
|
+
lines.append(line)
|
|
445
|
+
textBytes += lineBytes
|
|
446
|
+
} else {
|
|
447
|
+
truncated = true
|
|
448
|
+
}
|
|
449
|
+
records.append(ElementRecord(index: index, locator: locator, element: element, json: json, hashLine: hashLine))
|
|
450
|
+
if depth >= maxDepth {
|
|
451
|
+
if !axChildren(element).isEmpty { truncated = true }
|
|
452
|
+
continue
|
|
453
|
+
}
|
|
454
|
+
for (childIndex, child) in axChildren(element).enumerated() {
|
|
455
|
+
queue.append((child, locator + [childIndex], depth + 1))
|
|
456
|
+
}
|
|
457
|
+
}
|
|
458
|
+
let appData = try appJSON(app)
|
|
459
|
+
let windowHash = [
|
|
460
|
+
title ?? "",
|
|
461
|
+
windowId.map(String.init) ?? "",
|
|
462
|
+
frame.map { "\(Int($0.origin.x)),\(Int($0.origin.y)),\(Int($0.width)),\(Int($0.height))" } ?? "",
|
|
463
|
+
].joined(separator: "|")
|
|
464
|
+
let state = [appData["bundleId"] as? String ?? "", String(app.processIdentifier), windowHash, records.map(\.hashLine).joined(separator: "\n")].joined(separator: "\n")
|
|
465
|
+
return ObservationSnapshot(
|
|
466
|
+
app: app,
|
|
467
|
+
appJSON: appData,
|
|
468
|
+
frontmost: app.isActive,
|
|
469
|
+
window: rootWindow,
|
|
470
|
+
windowJSON: windowJSON,
|
|
471
|
+
stateHash: sha256(state),
|
|
472
|
+
treeText: lines.joined(separator: "\n"),
|
|
473
|
+
truncated: truncated,
|
|
474
|
+
elements: records
|
|
475
|
+
)
|
|
476
|
+
}
|
|
477
|
+
|
|
478
|
+
private func captureWindow(_ snapshot: ObservationSnapshot, path: String, required: Bool) async throws -> [String: Any]? {
|
|
479
|
+
guard CGPreflightScreenCaptureAccess() else {
|
|
480
|
+
if required { throw fail("COMPUTER_PERMISSION_REQUIRED", "macOS Screen Recording permission is required for screenshots") }
|
|
481
|
+
return nil
|
|
482
|
+
}
|
|
483
|
+
let content: SCShareableContent
|
|
484
|
+
do {
|
|
485
|
+
content = try await SCShareableContent.excludingDesktopWindows(false, onScreenWindowsOnly: true)
|
|
486
|
+
} catch {
|
|
487
|
+
if required { throw fail("COMPUTER_PROVIDER_FAILURE", "ScreenCaptureKit could not enumerate windows") }
|
|
488
|
+
return nil
|
|
489
|
+
}
|
|
490
|
+
let expectedId = (snapshot.windowJSON?["id"] as? NSNumber)?.uint32Value
|
|
491
|
+
let expectedTitle = snapshot.windowJSON?["title"] as? String
|
|
492
|
+
let windows = content.windows.filter { $0.owningApplication?.processID == snapshot.app.processIdentifier }
|
|
493
|
+
let selected = windows.first { window in
|
|
494
|
+
if let expectedId, window.windowID == expectedId { return true }
|
|
495
|
+
if let expectedTitle, !expectedTitle.isEmpty, window.title == expectedTitle { return true }
|
|
496
|
+
return false
|
|
497
|
+
} ?? windows.max { $0.frame.width * $0.frame.height < $1.frame.width * $1.frame.height }
|
|
498
|
+
guard let selected else {
|
|
499
|
+
if required { throw fail("COMPUTER_TARGET_UNAVAILABLE", "no capturable window belongs to the selected application") }
|
|
500
|
+
return nil
|
|
501
|
+
}
|
|
502
|
+
let configuration = SCStreamConfiguration()
|
|
503
|
+
configuration.width = max(1, Int(selected.frame.width.rounded()))
|
|
504
|
+
configuration.height = max(1, Int(selected.frame.height.rounded()))
|
|
505
|
+
configuration.showsCursor = false
|
|
506
|
+
let filter = SCContentFilter(desktopIndependentWindow: selected)
|
|
507
|
+
let image: CGImage
|
|
508
|
+
do {
|
|
509
|
+
image = try await SCScreenshotManager.captureImage(contentFilter: filter, configuration: configuration)
|
|
510
|
+
} catch {
|
|
511
|
+
if required { throw fail("COMPUTER_PROVIDER_FAILURE", "ScreenCaptureKit could not capture the selected window") }
|
|
512
|
+
return nil
|
|
513
|
+
}
|
|
514
|
+
let representation = NSBitmapImageRep(cgImage: image)
|
|
515
|
+
guard let data = representation.representation(using: .png, properties: [:]) else {
|
|
516
|
+
throw fail("COMPUTER_PROVIDER_FAILURE", "captured window could not be encoded as PNG")
|
|
517
|
+
}
|
|
518
|
+
do {
|
|
519
|
+
try data.write(to: URL(fileURLWithPath: path), options: .atomic)
|
|
520
|
+
} catch {
|
|
521
|
+
throw fail("COMPUTER_PROVIDER_FAILURE", "captured PNG could not be written")
|
|
522
|
+
}
|
|
523
|
+
return ["path": path, "width": image.width, "height": image.height]
|
|
524
|
+
}
|
|
525
|
+
|
|
526
|
+
private func observationJSON(_ snapshot: ObservationSnapshot, screenshot: [String: Any]?) -> [String: Any] {
|
|
527
|
+
var result: [String: Any] = [
|
|
528
|
+
"app": snapshot.appJSON,
|
|
529
|
+
"stateHash": snapshot.stateHash,
|
|
530
|
+
"frontmost": snapshot.frontmost,
|
|
531
|
+
"treeText": snapshot.treeText,
|
|
532
|
+
"truncated": snapshot.truncated,
|
|
533
|
+
"elements": snapshot.elements.map(\.json),
|
|
534
|
+
"permissions": [
|
|
535
|
+
"accessibility": permissionAccessibility(),
|
|
536
|
+
"screenRecording": permissionScreenRecording(),
|
|
537
|
+
],
|
|
538
|
+
]
|
|
539
|
+
if let window = snapshot.windowJSON { result["window"] = window }
|
|
540
|
+
if let screenshot { result["screenshot"] = screenshot }
|
|
541
|
+
return result
|
|
542
|
+
}
|
|
543
|
+
|
|
544
|
+
private func activate(_ app: NSRunningApplication, timeoutMs: Int) throws {
|
|
545
|
+
_ = app.activate(options: [.activateAllWindows])
|
|
546
|
+
let deadline = DispatchTime.now().uptimeNanoseconds + UInt64(timeoutMs) * 1_000_000
|
|
547
|
+
while !app.isActive {
|
|
548
|
+
if DispatchTime.now().uptimeNanoseconds >= deadline {
|
|
549
|
+
throw fail("COMPUTER_ACTION_BLOCKED", "the selected application did not become frontmost before the action timeout")
|
|
550
|
+
}
|
|
551
|
+
usleep(10_000)
|
|
552
|
+
}
|
|
553
|
+
}
|
|
554
|
+
|
|
555
|
+
private func targetedEventSource() throws -> CGEventSource {
|
|
556
|
+
guard let source = CGEventSource(stateID: .privateState) else {
|
|
557
|
+
throw fail("COMPUTER_ACTION_BLOCKED", "CoreGraphics input event source is unavailable")
|
|
558
|
+
}
|
|
559
|
+
return source
|
|
560
|
+
}
|
|
561
|
+
|
|
562
|
+
private func cgRect(_ frame: [String: Any]) throws -> CGRect {
|
|
563
|
+
CGRect(
|
|
564
|
+
x: try double(frame["x"], "window.x"),
|
|
565
|
+
y: try double(frame["y"], "window.y"),
|
|
566
|
+
width: try double(frame["width"], "window.width"),
|
|
567
|
+
height: try double(frame["height"], "window.height")
|
|
568
|
+
)
|
|
569
|
+
}
|
|
570
|
+
|
|
571
|
+
private func windowAtPoint(app: NSRunningApplication, point: CGPoint) throws -> (windowNumber: Int64, frame: CGRect) {
|
|
572
|
+
guard let windows = CGWindowListCopyWindowInfo(
|
|
573
|
+
[.optionOnScreenOnly, .excludeDesktopElements],
|
|
574
|
+
kCGNullWindowID
|
|
575
|
+
) as? [[String: Any]] else {
|
|
576
|
+
throw fail("COMPUTER_TARGET_UNAVAILABLE", "CoreGraphics window list is unavailable")
|
|
577
|
+
}
|
|
578
|
+
// CGWindowListCopyWindowInfo orders on-screen windows front to back, so the
|
|
579
|
+
// first match is the topmost window of the selected app at the point.
|
|
580
|
+
guard let window = windows.first(where: { candidate in
|
|
581
|
+
guard (candidate[kCGWindowOwnerPID as String] as? NSNumber)?.int32Value == app.processIdentifier,
|
|
582
|
+
(candidate[kCGWindowLayer as String] as? NSNumber)?.intValue == 0,
|
|
583
|
+
let bounds = candidate[kCGWindowBounds as String] as? [String: Any],
|
|
584
|
+
let frame = CGRect(dictionaryRepresentation: bounds as CFDictionary),
|
|
585
|
+
frame.contains(point) else { return false }
|
|
586
|
+
return true
|
|
587
|
+
}),
|
|
588
|
+
let windowNumber = (window[kCGWindowNumber as String] as? NSNumber)?.int64Value,
|
|
589
|
+
let bounds = window[kCGWindowBounds as String] as? [String: Any],
|
|
590
|
+
let frame = CGRect(dictionaryRepresentation: bounds as CFDictionary) else {
|
|
591
|
+
throw fail("COMPUTER_TARGET_UNAVAILABLE", "no on-screen window of the selected app contains the requested coordinate")
|
|
592
|
+
}
|
|
593
|
+
return (windowNumber, frame)
|
|
594
|
+
}
|
|
595
|
+
|
|
596
|
+
private func pointerTarget(app: NSRunningApplication, window: [String: Any]?, at point: CGPoint) throws -> TargetedPointerTarget {
|
|
597
|
+
if let windowNumber = (window?["id"] as? NSNumber)?.int64Value,
|
|
598
|
+
let frame = window?["frame"] as? [String: Any],
|
|
599
|
+
let windowFrame = try? cgRect(frame),
|
|
600
|
+
windowFrame.contains(point) {
|
|
601
|
+
return TargetedPointerTarget(
|
|
602
|
+
pid: app.processIdentifier,
|
|
603
|
+
windowNumber: windowNumber,
|
|
604
|
+
windowFrame: windowFrame
|
|
605
|
+
)
|
|
606
|
+
}
|
|
607
|
+
let resolved = try windowAtPoint(app: app, point: point)
|
|
608
|
+
return TargetedPointerTarget(
|
|
609
|
+
pid: app.processIdentifier,
|
|
610
|
+
windowNumber: resolved.windowNumber,
|
|
611
|
+
windowFrame: resolved.frame
|
|
612
|
+
)
|
|
613
|
+
}
|
|
614
|
+
|
|
615
|
+
private func pointerAction(_ body: () throws -> Void) throws {
|
|
616
|
+
do {
|
|
617
|
+
try body()
|
|
618
|
+
} catch let error as TargetedPointerError {
|
|
619
|
+
throw fail("COMPUTER_ACTION_BLOCKED", error.message)
|
|
620
|
+
}
|
|
621
|
+
}
|
|
622
|
+
|
|
623
|
+
private func windowPoint(_ action: [String: Any], window: [String: Any]?, xKey: String, yKey: String, coordinateSpace: String = "window") throws -> CGPoint {
|
|
624
|
+
let x = try double(action[xKey], "action.\(xKey)")
|
|
625
|
+
let y = try double(action[yKey], "action.\(yKey)")
|
|
626
|
+
if coordinateSpace == "screen" {
|
|
627
|
+
return CGPoint(x: x, y: y)
|
|
628
|
+
}
|
|
629
|
+
guard coordinateSpace == "window" else {
|
|
630
|
+
throw fail("COMPUTER_PROVIDER_FAILURE", "action coordinateSpace must be one of window, screen")
|
|
631
|
+
}
|
|
632
|
+
guard let frame = window?["frame"] as? [String: Any] else {
|
|
633
|
+
throw fail("COMPUTER_TARGET_UNAVAILABLE", "the observation has no current window coordinate space")
|
|
634
|
+
}
|
|
635
|
+
let width = try double(frame["width"], "window.width")
|
|
636
|
+
let height = try double(frame["height"], "window.height")
|
|
637
|
+
guard x >= 0, y >= 0, x <= width, y <= height else {
|
|
638
|
+
throw fail("COMPUTER_TARGET_UNAVAILABLE", "coordinate is outside the observed window")
|
|
639
|
+
}
|
|
640
|
+
let origin = try cgRect(frame)
|
|
641
|
+
return CGPoint(x: origin.origin.x + x, y: origin.origin.y + y)
|
|
642
|
+
}
|
|
643
|
+
|
|
644
|
+
private func mouseButton(_ value: String?) throws -> CGMouseButton {
|
|
645
|
+
switch value ?? "left" {
|
|
646
|
+
case "left": return .left
|
|
647
|
+
case "right": return .right
|
|
648
|
+
case "middle": return .center
|
|
649
|
+
default: throw fail("COMPUTER_PROVIDER_FAILURE", "unsupported mouse button")
|
|
650
|
+
}
|
|
651
|
+
}
|
|
652
|
+
|
|
653
|
+
private let keyCodes: [String: CGKeyCode] = [
|
|
654
|
+
"a": 0, "s": 1, "d": 2, "f": 3, "h": 4, "g": 5, "z": 6, "x": 7, "c": 8, "v": 9,
|
|
655
|
+
"b": 11, "q": 12, "w": 13, "e": 14, "r": 15, "y": 16, "t": 17, "1": 18, "2": 19,
|
|
656
|
+
"3": 20, "4": 21, "6": 22, "5": 23, "=": 24, "9": 25, "7": 26, "-": 27, "8": 28,
|
|
657
|
+
"0": 29, "]": 30, "o": 31, "u": 32, "[": 33, "i": 34, "p": 35, "return": 36,
|
|
658
|
+
"l": 37, "j": 38, "'": 39, "k": 40, ";": 41, "\\": 42, ",": 43, "/": 44, "n": 45,
|
|
659
|
+
"m": 46, ".": 47, "tab": 48, "space": 49, "delete": 51, "escape": 53,
|
|
660
|
+
"home": 115, "pageup": 116, "forwarddelete": 117, "end": 119, "pagedown": 121,
|
|
661
|
+
"left": 123, "right": 124, "down": 125, "up": 126,
|
|
662
|
+
]
|
|
663
|
+
|
|
664
|
+
private func flags(_ modifiers: [String]) throws -> CGEventFlags {
|
|
665
|
+
var result: CGEventFlags = []
|
|
666
|
+
for modifier in modifiers {
|
|
667
|
+
switch modifier {
|
|
668
|
+
case "command": result.insert(.maskCommand)
|
|
669
|
+
case "control": result.insert(.maskControl)
|
|
670
|
+
case "option": result.insert(.maskAlternate)
|
|
671
|
+
case "shift": result.insert(.maskShift)
|
|
672
|
+
default: throw fail("COMPUTER_PROVIDER_FAILURE", "unsupported key modifier")
|
|
673
|
+
}
|
|
674
|
+
}
|
|
675
|
+
return result
|
|
676
|
+
}
|
|
677
|
+
|
|
678
|
+
private func pressKey(_ key: String, modifiers: [String], app: NSRunningApplication) throws {
|
|
679
|
+
let normalized = key.lowercased()
|
|
680
|
+
guard let code = keyCodes[normalized] else {
|
|
681
|
+
throw fail("COMPUTER_PROVIDER_FAILURE", "unsupported key; use the documented key vocabulary")
|
|
682
|
+
}
|
|
683
|
+
let source = try targetedEventSource()
|
|
684
|
+
guard let down = CGEvent(keyboardEventSource: source, virtualKey: code, keyDown: true),
|
|
685
|
+
let up = CGEvent(keyboardEventSource: source, virtualKey: code, keyDown: false) else {
|
|
686
|
+
throw fail("COMPUTER_ACTION_BLOCKED", "CoreGraphics could not create keyboard events")
|
|
687
|
+
}
|
|
688
|
+
let eventFlags = try flags(modifiers)
|
|
689
|
+
down.flags = eventFlags
|
|
690
|
+
up.flags = eventFlags
|
|
691
|
+
down.postToPid(app.processIdentifier)
|
|
692
|
+
usleep(5_000)
|
|
693
|
+
up.postToPid(app.processIdentifier)
|
|
694
|
+
}
|
|
695
|
+
|
|
696
|
+
private func focusedElement(_ app: NSRunningApplication) -> AXUIElement? {
|
|
697
|
+
let appElement = AXUIElementCreateApplication(app.processIdentifier)
|
|
698
|
+
guard let focused = axCopy(appElement, kAXFocusedUIElementAttribute as CFString) as AnyObject?,
|
|
699
|
+
CFGetTypeID(focused) == AXUIElementGetTypeID() else { return nil }
|
|
700
|
+
return (focused as! AXUIElement)
|
|
701
|
+
}
|
|
702
|
+
|
|
703
|
+
private func setSelectedText(_ text: String, app: NSRunningApplication) -> Bool {
|
|
704
|
+
guard let focused = focusedElement(app) else { return false }
|
|
705
|
+
return AXUIElementSetAttributeValue(focused, kAXSelectedTextAttribute as CFString, text as CFTypeRef) == .success
|
|
706
|
+
}
|
|
707
|
+
|
|
708
|
+
private func typeTextWithKeyboard(_ text: String, app: NSRunningApplication) throws {
|
|
709
|
+
let source = try targetedEventSource()
|
|
710
|
+
guard let down = CGEvent(keyboardEventSource: source, virtualKey: 0, keyDown: true),
|
|
711
|
+
let up = CGEvent(keyboardEventSource: source, virtualKey: 0, keyDown: false) else {
|
|
712
|
+
throw fail("COMPUTER_ACTION_BLOCKED", "CoreGraphics could not create Unicode keyboard events")
|
|
713
|
+
}
|
|
714
|
+
let units = Array(text.utf16)
|
|
715
|
+
units.withUnsafeBufferPointer { buffer in
|
|
716
|
+
down.keyboardSetUnicodeString(stringLength: units.count, unicodeString: buffer.baseAddress!)
|
|
717
|
+
up.keyboardSetUnicodeString(stringLength: units.count, unicodeString: buffer.baseAddress!)
|
|
718
|
+
}
|
|
719
|
+
down.postToPid(app.processIdentifier)
|
|
720
|
+
usleep(5_000)
|
|
721
|
+
up.postToPid(app.processIdentifier)
|
|
722
|
+
}
|
|
723
|
+
|
|
724
|
+
private func targetRecord(_ request: [String: Any], snapshot: ObservationSnapshot) throws -> ElementRecord? {
|
|
725
|
+
guard let element = request["element"] as? [String: Any] else { return nil }
|
|
726
|
+
let locator = (element["locator"] as? [NSNumber])?.map(\.intValue) ?? []
|
|
727
|
+
guard let record = snapshot.elements.first(where: { $0.locator == locator }) else {
|
|
728
|
+
throw fail("COMPUTER_STALE_OBSERVATION", "the target element locator is no longer present")
|
|
729
|
+
}
|
|
730
|
+
return record
|
|
731
|
+
}
|
|
732
|
+
|
|
733
|
+
private func selectedIdentity(_ source: [String: Any], keys: [String]) -> [AnyHashable: Any] {
|
|
734
|
+
var identity: [AnyHashable: Any] = [:]
|
|
735
|
+
for key in keys {
|
|
736
|
+
if let value = source[key] { identity[key] = value }
|
|
737
|
+
}
|
|
738
|
+
return identity
|
|
739
|
+
}
|
|
740
|
+
|
|
741
|
+
private func targetMatches(_ expected: [String: Any], current: ElementRecord) -> Bool {
|
|
742
|
+
let keys = ["nativeIdentifier", "role", "subrole", "title", "label", "value", "enabled", "selected", "frame", "actions"]
|
|
743
|
+
return NSDictionary(dictionary: selectedIdentity(expected, keys: keys))
|
|
744
|
+
.isEqual(to: selectedIdentity(current.json, keys: keys))
|
|
745
|
+
}
|
|
746
|
+
|
|
747
|
+
private func windowMatches(_ expected: [String: Any]?, current: [String: Any]?) -> Bool {
|
|
748
|
+
guard let expected else { return current == nil }
|
|
749
|
+
guard let current else { return false }
|
|
750
|
+
let keys = ["id", "title", "frame"]
|
|
751
|
+
return NSDictionary(dictionary: selectedIdentity(expected, keys: keys))
|
|
752
|
+
.isEqual(to: selectedIdentity(current, keys: keys))
|
|
753
|
+
}
|
|
754
|
+
|
|
755
|
+
private func activationStateMatches(_ before: ObservationSnapshot, current: ObservationSnapshot) -> Bool {
|
|
756
|
+
guard windowMatches(before.windowJSON, current: current.windowJSON),
|
|
757
|
+
before.elements.count == current.elements.count else { return false }
|
|
758
|
+
return zip(before.elements, current.elements).allSatisfy { expected, actual in
|
|
759
|
+
expected.locator == actual.locator && targetMatches(expected.json, current: actual)
|
|
760
|
+
}
|
|
761
|
+
}
|
|
762
|
+
|
|
763
|
+
private func validateTarget(_ request: [String: Any], snapshot: ObservationSnapshot) throws -> ElementRecord? {
|
|
764
|
+
let record = try targetRecord(request, snapshot: snapshot)
|
|
765
|
+
if let record, let expectedElement = request["element"] as? [String: Any] {
|
|
766
|
+
guard windowMatches(request["window"] as? [String: Any], current: snapshot.windowJSON),
|
|
767
|
+
targetMatches(expectedElement, current: record) else {
|
|
768
|
+
throw fail("COMPUTER_STALE_OBSERVATION", "the target element or window changed after the referenced observation")
|
|
769
|
+
}
|
|
770
|
+
} else {
|
|
771
|
+
let expected = try string(request["expectedStateHash"], "request.expectedStateHash")
|
|
772
|
+
guard snapshot.stateHash == expected else {
|
|
773
|
+
throw fail("COMPUTER_STALE_OBSERVATION", "the application UI changed after the referenced observation")
|
|
774
|
+
}
|
|
775
|
+
}
|
|
776
|
+
return record
|
|
777
|
+
}
|
|
778
|
+
|
|
779
|
+
private func interactionValue(_ interaction: [String: Any], _ key: String, _ allowed: [String]) throws -> String {
|
|
780
|
+
let value = try string(interaction[key], "request.interaction.\(key)")
|
|
781
|
+
guard allowed.contains(value) else {
|
|
782
|
+
throw fail("COMPUTER_PROVIDER_FAILURE", "request.interaction.\(key) must be one of \(allowed.joined(separator: ", "))")
|
|
783
|
+
}
|
|
784
|
+
return value
|
|
785
|
+
}
|
|
786
|
+
|
|
787
|
+
private func requirePointerInput(_ policy: String) throws {
|
|
788
|
+
guard policy == "targeted" else {
|
|
789
|
+
throw fail("COMPUTER_ACTION_BLOCKED", "this action needs target-process pointer input, but interaction.pointerInputPolicy is deny")
|
|
790
|
+
}
|
|
791
|
+
}
|
|
792
|
+
|
|
793
|
+
private func requiresForegroundPermission(_ action: String) -> Bool {
|
|
794
|
+
action == (kAXRaiseAction as String)
|
|
795
|
+
}
|
|
796
|
+
|
|
797
|
+
private func inputContext(
|
|
798
|
+
app: NSRunningApplication,
|
|
799
|
+
request: [String: Any],
|
|
800
|
+
limits: [String: Any],
|
|
801
|
+
snapshot: ObservationSnapshot,
|
|
802
|
+
record: ElementRecord?,
|
|
803
|
+
focusPolicy: String,
|
|
804
|
+
keyboardPolicy: String,
|
|
805
|
+
actionKind: String,
|
|
806
|
+
timeoutMs: Int,
|
|
807
|
+
) throws -> (snapshot: ObservationSnapshot, record: ElementRecord?, activation: String) {
|
|
808
|
+
let keyboardAction = actionKind == "type-text" || actionKind == "press-key"
|
|
809
|
+
let effectiveFocusPolicy = keyboardAction && keyboardPolicy == "activate" ? "activate" : focusPolicy
|
|
810
|
+
guard effectiveFocusPolicy == "activate" else {
|
|
811
|
+
return (snapshot, record, "not-requested")
|
|
812
|
+
}
|
|
813
|
+
if app.isActive {
|
|
814
|
+
return (snapshot, record, "already-frontmost")
|
|
815
|
+
}
|
|
816
|
+
try activate(app, timeoutMs: timeoutMs)
|
|
817
|
+
let refreshed = try observeSnapshot(app: app, limits: limits)
|
|
818
|
+
let refreshedRecord: ElementRecord?
|
|
819
|
+
if request["element"] != nil {
|
|
820
|
+
refreshedRecord = try validateTarget(request, snapshot: refreshed)
|
|
821
|
+
} else if keyboardAction {
|
|
822
|
+
// Activation may move focus to the app's default control; typing targets
|
|
823
|
+
// the refreshed focused element. The app identity is already revalidated
|
|
824
|
+
// by the fresh observation, so full pre-activation state equality would
|
|
825
|
+
// only make reliable keyboard input fail.
|
|
826
|
+
refreshedRecord = nil
|
|
827
|
+
} else {
|
|
828
|
+
guard activationStateMatches(snapshot, current: refreshed) else {
|
|
829
|
+
throw fail("COMPUTER_STALE_OBSERVATION", "the application UI changed while the target application was activated")
|
|
830
|
+
}
|
|
831
|
+
refreshedRecord = nil
|
|
832
|
+
}
|
|
833
|
+
return (refreshed, refreshedRecord, "activated")
|
|
834
|
+
}
|
|
835
|
+
|
|
836
|
+
private func actionResult(
|
|
837
|
+
channel: String,
|
|
838
|
+
activation: String,
|
|
839
|
+
pointerInput: Bool,
|
|
840
|
+
pointerRouting: String = "none"
|
|
841
|
+
) -> [String: Any] {
|
|
842
|
+
[
|
|
843
|
+
"channel": channel,
|
|
844
|
+
"activation": activation,
|
|
845
|
+
"pointerInput": pointerInput,
|
|
846
|
+
"pointerRouting": pointerRouting,
|
|
847
|
+
]
|
|
848
|
+
}
|
|
849
|
+
|
|
850
|
+
private func performAction(_ request: [String: Any]) throws -> [String: Any] {
|
|
851
|
+
let appIdentity = try dictionary(request["app"], "request.app")
|
|
852
|
+
let selector: [String: Any] = [
|
|
853
|
+
"bundleId": try string(appIdentity["bundleId"], "app.bundleId"),
|
|
854
|
+
"pid": try int(appIdentity["pid"], "app.pid"),
|
|
855
|
+
]
|
|
856
|
+
let app = try resolveApp(selector)
|
|
857
|
+
let limits = try dictionary(request["limits"], "request.limits")
|
|
858
|
+
let snapshot = try observeSnapshot(app: app, limits: limits)
|
|
859
|
+
let action = try dictionary(request["action"], "request.action")
|
|
860
|
+
let kind = try string(action["kind"], "action.kind")
|
|
861
|
+
let actionTimeoutMs = try int(request["actionTimeoutMs"], "request.actionTimeoutMs")
|
|
862
|
+
guard actionTimeoutMs >= 1_000 && actionTimeoutMs <= 120_000 else {
|
|
863
|
+
throw fail("COMPUTER_PROVIDER_FAILURE", "request.actionTimeoutMs must be between 1000 and 120000")
|
|
864
|
+
}
|
|
865
|
+
let interaction = try dictionary(request["interaction"], "request.interaction")
|
|
866
|
+
let focusPolicy = try interactionValue(interaction, "focusPolicy", ["preserve", "activate"])
|
|
867
|
+
let keyboardPolicy = try interactionValue(interaction, "keyboardPolicy", ["preserve", "activate"])
|
|
868
|
+
let pointerInputPolicy = try interactionValue(interaction, "pointerInputPolicy", ["deny", "targeted"])
|
|
869
|
+
let record = try validateTarget(request, snapshot: snapshot)
|
|
870
|
+
|
|
871
|
+
switch kind {
|
|
872
|
+
case "click":
|
|
873
|
+
if let record, pressWithDescendantFallback(record.element) {
|
|
874
|
+
return actionResult(channel: "accessibility", activation: "not-requested", pointerInput: false)
|
|
875
|
+
}
|
|
876
|
+
let elementFallback = record != nil && bool(action["allowCoordinateFallback"])
|
|
877
|
+
let coordinateFallback = action["x"] != nil && action["y"] != nil
|
|
878
|
+
guard elementFallback || coordinateFallback else {
|
|
879
|
+
throw fail("COMPUTER_ELEMENT_UNAVAILABLE", "element does not support an actionable AXPress and coordinate fallback was not requested")
|
|
880
|
+
}
|
|
881
|
+
try requirePointerInput(pointerInputPolicy)
|
|
882
|
+
let current = try inputContext(
|
|
883
|
+
app: app,
|
|
884
|
+
request: request,
|
|
885
|
+
limits: limits,
|
|
886
|
+
snapshot: snapshot,
|
|
887
|
+
record: record,
|
|
888
|
+
focusPolicy: focusPolicy,
|
|
889
|
+
keyboardPolicy: keyboardPolicy,
|
|
890
|
+
actionKind: kind,
|
|
891
|
+
timeoutMs: actionTimeoutMs,
|
|
892
|
+
)
|
|
893
|
+
if let record = current.record, elementFallback, let frame = axFrame(record.element) {
|
|
894
|
+
let point = CGPoint(x: frame.midX, y: frame.midY)
|
|
895
|
+
let target = try pointerTarget(app: app, window: current.snapshot.windowJSON, at: point)
|
|
896
|
+
try pointerAction {
|
|
897
|
+
try targetedClick(
|
|
898
|
+
at: point,
|
|
899
|
+
button: try mouseButton(action["button"] as? String),
|
|
900
|
+
count: min(max((action["clickCount"] as? NSNumber)?.intValue ?? 1, 1), 3),
|
|
901
|
+
target: target
|
|
902
|
+
)
|
|
903
|
+
}
|
|
904
|
+
return actionResult(channel: "coordinates", activation: current.activation, pointerInput: true, pointerRouting: "target-process")
|
|
905
|
+
}
|
|
906
|
+
if action["x"] != nil, action["y"] != nil {
|
|
907
|
+
let coordinateSpace = action["coordinateSpace"] as? String ?? "window"
|
|
908
|
+
let point = try windowPoint(action, window: current.snapshot.windowJSON, xKey: "x", yKey: "y", coordinateSpace: coordinateSpace)
|
|
909
|
+
let target = try pointerTarget(app: app, window: current.snapshot.windowJSON, at: point)
|
|
910
|
+
try pointerAction {
|
|
911
|
+
try targetedClick(
|
|
912
|
+
at: point,
|
|
913
|
+
button: try mouseButton(action["button"] as? String),
|
|
914
|
+
count: min(max((action["clickCount"] as? NSNumber)?.intValue ?? 1, 1), 3),
|
|
915
|
+
target: target
|
|
916
|
+
)
|
|
917
|
+
}
|
|
918
|
+
return actionResult(channel: "coordinates", activation: current.activation, pointerInput: true, pointerRouting: "target-process")
|
|
919
|
+
}
|
|
920
|
+
throw fail("COMPUTER_ELEMENT_UNAVAILABLE", "coordinate fallback was requested but no current target coordinates were available")
|
|
921
|
+
case "set-value":
|
|
922
|
+
guard let record else { throw fail("COMPUTER_ELEMENT_UNAVAILABLE", "set-value requires an observed element") }
|
|
923
|
+
let value = try stringAllowingEmpty(action["value"], "action.value")
|
|
924
|
+
let result = AXUIElementSetAttributeValue(record.element, kAXValueAttribute as CFString, value as CFTypeRef)
|
|
925
|
+
guard result == .success else { throw fail("COMPUTER_ACTION_BLOCKED", "Accessibility value assignment was rejected") }
|
|
926
|
+
return actionResult(channel: "accessibility", activation: "not-requested", pointerInput: false)
|
|
927
|
+
case "type-text":
|
|
928
|
+
let text = try string(action["text"], "action.text")
|
|
929
|
+
if setSelectedText(text, app: app) {
|
|
930
|
+
return actionResult(channel: "accessibility", activation: "not-requested", pointerInput: false)
|
|
931
|
+
}
|
|
932
|
+
let current = try inputContext(
|
|
933
|
+
app: app,
|
|
934
|
+
request: request,
|
|
935
|
+
limits: limits,
|
|
936
|
+
snapshot: snapshot,
|
|
937
|
+
record: record,
|
|
938
|
+
focusPolicy: focusPolicy,
|
|
939
|
+
keyboardPolicy: keyboardPolicy,
|
|
940
|
+
actionKind: kind,
|
|
941
|
+
timeoutMs: actionTimeoutMs,
|
|
942
|
+
)
|
|
943
|
+
if current.activation == "activated", setSelectedText(text, app: app) {
|
|
944
|
+
return actionResult(channel: "accessibility", activation: current.activation, pointerInput: false)
|
|
945
|
+
}
|
|
946
|
+
try typeTextWithKeyboard(text, app: app)
|
|
947
|
+
return actionResult(channel: "keyboard", activation: current.activation, pointerInput: false)
|
|
948
|
+
case "press-key":
|
|
949
|
+
let current = try inputContext(
|
|
950
|
+
app: app,
|
|
951
|
+
request: request,
|
|
952
|
+
limits: limits,
|
|
953
|
+
snapshot: snapshot,
|
|
954
|
+
record: record,
|
|
955
|
+
focusPolicy: focusPolicy,
|
|
956
|
+
keyboardPolicy: keyboardPolicy,
|
|
957
|
+
actionKind: kind,
|
|
958
|
+
timeoutMs: actionTimeoutMs,
|
|
959
|
+
)
|
|
960
|
+
let modifiers = action["modifiers"] as? [String] ?? []
|
|
961
|
+
try pressKey(try string(action["key"], "action.key"), modifiers: modifiers, app: app)
|
|
962
|
+
return actionResult(channel: "keyboard", activation: current.activation, pointerInput: false)
|
|
963
|
+
case "scroll":
|
|
964
|
+
try requirePointerInput(pointerInputPolicy)
|
|
965
|
+
let current = try inputContext(
|
|
966
|
+
app: app,
|
|
967
|
+
request: request,
|
|
968
|
+
limits: limits,
|
|
969
|
+
snapshot: snapshot,
|
|
970
|
+
record: record,
|
|
971
|
+
focusPolicy: focusPolicy,
|
|
972
|
+
keyboardPolicy: keyboardPolicy,
|
|
973
|
+
actionKind: kind,
|
|
974
|
+
timeoutMs: actionTimeoutMs,
|
|
975
|
+
)
|
|
976
|
+
let point: CGPoint
|
|
977
|
+
if let record = current.record, let frame = axFrame(record.element) { point = CGPoint(x: frame.midX, y: frame.midY) }
|
|
978
|
+
else {
|
|
979
|
+
let coordinateSpace = action["coordinateSpace"] as? String ?? "window"
|
|
980
|
+
point = try windowPoint(action, window: current.snapshot.windowJSON, xKey: "x", yKey: "y", coordinateSpace: coordinateSpace)
|
|
981
|
+
}
|
|
982
|
+
let direction = try string(action["direction"], "action.direction")
|
|
983
|
+
let pages = min(max((action["pages"] as? NSNumber)?.intValue ?? 1, 1), 10)
|
|
984
|
+
let vertical = direction == "up" ? 10 * pages : direction == "down" ? -10 * pages : 0
|
|
985
|
+
let horizontal = direction == "left" ? 10 * pages : direction == "right" ? -10 * pages : 0
|
|
986
|
+
guard vertical != 0 || horizontal != 0 else { throw fail("COMPUTER_PROVIDER_FAILURE", "unsupported scroll direction") }
|
|
987
|
+
let target = try pointerTarget(app: app, window: current.snapshot.windowJSON, at: point)
|
|
988
|
+
try pointerAction {
|
|
989
|
+
try targetedScroll(
|
|
990
|
+
at: point,
|
|
991
|
+
vertical: Int32(vertical),
|
|
992
|
+
horizontal: Int32(horizontal),
|
|
993
|
+
target: target
|
|
994
|
+
)
|
|
995
|
+
}
|
|
996
|
+
return actionResult(channel: "coordinates", activation: current.activation, pointerInput: true, pointerRouting: "target-process")
|
|
997
|
+
case "drag":
|
|
998
|
+
try requirePointerInput(pointerInputPolicy)
|
|
999
|
+
let current = try inputContext(
|
|
1000
|
+
app: app,
|
|
1001
|
+
request: request,
|
|
1002
|
+
limits: limits,
|
|
1003
|
+
snapshot: snapshot,
|
|
1004
|
+
record: record,
|
|
1005
|
+
focusPolicy: focusPolicy,
|
|
1006
|
+
keyboardPolicy: keyboardPolicy,
|
|
1007
|
+
actionKind: kind,
|
|
1008
|
+
timeoutMs: actionTimeoutMs,
|
|
1009
|
+
)
|
|
1010
|
+
let coordinateSpace = action["coordinateSpace"] as? String ?? "window"
|
|
1011
|
+
let from = try windowPoint(action, window: current.snapshot.windowJSON, xKey: "fromX", yKey: "fromY", coordinateSpace: coordinateSpace)
|
|
1012
|
+
let to = try windowPoint(action, window: current.snapshot.windowJSON, xKey: "toX", yKey: "toY", coordinateSpace: coordinateSpace)
|
|
1013
|
+
let target = try pointerTarget(app: app, window: current.snapshot.windowJSON, at: from)
|
|
1014
|
+
try pointerAction {
|
|
1015
|
+
try targetedDrag(from: from, to: to, target: target)
|
|
1016
|
+
}
|
|
1017
|
+
return actionResult(channel: "coordinates", activation: current.activation, pointerInput: true, pointerRouting: "target-process")
|
|
1018
|
+
case "perform-action":
|
|
1019
|
+
guard let record else { throw fail("COMPUTER_ELEMENT_UNAVAILABLE", "perform-action requires an observed element") }
|
|
1020
|
+
let actionName = try string(action["action"], "action.action")
|
|
1021
|
+
let available = record.json["actions"] as? [String] ?? []
|
|
1022
|
+
guard available.contains(actionName) else { throw fail("COMPUTER_ELEMENT_UNAVAILABLE", "the element did not advertise the requested Accessibility action") }
|
|
1023
|
+
if requiresForegroundPermission(actionName), focusPolicy != "activate" {
|
|
1024
|
+
throw fail("COMPUTER_ACTION_BLOCKED", "the requested Accessibility action may raise the target window, but interaction.focusPolicy is preserve")
|
|
1025
|
+
}
|
|
1026
|
+
let current = requiresForegroundPermission(actionName)
|
|
1027
|
+
? try inputContext(
|
|
1028
|
+
app: app,
|
|
1029
|
+
request: request,
|
|
1030
|
+
limits: limits,
|
|
1031
|
+
snapshot: snapshot,
|
|
1032
|
+
record: record,
|
|
1033
|
+
focusPolicy: focusPolicy,
|
|
1034
|
+
keyboardPolicy: keyboardPolicy,
|
|
1035
|
+
actionKind: kind,
|
|
1036
|
+
timeoutMs: actionTimeoutMs
|
|
1037
|
+
)
|
|
1038
|
+
: (snapshot, record, "not-requested")
|
|
1039
|
+
guard let currentRecord = current.record else {
|
|
1040
|
+
throw fail("COMPUTER_ELEMENT_UNAVAILABLE", "perform-action target is unavailable after foreground validation")
|
|
1041
|
+
}
|
|
1042
|
+
let result = AXUIElementPerformAction(currentRecord.element, actionName as CFString)
|
|
1043
|
+
guard result == .success else { throw fail("COMPUTER_ACTION_BLOCKED", "Accessibility action was rejected") }
|
|
1044
|
+
return actionResult(channel: "accessibility", activation: current.activation, pointerInput: false)
|
|
1045
|
+
default:
|
|
1046
|
+
throw fail("COMPUTER_PROVIDER_FAILURE", "unsupported action kind")
|
|
1047
|
+
}
|
|
1048
|
+
}
|
|
1049
|
+
|
|
1050
|
+
private func handle(_ request: [String: Any]) async throws -> Any {
|
|
1051
|
+
guard (request["protocolVersion"] as? NSNumber)?.intValue == 1 else {
|
|
1052
|
+
throw fail("COMPUTER_PROVIDER_FAILURE", "unsupported helper protocol version")
|
|
1053
|
+
}
|
|
1054
|
+
let command = try string(request["command"], "command")
|
|
1055
|
+
switch command {
|
|
1056
|
+
case "health":
|
|
1057
|
+
return [
|
|
1058
|
+
"helperVersion": helperVersion,
|
|
1059
|
+
"accessibility": permissionAccessibility(),
|
|
1060
|
+
"screenRecording": permissionScreenRecording(),
|
|
1061
|
+
]
|
|
1062
|
+
case "list-apps":
|
|
1063
|
+
return try runningApps().map { app -> [String: Any] in
|
|
1064
|
+
var json = try appJSON(app)
|
|
1065
|
+
json["frontmost"] = app.isActive
|
|
1066
|
+
json["accessibility"] = permissionAccessibility()
|
|
1067
|
+
json["screenRecording"] = permissionScreenRecording()
|
|
1068
|
+
return json
|
|
1069
|
+
}
|
|
1070
|
+
case "resolve-app":
|
|
1071
|
+
return try appJSON(resolveApp(try dictionary(request["selector"], "selector")))
|
|
1072
|
+
case "open-settings":
|
|
1073
|
+
let kind = try string(request["kind"], "kind")
|
|
1074
|
+
let target: String
|
|
1075
|
+
switch kind {
|
|
1076
|
+
case "accessibility":
|
|
1077
|
+
target = "x-apple.systempreferences:com.apple.preference.security?Privacy_Accessibility"
|
|
1078
|
+
case "screen-recording":
|
|
1079
|
+
target = "x-apple.systempreferences:com.apple.preference.security?Privacy_ScreenCapture"
|
|
1080
|
+
default:
|
|
1081
|
+
throw fail("COMPUTER_PROVIDER_FAILURE", "unknown permission Settings pane")
|
|
1082
|
+
}
|
|
1083
|
+
guard let url = URL(string: target), NSWorkspace.shared.open(url) else {
|
|
1084
|
+
throw fail("COMPUTER_ACTION_BLOCKED", "macOS Settings could not be opened")
|
|
1085
|
+
}
|
|
1086
|
+
return NSNull()
|
|
1087
|
+
case "observe":
|
|
1088
|
+
let appIdentity = try dictionary(request["app"], "app")
|
|
1089
|
+
let app = try resolveApp([
|
|
1090
|
+
"bundleId": try string(appIdentity["bundleId"], "app.bundleId"),
|
|
1091
|
+
"pid": try int(appIdentity["pid"], "app.pid"),
|
|
1092
|
+
])
|
|
1093
|
+
let options = try dictionary(request["options"], "options")
|
|
1094
|
+
let snapshot = try observeSnapshot(app: app, limits: options)
|
|
1095
|
+
let mode = options["screenshot"] as? String ?? "optional"
|
|
1096
|
+
var screenshot: [String: Any]?
|
|
1097
|
+
if mode != "none" {
|
|
1098
|
+
let path = try string(options["screenshotPath"], "options.screenshotPath")
|
|
1099
|
+
screenshot = try await captureWindow(snapshot, path: path, required: mode == "required")
|
|
1100
|
+
}
|
|
1101
|
+
return observationJSON(snapshot, screenshot: screenshot)
|
|
1102
|
+
case "act":
|
|
1103
|
+
return try performAction(try dictionary(request["request"], "request"))
|
|
1104
|
+
default:
|
|
1105
|
+
throw fail("COMPUTER_PROVIDER_FAILURE", "unknown helper command")
|
|
1106
|
+
}
|
|
1107
|
+
}
|
|
1108
|
+
|
|
1109
|
+
private func emit(_ payload: [String: Any], exitCode: Int32) -> Never {
|
|
1110
|
+
let data = (try? JSONSerialization.data(withJSONObject: payload, options: [.sortedKeys])) ?? Data("{\"ok\":false,\"error\":{\"code\":\"COMPUTER_PROVIDER_FAILURE\",\"message\":\"response serialization failed\"}}".utf8)
|
|
1111
|
+
FileHandle.standardOutput.write(data)
|
|
1112
|
+
FileHandle.standardOutput.write(Data("\n".utf8))
|
|
1113
|
+
Darwin.exit(exitCode)
|
|
1114
|
+
}
|
|
1115
|
+
|
|
1116
|
+
@main
|
|
1117
|
+
private struct HelperMain {
|
|
1118
|
+
@MainActor
|
|
1119
|
+
static func main() async {
|
|
1120
|
+
do {
|
|
1121
|
+
_ = NSApplication.shared
|
|
1122
|
+
NSApplication.shared.setActivationPolicy(.prohibited)
|
|
1123
|
+
try requireHostTransport()
|
|
1124
|
+
if ProcessInfo.processInfo.arguments.contains("--cursor-overlay") {
|
|
1125
|
+
CursorOverlayRuntime.run()
|
|
1126
|
+
return
|
|
1127
|
+
}
|
|
1128
|
+
guard let data = try FileHandle.standardInput.readToEnd(), !data.isEmpty,
|
|
1129
|
+
let request = try JSONSerialization.jsonObject(with: data) as? [String: Any] else {
|
|
1130
|
+
throw fail("COMPUTER_PROVIDER_FAILURE", "stdin must contain one JSON request")
|
|
1131
|
+
}
|
|
1132
|
+
let value = try await handle(request)
|
|
1133
|
+
emit(["ok": true, "value": value], exitCode: 0)
|
|
1134
|
+
} catch let error as HelperError {
|
|
1135
|
+
emit(["ok": false, "error": ["code": error.code, "message": error.message]], exitCode: 2)
|
|
1136
|
+
} catch {
|
|
1137
|
+
emit(["ok": false, "error": ["code": "COMPUTER_PROVIDER_FAILURE", "message": String(describing: error).prefix(1000).description]], exitCode: 2)
|
|
1138
|
+
}
|
|
1139
|
+
}
|
|
1140
|
+
}
|