@vashp2029/glovebox 0.0.0-stage → 0.3.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +73 -0
- package/LICENSE +21 -0
- package/README.md +240 -2
- package/SECURITY.md +83 -0
- package/bin/glovebox.mjs +319 -0
- package/docs/agents.md +124 -0
- package/docs/api.md +150 -0
- package/docs/backend-protocol.md +118 -0
- package/docs/cli.md +150 -0
- package/docs/design.md +212 -0
- package/docs/recipes.md +162 -0
- package/docs/roadmap.md +167 -0
- package/docs/testing.md +79 -0
- package/native/macos/GloveboxHelper.swift +1661 -0
- package/package.json +28 -4
- package/skills/glovebox/SKILL.md +49 -0
- package/src/api.mjs +552 -0
- package/src/axformat.mjs +158 -0
- package/src/backends/index.mjs +27 -0
- package/src/backends/macos.mjs +106 -0
- package/src/config.mjs +92 -0
- package/src/daemon.mjs +127 -0
- package/src/geometry.mjs +38 -0
- package/src/kernel.mjs +217 -0
- package/src/keys.mjs +96 -0
- package/src/macos-helper.mjs +164 -0
- package/src/mcp.mjs +126 -0
- package/src/session.mjs +110 -0
- package/src/smoke.mjs +160 -0
- package/src/transform.mjs +131 -0
|
@@ -0,0 +1,1661 @@
|
|
|
1
|
+
// glovebox-helper: the macOS backend for glovebox.
|
|
2
|
+
//
|
|
3
|
+
// A small JSON-lines RPC server on a Unix socket (~/.glovebox/helper.sock),
|
|
4
|
+
// packaged as its own app so macOS grants Accessibility and Screen Recording
|
|
5
|
+
// to "Glovebox Helper" once, whichever agent or terminal starts it. Each request is
|
|
6
|
+
// {"id": n, "method": "...", "params": {...}} and each reply is
|
|
7
|
+
// {"id": n, "result": ...} or {"id": n, "error": {"message": "..."}}.
|
|
8
|
+
// Unsolicited events are {"event": "..."}.
|
|
9
|
+
//
|
|
10
|
+
// Coordinates on this side are always global screen points, top-left origin
|
|
11
|
+
// (the same space AX and CGEvent use). The Node side owns the mapping between
|
|
12
|
+
// those points and the image pixels a model sees.
|
|
13
|
+
//
|
|
14
|
+
// Build: glovebox setup (see src/macos-helper.mjs).
|
|
15
|
+
|
|
16
|
+
import AppKit
|
|
17
|
+
import ApplicationServices
|
|
18
|
+
import CoreGraphics
|
|
19
|
+
import Foundation
|
|
20
|
+
import ScreenCaptureKit
|
|
21
|
+
import UniformTypeIdentifiers
|
|
22
|
+
|
|
23
|
+
let helperVersion = "0.3.1"
|
|
24
|
+
// Marks events we synthesize so the Escape watcher ignores them.
|
|
25
|
+
let syntheticMarker: Int64 = 0x676C_6F76
|
|
26
|
+
|
|
27
|
+
// MARK: - Output
|
|
28
|
+
|
|
29
|
+
// Sends an unsolicited event to every connected client.
|
|
30
|
+
func emit(_ object: [String: Any]) {
|
|
31
|
+
for connection in Connection.all() { connection.send(object) }
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
struct HelperError: Error {
|
|
35
|
+
let message: String
|
|
36
|
+
let code: String
|
|
37
|
+
init(_ message: String, code: String = "failed") {
|
|
38
|
+
self.message = message
|
|
39
|
+
self.code = code
|
|
40
|
+
}
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
// MARK: - Param helpers
|
|
44
|
+
|
|
45
|
+
extension Dictionary where Key == String, Value == Any {
|
|
46
|
+
func int(_ key: String) -> Int? {
|
|
47
|
+
if let v = self[key] as? Int { return v }
|
|
48
|
+
if let v = self[key] as? Double { return Int(v) }
|
|
49
|
+
if let v = self[key] as? NSNumber { return v.intValue }
|
|
50
|
+
return nil
|
|
51
|
+
}
|
|
52
|
+
func double(_ key: String) -> Double? {
|
|
53
|
+
if let v = self[key] as? Double { return v }
|
|
54
|
+
if let v = self[key] as? Int { return Double(v) }
|
|
55
|
+
if let v = self[key] as? NSNumber { return v.doubleValue }
|
|
56
|
+
return nil
|
|
57
|
+
}
|
|
58
|
+
func string(_ key: String) -> String? { self[key] as? String }
|
|
59
|
+
func bool(_ key: String) -> Bool? { self[key] as? Bool }
|
|
60
|
+
func requireInt(_ key: String) throws -> Int {
|
|
61
|
+
guard let v = int(key) else { throw HelperError("missing integer param \(key)", code: "bad_request") }
|
|
62
|
+
return v
|
|
63
|
+
}
|
|
64
|
+
func requireDouble(_ key: String) throws -> Double {
|
|
65
|
+
guard let v = double(key) else { throw HelperError("missing number param \(key)", code: "bad_request") }
|
|
66
|
+
return v
|
|
67
|
+
}
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
// MARK: - Apps
|
|
71
|
+
|
|
72
|
+
func appInfo(_ app: NSRunningApplication) -> [String: Any] {
|
|
73
|
+
var info: [String: Any] = [
|
|
74
|
+
"name": app.localizedName ?? "",
|
|
75
|
+
"pid": Int(app.processIdentifier),
|
|
76
|
+
"running": true,
|
|
77
|
+
"active": app.isActive,
|
|
78
|
+
"hidden": app.isHidden,
|
|
79
|
+
]
|
|
80
|
+
if let id = app.bundleIdentifier { info["bundleId"] = id }
|
|
81
|
+
if let url = app.bundleURL { info["path"] = url.path }
|
|
82
|
+
return info
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
func installedApps() -> [[String: Any]] {
|
|
86
|
+
var roots = ["/Applications", "/Applications/Utilities", "/System/Applications", "/System/Applications/Utilities"]
|
|
87
|
+
roots.append(NSHomeDirectory() + "/Applications")
|
|
88
|
+
var seen = Set<String>()
|
|
89
|
+
var out: [[String: Any]] = []
|
|
90
|
+
let fm = FileManager.default
|
|
91
|
+
for root in roots {
|
|
92
|
+
guard let items = try? fm.contentsOfDirectory(atPath: root) else { continue }
|
|
93
|
+
for item in items where item.hasSuffix(".app") {
|
|
94
|
+
let path = root + "/" + item
|
|
95
|
+
guard let bundle = Bundle(path: path), let id = bundle.bundleIdentifier, !seen.contains(id) else { continue }
|
|
96
|
+
seen.insert(id)
|
|
97
|
+
let name = (bundle.object(forInfoDictionaryKey: "CFBundleDisplayName") as? String)
|
|
98
|
+
?? (bundle.object(forInfoDictionaryKey: "CFBundleName") as? String)
|
|
99
|
+
?? String(item.dropLast(4))
|
|
100
|
+
out.append(["name": name, "bundleId": id, "path": path, "running": false])
|
|
101
|
+
}
|
|
102
|
+
}
|
|
103
|
+
return out
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
func listApps() -> [[String: Any]] {
|
|
107
|
+
var out: [[String: Any]] = []
|
|
108
|
+
var seen = Set<String>()
|
|
109
|
+
for app in NSWorkspace.shared.runningApplications where app.activationPolicy == .regular {
|
|
110
|
+
out.append(appInfo(app))
|
|
111
|
+
if let id = app.bundleIdentifier { seen.insert(id) }
|
|
112
|
+
}
|
|
113
|
+
for app in installedApps() where !seen.contains(app["bundleId"] as? String ?? "") {
|
|
114
|
+
out.append(app)
|
|
115
|
+
}
|
|
116
|
+
return out
|
|
117
|
+
}
|
|
118
|
+
|
|
119
|
+
func resolveApp(_ query: String) throws -> [String: Any] {
|
|
120
|
+
let q = query.lowercased()
|
|
121
|
+
let running = NSWorkspace.shared.runningApplications.filter { $0.activationPolicy != .prohibited }
|
|
122
|
+
if q.hasPrefix("pid:"), let pid = Int32(q.dropFirst(4)),
|
|
123
|
+
let app = NSRunningApplication(processIdentifier: pid) {
|
|
124
|
+
return appInfo(app)
|
|
125
|
+
}
|
|
126
|
+
for app in running {
|
|
127
|
+
if app.bundleIdentifier?.lowercased() == q || app.localizedName?.lowercased() == q {
|
|
128
|
+
return appInfo(app)
|
|
129
|
+
}
|
|
130
|
+
}
|
|
131
|
+
let installed = installedApps()
|
|
132
|
+
for app in installed {
|
|
133
|
+
if (app["bundleId"] as? String)?.lowercased() == q || (app["name"] as? String)?.lowercased() == q {
|
|
134
|
+
return app
|
|
135
|
+
}
|
|
136
|
+
}
|
|
137
|
+
if query.hasSuffix(".app") || query.hasPrefix("/"), let bundle = Bundle(path: query), let id = bundle.bundleIdentifier {
|
|
138
|
+
return ["name": bundle.bundleURL.deletingPathExtension().lastPathComponent, "bundleId": id, "path": query, "running": false]
|
|
139
|
+
}
|
|
140
|
+
let near = (running.compactMap { $0.localizedName } + installed.compactMap { $0["name"] as? String })
|
|
141
|
+
.filter { $0.lowercased().contains(q) }
|
|
142
|
+
let hint = near.isEmpty ? "" : " Did you mean: \(Array(Set(near)).sorted().prefix(5).joined(separator: ", "))?"
|
|
143
|
+
throw HelperError("No app named \"\(query)\".\(hint)", code: "not_found")
|
|
144
|
+
}
|
|
145
|
+
|
|
146
|
+
// NSRunningApplication(processIdentifier:) can come back nil for a live app
|
|
147
|
+
// when called off the main thread, so a miss is checked against the process
|
|
148
|
+
// table and the workspace list before calling the app gone.
|
|
149
|
+
func runningApp(_ pid: Int) throws -> NSRunningApplication {
|
|
150
|
+
if let app = NSRunningApplication(processIdentifier: pid_t(pid)), !app.isTerminated { return app }
|
|
151
|
+
if kill(pid_t(pid), 0) == 0 {
|
|
152
|
+
var found: NSRunningApplication?
|
|
153
|
+
let look = {
|
|
154
|
+
found = NSRunningApplication(processIdentifier: pid_t(pid))
|
|
155
|
+
?? NSWorkspace.shared.runningApplications.first { $0.processIdentifier == pid_t(pid) }
|
|
156
|
+
}
|
|
157
|
+
if Thread.isMainThread { look() } else { DispatchQueue.main.sync(execute: look) }
|
|
158
|
+
if let app = found { return app }
|
|
159
|
+
}
|
|
160
|
+
throw HelperError("App with pid \(pid) is not running", code: "not_running")
|
|
161
|
+
}
|
|
162
|
+
|
|
163
|
+
func launch(_ params: [String: Any]) throws -> [String: Any] {
|
|
164
|
+
if let bundleId = params.string("bundleId"),
|
|
165
|
+
let app = NSWorkspace.shared.runningApplications.first(where: { $0.bundleIdentifier == bundleId }) {
|
|
166
|
+
return appInfo(app)
|
|
167
|
+
}
|
|
168
|
+
var url: URL?
|
|
169
|
+
if let path = params.string("path") { url = URL(fileURLWithPath: path) }
|
|
170
|
+
if url == nil, let bundleId = params.string("bundleId") {
|
|
171
|
+
url = NSWorkspace.shared.urlForApplication(withBundleIdentifier: bundleId)
|
|
172
|
+
}
|
|
173
|
+
guard let appURL = url else { throw HelperError("Cannot locate app to launch", code: "not_found") }
|
|
174
|
+
let config = NSWorkspace.OpenConfiguration()
|
|
175
|
+
config.activates = params.bool("activate") ?? true
|
|
176
|
+
let done = DispatchSemaphore(value: 0)
|
|
177
|
+
var launched: NSRunningApplication?
|
|
178
|
+
var failure: Error?
|
|
179
|
+
NSWorkspace.shared.openApplication(at: appURL, configuration: config) { app, error in
|
|
180
|
+
launched = app
|
|
181
|
+
failure = error
|
|
182
|
+
done.signal()
|
|
183
|
+
}
|
|
184
|
+
if done.wait(timeout: .now() + 20) == .timedOut { throw HelperError("Timed out launching \(appURL.lastPathComponent)") }
|
|
185
|
+
if let error = failure { throw HelperError("Launch failed: \(error.localizedDescription)") }
|
|
186
|
+
guard let app = launched else { throw HelperError("Launch returned no app") }
|
|
187
|
+
// Give the app a moment to put up a window.
|
|
188
|
+
let element = AXUIElementCreateApplication(app.processIdentifier)
|
|
189
|
+
for _ in 0..<50 {
|
|
190
|
+
if let windows: [AXUIElement] = axAttr(element, kAXWindowsAttribute), !windows.isEmpty { break }
|
|
191
|
+
Thread.sleep(forTimeInterval: 0.1)
|
|
192
|
+
}
|
|
193
|
+
return appInfo(app)
|
|
194
|
+
}
|
|
195
|
+
|
|
196
|
+
func activate(_ pid: Int) throws {
|
|
197
|
+
let app = try runningApp(pid)
|
|
198
|
+
if app.isActive { return }
|
|
199
|
+
if app.isHidden { app.unhide() }
|
|
200
|
+
let element = AXUIElementCreateApplication(pid_t(pid))
|
|
201
|
+
AXUIElementSetAttributeValue(element, kAXFrontmostAttribute as CFString, kCFBooleanTrue)
|
|
202
|
+
for _ in 0..<20 {
|
|
203
|
+
if app.isActive { return }
|
|
204
|
+
Thread.sleep(forTimeInterval: 0.05)
|
|
205
|
+
}
|
|
206
|
+
if let url = app.bundleURL {
|
|
207
|
+
let config = NSWorkspace.OpenConfiguration()
|
|
208
|
+
config.activates = true
|
|
209
|
+
NSWorkspace.shared.openApplication(at: url, configuration: config, completionHandler: nil)
|
|
210
|
+
} else {
|
|
211
|
+
app.activate(options: [.activateAllWindows])
|
|
212
|
+
}
|
|
213
|
+
for _ in 0..<40 {
|
|
214
|
+
if app.isActive { return }
|
|
215
|
+
Thread.sleep(forTimeInterval: 0.05)
|
|
216
|
+
}
|
|
217
|
+
throw HelperError("Could not bring \(app.localizedName ?? "app") to the front", code: "activate_failed")
|
|
218
|
+
}
|
|
219
|
+
|
|
220
|
+
// MARK: - AX basics
|
|
221
|
+
|
|
222
|
+
func axAttr<T>(_ element: AXUIElement, _ name: String) -> T? {
|
|
223
|
+
var value: CFTypeRef?
|
|
224
|
+
guard AXUIElementCopyAttributeValue(element, name as CFString, &value) == .success, let v = value else { return nil }
|
|
225
|
+
return v as? T
|
|
226
|
+
}
|
|
227
|
+
|
|
228
|
+
func axPoint(_ value: CFTypeRef?) -> CGPoint? {
|
|
229
|
+
guard let v = value, CFGetTypeID(v) == AXValueGetTypeID() else { return nil }
|
|
230
|
+
var point = CGPoint.zero
|
|
231
|
+
return AXValueGetValue(v as! AXValue, .cgPoint, &point) ? point : nil
|
|
232
|
+
}
|
|
233
|
+
|
|
234
|
+
func axSize(_ value: CFTypeRef?) -> CGSize? {
|
|
235
|
+
guard let v = value, CFGetTypeID(v) == AXValueGetTypeID() else { return nil }
|
|
236
|
+
var size = CGSize.zero
|
|
237
|
+
return AXValueGetValue(v as! AXValue, .cgSize, &size) ? size : nil
|
|
238
|
+
}
|
|
239
|
+
|
|
240
|
+
func axFrame(_ element: AXUIElement) -> CGRect? {
|
|
241
|
+
var pos: CFTypeRef?
|
|
242
|
+
var size: CFTypeRef?
|
|
243
|
+
AXUIElementCopyAttributeValue(element, kAXPositionAttribute as CFString, &pos)
|
|
244
|
+
AXUIElementCopyAttributeValue(element, kAXSizeAttribute as CFString, &size)
|
|
245
|
+
guard let p = axPoint(pos), let s = axSize(size) else { return nil }
|
|
246
|
+
return CGRect(origin: p, size: s)
|
|
247
|
+
}
|
|
248
|
+
|
|
249
|
+
func rectJSON(_ r: CGRect) -> [Double] {
|
|
250
|
+
[Double(r.origin.x), Double(r.origin.y), Double(r.size.width), Double(r.size.height)]
|
|
251
|
+
}
|
|
252
|
+
|
|
253
|
+
func stringify(_ value: Any?, limit: Int) -> String? {
|
|
254
|
+
guard let value = value else { return nil }
|
|
255
|
+
var text: String?
|
|
256
|
+
switch value {
|
|
257
|
+
case let s as String: text = s
|
|
258
|
+
case let n as NSNumber: text = n.stringValue
|
|
259
|
+
case let a as NSAttributedString: text = a.string
|
|
260
|
+
case let url as URL: text = url.absoluteString
|
|
261
|
+
default:
|
|
262
|
+
let cf = value as CFTypeRef
|
|
263
|
+
if CFGetTypeID(cf) == AXValueGetTypeID() { return nil }
|
|
264
|
+
if CFGetTypeID(cf) == AXUIElementGetTypeID() { return nil }
|
|
265
|
+
text = nil
|
|
266
|
+
}
|
|
267
|
+
guard var t = text, !t.isEmpty else { return nil }
|
|
268
|
+
if t.count > limit { t = String(t.prefix(limit)) + "…" }
|
|
269
|
+
return t
|
|
270
|
+
}
|
|
271
|
+
|
|
272
|
+
// MARK: - Element tables (stable indices per app)
|
|
273
|
+
|
|
274
|
+
final class ElementTable {
|
|
275
|
+
var elements: [AXUIElement] = []
|
|
276
|
+
var buckets: [CFHashCode: [Int]] = [:]
|
|
277
|
+
|
|
278
|
+
func index(of element: AXUIElement) -> Int {
|
|
279
|
+
let hash = CFHash(element)
|
|
280
|
+
if let candidates = buckets[hash] {
|
|
281
|
+
for i in candidates where CFEqual(elements[i], element) { return i }
|
|
282
|
+
}
|
|
283
|
+
elements.append(element)
|
|
284
|
+
let i = elements.count - 1
|
|
285
|
+
buckets[hash, default: []].append(i)
|
|
286
|
+
return i
|
|
287
|
+
}
|
|
288
|
+
|
|
289
|
+
func element(_ index: Int) throws -> AXUIElement {
|
|
290
|
+
guard index >= 0 && index < elements.count else {
|
|
291
|
+
throw HelperError("No element \(index). Read the accessibility tree again.", code: "stale_index")
|
|
292
|
+
}
|
|
293
|
+
return elements[index]
|
|
294
|
+
}
|
|
295
|
+
}
|
|
296
|
+
|
|
297
|
+
var tables: [Int: ElementTable] = [:]
|
|
298
|
+
|
|
299
|
+
func table(_ pid: Int) -> ElementTable {
|
|
300
|
+
if let t = tables[pid] { return t }
|
|
301
|
+
let t = ElementTable()
|
|
302
|
+
tables[pid] = t
|
|
303
|
+
return t
|
|
304
|
+
}
|
|
305
|
+
|
|
306
|
+
// MARK: - Windows
|
|
307
|
+
|
|
308
|
+
func cgWindows(pid: Int) -> [[String: Any]] {
|
|
309
|
+
guard let info = CGWindowListCopyWindowInfo([.optionAll, .excludeDesktopElements], kCGNullWindowID) as? [[String: Any]] else { return [] }
|
|
310
|
+
return info.filter { ($0[kCGWindowOwnerPID as String] as? Int) == pid && ($0[kCGWindowLayer as String] as? Int) == 0 }
|
|
311
|
+
}
|
|
312
|
+
|
|
313
|
+
func cgWindowID(pid: Int, frame: CGRect) -> Int? {
|
|
314
|
+
var best: (Int, CGFloat)?
|
|
315
|
+
for w in cgWindows(pid: pid) {
|
|
316
|
+
guard let id = w[kCGWindowNumber as String] as? Int,
|
|
317
|
+
let b = w[kCGWindowBounds as String] as? [String: Any],
|
|
318
|
+
let rect = CGRect(dictionaryRepresentation: b as CFDictionary) else { continue }
|
|
319
|
+
let d = abs(rect.minX - frame.minX) + abs(rect.minY - frame.minY) + abs(rect.width - frame.width) + abs(rect.height - frame.height)
|
|
320
|
+
if best == nil || d < best!.1 { best = (id, d) }
|
|
321
|
+
}
|
|
322
|
+
if let b = best, b.1 < 8 { return b.0 }
|
|
323
|
+
return nil
|
|
324
|
+
}
|
|
325
|
+
|
|
326
|
+
func backingScale(for frame: CGRect) -> Double {
|
|
327
|
+
// NSScreen frames are bottom-left origin; convert the window's center.
|
|
328
|
+
guard let primary = NSScreen.screens.first else { return 2 }
|
|
329
|
+
let center = CGPoint(x: frame.midX, y: primary.frame.height - frame.midY)
|
|
330
|
+
for screen in NSScreen.screens where screen.frame.contains(center) {
|
|
331
|
+
return Double(screen.backingScaleFactor)
|
|
332
|
+
}
|
|
333
|
+
return Double(primary.backingScaleFactor)
|
|
334
|
+
}
|
|
335
|
+
|
|
336
|
+
func axWindows(_ pid: Int) -> [AXUIElement] {
|
|
337
|
+
let app = AXUIElementCreateApplication(pid_t(pid))
|
|
338
|
+
return axAttr(app, kAXWindowsAttribute) ?? []
|
|
339
|
+
}
|
|
340
|
+
|
|
341
|
+
func windowsInfo(_ pid: Int) -> [[String: Any]] {
|
|
342
|
+
let app = AXUIElementCreateApplication(pid_t(pid))
|
|
343
|
+
let focused: AXUIElement? = axAttr(app, kAXFocusedWindowAttribute)
|
|
344
|
+
let main: AXUIElement? = axAttr(app, kAXMainWindowAttribute)
|
|
345
|
+
var out: [[String: Any]] = []
|
|
346
|
+
for (i, w) in axWindows(pid).enumerated() {
|
|
347
|
+
guard let frame = axFrame(w) else { continue }
|
|
348
|
+
var info: [String: Any] = [
|
|
349
|
+
"index": i,
|
|
350
|
+
"title": (axAttr(w, kAXTitleAttribute) as String?) ?? "",
|
|
351
|
+
"frame": rectJSON(frame),
|
|
352
|
+
"scale": backingScale(for: frame),
|
|
353
|
+
"minimized": (axAttr(w, kAXMinimizedAttribute) as Bool?) ?? false,
|
|
354
|
+
"focused": focused.map { CFEqual($0, w) } ?? false,
|
|
355
|
+
"main": main.map { CFEqual($0, w) } ?? false,
|
|
356
|
+
]
|
|
357
|
+
if let role: String = axAttr(w, kAXSubroleAttribute) { info["subrole"] = role }
|
|
358
|
+
if let id = cgWindowID(pid: pid, frame: frame) { info["windowId"] = id }
|
|
359
|
+
out.append(info)
|
|
360
|
+
}
|
|
361
|
+
return out
|
|
362
|
+
}
|
|
363
|
+
|
|
364
|
+
// Picks the window an action should target: an explicit index, else focused, main, first.
|
|
365
|
+
func targetWindow(_ pid: Int, _ params: [String: Any]) throws -> (AXUIElement, Int) {
|
|
366
|
+
let windows = axWindows(pid)
|
|
367
|
+
if windows.isEmpty { throw HelperError("The app has no open windows", code: "no_window") }
|
|
368
|
+
if let i = params.int("window") {
|
|
369
|
+
guard i >= 0 && i < windows.count else { throw HelperError("No window \(i); the app has \(windows.count)", code: "no_window") }
|
|
370
|
+
return (windows[i], i)
|
|
371
|
+
}
|
|
372
|
+
let app = AXUIElementCreateApplication(pid_t(pid))
|
|
373
|
+
for attr in [kAXFocusedWindowAttribute, kAXMainWindowAttribute] {
|
|
374
|
+
if let w: AXUIElement = axAttr(app, attr), let i = windows.firstIndex(where: { CFEqual($0, w) }) {
|
|
375
|
+
return (w, i)
|
|
376
|
+
}
|
|
377
|
+
}
|
|
378
|
+
return (windows[0], 0)
|
|
379
|
+
}
|
|
380
|
+
|
|
381
|
+
// MARK: - Accessibility tree
|
|
382
|
+
|
|
383
|
+
let treeAttributes: [String] = [
|
|
384
|
+
kAXRoleAttribute, kAXSubroleAttribute, kAXTitleAttribute, kAXValueAttribute, kAXDescriptionAttribute,
|
|
385
|
+
kAXHelpAttribute, kAXPlaceholderValueAttribute, kAXPositionAttribute, kAXSizeAttribute, kAXChildrenAttribute,
|
|
386
|
+
kAXEnabledAttribute, kAXFocusedAttribute, kAXSelectedAttribute, kAXIdentifierAttribute,
|
|
387
|
+
kAXRoleDescriptionAttribute,
|
|
388
|
+
]
|
|
389
|
+
|
|
390
|
+
func axTree(_ params: [String: Any]) throws -> [String: Any] {
|
|
391
|
+
let pid = try params.requireInt("pid")
|
|
392
|
+
_ = try runningApp(pid)
|
|
393
|
+
let maxNodes = params.int("maxNodes") ?? 800
|
|
394
|
+
let maxDepth = params.int("maxDepth") ?? 40
|
|
395
|
+
let maxText = params.int("maxText") ?? 160
|
|
396
|
+
let includeMenus = params.bool("menus") ?? false
|
|
397
|
+
let (window, windowIndex) = try targetWindow(pid, params)
|
|
398
|
+
let t = table(pid)
|
|
399
|
+
var nodes: [[String: Any]] = []
|
|
400
|
+
var truncated = false
|
|
401
|
+
|
|
402
|
+
let appElement = AXUIElementCreateApplication(pid_t(pid))
|
|
403
|
+
let focusedElement: AXUIElement? = axAttr(appElement, kAXFocusedUIElementAttribute)
|
|
404
|
+
|
|
405
|
+
func visit(_ element: AXUIElement, _ depth: Int) {
|
|
406
|
+
if nodes.count >= maxNodes { truncated = true; return }
|
|
407
|
+
var raw: CFArray?
|
|
408
|
+
let err = AXUIElementCopyMultipleAttributeValues(element, treeAttributes as CFArray, AXCopyMultipleAttributeOptions(rawValue: 0), &raw)
|
|
409
|
+
guard err == .success, let values = raw as? [Any] else { return }
|
|
410
|
+
func v(_ i: Int) -> Any? {
|
|
411
|
+
let value = values[i]
|
|
412
|
+
if CFGetTypeID(value as CFTypeRef) == AXValueGetTypeID() {
|
|
413
|
+
// Errors come back as AXValue of type kAXValueAXErrorType.
|
|
414
|
+
if AXValueGetType(value as! AXValue) == .axError { return nil }
|
|
415
|
+
}
|
|
416
|
+
return value is NSNull ? nil : value
|
|
417
|
+
}
|
|
418
|
+
var node: [String: Any] = ["i": t.index(of: element), "d": depth]
|
|
419
|
+
node["role"] = (v(0) as? String) ?? "AXUnknown"
|
|
420
|
+
if let s = v(1) as? String { node["subrole"] = s }
|
|
421
|
+
if let s = stringify(v(2), limit: maxText) { node["title"] = s }
|
|
422
|
+
if let s = stringify(v(3), limit: maxText) { node["value"] = s }
|
|
423
|
+
if let s = stringify(v(4), limit: maxText) { node["desc"] = s }
|
|
424
|
+
if let s = stringify(v(5), limit: maxText) { node["help"] = s }
|
|
425
|
+
if let s = stringify(v(6), limit: maxText) { node["placeholder"] = s }
|
|
426
|
+
if let p = axPoint(v(7) as CFTypeRef?), let s = axSize(v(8) as CFTypeRef?) {
|
|
427
|
+
node["frame"] = rectJSON(CGRect(origin: p, size: s))
|
|
428
|
+
}
|
|
429
|
+
if let enabled = v(10) as? Bool, !enabled { node["disabled"] = true }
|
|
430
|
+
if let focused = v(11) as? Bool, focused { node["focused"] = true }
|
|
431
|
+
else if let f = focusedElement, CFEqual(f, element) { node["focused"] = true }
|
|
432
|
+
if let selected = v(12) as? Bool, selected { node["selected"] = true }
|
|
433
|
+
if let s = v(13) as? String, !s.isEmpty { node["id"] = s }
|
|
434
|
+
var names: CFArray?
|
|
435
|
+
if AXUIElementCopyActionNames(element, &names) == .success, let actions = names as? [String] {
|
|
436
|
+
let useful = actions.filter { $0 != "AXScrollToVisible" && $0 != "AXShowDefaultUI" && $0 != "AXShowAlternateUI" }
|
|
437
|
+
if !useful.isEmpty { node["actions"] = useful }
|
|
438
|
+
}
|
|
439
|
+
var settable: DarwinBoolean = false
|
|
440
|
+
if AXUIElementIsAttributeSettable(element, kAXValueAttribute as CFString, &settable) == .success, settable.boolValue {
|
|
441
|
+
node["settable"] = true
|
|
442
|
+
}
|
|
443
|
+
nodes.append(node)
|
|
444
|
+
if depth >= maxDepth { return }
|
|
445
|
+
if let children = v(9) as? [AXUIElement] {
|
|
446
|
+
for child in children {
|
|
447
|
+
if nodes.count >= maxNodes { truncated = true; return }
|
|
448
|
+
visit(child, depth + 1)
|
|
449
|
+
}
|
|
450
|
+
}
|
|
451
|
+
}
|
|
452
|
+
|
|
453
|
+
AXUIElementSetMessagingTimeout(appElement, 2.0)
|
|
454
|
+
visit(window, 0)
|
|
455
|
+
if includeMenus, let bar: AXUIElement = axAttr(appElement, kAXMenuBarAttribute) {
|
|
456
|
+
visit(bar, 0)
|
|
457
|
+
}
|
|
458
|
+
guard let frame = axFrame(window) else { throw HelperError("Window has no frame", code: "no_window") }
|
|
459
|
+
var windowInfo: [String: Any] = [
|
|
460
|
+
"index": windowIndex,
|
|
461
|
+
"title": (axAttr(window, kAXTitleAttribute) as String?) ?? "",
|
|
462
|
+
"frame": rectJSON(frame),
|
|
463
|
+
"scale": backingScale(for: frame),
|
|
464
|
+
]
|
|
465
|
+
if let id = cgWindowID(pid: pid, frame: frame) { windowInfo["windowId"] = id }
|
|
466
|
+
return ["window": windowInfo, "nodes": nodes, "truncated": truncated]
|
|
467
|
+
}
|
|
468
|
+
|
|
469
|
+
// MARK: - Element actions
|
|
470
|
+
|
|
471
|
+
func elementFrame(_ params: [String: Any]) throws -> [String: Any] {
|
|
472
|
+
let pid = try params.requireInt("pid")
|
|
473
|
+
let element = try table(pid).element(try params.requireInt("index"))
|
|
474
|
+
guard let frame = axFrame(element) else { throw HelperError("Element has no on-screen frame", code: "no_frame") }
|
|
475
|
+
return ["frame": rectJSON(frame)]
|
|
476
|
+
}
|
|
477
|
+
|
|
478
|
+
func performAction(_ params: [String: Any]) throws -> [String: Any] {
|
|
479
|
+
let pid = try params.requireInt("pid")
|
|
480
|
+
let element = try table(pid).element(try params.requireInt("index"))
|
|
481
|
+
let action = params.string("action") ?? kAXPressAction
|
|
482
|
+
let err = AXUIElementPerformAction(element, action as CFString)
|
|
483
|
+
if err != .success {
|
|
484
|
+
throw HelperError("\(action) failed (AXError \(err.rawValue))", code: "action_failed")
|
|
485
|
+
}
|
|
486
|
+
return [:]
|
|
487
|
+
}
|
|
488
|
+
|
|
489
|
+
func actionNames(_ params: [String: Any]) throws -> [String: Any] {
|
|
490
|
+
let pid = try params.requireInt("pid")
|
|
491
|
+
let element = try table(pid).element(try params.requireInt("index"))
|
|
492
|
+
var names: CFArray?
|
|
493
|
+
AXUIElementCopyActionNames(element, &names)
|
|
494
|
+
return ["actions": (names as? [String]) ?? []]
|
|
495
|
+
}
|
|
496
|
+
|
|
497
|
+
func setValue(_ params: [String: Any]) throws -> [String: Any] {
|
|
498
|
+
let pid = try params.requireInt("pid")
|
|
499
|
+
let element = try table(pid).element(try params.requireInt("index"))
|
|
500
|
+
let attribute = params.string("attribute") ?? kAXValueAttribute
|
|
501
|
+
// Finder takes a new value on a file-name field but only relabels the row:
|
|
502
|
+
// the file keeps its old name. Only the rename editor (focused) commits.
|
|
503
|
+
if attribute == kAXValueAttribute, NSRunningApplication(processIdentifier: pid_t(pid))?.bundleIdentifier == "com.apple.finder",
|
|
504
|
+
(axAttr(element, kAXRoleAttribute) as String?) == kAXTextFieldRole, (axAttr(element, kAXFocusedAttribute) as Bool?) != true {
|
|
505
|
+
throw HelperError("Finder would show the new name without renaming the file. Rename instead: click the item, pressKey(\"Return\"), typeText(newName), pressKey(\"Return\").", code: "label_only")
|
|
506
|
+
}
|
|
507
|
+
var value: CFTypeRef
|
|
508
|
+
if let s = params["value"] as? String { value = s as CFString }
|
|
509
|
+
else if let b = params["value"] as? Bool { value = (b ? kCFBooleanTrue : kCFBooleanFalse)! }
|
|
510
|
+
else if let n = params["value"] as? NSNumber { value = n }
|
|
511
|
+
else { throw HelperError("setValue needs a string, number, or boolean", code: "bad_request") }
|
|
512
|
+
let err = AXUIElementSetAttributeValue(element, attribute as CFString, value)
|
|
513
|
+
if err != .success { throw HelperError("Setting \(attribute) failed (AXError \(err.rawValue))", code: "action_failed") }
|
|
514
|
+
return [:]
|
|
515
|
+
}
|
|
516
|
+
|
|
517
|
+
func selectText(_ params: [String: Any]) throws -> [String: Any] {
|
|
518
|
+
let pid = try params.requireInt("pid")
|
|
519
|
+
let element = try table(pid).element(try params.requireInt("index"))
|
|
520
|
+
AXUIElementSetAttributeValue(element, kAXFocusedAttribute as CFString, kCFBooleanTrue)
|
|
521
|
+
var range = CFRange(location: params.int("start") ?? 0, length: params.int("length") ?? 0)
|
|
522
|
+
if params.int("length") == nil, let value: String = axAttr(element, kAXValueAttribute) {
|
|
523
|
+
range.length = (value as NSString).length - range.location
|
|
524
|
+
}
|
|
525
|
+
guard let axRange = AXValueCreate(.cfRange, &range) else { throw HelperError("Bad range") }
|
|
526
|
+
let err = AXUIElementSetAttributeValue(element, kAXSelectedTextRangeAttribute as CFString, axRange)
|
|
527
|
+
if err != .success { throw HelperError("Selecting text failed (AXError \(err.rawValue))", code: "action_failed") }
|
|
528
|
+
return [:]
|
|
529
|
+
}
|
|
530
|
+
|
|
531
|
+
// MARK: - Input
|
|
532
|
+
|
|
533
|
+
let eventSource: CGEventSource? = {
|
|
534
|
+
let source = CGEventSource(stateID: .hidSystemState)
|
|
535
|
+
source?.userData = syntheticMarker
|
|
536
|
+
return source
|
|
537
|
+
}()
|
|
538
|
+
|
|
539
|
+
// Mouse events. On the background route they go to the target process,
|
|
540
|
+
// stamped with its window; otherwise to the system at the screen point.
|
|
541
|
+
func post(_ event: CGEvent?) {
|
|
542
|
+
guard let e = event else { return }
|
|
543
|
+
e.setIntegerValueField(.eventSourceUserData, value: syntheticMarker)
|
|
544
|
+
if let route = backgroundRoute {
|
|
545
|
+
route.postMouse(e)
|
|
546
|
+
} else {
|
|
547
|
+
e.post(tap: .cghidEventTap)
|
|
548
|
+
}
|
|
549
|
+
}
|
|
550
|
+
|
|
551
|
+
// Keyboard events go to the target app's process when it is in the
|
|
552
|
+
// background, so typing does not depend on (or steal) the user's focus.
|
|
553
|
+
var keyTargetPid: pid_t?
|
|
554
|
+
|
|
555
|
+
func postKey(_ event: CGEvent?) {
|
|
556
|
+
guard let e = event else { return }
|
|
557
|
+
e.setIntegerValueField(.eventSourceUserData, value: syntheticMarker)
|
|
558
|
+
if let route = backgroundRoute {
|
|
559
|
+
route.postKey(e)
|
|
560
|
+
} else if let pid = keyTargetPid, let app = NSRunningApplication(processIdentifier: pid), !app.isActive {
|
|
561
|
+
e.postToPid(pid)
|
|
562
|
+
} else {
|
|
563
|
+
e.post(tap: .cghidEventTap)
|
|
564
|
+
}
|
|
565
|
+
}
|
|
566
|
+
|
|
567
|
+
func withKeyTarget<T>(_ params: [String: Any], _ body: () throws -> T) rethrows -> T {
|
|
568
|
+
keyTargetPid = params.int("pid").map { pid_t($0) }
|
|
569
|
+
defer { keyTargetPid = nil }
|
|
570
|
+
return try body()
|
|
571
|
+
}
|
|
572
|
+
|
|
573
|
+
// The element under a screen point, registered in the app's element table.
|
|
574
|
+
func hitTest(_ params: [String: Any]) throws -> [String: Any] {
|
|
575
|
+
let pid = try params.requireInt("pid")
|
|
576
|
+
let app = AXUIElementCreateApplication(pid_t(pid))
|
|
577
|
+
var hit: AXUIElement?
|
|
578
|
+
let err = AXUIElementCopyElementAtPosition(app, Float(try params.requireDouble("x")), Float(try params.requireDouble("y")), &hit)
|
|
579
|
+
guard err == .success, let element = hit else { return [:] }
|
|
580
|
+
var elementPid: pid_t = 0
|
|
581
|
+
AXUIElementGetPid(element, &elementPid)
|
|
582
|
+
if Int(elementPid) != pid { return ["foreign": true] }
|
|
583
|
+
var out: [String: Any] = ["index": table(pid).index(of: element), "role": (axAttr(element, kAXRoleAttribute) as String?) ?? "AXUnknown"]
|
|
584
|
+
var names: CFArray?
|
|
585
|
+
if AXUIElementCopyActionNames(element, &names) == .success, let actions = names as? [String] { out["actions"] = actions }
|
|
586
|
+
if let frame = axFrame(element) { out["frame"] = rectJSON(frame) }
|
|
587
|
+
return out
|
|
588
|
+
}
|
|
589
|
+
|
|
590
|
+
let keyCodes: [String: CGKeyCode] = [
|
|
591
|
+
"a": 0, "s": 1, "d": 2, "f": 3, "h": 4, "g": 5, "z": 6, "x": 7, "c": 8, "v": 9, "b": 11, "q": 12,
|
|
592
|
+
"w": 13, "e": 14, "r": 15, "y": 16, "t": 17, "1": 18, "2": 19, "3": 20, "4": 21, "6": 22, "5": 23,
|
|
593
|
+
"=": 24, "equal": 24, "9": 25, "7": 26, "-": 27, "minus": 27, "8": 28, "0": 29, "]": 30, "bracketright": 30,
|
|
594
|
+
"o": 31, "u": 32, "[": 33, "bracketleft": 33, "i": 34, "p": 35, "Return": 36, "l": 37, "j": 38,
|
|
595
|
+
"'": 39, "apostrophe": 39, "k": 40, ";": 41, "semicolon": 41, "\\": 42, "backslash": 42, ",": 43, "comma": 43,
|
|
596
|
+
"/": 44, "slash": 44, "n": 45, "m": 46, ".": 47, "period": 47, "Tab": 48, "space": 49, " ": 49,
|
|
597
|
+
"`": 50, "grave": 50, "BackSpace": 51, "Escape": 53, "Caps_Lock": 57, "KP_Enter": 76,
|
|
598
|
+
"F1": 122, "F2": 120, "F3": 99, "F4": 118, "F5": 96, "F6": 97, "F7": 98, "F8": 100, "F9": 101,
|
|
599
|
+
"F10": 109, "F11": 103, "F12": 111, "F13": 105, "F14": 107, "F15": 113, "F16": 106, "F17": 64,
|
|
600
|
+
"F18": 79, "F19": 80, "F20": 90, "Help": 114, "Insert": 114, "Home": 115, "Page_Up": 116, "Delete": 117,
|
|
601
|
+
"End": 119, "Page_Down": 121, "Left": 123, "Right": 124, "Down": 125, "Up": 126,
|
|
602
|
+
"ctrl": 59, "shift": 56, "alt": 58, "super": 55, "fn": 63,
|
|
603
|
+
]
|
|
604
|
+
|
|
605
|
+
// Shifted characters on a US layout, so "ctrl+shift+?" and "?" both work.
|
|
606
|
+
let shifted: [String: String] = [
|
|
607
|
+
"!": "1", "@": "2", "#": "3", "$": "4", "%": "5", "^": "6", "&": "7", "*": "8", "(": "9", ")": "0",
|
|
608
|
+
"_": "-", "+": "=", "plus": "=", "{": "[", "}": "]", "|": "\\", ":": ";", "\"": "'", "<": ",", ">": ".",
|
|
609
|
+
"?": "/", "~": "`",
|
|
610
|
+
]
|
|
611
|
+
|
|
612
|
+
func flags(_ mods: [String]) -> CGEventFlags {
|
|
613
|
+
var f: CGEventFlags = []
|
|
614
|
+
for m in mods {
|
|
615
|
+
switch m {
|
|
616
|
+
case "ctrl": f.insert(.maskControl)
|
|
617
|
+
case "shift": f.insert(.maskShift)
|
|
618
|
+
case "alt": f.insert(.maskAlternate)
|
|
619
|
+
case "super": f.insert(.maskCommand)
|
|
620
|
+
case "fn": f.insert(.maskSecondaryFn)
|
|
621
|
+
default: break
|
|
622
|
+
}
|
|
623
|
+
}
|
|
624
|
+
return f
|
|
625
|
+
}
|
|
626
|
+
|
|
627
|
+
func pressKeys(_ params: [String: Any]) throws -> [String: Any] {
|
|
628
|
+
guard let chords = params["chords"] as? [[String: Any]] else { throw HelperError("missing chords", code: "bad_request") }
|
|
629
|
+
for chord in chords {
|
|
630
|
+
guard var key = chord["key"] as? String else { continue }
|
|
631
|
+
var mods = (chord["mods"] as? [String]) ?? []
|
|
632
|
+
if let base = shifted[key] {
|
|
633
|
+
key = base
|
|
634
|
+
if !mods.contains("shift") { mods.append("shift") }
|
|
635
|
+
}
|
|
636
|
+
let lower = key.count == 1 ? key.lowercased() : key
|
|
637
|
+
if key.count == 1 && key != lower && !mods.contains("shift") { mods.append("shift") }
|
|
638
|
+
if let code = keyCodes[lower] {
|
|
639
|
+
let f = flags(mods)
|
|
640
|
+
let down = CGEvent(keyboardEventSource: eventSource, virtualKey: code, keyDown: true)
|
|
641
|
+
let up = CGEvent(keyboardEventSource: eventSource, virtualKey: code, keyDown: false)
|
|
642
|
+
down?.flags = f
|
|
643
|
+
up?.flags = f
|
|
644
|
+
postKey(down)
|
|
645
|
+
usleep(8000)
|
|
646
|
+
postKey(up)
|
|
647
|
+
} else if mods.isEmpty {
|
|
648
|
+
try typeText(["text": key])
|
|
649
|
+
} else {
|
|
650
|
+
throw HelperError("No key code for \"\(key)\" with modifiers", code: "bad_key")
|
|
651
|
+
}
|
|
652
|
+
usleep(25000)
|
|
653
|
+
}
|
|
654
|
+
return [:]
|
|
655
|
+
}
|
|
656
|
+
|
|
657
|
+
@discardableResult
|
|
658
|
+
func typeText(_ params: [String: Any]) throws -> [String: Any] {
|
|
659
|
+
guard let text = params.string("text") else { throw HelperError("missing text", code: "bad_request") }
|
|
660
|
+
var chunk: [UniChar] = []
|
|
661
|
+
func flush() {
|
|
662
|
+
guard !chunk.isEmpty else { return }
|
|
663
|
+
let down = CGEvent(keyboardEventSource: eventSource, virtualKey: 0, keyDown: true)
|
|
664
|
+
let up = CGEvent(keyboardEventSource: eventSource, virtualKey: 0, keyDown: false)
|
|
665
|
+
down?.keyboardSetUnicodeString(stringLength: chunk.count, unicodeString: chunk)
|
|
666
|
+
up?.keyboardSetUnicodeString(stringLength: chunk.count, unicodeString: chunk)
|
|
667
|
+
postKey(down)
|
|
668
|
+
usleep(4000)
|
|
669
|
+
postKey(up)
|
|
670
|
+
usleep(12000)
|
|
671
|
+
chunk.removeAll()
|
|
672
|
+
}
|
|
673
|
+
for scalar in text.unicodeScalars {
|
|
674
|
+
if scalar == "\n" || scalar == "\r" || scalar == "\t" {
|
|
675
|
+
flush()
|
|
676
|
+
let code: CGKeyCode = scalar == "\t" ? 48 : 36
|
|
677
|
+
postKey(CGEvent(keyboardEventSource: eventSource, virtualKey: code, keyDown: true))
|
|
678
|
+
usleep(8000)
|
|
679
|
+
postKey(CGEvent(keyboardEventSource: eventSource, virtualKey: code, keyDown: false))
|
|
680
|
+
usleep(20000)
|
|
681
|
+
continue
|
|
682
|
+
}
|
|
683
|
+
// One character per event: some apps drop the tail of multi-character events.
|
|
684
|
+
chunk.append(contentsOf: Array(String(scalar).utf16))
|
|
685
|
+
flush()
|
|
686
|
+
}
|
|
687
|
+
flush()
|
|
688
|
+
return [:]
|
|
689
|
+
}
|
|
690
|
+
|
|
691
|
+
func mouseButton(_ name: String?) -> (CGMouseButton, CGEventType, CGEventType, CGEventType) {
|
|
692
|
+
switch name {
|
|
693
|
+
case "right": return (.right, .rightMouseDown, .rightMouseUp, .rightMouseDragged)
|
|
694
|
+
case "middle": return (.center, .otherMouseDown, .otherMouseUp, .otherMouseDragged)
|
|
695
|
+
default: return (.left, .leftMouseDown, .leftMouseUp, .leftMouseDragged)
|
|
696
|
+
}
|
|
697
|
+
}
|
|
698
|
+
|
|
699
|
+
func click(_ params: [String: Any]) throws -> [String: Any] {
|
|
700
|
+
let point = CGPoint(x: try params.requireDouble("x"), y: try params.requireDouble("y"))
|
|
701
|
+
let count = max(1, params.int("count") ?? 1)
|
|
702
|
+
let (button, downType, upType, _) = mouseButton(params.string("button"))
|
|
703
|
+
let f = flags((params["mods"] as? [String]) ?? [])
|
|
704
|
+
post(CGEvent(mouseEventSource: eventSource, mouseType: .mouseMoved, mouseCursorPosition: point, mouseButton: .left))
|
|
705
|
+
usleep(30000)
|
|
706
|
+
backgroundRoute?.primeChromiumClick()
|
|
707
|
+
for n in 1...count {
|
|
708
|
+
let down = CGEvent(mouseEventSource: eventSource, mouseType: downType, mouseCursorPosition: point, mouseButton: button)
|
|
709
|
+
let up = CGEvent(mouseEventSource: eventSource, mouseType: upType, mouseCursorPosition: point, mouseButton: button)
|
|
710
|
+
down?.setIntegerValueField(.mouseEventClickState, value: Int64(n))
|
|
711
|
+
up?.setIntegerValueField(.mouseEventClickState, value: Int64(n))
|
|
712
|
+
down?.flags = f
|
|
713
|
+
up?.flags = f
|
|
714
|
+
post(down)
|
|
715
|
+
// An NSButton's mouseDown polls for the matching up; too tight a gap races it.
|
|
716
|
+
usleep(28000)
|
|
717
|
+
post(up)
|
|
718
|
+
usleep(count > 1 ? 80000 : 30000)
|
|
719
|
+
}
|
|
720
|
+
return [:]
|
|
721
|
+
}
|
|
722
|
+
|
|
723
|
+
func moveMouse(_ params: [String: Any]) throws -> [String: Any] {
|
|
724
|
+
let point = CGPoint(x: try params.requireDouble("x"), y: try params.requireDouble("y"))
|
|
725
|
+
post(CGEvent(mouseEventSource: eventSource, mouseType: .mouseMoved, mouseCursorPosition: point, mouseButton: .left))
|
|
726
|
+
return [:]
|
|
727
|
+
}
|
|
728
|
+
|
|
729
|
+
func scroll(_ params: [String: Any]) throws -> [String: Any] {
|
|
730
|
+
let point = CGPoint(x: try params.requireDouble("x"), y: try params.requireDouble("y"))
|
|
731
|
+
post(CGEvent(mouseEventSource: eventSource, mouseType: .mouseMoved, mouseCursorPosition: point, mouseButton: .left))
|
|
732
|
+
usleep(30000)
|
|
733
|
+
// Positive dy scrolls content down (toward the end), like a trackpad swipe up.
|
|
734
|
+
let dy = Int32(-(params.int("dy") ?? 0))
|
|
735
|
+
let dx = Int32(-(params.int("dx") ?? 0))
|
|
736
|
+
let steps = max(abs(dy), abs(dx), 1)
|
|
737
|
+
for i in 0..<steps {
|
|
738
|
+
let sy: Int32 = i < abs(dy) ? (dy > 0 ? 1 : -1) : 0
|
|
739
|
+
let sx: Int32 = i < abs(dx) ? (dx > 0 ? 1 : -1) : 0
|
|
740
|
+
let e = CGEvent(scrollWheelEvent2Source: eventSource, units: .line, wheelCount: 2, wheel1: sy, wheel2: sx, wheel3: 0)
|
|
741
|
+
e?.location = point
|
|
742
|
+
// Some clients (Citrix Viewer) read only the point and fixed-point
|
|
743
|
+
// deltas, which the line-based constructor leaves at 0 for axis 2.
|
|
744
|
+
if sx != 0 {
|
|
745
|
+
e?.setIntegerValueField(.scrollWheelEventPointDeltaAxis2, value: Int64(sx) * 10)
|
|
746
|
+
e?.setDoubleValueField(.scrollWheelEventFixedPtDeltaAxis2, value: Double(sx))
|
|
747
|
+
}
|
|
748
|
+
if let route = backgroundRoute, let event = e {
|
|
749
|
+
event.setIntegerValueField(.eventSourceUserData, value: syntheticMarker)
|
|
750
|
+
route.postScroll(event)
|
|
751
|
+
} else {
|
|
752
|
+
post(e)
|
|
753
|
+
}
|
|
754
|
+
usleep(15000)
|
|
755
|
+
}
|
|
756
|
+
return [:]
|
|
757
|
+
}
|
|
758
|
+
|
|
759
|
+
func drag(_ params: [String: Any]) throws -> [String: Any] {
|
|
760
|
+
guard let raw = params["path"] as? [[Double]], raw.count >= 2 else { throw HelperError("drag needs at least two points", code: "bad_request") }
|
|
761
|
+
let points = raw.map { CGPoint(x: $0[0], y: $0[1]) }
|
|
762
|
+
let (button, downType, upType, dragType) = mouseButton(params.string("button"))
|
|
763
|
+
post(CGEvent(mouseEventSource: eventSource, mouseType: .mouseMoved, mouseCursorPosition: points[0], mouseButton: button))
|
|
764
|
+
usleep(40000)
|
|
765
|
+
// Background delivery carries no click count unless we set one, and AppKit
|
|
766
|
+
// ignores a mouseDown without it.
|
|
767
|
+
func withClickState(_ e: CGEvent?) -> CGEvent? {
|
|
768
|
+
e?.setIntegerValueField(.mouseEventClickState, value: 1)
|
|
769
|
+
return e
|
|
770
|
+
}
|
|
771
|
+
post(withClickState(CGEvent(mouseEventSource: eventSource, mouseType: downType, mouseCursorPosition: points[0], mouseButton: button)))
|
|
772
|
+
usleep(60000)
|
|
773
|
+
for i in 1..<points.count {
|
|
774
|
+
let a = points[i - 1], b = points[i]
|
|
775
|
+
let steps = max(4, Int(hypot(b.x - a.x, b.y - a.y) / 12))
|
|
776
|
+
for s in 1...steps {
|
|
777
|
+
let t = CGFloat(s) / CGFloat(steps)
|
|
778
|
+
let p = CGPoint(x: a.x + (b.x - a.x) * t, y: a.y + (b.y - a.y) * t)
|
|
779
|
+
post(withClickState(CGEvent(mouseEventSource: eventSource, mouseType: dragType, mouseCursorPosition: p, mouseButton: button)))
|
|
780
|
+
usleep(8000)
|
|
781
|
+
}
|
|
782
|
+
}
|
|
783
|
+
usleep(40000)
|
|
784
|
+
post(withClickState(CGEvent(mouseEventSource: eventSource, mouseType: upType, mouseCursorPosition: points.last!, mouseButton: button)))
|
|
785
|
+
return [:]
|
|
786
|
+
}
|
|
787
|
+
|
|
788
|
+
// MARK: - Background input
|
|
789
|
+
//
|
|
790
|
+
// Delivers input to an app that stays in the background: its window is made
|
|
791
|
+
// key inside the app without activating it, and events are posted to the
|
|
792
|
+
// process with the window stamped on them. The real cursor never moves and the
|
|
793
|
+
// user's frontmost app keeps focus. This uses private SkyLight functions that
|
|
794
|
+
// Codex, Claude desktop, cua-driver, Peekaboo, and yabai also rely on; each is
|
|
795
|
+
// looked up at runtime, and when one is missing the caller falls back to the
|
|
796
|
+
// foreground route.
|
|
797
|
+
|
|
798
|
+
enum SkyLight {
|
|
799
|
+
typealias PostRecord = @convention(c) (UnsafeRawPointer, UnsafeRawPointer) -> Int32
|
|
800
|
+
typealias PostToPid = @convention(c) (pid_t, CGEvent) -> Void
|
|
801
|
+
typealias SetWindowLocation = @convention(c) (CGEvent, CGPoint) -> Void
|
|
802
|
+
typealias SetIntegerField = @convention(c) (CGEvent, UInt32, Int64) -> Void
|
|
803
|
+
typealias MainConnection = @convention(c) () -> UInt32
|
|
804
|
+
typealias WindowOwner = @convention(c) (UInt32, UInt32, UnsafeMutablePointer<UInt32>) -> Int32
|
|
805
|
+
typealias ConnectionPSN = @convention(c) (UInt32, UnsafeMutableRawPointer) -> Int32
|
|
806
|
+
typealias ProcessForPID = @convention(c) (pid_t, UnsafeMutableRawPointer) -> Int32
|
|
807
|
+
|
|
808
|
+
static let handle = dlopen("/System/Library/PrivateFrameworks/SkyLight.framework/SkyLight", RTLD_LAZY)
|
|
809
|
+
|
|
810
|
+
static func symbol<T>(_ name: String, _: T.Type) -> T? {
|
|
811
|
+
_ = handle
|
|
812
|
+
// RTLD_DEFAULT is ((void *)-2) and is not imported into Swift.
|
|
813
|
+
guard let pointer = dlsym(UnsafeMutableRawPointer(bitPattern: -2), name) else { return nil }
|
|
814
|
+
return unsafeBitCast(pointer, to: T.self)
|
|
815
|
+
}
|
|
816
|
+
|
|
817
|
+
static let postRecord = symbol("SLPSPostEventRecordTo", PostRecord.self)
|
|
818
|
+
static let postToPid = symbol("SLEventPostToPid", PostToPid.self)
|
|
819
|
+
static let setWindowLocation = symbol("CGEventSetWindowLocation", SetWindowLocation.self)
|
|
820
|
+
static let setIntegerField = symbol("CGEventSetIntegerValueField", SetIntegerField.self)
|
|
821
|
+
static let mainConnection = symbol("SLSMainConnectionID", MainConnection.self) ?? symbol("CGSMainConnectionID", MainConnection.self)
|
|
822
|
+
static let windowOwner = symbol("SLSGetWindowOwner", WindowOwner.self)
|
|
823
|
+
static let connectionPSN = symbol("SLSGetConnectionPSN", ConnectionPSN.self)
|
|
824
|
+
static let processForPID = symbol("GetProcessForPID", ProcessForPID.self)
|
|
825
|
+
|
|
826
|
+
// What the background route needs: records to make a window key, and a
|
|
827
|
+
// way to stamp window fields. Posting falls back to CGEventPostToPid.
|
|
828
|
+
static var available: Bool { postRecord != nil && setIntegerField != nil && (windowOwner != nil || processForPID != nil) }
|
|
829
|
+
|
|
830
|
+
static var capabilities: [String: Bool] {
|
|
831
|
+
[
|
|
832
|
+
"postEventRecord": postRecord != nil,
|
|
833
|
+
"postToPid": postToPid != nil,
|
|
834
|
+
"setWindowLocation": setWindowLocation != nil,
|
|
835
|
+
"setIntegerField": setIntegerField != nil,
|
|
836
|
+
"processSerialNumber": windowOwner != nil || processForPID != nil,
|
|
837
|
+
]
|
|
838
|
+
}
|
|
839
|
+
|
|
840
|
+
// The 8-byte ProcessSerialNumber of the process that owns `windowId`.
|
|
841
|
+
static func psn(pid: pid_t, windowId: UInt32) -> [UInt8]? {
|
|
842
|
+
var psn = [UInt8](repeating: 0, count: 8)
|
|
843
|
+
if let owner = windowOwner, let toPSN = connectionPSN, let main = mainConnection {
|
|
844
|
+
var cid: UInt32 = 0
|
|
845
|
+
if owner(main(), windowId, &cid) == 0, cid != 0,
|
|
846
|
+
psn.withUnsafeMutableBytes({ toPSN(cid, $0.baseAddress!) }) == 0 {
|
|
847
|
+
return psn
|
|
848
|
+
}
|
|
849
|
+
}
|
|
850
|
+
if let forPid = processForPID, psn.withUnsafeMutableBytes({ forPid(pid, $0.baseAddress!) }) == 0 {
|
|
851
|
+
return psn
|
|
852
|
+
}
|
|
853
|
+
return nil
|
|
854
|
+
}
|
|
855
|
+
|
|
856
|
+
// 248-byte event records. Layout from yabai and cua-driver (MIT).
|
|
857
|
+
static func focusRecord(_ windowId: UInt32, focus: Bool) -> [UInt8] {
|
|
858
|
+
var record = [UInt8](repeating: 0, count: 0xF8)
|
|
859
|
+
record[0x04] = 0xF8
|
|
860
|
+
record[0x08] = 0x0D
|
|
861
|
+
withUnsafeBytes(of: windowId.littleEndian) { record.replaceSubrange(0x3C..<0x40, with: $0) }
|
|
862
|
+
record[0x8A] = focus ? 0x01 : 0x02
|
|
863
|
+
return record
|
|
864
|
+
}
|
|
865
|
+
|
|
866
|
+
static func makeKeyRecord(_ windowId: UInt32, kind: UInt8) -> [UInt8] {
|
|
867
|
+
var record = [UInt8](repeating: 0, count: 0xF8)
|
|
868
|
+
record[0x04] = 0xF8
|
|
869
|
+
record[0x08] = kind
|
|
870
|
+
record[0x3A] = 0x10
|
|
871
|
+
for i in 0x20..<0x30 { record[i] = 0xFF }
|
|
872
|
+
withUnsafeBytes(of: windowId.littleEndian) { record.replaceSubrange(0x3C..<0x40, with: $0) }
|
|
873
|
+
return record
|
|
874
|
+
}
|
|
875
|
+
|
|
876
|
+
@discardableResult
|
|
877
|
+
static func send(_ record: [UInt8], to psn: [UInt8]) -> Bool {
|
|
878
|
+
guard let post = postRecord else { return false }
|
|
879
|
+
return psn.withUnsafeBytes { p in record.withUnsafeBytes { r in post(p.baseAddress!, r.baseAddress!) } } == 0
|
|
880
|
+
}
|
|
881
|
+
}
|
|
882
|
+
|
|
883
|
+
func bundleUsesChromium(_ pid: pid_t) -> Bool {
|
|
884
|
+
guard let url = NSRunningApplication(processIdentifier: pid)?.bundleURL else { return false }
|
|
885
|
+
let frameworks = url.appendingPathComponent("Contents/Frameworks")
|
|
886
|
+
let names = (try? FileManager.default.contentsOfDirectory(atPath: frameworks.path)) ?? []
|
|
887
|
+
if names.contains(where: { $0.hasPrefix("Electron Framework") || $0.hasPrefix("Chromium Embedded Framework") || $0.contains("Chromium Framework") }) {
|
|
888
|
+
return true
|
|
889
|
+
}
|
|
890
|
+
let chromiumBrowsers: Set<String> = ["com.google.Chrome", "com.microsoft.edgemac", "com.brave.Browser", "company.thebrowser.Browser", "com.vivaldi.Vivaldi", "com.operasoftware.Opera"]
|
|
891
|
+
return chromiumBrowsers.contains(NSRunningApplication(processIdentifier: pid)?.bundleIdentifier ?? "")
|
|
892
|
+
}
|
|
893
|
+
|
|
894
|
+
// The CGWindowID of the app's key window as Accessibility reports it.
|
|
895
|
+
func keyWindowId(_ pid: pid_t) -> UInt32? {
|
|
896
|
+
let app = AXUIElementCreateApplication(pid)
|
|
897
|
+
guard let window: AXUIElement = axAttr(app, kAXFocusedWindowAttribute), let frame = axFrame(window),
|
|
898
|
+
let id = cgWindowID(pid: Int(pid), frame: frame) else { return nil }
|
|
899
|
+
return UInt32(id)
|
|
900
|
+
}
|
|
901
|
+
|
|
902
|
+
func cgWindowBounds(_ windowId: UInt32) -> CGRect? {
|
|
903
|
+
guard let info = CGWindowListCopyWindowInfo([.optionIncludingWindow], CGWindowID(windowId)) as? [[String: Any]],
|
|
904
|
+
let bounds = info.first?[kCGWindowBounds as String] as? [String: Any] else { return nil }
|
|
905
|
+
return CGRect(dictionaryRepresentation: bounds as CFDictionary)
|
|
906
|
+
}
|
|
907
|
+
|
|
908
|
+
final class BackgroundRoute {
|
|
909
|
+
let pid: pid_t
|
|
910
|
+
let windowId: UInt32
|
|
911
|
+
let origin: CGPoint
|
|
912
|
+
let chromium: Bool
|
|
913
|
+
let localLocation: Bool
|
|
914
|
+
let gesture = Int64.random(in: 1...Int64(Int32.max))
|
|
915
|
+
|
|
916
|
+
init(pid: pid_t, windowId: UInt32, origin: CGPoint, localLocation: Bool) {
|
|
917
|
+
self.pid = pid
|
|
918
|
+
self.windowId = windowId
|
|
919
|
+
self.origin = origin
|
|
920
|
+
self.chromium = bundleUsesChromium(pid)
|
|
921
|
+
self.localLocation = localLocation
|
|
922
|
+
}
|
|
923
|
+
|
|
924
|
+
private func stamp(_ e: CGEvent) {
|
|
925
|
+
guard let set = SkyLight.setIntegerField else { return }
|
|
926
|
+
set(e, 40, Int64(pid)) // kCGEventTargetUnixProcessID
|
|
927
|
+
set(e, 51, Int64(windowId)) // window number
|
|
928
|
+
set(e, 91, Int64(windowId)) // kCGMouseEventWindowUnderMousePointer
|
|
929
|
+
set(e, 92, Int64(windowId)) // ...ThatCanHandleThisEvent
|
|
930
|
+
set(e, 58, gesture) // groups one gesture's events
|
|
931
|
+
if let setLocation = SkyLight.setWindowLocation {
|
|
932
|
+
let p = e.location
|
|
933
|
+
setLocation(e, localLocation ? CGPoint(x: p.x - origin.x, y: p.y - origin.y) : p)
|
|
934
|
+
}
|
|
935
|
+
}
|
|
936
|
+
|
|
937
|
+
// Chromium and Electron want the SkyLight route; AppKit takes the public
|
|
938
|
+
// one. Never both: AppKit controls would see each click twice.
|
|
939
|
+
func postMouse(_ e: CGEvent) {
|
|
940
|
+
stamp(e)
|
|
941
|
+
if chromium, let post = SkyLight.postToPid { post(pid, e) } else { e.postToPid(pid) }
|
|
942
|
+
}
|
|
943
|
+
|
|
944
|
+
func postKey(_ e: CGEvent) {
|
|
945
|
+
if let post = SkyLight.postToPid { post(pid, e) } else { e.postToPid(pid) }
|
|
946
|
+
}
|
|
947
|
+
|
|
948
|
+
// Scroll wheel events reach background AppKit views only through both
|
|
949
|
+
// routes (cua-driver does the same); a doubled scroll tick is harmless.
|
|
950
|
+
func postScroll(_ e: CGEvent) {
|
|
951
|
+
stamp(e)
|
|
952
|
+
e.postToPid(pid)
|
|
953
|
+
if let post = SkyLight.postToPid { post(pid, e) }
|
|
954
|
+
}
|
|
955
|
+
|
|
956
|
+
// Chromium ignores a background click until it has seen user activation;
|
|
957
|
+
// an off-screen click satisfies that without touching anything.
|
|
958
|
+
func primeChromiumClick() {
|
|
959
|
+
guard chromium else { return }
|
|
960
|
+
let off = CGPoint(x: -1, y: -1)
|
|
961
|
+
postMouse(CGEvent(mouseEventSource: eventSource, mouseType: .leftMouseDown, mouseCursorPosition: off, mouseButton: .left)!)
|
|
962
|
+
usleep(10000)
|
|
963
|
+
postMouse(CGEvent(mouseEventSource: eventSource, mouseType: .leftMouseUp, mouseCursorPosition: off, mouseButton: .left)!)
|
|
964
|
+
usleep(100000)
|
|
965
|
+
}
|
|
966
|
+
}
|
|
967
|
+
|
|
968
|
+
var backgroundRoute: BackgroundRoute?
|
|
969
|
+
|
|
970
|
+
// Makes `windowId` the key window inside its app without activating the app.
|
|
971
|
+
// Sends nothing to any other process. Returns whether AX confirms it.
|
|
972
|
+
func makeWindowKey(pid: pid_t, windowId: UInt32, settle: UInt32 = 120_000) -> Bool {
|
|
973
|
+
guard let psn = SkyLight.psn(pid: pid, windowId: windowId) else { return false }
|
|
974
|
+
if let current = keyWindowId(pid), current != windowId {
|
|
975
|
+
SkyLight.send(SkyLight.focusRecord(current, focus: false), to: psn)
|
|
976
|
+
usleep(settle)
|
|
977
|
+
}
|
|
978
|
+
SkyLight.send(SkyLight.focusRecord(windowId, focus: true), to: psn)
|
|
979
|
+
usleep(settle)
|
|
980
|
+
SkyLight.send(SkyLight.makeKeyRecord(windowId, kind: 0x01), to: psn)
|
|
981
|
+
SkyLight.send(SkyLight.makeKeyRecord(windowId, kind: 0x02), to: psn)
|
|
982
|
+
for _ in 0..<10 {
|
|
983
|
+
if keyWindowId(pid) == windowId { return true }
|
|
984
|
+
usleep(40000)
|
|
985
|
+
}
|
|
986
|
+
return false
|
|
987
|
+
}
|
|
988
|
+
|
|
989
|
+
enum InputKind { case keys, mouse }
|
|
990
|
+
|
|
991
|
+
// True when keyboard focus is in a small uncapturable window inside
|
|
992
|
+
// `windowId`, such as Finder's rename field editor. Keys already go there, and
|
|
993
|
+
// making the parent key again would end the edit. A background app reports no
|
|
994
|
+
// focused window, so look for a focused child instead.
|
|
995
|
+
func focusIsInside(pid: pid_t, windowId: UInt32) -> Bool {
|
|
996
|
+
guard let bounds = cgWindowBounds(windowId)?.insetBy(dx: -2, dy: -2) else { return false }
|
|
997
|
+
for w in axWindows(Int(pid)) {
|
|
998
|
+
guard let frame = axFrame(w), frame.height < 120, bounds.contains(frame),
|
|
999
|
+
cgWindowID(pid: Int(pid), frame: frame) == nil else { continue }
|
|
1000
|
+
// Finder's editor "window" is the focused text field itself.
|
|
1001
|
+
let children: [AXUIElement] = axAttr(w, kAXChildrenAttribute) ?? []
|
|
1002
|
+
if ([w] + children).contains(where: { (axAttr($0, kAXFocusedAttribute) as Bool?) == true }) { return true }
|
|
1003
|
+
}
|
|
1004
|
+
return false
|
|
1005
|
+
}
|
|
1006
|
+
|
|
1007
|
+
// Runs `body` with input routed to the background app named in params, and
|
|
1008
|
+
// restores the user's frontmost app if the target took focus anyway.
|
|
1009
|
+
func withInputRoute(_ params: [String: Any], _ kind: InputKind, _ body: () throws -> [String: Any]) throws -> [String: Any] {
|
|
1010
|
+
guard params.string("mode") == "background" else { return try body() }
|
|
1011
|
+
guard SkyLight.available else {
|
|
1012
|
+
throw HelperError("Background input is unavailable on this macOS (private SkyLight functions missing).", code: "background_unavailable")
|
|
1013
|
+
}
|
|
1014
|
+
let pid = pid_t(try params.requireInt("pid"))
|
|
1015
|
+
let windowId = UInt32(try params.requireInt("windowId"))
|
|
1016
|
+
guard let bounds = cgWindowBounds(windowId) else { throw HelperError("Window \(windowId) is gone", code: "no_window") }
|
|
1017
|
+
let front = NSWorkspace.shared.frontmostApplication
|
|
1018
|
+
var keyed = true
|
|
1019
|
+
if kind == .keys {
|
|
1020
|
+
keyed = focusIsInside(pid: pid, windowId: windowId) || makeWindowKey(pid: pid, windowId: windowId)
|
|
1021
|
+
if !keyed {
|
|
1022
|
+
throw HelperError("Could not make the window key in the background; use foreground mode for this app.", code: "background_unconfirmed")
|
|
1023
|
+
}
|
|
1024
|
+
} else if let psn = SkyLight.psn(pid: pid, windowId: windowId) {
|
|
1025
|
+
// Target-only focus record so AppKit routes the click to this window.
|
|
1026
|
+
SkyLight.send(SkyLight.focusRecord(windowId, focus: true), to: psn)
|
|
1027
|
+
usleep(50000)
|
|
1028
|
+
}
|
|
1029
|
+
let route = BackgroundRoute(pid: pid, windowId: windowId, origin: bounds.origin, localLocation: params.string("windowLocation") != "screen")
|
|
1030
|
+
backgroundRoute = route
|
|
1031
|
+
defer { backgroundRoute = nil }
|
|
1032
|
+
var result = try body()
|
|
1033
|
+
usleep(60000)
|
|
1034
|
+
var restored = false
|
|
1035
|
+
if let front = front, front.processIdentifier != pid,
|
|
1036
|
+
NSWorkspace.shared.frontmostApplication?.processIdentifier == pid {
|
|
1037
|
+
// The target took focus anyway: give it back to the user's app.
|
|
1038
|
+
AXUIElementSetAttributeValue(AXUIElementCreateApplication(front.processIdentifier), kAXFrontmostAttribute as CFString, kCFBooleanTrue)
|
|
1039
|
+
restored = true
|
|
1040
|
+
}
|
|
1041
|
+
result["route"] = route.chromium ? "background-skylight" : "background"
|
|
1042
|
+
result["focusRestored"] = restored
|
|
1043
|
+
return result
|
|
1044
|
+
}
|
|
1045
|
+
|
|
1046
|
+
// MARK: - Agent cursor
|
|
1047
|
+
//
|
|
1048
|
+
// A drawn pointer that shows where the agent acts. It sits just above the
|
|
1049
|
+
// target window (not above everything), ignores the mouse, and never moves
|
|
1050
|
+
// the real cursor.
|
|
1051
|
+
|
|
1052
|
+
final class AgentCursor {
|
|
1053
|
+
static let shared = AgentCursor()
|
|
1054
|
+
var enabled = true
|
|
1055
|
+
private var window: NSWindow?
|
|
1056
|
+
private var hideTimer: Timer?
|
|
1057
|
+
private let size = CGSize(width: 36, height: 36)
|
|
1058
|
+
|
|
1059
|
+
private func makeWindow() -> NSWindow {
|
|
1060
|
+
let w = NSWindow(contentRect: NSRect(origin: .zero, size: size), styleMask: .borderless, backing: .buffered, defer: false)
|
|
1061
|
+
w.isOpaque = false
|
|
1062
|
+
w.backgroundColor = .clear
|
|
1063
|
+
w.hasShadow = false
|
|
1064
|
+
w.ignoresMouseEvents = true
|
|
1065
|
+
w.level = .normal
|
|
1066
|
+
w.collectionBehavior = [.canJoinAllSpaces, .fullScreenAuxiliary, .stationary, .ignoresCycle]
|
|
1067
|
+
w.sharingType = .readOnly
|
|
1068
|
+
w.title = "glovebox cursor"
|
|
1069
|
+
let view = NSImageView(frame: NSRect(origin: .zero, size: size))
|
|
1070
|
+
let config = NSImage.SymbolConfiguration(pointSize: 22, weight: .regular).applying(.init(paletteColors: [.white, .systemPurple]))
|
|
1071
|
+
view.image = NSImage(systemSymbolName: "cursorarrow", accessibilityDescription: "agent cursor")?.withSymbolConfiguration(config)
|
|
1072
|
+
view.imageAlignment = .alignTopLeft
|
|
1073
|
+
w.contentView = view
|
|
1074
|
+
return w
|
|
1075
|
+
}
|
|
1076
|
+
|
|
1077
|
+
// Moves the cursor to a screen point (top-left origin) above `windowId`.
|
|
1078
|
+
// Blocks until the move animation finishes so the action lines up with it.
|
|
1079
|
+
func move(to point: CGPoint, above windowId: Int?) {
|
|
1080
|
+
guard enabled else { return }
|
|
1081
|
+
var from = NSPoint.zero
|
|
1082
|
+
var target = NSPoint.zero
|
|
1083
|
+
var duration: TimeInterval = 0
|
|
1084
|
+
DispatchQueue.main.sync {
|
|
1085
|
+
let w = self.window ?? self.makeWindow()
|
|
1086
|
+
self.window = w
|
|
1087
|
+
guard let primary = NSScreen.screens.first else { return }
|
|
1088
|
+
// AppKit frames are bottom-left origin; the arrow tip is the frame's top-left.
|
|
1089
|
+
target = NSPoint(x: point.x - 4, y: primary.frame.height - point.y - self.size.height + 3)
|
|
1090
|
+
from = w.isVisible ? w.frame.origin : target
|
|
1091
|
+
let distance = hypot(target.x - from.x, target.y - from.y)
|
|
1092
|
+
duration = w.isVisible ? min(0.45, 0.12 + Double(distance) / 2400) : 0
|
|
1093
|
+
if let id = windowId { w.order(.above, relativeTo: id) } else { w.orderFrontRegardless() }
|
|
1094
|
+
if duration == 0 { w.setFrameOrigin(target) }
|
|
1095
|
+
self.hideTimer?.invalidate()
|
|
1096
|
+
self.hideTimer = Timer.scheduledTimer(withTimeInterval: 4, repeats: false) { _ in self.hide() }
|
|
1097
|
+
}
|
|
1098
|
+
guard duration > 0 else { return }
|
|
1099
|
+
// Stepped by hand: an animator() frame move left the window where it was.
|
|
1100
|
+
let steps = max(1, Int(duration * 60))
|
|
1101
|
+
for step in 1...steps {
|
|
1102
|
+
let t = Double(step) / Double(steps)
|
|
1103
|
+
let eased = t * t * (3 - 2 * t)
|
|
1104
|
+
let next = NSPoint(x: from.x + (target.x - from.x) * eased, y: from.y + (target.y - from.y) * eased)
|
|
1105
|
+
DispatchQueue.main.sync { self.window?.setFrameOrigin(next) }
|
|
1106
|
+
usleep(useconds_t(duration / Double(steps) * 1_000_000))
|
|
1107
|
+
}
|
|
1108
|
+
}
|
|
1109
|
+
|
|
1110
|
+
func hide() {
|
|
1111
|
+
let work: () -> Void = { self.window?.orderOut(nil) }
|
|
1112
|
+
if Thread.isMainThread { work() } else { DispatchQueue.main.async(execute: work) }
|
|
1113
|
+
}
|
|
1114
|
+
}
|
|
1115
|
+
|
|
1116
|
+
func cursorCommand(_ params: [String: Any]) throws -> [String: Any] {
|
|
1117
|
+
if let enabled = params.bool("enabled") {
|
|
1118
|
+
AgentCursor.shared.enabled = enabled
|
|
1119
|
+
if !enabled { AgentCursor.shared.hide() }
|
|
1120
|
+
}
|
|
1121
|
+
if params.bool("hide") == true { AgentCursor.shared.hide() }
|
|
1122
|
+
if let x = params.double("x"), let y = params.double("y") {
|
|
1123
|
+
AgentCursor.shared.move(to: CGPoint(x: x, y: y), above: params.int("windowId"))
|
|
1124
|
+
}
|
|
1125
|
+
return ["enabled": AgentCursor.shared.enabled]
|
|
1126
|
+
}
|
|
1127
|
+
|
|
1128
|
+
// MARK: - Screenshots
|
|
1129
|
+
|
|
1130
|
+
func encode(_ image: CGImage, format: String, quality: Double) throws -> Data {
|
|
1131
|
+
let data = NSMutableData()
|
|
1132
|
+
let type = format == "jpeg" ? UTType.jpeg.identifier : UTType.png.identifier
|
|
1133
|
+
guard let dest = CGImageDestinationCreateWithData(data, type as CFString, 1, nil) else { throw HelperError("Cannot encode image") }
|
|
1134
|
+
let options: [CFString: Any] = format == "jpeg" ? [kCGImageDestinationLossyCompressionQuality: quality] : [:]
|
|
1135
|
+
CGImageDestinationAddImage(dest, image, options as CFDictionary)
|
|
1136
|
+
guard CGImageDestinationFinalize(dest) else { throw HelperError("Cannot encode image") }
|
|
1137
|
+
return data as Data
|
|
1138
|
+
}
|
|
1139
|
+
|
|
1140
|
+
func screenshot(_ params: [String: Any]) throws -> [String: Any] {
|
|
1141
|
+
let windowId = try params.requireInt("windowId")
|
|
1142
|
+
let width = try params.requireInt("width")
|
|
1143
|
+
let height = try params.requireInt("height")
|
|
1144
|
+
let format = params.string("format") ?? "png"
|
|
1145
|
+
let quality = params.double("quality") ?? 0.8
|
|
1146
|
+
|
|
1147
|
+
// CGPreflightScreenCaptureAccess can report false for a granted ad-hoc
|
|
1148
|
+
// signed app, so ScreenCaptureKit itself is the source of truth.
|
|
1149
|
+
let done = DispatchSemaphore(value: 0)
|
|
1150
|
+
var captured: CGImage?
|
|
1151
|
+
var failure: Error?
|
|
1152
|
+
Task {
|
|
1153
|
+
do {
|
|
1154
|
+
let content = try await SCShareableContent.excludingDesktopWindows(false, onScreenWindowsOnly: false)
|
|
1155
|
+
guard let window = content.windows.first(where: { Int($0.windowID) == windowId }) else {
|
|
1156
|
+
throw HelperError("Window \(windowId) is not capturable (closed, or on another Space?)", code: "no_window")
|
|
1157
|
+
}
|
|
1158
|
+
let filter = SCContentFilter(desktopIndependentWindow: window)
|
|
1159
|
+
let config = SCStreamConfiguration()
|
|
1160
|
+
config.width = width
|
|
1161
|
+
config.height = height
|
|
1162
|
+
config.showsCursor = false
|
|
1163
|
+
config.scalesToFit = true
|
|
1164
|
+
if #available(macOS 14.0, *) { config.ignoreShadowsSingleWindow = true }
|
|
1165
|
+
captured = try await SCScreenshotManager.captureImage(contentFilter: filter, configuration: config)
|
|
1166
|
+
} catch {
|
|
1167
|
+
failure = error
|
|
1168
|
+
}
|
|
1169
|
+
done.signal()
|
|
1170
|
+
}
|
|
1171
|
+
if done.wait(timeout: .now() + 15) == .timedOut { throw HelperError("Screenshot timed out") }
|
|
1172
|
+
if let error = failure as? HelperError { throw error }
|
|
1173
|
+
if let error = failure {
|
|
1174
|
+
if !screenCaptureAllowed() {
|
|
1175
|
+
throw HelperError("Screen Recording permission is missing for Glovebox Helper. Run `glovebox doctor`.", code: "permission")
|
|
1176
|
+
}
|
|
1177
|
+
throw HelperError("Screenshot failed: \(error.localizedDescription)")
|
|
1178
|
+
}
|
|
1179
|
+
guard let image = captured else { throw HelperError("Screenshot returned nothing") }
|
|
1180
|
+
let data = try encode(image, format: format, quality: quality)
|
|
1181
|
+
return ["data": data.base64EncodedString(), "width": image.width, "height": image.height, "mimeType": format == "jpeg" ? "image/jpeg" : "image/png"]
|
|
1182
|
+
}
|
|
1183
|
+
|
|
1184
|
+
// True when ScreenCaptureKit will hand us window content.
|
|
1185
|
+
func screenCaptureAllowed() -> Bool {
|
|
1186
|
+
if CGPreflightScreenCaptureAccess() { return true }
|
|
1187
|
+
let done = DispatchSemaphore(value: 0)
|
|
1188
|
+
var ok = false
|
|
1189
|
+
Task {
|
|
1190
|
+
ok = (try? await SCShareableContent.excludingDesktopWindows(false, onScreenWindowsOnly: true)) != nil
|
|
1191
|
+
done.signal()
|
|
1192
|
+
}
|
|
1193
|
+
return done.wait(timeout: .now() + 5) == .success && ok
|
|
1194
|
+
}
|
|
1195
|
+
|
|
1196
|
+
// MARK: - Stopping scripts
|
|
1197
|
+
//
|
|
1198
|
+
// Two ways for the user to stop a running script, both broadcast as an
|
|
1199
|
+
// "escape" event that clients turn into an abort:
|
|
1200
|
+
// - a physical Escape press (synthetic events we post carry syntheticMarker
|
|
1201
|
+
// and are ignored);
|
|
1202
|
+
// - "Stop Running Scripts" in the menu-bar item shown while agents are connected.
|
|
1203
|
+
|
|
1204
|
+
var watchTap: CFMachPort?
|
|
1205
|
+
var keyMonitor: Any?
|
|
1206
|
+
var lastEscape = Date.distantPast
|
|
1207
|
+
|
|
1208
|
+
func userPressedEscape() {
|
|
1209
|
+
// The tap and the monitor can both see one press.
|
|
1210
|
+
if Date().timeIntervalSince(lastEscape) < 0.2 { return }
|
|
1211
|
+
lastEscape = Date()
|
|
1212
|
+
emit(["event": "escape"])
|
|
1213
|
+
}
|
|
1214
|
+
|
|
1215
|
+
// A global key monitor. macOS delivers key events to it only with Input
|
|
1216
|
+
// Monitoring granted; Accessibility alone installs it but it stays silent.
|
|
1217
|
+
func startEscapeMonitor() {
|
|
1218
|
+
guard keyMonitor == nil, AXIsProcessTrusted() else { return }
|
|
1219
|
+
let install = {
|
|
1220
|
+
keyMonitor = NSEvent.addGlobalMonitorForEvents(matching: .keyDown) { event in
|
|
1221
|
+
guard event.keyCode == 53 else { return }
|
|
1222
|
+
if let cg = event.cgEvent, cg.getIntegerValueField(.eventSourceUserData) == syntheticMarker { return }
|
|
1223
|
+
userPressedEscape()
|
|
1224
|
+
}
|
|
1225
|
+
}
|
|
1226
|
+
if Thread.isMainThread { install() } else { DispatchQueue.main.sync(execute: install) }
|
|
1227
|
+
}
|
|
1228
|
+
|
|
1229
|
+
// An event tap also sees keys sent to apps that bypass the monitor, but
|
|
1230
|
+
// macOS only feeds it with Input Monitoring; without it the tap is silent.
|
|
1231
|
+
func startEscapeTap() {
|
|
1232
|
+
guard watchTap == nil, CGPreflightListenEventAccess() else { return }
|
|
1233
|
+
let mask = CGEventMask(1 << CGEventType.keyDown.rawValue)
|
|
1234
|
+
let callback: CGEventTapCallBack = { _, type, event, _ in
|
|
1235
|
+
if type == .tapDisabledByTimeout || type == .tapDisabledByUserInput {
|
|
1236
|
+
if let tap = watchTap { CGEvent.tapEnable(tap: tap, enable: true) }
|
|
1237
|
+
return Unmanaged.passUnretained(event)
|
|
1238
|
+
}
|
|
1239
|
+
if type == .keyDown,
|
|
1240
|
+
event.getIntegerValueField(.keyboardEventKeycode) == 53,
|
|
1241
|
+
event.getIntegerValueField(.eventSourceUserData) != syntheticMarker {
|
|
1242
|
+
userPressedEscape()
|
|
1243
|
+
}
|
|
1244
|
+
return Unmanaged.passUnretained(event)
|
|
1245
|
+
}
|
|
1246
|
+
guard let tap = CGEvent.tapCreate(tap: .cgSessionEventTap, place: .headInsertEventTap, options: .listenOnly,
|
|
1247
|
+
eventsOfInterest: mask, callback: callback, userInfo: nil) else { return }
|
|
1248
|
+
watchTap = tap
|
|
1249
|
+
let source = CFMachPortCreateRunLoopSource(nil, tap, 0)
|
|
1250
|
+
CFRunLoopAddSource(CFRunLoopGetMain(), source, .commonModes)
|
|
1251
|
+
CGEvent.tapEnable(tap: tap, enable: true)
|
|
1252
|
+
}
|
|
1253
|
+
|
|
1254
|
+
@discardableResult
|
|
1255
|
+
func startEscapeWatcher() -> Bool {
|
|
1256
|
+
startEscapeMonitor()
|
|
1257
|
+
startEscapeTap()
|
|
1258
|
+
return keyMonitor != nil || watchTap != nil
|
|
1259
|
+
}
|
|
1260
|
+
|
|
1261
|
+
func escapeWatcherActive() -> Bool {
|
|
1262
|
+
startEscapeWatcher()
|
|
1263
|
+
}
|
|
1264
|
+
|
|
1265
|
+
// MARK: - Menu-bar item
|
|
1266
|
+
|
|
1267
|
+
final class StatusMenu: NSObject, NSMenuDelegate {
|
|
1268
|
+
var item: NSStatusItem?
|
|
1269
|
+
|
|
1270
|
+
// Called on the main thread whenever clients connect or disconnect.
|
|
1271
|
+
func refresh() {
|
|
1272
|
+
let connections = Connection.all()
|
|
1273
|
+
if connections.isEmpty {
|
|
1274
|
+
if let item = item { NSStatusBar.system.removeStatusItem(item) }
|
|
1275
|
+
item = nil
|
|
1276
|
+
return
|
|
1277
|
+
}
|
|
1278
|
+
if item == nil {
|
|
1279
|
+
let created = NSStatusBar.system.statusItem(withLength: NSStatusItem.variableLength)
|
|
1280
|
+
if let image = NSImage(systemSymbolName: "hand.raised.fill", accessibilityDescription: "glovebox") {
|
|
1281
|
+
image.isTemplate = true
|
|
1282
|
+
created.button?.image = image
|
|
1283
|
+
} else {
|
|
1284
|
+
created.button?.title = "glovebox"
|
|
1285
|
+
}
|
|
1286
|
+
let menu = NSMenu()
|
|
1287
|
+
menu.delegate = self
|
|
1288
|
+
created.menu = menu
|
|
1289
|
+
item = created
|
|
1290
|
+
}
|
|
1291
|
+
item?.button?.toolTip = "glovebox: \(connections.count) agent session(s) connected"
|
|
1292
|
+
}
|
|
1293
|
+
|
|
1294
|
+
func menuNeedsUpdate(_ menu: NSMenu) {
|
|
1295
|
+
menu.removeAllItems()
|
|
1296
|
+
let connections = Connection.all()
|
|
1297
|
+
menu.addItem(withTitle: "glovebox — \(connections.count) connected", action: nil, keyEquivalent: "")
|
|
1298
|
+
for c in connections {
|
|
1299
|
+
let apps = c.approved.sorted().joined(separator: ", ")
|
|
1300
|
+
menu.addItem(withTitle: " \(c.client)\(apps.isEmpty ? "" : ": \(apps)")", action: nil, keyEquivalent: "")
|
|
1301
|
+
}
|
|
1302
|
+
menu.addItem(.separator())
|
|
1303
|
+
let stop = NSMenuItem(title: "Stop Running Scripts", action: #selector(stopScripts), keyEquivalent: "")
|
|
1304
|
+
stop.target = self
|
|
1305
|
+
menu.addItem(stop)
|
|
1306
|
+
let revoke = NSMenuItem(title: "Revoke Session Approvals", action: #selector(revokeApprovals), keyEquivalent: "")
|
|
1307
|
+
revoke.target = self
|
|
1308
|
+
menu.addItem(revoke)
|
|
1309
|
+
let cursor = NSMenuItem(title: "Hide Agent Cursor", action: #selector(toggleCursor), keyEquivalent: "")
|
|
1310
|
+
cursor.target = self
|
|
1311
|
+
cursor.state = AgentCursor.shared.enabled ? .off : .on
|
|
1312
|
+
menu.addItem(cursor)
|
|
1313
|
+
let quit = NSMenuItem(title: "Quit Glovebox Helper", action: #selector(quitHelper), keyEquivalent: "")
|
|
1314
|
+
quit.target = self
|
|
1315
|
+
menu.addItem(quit)
|
|
1316
|
+
}
|
|
1317
|
+
|
|
1318
|
+
@objc func stopScripts() { emit(["event": "escape"]) }
|
|
1319
|
+
|
|
1320
|
+
@objc func toggleCursor() {
|
|
1321
|
+
AgentCursor.shared.enabled.toggle()
|
|
1322
|
+
if !AgentCursor.shared.enabled { AgentCursor.shared.hide() }
|
|
1323
|
+
}
|
|
1324
|
+
|
|
1325
|
+
@objc func revokeApprovals() {
|
|
1326
|
+
work.async { for c in Connection.all() { c.approved.removeAll() } }
|
|
1327
|
+
}
|
|
1328
|
+
|
|
1329
|
+
@objc func quitHelper() {
|
|
1330
|
+
unlink(socketPath)
|
|
1331
|
+
exit(0)
|
|
1332
|
+
}
|
|
1333
|
+
}
|
|
1334
|
+
|
|
1335
|
+
let statusMenu = StatusMenu()
|
|
1336
|
+
|
|
1337
|
+
func refreshStatusMenu() {
|
|
1338
|
+
DispatchQueue.main.async { statusMenu.refresh() }
|
|
1339
|
+
}
|
|
1340
|
+
|
|
1341
|
+
// MARK: - Approval policy
|
|
1342
|
+
//
|
|
1343
|
+
// The approval gate lives here, at the process that holds the Accessibility
|
|
1344
|
+
// grant, so a client cannot skip it by talking to the socket directly.
|
|
1345
|
+
|
|
1346
|
+
let gloveboxHome: String = ProcessInfo.processInfo.environment["GLOVEBOX_HOME"] ?? (NSHomeDirectory() + "/.glovebox")
|
|
1347
|
+
let configPath = gloveboxHome + "/config.json"
|
|
1348
|
+
|
|
1349
|
+
// Apps that can run arbitrary commands, hold secrets, or change system
|
|
1350
|
+
// security. These are never offered in a dialog; only an explicit `allow`
|
|
1351
|
+
// entry in config.json unlocks one.
|
|
1352
|
+
let blockedByDefault: Set<String> = [
|
|
1353
|
+
"com.apple.terminal", "com.googlecode.iterm2", "com.mitchellh.ghostty", "dev.warp.warp-stable",
|
|
1354
|
+
"com.github.wez.wezterm", "org.alacritty", "net.kovidgoyal.kitty", "org.tabby", "co.zeit.hyper",
|
|
1355
|
+
"com.apple.systempreferences", "com.apple.keychainaccess", "com.apple.passwords",
|
|
1356
|
+
"com.1password.1password", "com.agilebits.onepassword7", "com.bitwarden.desktop", "com.lastpass.lastpassmacdesktop",
|
|
1357
|
+
"com.apple.scripteditor2", "com.apple.automator", "io.github.vpatel9202.glovebox.helper",
|
|
1358
|
+
]
|
|
1359
|
+
|
|
1360
|
+
func readConfig() -> [String: Any] {
|
|
1361
|
+
guard let data = FileManager.default.contents(atPath: configPath),
|
|
1362
|
+
let obj = try? JSONSerialization.jsonObject(with: data) as? [String: Any] else { return [:] }
|
|
1363
|
+
return obj
|
|
1364
|
+
}
|
|
1365
|
+
|
|
1366
|
+
func matches(_ list: [String], _ bundleId: String, _ name: String) -> Bool {
|
|
1367
|
+
let id = bundleId.lowercased(), n = name.lowercased()
|
|
1368
|
+
return list.contains { let e = $0.lowercased(); return e == id || e == n }
|
|
1369
|
+
}
|
|
1370
|
+
|
|
1371
|
+
func persistAllow(_ bundleId: String) {
|
|
1372
|
+
var config = readConfig()
|
|
1373
|
+
var allow = (config["allow"] as? [String]) ?? []
|
|
1374
|
+
if !allow.contains(where: { $0.lowercased() == bundleId.lowercased() }) { allow.append(bundleId) }
|
|
1375
|
+
config["allow"] = allow
|
|
1376
|
+
try? FileManager.default.createDirectory(atPath: gloveboxHome, withIntermediateDirectories: true, attributes: [.posixPermissions: 0o700])
|
|
1377
|
+
if let data = try? JSONSerialization.data(withJSONObject: config, options: [.prettyPrinted, .sortedKeys]) {
|
|
1378
|
+
FileManager.default.createFile(atPath: configPath, contents: data, attributes: [.posixPermissions: 0o600])
|
|
1379
|
+
}
|
|
1380
|
+
}
|
|
1381
|
+
|
|
1382
|
+
enum Choice { case session, always, deny }
|
|
1383
|
+
|
|
1384
|
+
func askUser(client: String, app: String, bundleId: String) -> Choice {
|
|
1385
|
+
var choice = Choice.deny
|
|
1386
|
+
DispatchQueue.main.sync {
|
|
1387
|
+
let alert = NSAlert()
|
|
1388
|
+
alert.messageText = "Allow \(client) to control \(app)?"
|
|
1389
|
+
alert.informativeText = "An agent using glovebox wants to click, type, and read the screen in \(app) (\(bundleId)). Every action is logged in \(gloveboxHome)/actions.log. Press Escape any time to stop a running script."
|
|
1390
|
+
alert.alertStyle = .warning
|
|
1391
|
+
alert.addButton(withTitle: "Allow for This Session")
|
|
1392
|
+
alert.addButton(withTitle: "Always Allow")
|
|
1393
|
+
alert.addButton(withTitle: "Deny")
|
|
1394
|
+
NSApp.activate(ignoringOtherApps: true)
|
|
1395
|
+
alert.window.level = .floating
|
|
1396
|
+
switch alert.runModal() {
|
|
1397
|
+
case .alertFirstButtonReturn: choice = .session
|
|
1398
|
+
case .alertSecondButtonReturn: choice = .always
|
|
1399
|
+
default: choice = .deny
|
|
1400
|
+
}
|
|
1401
|
+
}
|
|
1402
|
+
return choice
|
|
1403
|
+
}
|
|
1404
|
+
|
|
1405
|
+
func authorize(_ connection: Connection, bundleId: String, name: String) throws {
|
|
1406
|
+
if connection.approved.contains(bundleId) { return }
|
|
1407
|
+
let config = readConfig()
|
|
1408
|
+
let allow = (config["allow"] as? [String]) ?? []
|
|
1409
|
+
let deny = (config["deny"] as? [String]) ?? []
|
|
1410
|
+
let mode = (ProcessInfo.processInfo.environment["GLOVEBOX_APPROVAL"] ?? config["approval"] as? String) ?? "ask"
|
|
1411
|
+
if matches(deny, bundleId, name) {
|
|
1412
|
+
throw HelperError("\(name) is on the deny list in \(configPath).", code: "denied")
|
|
1413
|
+
}
|
|
1414
|
+
if matches(allow, bundleId, name) {
|
|
1415
|
+
connection.approved.insert(bundleId)
|
|
1416
|
+
return
|
|
1417
|
+
}
|
|
1418
|
+
if blockedByDefault.contains(bundleId.lowercased()) {
|
|
1419
|
+
throw HelperError("\(name) is blocked by default because it can run commands or holds secrets. To allow it anyway, run: glovebox allow \(bundleId)", code: "blocked")
|
|
1420
|
+
}
|
|
1421
|
+
switch mode {
|
|
1422
|
+
case "allow-all":
|
|
1423
|
+
connection.approved.insert(bundleId)
|
|
1424
|
+
case "allowlist":
|
|
1425
|
+
throw HelperError("\(name) is not in the allowlist. Run: glovebox allow \"\(name)\"", code: "not_allowed")
|
|
1426
|
+
default:
|
|
1427
|
+
switch askUser(client: connection.client, app: name, bundleId: bundleId) {
|
|
1428
|
+
case .session: connection.approved.insert(bundleId)
|
|
1429
|
+
case .always: connection.approved.insert(bundleId); persistAllow(bundleId)
|
|
1430
|
+
case .deny: throw HelperError("The user denied access to \(name).", code: "denied")
|
|
1431
|
+
}
|
|
1432
|
+
}
|
|
1433
|
+
}
|
|
1434
|
+
|
|
1435
|
+
// Methods that only read the app list or the helper's own state.
|
|
1436
|
+
let ungatedMethods: Set<String> = ["hello", "ping", "permissions", "listApps", "resolveApp", "forget", "cursor"]
|
|
1437
|
+
|
|
1438
|
+
func gate(_ connection: Connection, _ method: String, _ params: [String: Any]) throws {
|
|
1439
|
+
if ungatedMethods.contains(method) { return }
|
|
1440
|
+
if method == "launch" {
|
|
1441
|
+
let bundleId = params.string("bundleId") ?? params.string("path").flatMap { Bundle(path: $0)?.bundleIdentifier } ?? ""
|
|
1442
|
+
let name = params.string("name") ?? bundleId
|
|
1443
|
+
if bundleId.isEmpty { throw HelperError("launch needs a bundleId", code: "bad_request") }
|
|
1444
|
+
try authorize(connection, bundleId: bundleId, name: name)
|
|
1445
|
+
return
|
|
1446
|
+
}
|
|
1447
|
+
guard let pid = params.int("pid") else { throw HelperError("\(method) needs a pid", code: "bad_request") }
|
|
1448
|
+
let app = try runningApp(pid)
|
|
1449
|
+
guard let bundleId = app.bundleIdentifier else { throw HelperError("App has no bundle identifier", code: "denied") }
|
|
1450
|
+
try authorize(connection, bundleId: bundleId, name: app.localizedName ?? bundleId)
|
|
1451
|
+
}
|
|
1452
|
+
|
|
1453
|
+
// MARK: - Dispatch
|
|
1454
|
+
|
|
1455
|
+
func permissions(_ params: [String: Any]) -> [String: Any] {
|
|
1456
|
+
let prompt = params.bool("prompt") ?? false
|
|
1457
|
+
let options = [kAXTrustedCheckOptionPrompt.takeUnretainedValue() as String: prompt] as CFDictionary
|
|
1458
|
+
let ax = AXIsProcessTrustedWithOptions(options)
|
|
1459
|
+
var input = CGPreflightListenEventAccess()
|
|
1460
|
+
if prompt && params.bool("inputMonitoring") == true && !input { input = CGRequestListenEventAccess() }
|
|
1461
|
+
var screen = screenCaptureAllowed()
|
|
1462
|
+
if prompt && !screen { screen = CGRequestScreenCaptureAccess() }
|
|
1463
|
+
return ["accessibility": ax, "screenRecording": screen, "escapeWatcher": escapeWatcherActive(), "inputMonitoring": input]
|
|
1464
|
+
}
|
|
1465
|
+
|
|
1466
|
+
func handle(_ connection: Connection, _ method: String, _ params: [String: Any]) throws -> Any {
|
|
1467
|
+
try gate(connection, method, params)
|
|
1468
|
+
switch method {
|
|
1469
|
+
case "hello":
|
|
1470
|
+
connection.client = params.string("client") ?? connection.client
|
|
1471
|
+
return ["version": helperVersion, "platform": "macos", "pid": Int(getpid())]
|
|
1472
|
+
case "ping": return ["version": helperVersion, "platform": "macos", "pid": Int(getpid()), "background": SkyLight.capabilities]
|
|
1473
|
+
case "permissions": return permissions(params)
|
|
1474
|
+
case "listApps": return listApps()
|
|
1475
|
+
case "resolveApp": return try resolveApp(params.string("query") ?? "")
|
|
1476
|
+
case "launch": return try launch(params)
|
|
1477
|
+
case "activate": try activate(try params.requireInt("pid")); return [:]
|
|
1478
|
+
case "isActive": return ["active": try runningApp(try params.requireInt("pid")).isActive]
|
|
1479
|
+
case "windows": return windowsInfo(try params.requireInt("pid"))
|
|
1480
|
+
case "axTree": return try axTree(params)
|
|
1481
|
+
case "hitTest": return try hitTest(params)
|
|
1482
|
+
case "elementFrame": return try elementFrame(params)
|
|
1483
|
+
case "performAction": return try performAction(params)
|
|
1484
|
+
case "actionNames": return try actionNames(params)
|
|
1485
|
+
case "setValue": return try setValue(params)
|
|
1486
|
+
case "selectText": return try selectText(params)
|
|
1487
|
+
case "focusElement":
|
|
1488
|
+
var p = params
|
|
1489
|
+
p["attribute"] = kAXFocusedAttribute
|
|
1490
|
+
p["value"] = true
|
|
1491
|
+
return try setValue(p)
|
|
1492
|
+
case "keys": return try withInputRoute(params, .keys) { try withKeyTarget(params) { try pressKeys(params) } }
|
|
1493
|
+
case "type": return try withInputRoute(params, .keys) { try withKeyTarget(params) { try typeText(params) } }
|
|
1494
|
+
case "click": return try withInputRoute(params, .mouse) { try click(params) }
|
|
1495
|
+
case "cursor": return try cursorCommand(params)
|
|
1496
|
+
case "makeKey":
|
|
1497
|
+
let ok = makeWindowKey(pid: pid_t(try params.requireInt("pid")), windowId: UInt32(try params.requireInt("windowId")))
|
|
1498
|
+
return ["keyed": ok]
|
|
1499
|
+
case "move": return try moveMouse(params)
|
|
1500
|
+
case "scroll": return try withInputRoute(params, .mouse) { try scroll(params) }
|
|
1501
|
+
case "drag": return try withInputRoute(params, .mouse) { try drag(params) }
|
|
1502
|
+
case "screenshot": return try screenshot(params)
|
|
1503
|
+
case "forget": tables[try params.requireInt("pid")] = nil; return [:]
|
|
1504
|
+
default: throw HelperError("Unknown method \(method)", code: "bad_request")
|
|
1505
|
+
}
|
|
1506
|
+
}
|
|
1507
|
+
|
|
1508
|
+
// MARK: - Socket server
|
|
1509
|
+
|
|
1510
|
+
// All GUI work runs on one queue so actions from different clients never interleave.
|
|
1511
|
+
let work = DispatchQueue(label: "glovebox.work")
|
|
1512
|
+
|
|
1513
|
+
final class Connection {
|
|
1514
|
+
static var lock = NSLock()
|
|
1515
|
+
static var open: [ObjectIdentifier: Connection] = [:]
|
|
1516
|
+
static var lastClosed = Date()
|
|
1517
|
+
|
|
1518
|
+
static func all() -> [Connection] {
|
|
1519
|
+
lock.lock(); defer { lock.unlock() }
|
|
1520
|
+
return Array(open.values)
|
|
1521
|
+
}
|
|
1522
|
+
|
|
1523
|
+
let fd: Int32
|
|
1524
|
+
let writeLock = NSLock()
|
|
1525
|
+
var client = "An agent"
|
|
1526
|
+
private let approvedLock = NSLock()
|
|
1527
|
+
private var approvedApps = Set<String>()
|
|
1528
|
+
// Read from the menu on the main thread and written on the work queue.
|
|
1529
|
+
var approved: Set<String> {
|
|
1530
|
+
get { approvedLock.lock(); defer { approvedLock.unlock() }; return approvedApps }
|
|
1531
|
+
set { approvedLock.lock(); approvedApps = newValue; approvedLock.unlock() }
|
|
1532
|
+
}
|
|
1533
|
+
|
|
1534
|
+
init(fd: Int32) { self.fd = fd }
|
|
1535
|
+
|
|
1536
|
+
func send(_ object: [String: Any]) {
|
|
1537
|
+
guard var data = try? JSONSerialization.data(withJSONObject: object, options: []) else { return }
|
|
1538
|
+
data.append(0x0A)
|
|
1539
|
+
writeLock.lock(); defer { writeLock.unlock() }
|
|
1540
|
+
data.withUnsafeBytes { raw in
|
|
1541
|
+
var offset = 0
|
|
1542
|
+
while offset < raw.count {
|
|
1543
|
+
let n = Darwin.write(fd, raw.baseAddress! + offset, raw.count - offset)
|
|
1544
|
+
if n <= 0 { return }
|
|
1545
|
+
offset += n
|
|
1546
|
+
}
|
|
1547
|
+
}
|
|
1548
|
+
}
|
|
1549
|
+
|
|
1550
|
+
func run() {
|
|
1551
|
+
Connection.lock.lock(); Connection.open[ObjectIdentifier(self)] = self; Connection.lock.unlock()
|
|
1552
|
+
refreshStatusMenu()
|
|
1553
|
+
var buffer = Data()
|
|
1554
|
+
var chunk = [UInt8](repeating: 0, count: 65536)
|
|
1555
|
+
while true {
|
|
1556
|
+
let n = read(fd, &chunk, chunk.count)
|
|
1557
|
+
if n <= 0 { break }
|
|
1558
|
+
buffer.append(chunk, count: n)
|
|
1559
|
+
while let newline = buffer.firstIndex(of: 0x0A) {
|
|
1560
|
+
let line = buffer.subdata(in: buffer.startIndex..<newline)
|
|
1561
|
+
buffer.removeSubrange(buffer.startIndex...newline)
|
|
1562
|
+
guard !line.isEmpty,
|
|
1563
|
+
let obj = try? JSONSerialization.jsonObject(with: line) as? [String: Any],
|
|
1564
|
+
let method = obj["method"] as? String else { continue }
|
|
1565
|
+
let id = obj["id"] ?? NSNull()
|
|
1566
|
+
let params = (obj["params"] as? [String: Any]) ?? [:]
|
|
1567
|
+
work.async {
|
|
1568
|
+
do {
|
|
1569
|
+
let result = try handle(self, method, params)
|
|
1570
|
+
self.send(["id": id, "result": result])
|
|
1571
|
+
} catch let error as HelperError {
|
|
1572
|
+
self.send(["id": id, "error": ["message": error.message, "code": error.code]])
|
|
1573
|
+
} catch {
|
|
1574
|
+
self.send(["id": id, "error": ["message": "\(error)", "code": "failed"]])
|
|
1575
|
+
}
|
|
1576
|
+
}
|
|
1577
|
+
}
|
|
1578
|
+
}
|
|
1579
|
+
close(fd)
|
|
1580
|
+
Connection.lock.lock()
|
|
1581
|
+
Connection.open[ObjectIdentifier(self)] = nil
|
|
1582
|
+
Connection.lastClosed = Date()
|
|
1583
|
+
Connection.lock.unlock()
|
|
1584
|
+
refreshStatusMenu()
|
|
1585
|
+
}
|
|
1586
|
+
}
|
|
1587
|
+
|
|
1588
|
+
func socketAddress(_ path: String) -> sockaddr_un {
|
|
1589
|
+
var addr = sockaddr_un()
|
|
1590
|
+
addr.sun_family = sa_family_t(AF_UNIX)
|
|
1591
|
+
let bytes = Array(path.utf8)
|
|
1592
|
+
withUnsafeMutableBytes(of: &addr.sun_path) { raw in
|
|
1593
|
+
for (i, b) in bytes.prefix(raw.count - 1).enumerated() { raw[i] = b }
|
|
1594
|
+
}
|
|
1595
|
+
return addr
|
|
1596
|
+
}
|
|
1597
|
+
|
|
1598
|
+
func listen(at path: String) -> Int32 {
|
|
1599
|
+
// A live helper already owns the socket: let it keep serving.
|
|
1600
|
+
let probe = socket(AF_UNIX, SOCK_STREAM, 0)
|
|
1601
|
+
var addr = socketAddress(path)
|
|
1602
|
+
let alive = withUnsafePointer(to: &addr) {
|
|
1603
|
+
$0.withMemoryRebound(to: sockaddr.self, capacity: 1) { connect(probe, $0, socklen_t(MemoryLayout<sockaddr_un>.size)) }
|
|
1604
|
+
} == 0
|
|
1605
|
+
close(probe)
|
|
1606
|
+
if alive {
|
|
1607
|
+
FileHandle.standardError.write("glovebox-helper: already running\n".data(using: .utf8)!)
|
|
1608
|
+
exit(0)
|
|
1609
|
+
}
|
|
1610
|
+
unlink(path)
|
|
1611
|
+
let fd = socket(AF_UNIX, SOCK_STREAM, 0)
|
|
1612
|
+
let bound = withUnsafePointer(to: &addr) {
|
|
1613
|
+
$0.withMemoryRebound(to: sockaddr.self, capacity: 1) { bind(fd, $0, socklen_t(MemoryLayout<sockaddr_un>.size)) }
|
|
1614
|
+
}
|
|
1615
|
+
if bound != 0 || Darwin.listen(fd, 16) != 0 {
|
|
1616
|
+
FileHandle.standardError.write("glovebox-helper: cannot listen on \(path)\n".data(using: .utf8)!)
|
|
1617
|
+
exit(1)
|
|
1618
|
+
}
|
|
1619
|
+
chmod(path, 0o600)
|
|
1620
|
+
return fd
|
|
1621
|
+
}
|
|
1622
|
+
|
|
1623
|
+
func acceptLoop(_ server: Int32) {
|
|
1624
|
+
while true {
|
|
1625
|
+
let fd = accept(server, nil, nil)
|
|
1626
|
+
if fd < 0 { continue }
|
|
1627
|
+
// Only the user who owns this helper may drive it.
|
|
1628
|
+
var uid: uid_t = 0
|
|
1629
|
+
var gid: gid_t = 0
|
|
1630
|
+
if getpeereid(fd, &uid, &gid) != 0 || uid != getuid() {
|
|
1631
|
+
close(fd)
|
|
1632
|
+
continue
|
|
1633
|
+
}
|
|
1634
|
+
var on: Int32 = 1
|
|
1635
|
+
setsockopt(fd, SOL_SOCKET, SO_NOSIGPIPE, &on, socklen_t(MemoryLayout<Int32>.size))
|
|
1636
|
+
let connection = Connection(fd: fd)
|
|
1637
|
+
Thread.detachNewThread { connection.run() }
|
|
1638
|
+
}
|
|
1639
|
+
}
|
|
1640
|
+
|
|
1641
|
+
let socketPath = ProcessInfo.processInfo.environment["GLOVEBOX_SOCKET"] ?? (gloveboxHome + "/helper.sock")
|
|
1642
|
+
let idleQuitSeconds: TimeInterval = 30 * 60
|
|
1643
|
+
|
|
1644
|
+
try? FileManager.default.createDirectory(atPath: gloveboxHome, withIntermediateDirectories: true, attributes: [.posixPermissions: 0o700])
|
|
1645
|
+
_ = NSApplication.shared
|
|
1646
|
+
NSApp.setActivationPolicy(.accessory)
|
|
1647
|
+
signal(SIGPIPE, SIG_IGN)
|
|
1648
|
+
if CommandLine.arguments.contains("--request-permissions") {
|
|
1649
|
+
_ = permissions(["prompt": true])
|
|
1650
|
+
}
|
|
1651
|
+
_ = startEscapeWatcher()
|
|
1652
|
+
let server = listen(at: socketPath)
|
|
1653
|
+
Thread.detachNewThread { acceptLoop(server) }
|
|
1654
|
+
// Quit after a long stretch with no clients; the next client relaunches us.
|
|
1655
|
+
Timer.scheduledTimer(withTimeInterval: 60, repeats: true) { _ in
|
|
1656
|
+
Connection.lock.lock()
|
|
1657
|
+
let idle = Connection.open.isEmpty && Date().timeIntervalSince(Connection.lastClosed) > idleQuitSeconds
|
|
1658
|
+
Connection.lock.unlock()
|
|
1659
|
+
if idle { unlink(socketPath); exit(0) }
|
|
1660
|
+
}
|
|
1661
|
+
NSApp.run()
|