@vashp2029/glovebox 0.0.0-stage → 0.3.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,1661 @@
1
+ // glovebox-helper: the macOS backend for glovebox.
2
+ //
3
+ // A small JSON-lines RPC server on a Unix socket (~/.glovebox/helper.sock),
4
+ // packaged as its own app so macOS grants Accessibility and Screen Recording
5
+ // to "Glovebox Helper" once, whichever agent or terminal starts it. Each request is
6
+ // {"id": n, "method": "...", "params": {...}} and each reply is
7
+ // {"id": n, "result": ...} or {"id": n, "error": {"message": "..."}}.
8
+ // Unsolicited events are {"event": "..."}.
9
+ //
10
+ // Coordinates on this side are always global screen points, top-left origin
11
+ // (the same space AX and CGEvent use). The Node side owns the mapping between
12
+ // those points and the image pixels a model sees.
13
+ //
14
+ // Build: glovebox setup (see src/macos-helper.mjs).
15
+
16
+ import AppKit
17
+ import ApplicationServices
18
+ import CoreGraphics
19
+ import Foundation
20
+ import ScreenCaptureKit
21
+ import UniformTypeIdentifiers
22
+
23
+ let helperVersion = "0.3.1"
24
+ // Marks events we synthesize so the Escape watcher ignores them.
25
+ let syntheticMarker: Int64 = 0x676C_6F76
26
+
27
+ // MARK: - Output
28
+
29
+ // Sends an unsolicited event to every connected client.
30
+ func emit(_ object: [String: Any]) {
31
+ for connection in Connection.all() { connection.send(object) }
32
+ }
33
+
34
+ struct HelperError: Error {
35
+ let message: String
36
+ let code: String
37
+ init(_ message: String, code: String = "failed") {
38
+ self.message = message
39
+ self.code = code
40
+ }
41
+ }
42
+
43
+ // MARK: - Param helpers
44
+
45
+ extension Dictionary where Key == String, Value == Any {
46
+ func int(_ key: String) -> Int? {
47
+ if let v = self[key] as? Int { return v }
48
+ if let v = self[key] as? Double { return Int(v) }
49
+ if let v = self[key] as? NSNumber { return v.intValue }
50
+ return nil
51
+ }
52
+ func double(_ key: String) -> Double? {
53
+ if let v = self[key] as? Double { return v }
54
+ if let v = self[key] as? Int { return Double(v) }
55
+ if let v = self[key] as? NSNumber { return v.doubleValue }
56
+ return nil
57
+ }
58
+ func string(_ key: String) -> String? { self[key] as? String }
59
+ func bool(_ key: String) -> Bool? { self[key] as? Bool }
60
+ func requireInt(_ key: String) throws -> Int {
61
+ guard let v = int(key) else { throw HelperError("missing integer param \(key)", code: "bad_request") }
62
+ return v
63
+ }
64
+ func requireDouble(_ key: String) throws -> Double {
65
+ guard let v = double(key) else { throw HelperError("missing number param \(key)", code: "bad_request") }
66
+ return v
67
+ }
68
+ }
69
+
70
+ // MARK: - Apps
71
+
72
+ func appInfo(_ app: NSRunningApplication) -> [String: Any] {
73
+ var info: [String: Any] = [
74
+ "name": app.localizedName ?? "",
75
+ "pid": Int(app.processIdentifier),
76
+ "running": true,
77
+ "active": app.isActive,
78
+ "hidden": app.isHidden,
79
+ ]
80
+ if let id = app.bundleIdentifier { info["bundleId"] = id }
81
+ if let url = app.bundleURL { info["path"] = url.path }
82
+ return info
83
+ }
84
+
85
+ func installedApps() -> [[String: Any]] {
86
+ var roots = ["/Applications", "/Applications/Utilities", "/System/Applications", "/System/Applications/Utilities"]
87
+ roots.append(NSHomeDirectory() + "/Applications")
88
+ var seen = Set<String>()
89
+ var out: [[String: Any]] = []
90
+ let fm = FileManager.default
91
+ for root in roots {
92
+ guard let items = try? fm.contentsOfDirectory(atPath: root) else { continue }
93
+ for item in items where item.hasSuffix(".app") {
94
+ let path = root + "/" + item
95
+ guard let bundle = Bundle(path: path), let id = bundle.bundleIdentifier, !seen.contains(id) else { continue }
96
+ seen.insert(id)
97
+ let name = (bundle.object(forInfoDictionaryKey: "CFBundleDisplayName") as? String)
98
+ ?? (bundle.object(forInfoDictionaryKey: "CFBundleName") as? String)
99
+ ?? String(item.dropLast(4))
100
+ out.append(["name": name, "bundleId": id, "path": path, "running": false])
101
+ }
102
+ }
103
+ return out
104
+ }
105
+
106
+ func listApps() -> [[String: Any]] {
107
+ var out: [[String: Any]] = []
108
+ var seen = Set<String>()
109
+ for app in NSWorkspace.shared.runningApplications where app.activationPolicy == .regular {
110
+ out.append(appInfo(app))
111
+ if let id = app.bundleIdentifier { seen.insert(id) }
112
+ }
113
+ for app in installedApps() where !seen.contains(app["bundleId"] as? String ?? "") {
114
+ out.append(app)
115
+ }
116
+ return out
117
+ }
118
+
119
+ func resolveApp(_ query: String) throws -> [String: Any] {
120
+ let q = query.lowercased()
121
+ let running = NSWorkspace.shared.runningApplications.filter { $0.activationPolicy != .prohibited }
122
+ if q.hasPrefix("pid:"), let pid = Int32(q.dropFirst(4)),
123
+ let app = NSRunningApplication(processIdentifier: pid) {
124
+ return appInfo(app)
125
+ }
126
+ for app in running {
127
+ if app.bundleIdentifier?.lowercased() == q || app.localizedName?.lowercased() == q {
128
+ return appInfo(app)
129
+ }
130
+ }
131
+ let installed = installedApps()
132
+ for app in installed {
133
+ if (app["bundleId"] as? String)?.lowercased() == q || (app["name"] as? String)?.lowercased() == q {
134
+ return app
135
+ }
136
+ }
137
+ if query.hasSuffix(".app") || query.hasPrefix("/"), let bundle = Bundle(path: query), let id = bundle.bundleIdentifier {
138
+ return ["name": bundle.bundleURL.deletingPathExtension().lastPathComponent, "bundleId": id, "path": query, "running": false]
139
+ }
140
+ let near = (running.compactMap { $0.localizedName } + installed.compactMap { $0["name"] as? String })
141
+ .filter { $0.lowercased().contains(q) }
142
+ let hint = near.isEmpty ? "" : " Did you mean: \(Array(Set(near)).sorted().prefix(5).joined(separator: ", "))?"
143
+ throw HelperError("No app named \"\(query)\".\(hint)", code: "not_found")
144
+ }
145
+
146
+ // NSRunningApplication(processIdentifier:) can come back nil for a live app
147
+ // when called off the main thread, so a miss is checked against the process
148
+ // table and the workspace list before calling the app gone.
149
+ func runningApp(_ pid: Int) throws -> NSRunningApplication {
150
+ if let app = NSRunningApplication(processIdentifier: pid_t(pid)), !app.isTerminated { return app }
151
+ if kill(pid_t(pid), 0) == 0 {
152
+ var found: NSRunningApplication?
153
+ let look = {
154
+ found = NSRunningApplication(processIdentifier: pid_t(pid))
155
+ ?? NSWorkspace.shared.runningApplications.first { $0.processIdentifier == pid_t(pid) }
156
+ }
157
+ if Thread.isMainThread { look() } else { DispatchQueue.main.sync(execute: look) }
158
+ if let app = found { return app }
159
+ }
160
+ throw HelperError("App with pid \(pid) is not running", code: "not_running")
161
+ }
162
+
163
+ func launch(_ params: [String: Any]) throws -> [String: Any] {
164
+ if let bundleId = params.string("bundleId"),
165
+ let app = NSWorkspace.shared.runningApplications.first(where: { $0.bundleIdentifier == bundleId }) {
166
+ return appInfo(app)
167
+ }
168
+ var url: URL?
169
+ if let path = params.string("path") { url = URL(fileURLWithPath: path) }
170
+ if url == nil, let bundleId = params.string("bundleId") {
171
+ url = NSWorkspace.shared.urlForApplication(withBundleIdentifier: bundleId)
172
+ }
173
+ guard let appURL = url else { throw HelperError("Cannot locate app to launch", code: "not_found") }
174
+ let config = NSWorkspace.OpenConfiguration()
175
+ config.activates = params.bool("activate") ?? true
176
+ let done = DispatchSemaphore(value: 0)
177
+ var launched: NSRunningApplication?
178
+ var failure: Error?
179
+ NSWorkspace.shared.openApplication(at: appURL, configuration: config) { app, error in
180
+ launched = app
181
+ failure = error
182
+ done.signal()
183
+ }
184
+ if done.wait(timeout: .now() + 20) == .timedOut { throw HelperError("Timed out launching \(appURL.lastPathComponent)") }
185
+ if let error = failure { throw HelperError("Launch failed: \(error.localizedDescription)") }
186
+ guard let app = launched else { throw HelperError("Launch returned no app") }
187
+ // Give the app a moment to put up a window.
188
+ let element = AXUIElementCreateApplication(app.processIdentifier)
189
+ for _ in 0..<50 {
190
+ if let windows: [AXUIElement] = axAttr(element, kAXWindowsAttribute), !windows.isEmpty { break }
191
+ Thread.sleep(forTimeInterval: 0.1)
192
+ }
193
+ return appInfo(app)
194
+ }
195
+
196
+ func activate(_ pid: Int) throws {
197
+ let app = try runningApp(pid)
198
+ if app.isActive { return }
199
+ if app.isHidden { app.unhide() }
200
+ let element = AXUIElementCreateApplication(pid_t(pid))
201
+ AXUIElementSetAttributeValue(element, kAXFrontmostAttribute as CFString, kCFBooleanTrue)
202
+ for _ in 0..<20 {
203
+ if app.isActive { return }
204
+ Thread.sleep(forTimeInterval: 0.05)
205
+ }
206
+ if let url = app.bundleURL {
207
+ let config = NSWorkspace.OpenConfiguration()
208
+ config.activates = true
209
+ NSWorkspace.shared.openApplication(at: url, configuration: config, completionHandler: nil)
210
+ } else {
211
+ app.activate(options: [.activateAllWindows])
212
+ }
213
+ for _ in 0..<40 {
214
+ if app.isActive { return }
215
+ Thread.sleep(forTimeInterval: 0.05)
216
+ }
217
+ throw HelperError("Could not bring \(app.localizedName ?? "app") to the front", code: "activate_failed")
218
+ }
219
+
220
+ // MARK: - AX basics
221
+
222
+ func axAttr<T>(_ element: AXUIElement, _ name: String) -> T? {
223
+ var value: CFTypeRef?
224
+ guard AXUIElementCopyAttributeValue(element, name as CFString, &value) == .success, let v = value else { return nil }
225
+ return v as? T
226
+ }
227
+
228
+ func axPoint(_ value: CFTypeRef?) -> CGPoint? {
229
+ guard let v = value, CFGetTypeID(v) == AXValueGetTypeID() else { return nil }
230
+ var point = CGPoint.zero
231
+ return AXValueGetValue(v as! AXValue, .cgPoint, &point) ? point : nil
232
+ }
233
+
234
+ func axSize(_ value: CFTypeRef?) -> CGSize? {
235
+ guard let v = value, CFGetTypeID(v) == AXValueGetTypeID() else { return nil }
236
+ var size = CGSize.zero
237
+ return AXValueGetValue(v as! AXValue, .cgSize, &size) ? size : nil
238
+ }
239
+
240
+ func axFrame(_ element: AXUIElement) -> CGRect? {
241
+ var pos: CFTypeRef?
242
+ var size: CFTypeRef?
243
+ AXUIElementCopyAttributeValue(element, kAXPositionAttribute as CFString, &pos)
244
+ AXUIElementCopyAttributeValue(element, kAXSizeAttribute as CFString, &size)
245
+ guard let p = axPoint(pos), let s = axSize(size) else { return nil }
246
+ return CGRect(origin: p, size: s)
247
+ }
248
+
249
+ func rectJSON(_ r: CGRect) -> [Double] {
250
+ [Double(r.origin.x), Double(r.origin.y), Double(r.size.width), Double(r.size.height)]
251
+ }
252
+
253
+ func stringify(_ value: Any?, limit: Int) -> String? {
254
+ guard let value = value else { return nil }
255
+ var text: String?
256
+ switch value {
257
+ case let s as String: text = s
258
+ case let n as NSNumber: text = n.stringValue
259
+ case let a as NSAttributedString: text = a.string
260
+ case let url as URL: text = url.absoluteString
261
+ default:
262
+ let cf = value as CFTypeRef
263
+ if CFGetTypeID(cf) == AXValueGetTypeID() { return nil }
264
+ if CFGetTypeID(cf) == AXUIElementGetTypeID() { return nil }
265
+ text = nil
266
+ }
267
+ guard var t = text, !t.isEmpty else { return nil }
268
+ if t.count > limit { t = String(t.prefix(limit)) + "…" }
269
+ return t
270
+ }
271
+
272
+ // MARK: - Element tables (stable indices per app)
273
+
274
+ final class ElementTable {
275
+ var elements: [AXUIElement] = []
276
+ var buckets: [CFHashCode: [Int]] = [:]
277
+
278
+ func index(of element: AXUIElement) -> Int {
279
+ let hash = CFHash(element)
280
+ if let candidates = buckets[hash] {
281
+ for i in candidates where CFEqual(elements[i], element) { return i }
282
+ }
283
+ elements.append(element)
284
+ let i = elements.count - 1
285
+ buckets[hash, default: []].append(i)
286
+ return i
287
+ }
288
+
289
+ func element(_ index: Int) throws -> AXUIElement {
290
+ guard index >= 0 && index < elements.count else {
291
+ throw HelperError("No element \(index). Read the accessibility tree again.", code: "stale_index")
292
+ }
293
+ return elements[index]
294
+ }
295
+ }
296
+
297
+ var tables: [Int: ElementTable] = [:]
298
+
299
+ func table(_ pid: Int) -> ElementTable {
300
+ if let t = tables[pid] { return t }
301
+ let t = ElementTable()
302
+ tables[pid] = t
303
+ return t
304
+ }
305
+
306
+ // MARK: - Windows
307
+
308
+ func cgWindows(pid: Int) -> [[String: Any]] {
309
+ guard let info = CGWindowListCopyWindowInfo([.optionAll, .excludeDesktopElements], kCGNullWindowID) as? [[String: Any]] else { return [] }
310
+ return info.filter { ($0[kCGWindowOwnerPID as String] as? Int) == pid && ($0[kCGWindowLayer as String] as? Int) == 0 }
311
+ }
312
+
313
+ func cgWindowID(pid: Int, frame: CGRect) -> Int? {
314
+ var best: (Int, CGFloat)?
315
+ for w in cgWindows(pid: pid) {
316
+ guard let id = w[kCGWindowNumber as String] as? Int,
317
+ let b = w[kCGWindowBounds as String] as? [String: Any],
318
+ let rect = CGRect(dictionaryRepresentation: b as CFDictionary) else { continue }
319
+ let d = abs(rect.minX - frame.minX) + abs(rect.minY - frame.minY) + abs(rect.width - frame.width) + abs(rect.height - frame.height)
320
+ if best == nil || d < best!.1 { best = (id, d) }
321
+ }
322
+ if let b = best, b.1 < 8 { return b.0 }
323
+ return nil
324
+ }
325
+
326
+ func backingScale(for frame: CGRect) -> Double {
327
+ // NSScreen frames are bottom-left origin; convert the window's center.
328
+ guard let primary = NSScreen.screens.first else { return 2 }
329
+ let center = CGPoint(x: frame.midX, y: primary.frame.height - frame.midY)
330
+ for screen in NSScreen.screens where screen.frame.contains(center) {
331
+ return Double(screen.backingScaleFactor)
332
+ }
333
+ return Double(primary.backingScaleFactor)
334
+ }
335
+
336
+ func axWindows(_ pid: Int) -> [AXUIElement] {
337
+ let app = AXUIElementCreateApplication(pid_t(pid))
338
+ return axAttr(app, kAXWindowsAttribute) ?? []
339
+ }
340
+
341
+ func windowsInfo(_ pid: Int) -> [[String: Any]] {
342
+ let app = AXUIElementCreateApplication(pid_t(pid))
343
+ let focused: AXUIElement? = axAttr(app, kAXFocusedWindowAttribute)
344
+ let main: AXUIElement? = axAttr(app, kAXMainWindowAttribute)
345
+ var out: [[String: Any]] = []
346
+ for (i, w) in axWindows(pid).enumerated() {
347
+ guard let frame = axFrame(w) else { continue }
348
+ var info: [String: Any] = [
349
+ "index": i,
350
+ "title": (axAttr(w, kAXTitleAttribute) as String?) ?? "",
351
+ "frame": rectJSON(frame),
352
+ "scale": backingScale(for: frame),
353
+ "minimized": (axAttr(w, kAXMinimizedAttribute) as Bool?) ?? false,
354
+ "focused": focused.map { CFEqual($0, w) } ?? false,
355
+ "main": main.map { CFEqual($0, w) } ?? false,
356
+ ]
357
+ if let role: String = axAttr(w, kAXSubroleAttribute) { info["subrole"] = role }
358
+ if let id = cgWindowID(pid: pid, frame: frame) { info["windowId"] = id }
359
+ out.append(info)
360
+ }
361
+ return out
362
+ }
363
+
364
+ // Picks the window an action should target: an explicit index, else focused, main, first.
365
+ func targetWindow(_ pid: Int, _ params: [String: Any]) throws -> (AXUIElement, Int) {
366
+ let windows = axWindows(pid)
367
+ if windows.isEmpty { throw HelperError("The app has no open windows", code: "no_window") }
368
+ if let i = params.int("window") {
369
+ guard i >= 0 && i < windows.count else { throw HelperError("No window \(i); the app has \(windows.count)", code: "no_window") }
370
+ return (windows[i], i)
371
+ }
372
+ let app = AXUIElementCreateApplication(pid_t(pid))
373
+ for attr in [kAXFocusedWindowAttribute, kAXMainWindowAttribute] {
374
+ if let w: AXUIElement = axAttr(app, attr), let i = windows.firstIndex(where: { CFEqual($0, w) }) {
375
+ return (w, i)
376
+ }
377
+ }
378
+ return (windows[0], 0)
379
+ }
380
+
381
+ // MARK: - Accessibility tree
382
+
383
+ let treeAttributes: [String] = [
384
+ kAXRoleAttribute, kAXSubroleAttribute, kAXTitleAttribute, kAXValueAttribute, kAXDescriptionAttribute,
385
+ kAXHelpAttribute, kAXPlaceholderValueAttribute, kAXPositionAttribute, kAXSizeAttribute, kAXChildrenAttribute,
386
+ kAXEnabledAttribute, kAXFocusedAttribute, kAXSelectedAttribute, kAXIdentifierAttribute,
387
+ kAXRoleDescriptionAttribute,
388
+ ]
389
+
390
+ func axTree(_ params: [String: Any]) throws -> [String: Any] {
391
+ let pid = try params.requireInt("pid")
392
+ _ = try runningApp(pid)
393
+ let maxNodes = params.int("maxNodes") ?? 800
394
+ let maxDepth = params.int("maxDepth") ?? 40
395
+ let maxText = params.int("maxText") ?? 160
396
+ let includeMenus = params.bool("menus") ?? false
397
+ let (window, windowIndex) = try targetWindow(pid, params)
398
+ let t = table(pid)
399
+ var nodes: [[String: Any]] = []
400
+ var truncated = false
401
+
402
+ let appElement = AXUIElementCreateApplication(pid_t(pid))
403
+ let focusedElement: AXUIElement? = axAttr(appElement, kAXFocusedUIElementAttribute)
404
+
405
+ func visit(_ element: AXUIElement, _ depth: Int) {
406
+ if nodes.count >= maxNodes { truncated = true; return }
407
+ var raw: CFArray?
408
+ let err = AXUIElementCopyMultipleAttributeValues(element, treeAttributes as CFArray, AXCopyMultipleAttributeOptions(rawValue: 0), &raw)
409
+ guard err == .success, let values = raw as? [Any] else { return }
410
+ func v(_ i: Int) -> Any? {
411
+ let value = values[i]
412
+ if CFGetTypeID(value as CFTypeRef) == AXValueGetTypeID() {
413
+ // Errors come back as AXValue of type kAXValueAXErrorType.
414
+ if AXValueGetType(value as! AXValue) == .axError { return nil }
415
+ }
416
+ return value is NSNull ? nil : value
417
+ }
418
+ var node: [String: Any] = ["i": t.index(of: element), "d": depth]
419
+ node["role"] = (v(0) as? String) ?? "AXUnknown"
420
+ if let s = v(1) as? String { node["subrole"] = s }
421
+ if let s = stringify(v(2), limit: maxText) { node["title"] = s }
422
+ if let s = stringify(v(3), limit: maxText) { node["value"] = s }
423
+ if let s = stringify(v(4), limit: maxText) { node["desc"] = s }
424
+ if let s = stringify(v(5), limit: maxText) { node["help"] = s }
425
+ if let s = stringify(v(6), limit: maxText) { node["placeholder"] = s }
426
+ if let p = axPoint(v(7) as CFTypeRef?), let s = axSize(v(8) as CFTypeRef?) {
427
+ node["frame"] = rectJSON(CGRect(origin: p, size: s))
428
+ }
429
+ if let enabled = v(10) as? Bool, !enabled { node["disabled"] = true }
430
+ if let focused = v(11) as? Bool, focused { node["focused"] = true }
431
+ else if let f = focusedElement, CFEqual(f, element) { node["focused"] = true }
432
+ if let selected = v(12) as? Bool, selected { node["selected"] = true }
433
+ if let s = v(13) as? String, !s.isEmpty { node["id"] = s }
434
+ var names: CFArray?
435
+ if AXUIElementCopyActionNames(element, &names) == .success, let actions = names as? [String] {
436
+ let useful = actions.filter { $0 != "AXScrollToVisible" && $0 != "AXShowDefaultUI" && $0 != "AXShowAlternateUI" }
437
+ if !useful.isEmpty { node["actions"] = useful }
438
+ }
439
+ var settable: DarwinBoolean = false
440
+ if AXUIElementIsAttributeSettable(element, kAXValueAttribute as CFString, &settable) == .success, settable.boolValue {
441
+ node["settable"] = true
442
+ }
443
+ nodes.append(node)
444
+ if depth >= maxDepth { return }
445
+ if let children = v(9) as? [AXUIElement] {
446
+ for child in children {
447
+ if nodes.count >= maxNodes { truncated = true; return }
448
+ visit(child, depth + 1)
449
+ }
450
+ }
451
+ }
452
+
453
+ AXUIElementSetMessagingTimeout(appElement, 2.0)
454
+ visit(window, 0)
455
+ if includeMenus, let bar: AXUIElement = axAttr(appElement, kAXMenuBarAttribute) {
456
+ visit(bar, 0)
457
+ }
458
+ guard let frame = axFrame(window) else { throw HelperError("Window has no frame", code: "no_window") }
459
+ var windowInfo: [String: Any] = [
460
+ "index": windowIndex,
461
+ "title": (axAttr(window, kAXTitleAttribute) as String?) ?? "",
462
+ "frame": rectJSON(frame),
463
+ "scale": backingScale(for: frame),
464
+ ]
465
+ if let id = cgWindowID(pid: pid, frame: frame) { windowInfo["windowId"] = id }
466
+ return ["window": windowInfo, "nodes": nodes, "truncated": truncated]
467
+ }
468
+
469
+ // MARK: - Element actions
470
+
471
+ func elementFrame(_ params: [String: Any]) throws -> [String: Any] {
472
+ let pid = try params.requireInt("pid")
473
+ let element = try table(pid).element(try params.requireInt("index"))
474
+ guard let frame = axFrame(element) else { throw HelperError("Element has no on-screen frame", code: "no_frame") }
475
+ return ["frame": rectJSON(frame)]
476
+ }
477
+
478
+ func performAction(_ params: [String: Any]) throws -> [String: Any] {
479
+ let pid = try params.requireInt("pid")
480
+ let element = try table(pid).element(try params.requireInt("index"))
481
+ let action = params.string("action") ?? kAXPressAction
482
+ let err = AXUIElementPerformAction(element, action as CFString)
483
+ if err != .success {
484
+ throw HelperError("\(action) failed (AXError \(err.rawValue))", code: "action_failed")
485
+ }
486
+ return [:]
487
+ }
488
+
489
+ func actionNames(_ params: [String: Any]) throws -> [String: Any] {
490
+ let pid = try params.requireInt("pid")
491
+ let element = try table(pid).element(try params.requireInt("index"))
492
+ var names: CFArray?
493
+ AXUIElementCopyActionNames(element, &names)
494
+ return ["actions": (names as? [String]) ?? []]
495
+ }
496
+
497
+ func setValue(_ params: [String: Any]) throws -> [String: Any] {
498
+ let pid = try params.requireInt("pid")
499
+ let element = try table(pid).element(try params.requireInt("index"))
500
+ let attribute = params.string("attribute") ?? kAXValueAttribute
501
+ // Finder takes a new value on a file-name field but only relabels the row:
502
+ // the file keeps its old name. Only the rename editor (focused) commits.
503
+ if attribute == kAXValueAttribute, NSRunningApplication(processIdentifier: pid_t(pid))?.bundleIdentifier == "com.apple.finder",
504
+ (axAttr(element, kAXRoleAttribute) as String?) == kAXTextFieldRole, (axAttr(element, kAXFocusedAttribute) as Bool?) != true {
505
+ throw HelperError("Finder would show the new name without renaming the file. Rename instead: click the item, pressKey(\"Return\"), typeText(newName), pressKey(\"Return\").", code: "label_only")
506
+ }
507
+ var value: CFTypeRef
508
+ if let s = params["value"] as? String { value = s as CFString }
509
+ else if let b = params["value"] as? Bool { value = (b ? kCFBooleanTrue : kCFBooleanFalse)! }
510
+ else if let n = params["value"] as? NSNumber { value = n }
511
+ else { throw HelperError("setValue needs a string, number, or boolean", code: "bad_request") }
512
+ let err = AXUIElementSetAttributeValue(element, attribute as CFString, value)
513
+ if err != .success { throw HelperError("Setting \(attribute) failed (AXError \(err.rawValue))", code: "action_failed") }
514
+ return [:]
515
+ }
516
+
517
+ func selectText(_ params: [String: Any]) throws -> [String: Any] {
518
+ let pid = try params.requireInt("pid")
519
+ let element = try table(pid).element(try params.requireInt("index"))
520
+ AXUIElementSetAttributeValue(element, kAXFocusedAttribute as CFString, kCFBooleanTrue)
521
+ var range = CFRange(location: params.int("start") ?? 0, length: params.int("length") ?? 0)
522
+ if params.int("length") == nil, let value: String = axAttr(element, kAXValueAttribute) {
523
+ range.length = (value as NSString).length - range.location
524
+ }
525
+ guard let axRange = AXValueCreate(.cfRange, &range) else { throw HelperError("Bad range") }
526
+ let err = AXUIElementSetAttributeValue(element, kAXSelectedTextRangeAttribute as CFString, axRange)
527
+ if err != .success { throw HelperError("Selecting text failed (AXError \(err.rawValue))", code: "action_failed") }
528
+ return [:]
529
+ }
530
+
531
+ // MARK: - Input
532
+
533
+ let eventSource: CGEventSource? = {
534
+ let source = CGEventSource(stateID: .hidSystemState)
535
+ source?.userData = syntheticMarker
536
+ return source
537
+ }()
538
+
539
+ // Mouse events. On the background route they go to the target process,
540
+ // stamped with its window; otherwise to the system at the screen point.
541
+ func post(_ event: CGEvent?) {
542
+ guard let e = event else { return }
543
+ e.setIntegerValueField(.eventSourceUserData, value: syntheticMarker)
544
+ if let route = backgroundRoute {
545
+ route.postMouse(e)
546
+ } else {
547
+ e.post(tap: .cghidEventTap)
548
+ }
549
+ }
550
+
551
+ // Keyboard events go to the target app's process when it is in the
552
+ // background, so typing does not depend on (or steal) the user's focus.
553
+ var keyTargetPid: pid_t?
554
+
555
+ func postKey(_ event: CGEvent?) {
556
+ guard let e = event else { return }
557
+ e.setIntegerValueField(.eventSourceUserData, value: syntheticMarker)
558
+ if let route = backgroundRoute {
559
+ route.postKey(e)
560
+ } else if let pid = keyTargetPid, let app = NSRunningApplication(processIdentifier: pid), !app.isActive {
561
+ e.postToPid(pid)
562
+ } else {
563
+ e.post(tap: .cghidEventTap)
564
+ }
565
+ }
566
+
567
+ func withKeyTarget<T>(_ params: [String: Any], _ body: () throws -> T) rethrows -> T {
568
+ keyTargetPid = params.int("pid").map { pid_t($0) }
569
+ defer { keyTargetPid = nil }
570
+ return try body()
571
+ }
572
+
573
+ // The element under a screen point, registered in the app's element table.
574
+ func hitTest(_ params: [String: Any]) throws -> [String: Any] {
575
+ let pid = try params.requireInt("pid")
576
+ let app = AXUIElementCreateApplication(pid_t(pid))
577
+ var hit: AXUIElement?
578
+ let err = AXUIElementCopyElementAtPosition(app, Float(try params.requireDouble("x")), Float(try params.requireDouble("y")), &hit)
579
+ guard err == .success, let element = hit else { return [:] }
580
+ var elementPid: pid_t = 0
581
+ AXUIElementGetPid(element, &elementPid)
582
+ if Int(elementPid) != pid { return ["foreign": true] }
583
+ var out: [String: Any] = ["index": table(pid).index(of: element), "role": (axAttr(element, kAXRoleAttribute) as String?) ?? "AXUnknown"]
584
+ var names: CFArray?
585
+ if AXUIElementCopyActionNames(element, &names) == .success, let actions = names as? [String] { out["actions"] = actions }
586
+ if let frame = axFrame(element) { out["frame"] = rectJSON(frame) }
587
+ return out
588
+ }
589
+
590
+ let keyCodes: [String: CGKeyCode] = [
591
+ "a": 0, "s": 1, "d": 2, "f": 3, "h": 4, "g": 5, "z": 6, "x": 7, "c": 8, "v": 9, "b": 11, "q": 12,
592
+ "w": 13, "e": 14, "r": 15, "y": 16, "t": 17, "1": 18, "2": 19, "3": 20, "4": 21, "6": 22, "5": 23,
593
+ "=": 24, "equal": 24, "9": 25, "7": 26, "-": 27, "minus": 27, "8": 28, "0": 29, "]": 30, "bracketright": 30,
594
+ "o": 31, "u": 32, "[": 33, "bracketleft": 33, "i": 34, "p": 35, "Return": 36, "l": 37, "j": 38,
595
+ "'": 39, "apostrophe": 39, "k": 40, ";": 41, "semicolon": 41, "\\": 42, "backslash": 42, ",": 43, "comma": 43,
596
+ "/": 44, "slash": 44, "n": 45, "m": 46, ".": 47, "period": 47, "Tab": 48, "space": 49, " ": 49,
597
+ "`": 50, "grave": 50, "BackSpace": 51, "Escape": 53, "Caps_Lock": 57, "KP_Enter": 76,
598
+ "F1": 122, "F2": 120, "F3": 99, "F4": 118, "F5": 96, "F6": 97, "F7": 98, "F8": 100, "F9": 101,
599
+ "F10": 109, "F11": 103, "F12": 111, "F13": 105, "F14": 107, "F15": 113, "F16": 106, "F17": 64,
600
+ "F18": 79, "F19": 80, "F20": 90, "Help": 114, "Insert": 114, "Home": 115, "Page_Up": 116, "Delete": 117,
601
+ "End": 119, "Page_Down": 121, "Left": 123, "Right": 124, "Down": 125, "Up": 126,
602
+ "ctrl": 59, "shift": 56, "alt": 58, "super": 55, "fn": 63,
603
+ ]
604
+
605
+ // Shifted characters on a US layout, so "ctrl+shift+?" and "?" both work.
606
+ let shifted: [String: String] = [
607
+ "!": "1", "@": "2", "#": "3", "$": "4", "%": "5", "^": "6", "&": "7", "*": "8", "(": "9", ")": "0",
608
+ "_": "-", "+": "=", "plus": "=", "{": "[", "}": "]", "|": "\\", ":": ";", "\"": "'", "<": ",", ">": ".",
609
+ "?": "/", "~": "`",
610
+ ]
611
+
612
+ func flags(_ mods: [String]) -> CGEventFlags {
613
+ var f: CGEventFlags = []
614
+ for m in mods {
615
+ switch m {
616
+ case "ctrl": f.insert(.maskControl)
617
+ case "shift": f.insert(.maskShift)
618
+ case "alt": f.insert(.maskAlternate)
619
+ case "super": f.insert(.maskCommand)
620
+ case "fn": f.insert(.maskSecondaryFn)
621
+ default: break
622
+ }
623
+ }
624
+ return f
625
+ }
626
+
627
+ func pressKeys(_ params: [String: Any]) throws -> [String: Any] {
628
+ guard let chords = params["chords"] as? [[String: Any]] else { throw HelperError("missing chords", code: "bad_request") }
629
+ for chord in chords {
630
+ guard var key = chord["key"] as? String else { continue }
631
+ var mods = (chord["mods"] as? [String]) ?? []
632
+ if let base = shifted[key] {
633
+ key = base
634
+ if !mods.contains("shift") { mods.append("shift") }
635
+ }
636
+ let lower = key.count == 1 ? key.lowercased() : key
637
+ if key.count == 1 && key != lower && !mods.contains("shift") { mods.append("shift") }
638
+ if let code = keyCodes[lower] {
639
+ let f = flags(mods)
640
+ let down = CGEvent(keyboardEventSource: eventSource, virtualKey: code, keyDown: true)
641
+ let up = CGEvent(keyboardEventSource: eventSource, virtualKey: code, keyDown: false)
642
+ down?.flags = f
643
+ up?.flags = f
644
+ postKey(down)
645
+ usleep(8000)
646
+ postKey(up)
647
+ } else if mods.isEmpty {
648
+ try typeText(["text": key])
649
+ } else {
650
+ throw HelperError("No key code for \"\(key)\" with modifiers", code: "bad_key")
651
+ }
652
+ usleep(25000)
653
+ }
654
+ return [:]
655
+ }
656
+
657
+ @discardableResult
658
+ func typeText(_ params: [String: Any]) throws -> [String: Any] {
659
+ guard let text = params.string("text") else { throw HelperError("missing text", code: "bad_request") }
660
+ var chunk: [UniChar] = []
661
+ func flush() {
662
+ guard !chunk.isEmpty else { return }
663
+ let down = CGEvent(keyboardEventSource: eventSource, virtualKey: 0, keyDown: true)
664
+ let up = CGEvent(keyboardEventSource: eventSource, virtualKey: 0, keyDown: false)
665
+ down?.keyboardSetUnicodeString(stringLength: chunk.count, unicodeString: chunk)
666
+ up?.keyboardSetUnicodeString(stringLength: chunk.count, unicodeString: chunk)
667
+ postKey(down)
668
+ usleep(4000)
669
+ postKey(up)
670
+ usleep(12000)
671
+ chunk.removeAll()
672
+ }
673
+ for scalar in text.unicodeScalars {
674
+ if scalar == "\n" || scalar == "\r" || scalar == "\t" {
675
+ flush()
676
+ let code: CGKeyCode = scalar == "\t" ? 48 : 36
677
+ postKey(CGEvent(keyboardEventSource: eventSource, virtualKey: code, keyDown: true))
678
+ usleep(8000)
679
+ postKey(CGEvent(keyboardEventSource: eventSource, virtualKey: code, keyDown: false))
680
+ usleep(20000)
681
+ continue
682
+ }
683
+ // One character per event: some apps drop the tail of multi-character events.
684
+ chunk.append(contentsOf: Array(String(scalar).utf16))
685
+ flush()
686
+ }
687
+ flush()
688
+ return [:]
689
+ }
690
+
691
+ func mouseButton(_ name: String?) -> (CGMouseButton, CGEventType, CGEventType, CGEventType) {
692
+ switch name {
693
+ case "right": return (.right, .rightMouseDown, .rightMouseUp, .rightMouseDragged)
694
+ case "middle": return (.center, .otherMouseDown, .otherMouseUp, .otherMouseDragged)
695
+ default: return (.left, .leftMouseDown, .leftMouseUp, .leftMouseDragged)
696
+ }
697
+ }
698
+
699
+ func click(_ params: [String: Any]) throws -> [String: Any] {
700
+ let point = CGPoint(x: try params.requireDouble("x"), y: try params.requireDouble("y"))
701
+ let count = max(1, params.int("count") ?? 1)
702
+ let (button, downType, upType, _) = mouseButton(params.string("button"))
703
+ let f = flags((params["mods"] as? [String]) ?? [])
704
+ post(CGEvent(mouseEventSource: eventSource, mouseType: .mouseMoved, mouseCursorPosition: point, mouseButton: .left))
705
+ usleep(30000)
706
+ backgroundRoute?.primeChromiumClick()
707
+ for n in 1...count {
708
+ let down = CGEvent(mouseEventSource: eventSource, mouseType: downType, mouseCursorPosition: point, mouseButton: button)
709
+ let up = CGEvent(mouseEventSource: eventSource, mouseType: upType, mouseCursorPosition: point, mouseButton: button)
710
+ down?.setIntegerValueField(.mouseEventClickState, value: Int64(n))
711
+ up?.setIntegerValueField(.mouseEventClickState, value: Int64(n))
712
+ down?.flags = f
713
+ up?.flags = f
714
+ post(down)
715
+ // An NSButton's mouseDown polls for the matching up; too tight a gap races it.
716
+ usleep(28000)
717
+ post(up)
718
+ usleep(count > 1 ? 80000 : 30000)
719
+ }
720
+ return [:]
721
+ }
722
+
723
+ func moveMouse(_ params: [String: Any]) throws -> [String: Any] {
724
+ let point = CGPoint(x: try params.requireDouble("x"), y: try params.requireDouble("y"))
725
+ post(CGEvent(mouseEventSource: eventSource, mouseType: .mouseMoved, mouseCursorPosition: point, mouseButton: .left))
726
+ return [:]
727
+ }
728
+
729
+ func scroll(_ params: [String: Any]) throws -> [String: Any] {
730
+ let point = CGPoint(x: try params.requireDouble("x"), y: try params.requireDouble("y"))
731
+ post(CGEvent(mouseEventSource: eventSource, mouseType: .mouseMoved, mouseCursorPosition: point, mouseButton: .left))
732
+ usleep(30000)
733
+ // Positive dy scrolls content down (toward the end), like a trackpad swipe up.
734
+ let dy = Int32(-(params.int("dy") ?? 0))
735
+ let dx = Int32(-(params.int("dx") ?? 0))
736
+ let steps = max(abs(dy), abs(dx), 1)
737
+ for i in 0..<steps {
738
+ let sy: Int32 = i < abs(dy) ? (dy > 0 ? 1 : -1) : 0
739
+ let sx: Int32 = i < abs(dx) ? (dx > 0 ? 1 : -1) : 0
740
+ let e = CGEvent(scrollWheelEvent2Source: eventSource, units: .line, wheelCount: 2, wheel1: sy, wheel2: sx, wheel3: 0)
741
+ e?.location = point
742
+ // Some clients (Citrix Viewer) read only the point and fixed-point
743
+ // deltas, which the line-based constructor leaves at 0 for axis 2.
744
+ if sx != 0 {
745
+ e?.setIntegerValueField(.scrollWheelEventPointDeltaAxis2, value: Int64(sx) * 10)
746
+ e?.setDoubleValueField(.scrollWheelEventFixedPtDeltaAxis2, value: Double(sx))
747
+ }
748
+ if let route = backgroundRoute, let event = e {
749
+ event.setIntegerValueField(.eventSourceUserData, value: syntheticMarker)
750
+ route.postScroll(event)
751
+ } else {
752
+ post(e)
753
+ }
754
+ usleep(15000)
755
+ }
756
+ return [:]
757
+ }
758
+
759
+ func drag(_ params: [String: Any]) throws -> [String: Any] {
760
+ guard let raw = params["path"] as? [[Double]], raw.count >= 2 else { throw HelperError("drag needs at least two points", code: "bad_request") }
761
+ let points = raw.map { CGPoint(x: $0[0], y: $0[1]) }
762
+ let (button, downType, upType, dragType) = mouseButton(params.string("button"))
763
+ post(CGEvent(mouseEventSource: eventSource, mouseType: .mouseMoved, mouseCursorPosition: points[0], mouseButton: button))
764
+ usleep(40000)
765
+ // Background delivery carries no click count unless we set one, and AppKit
766
+ // ignores a mouseDown without it.
767
+ func withClickState(_ e: CGEvent?) -> CGEvent? {
768
+ e?.setIntegerValueField(.mouseEventClickState, value: 1)
769
+ return e
770
+ }
771
+ post(withClickState(CGEvent(mouseEventSource: eventSource, mouseType: downType, mouseCursorPosition: points[0], mouseButton: button)))
772
+ usleep(60000)
773
+ for i in 1..<points.count {
774
+ let a = points[i - 1], b = points[i]
775
+ let steps = max(4, Int(hypot(b.x - a.x, b.y - a.y) / 12))
776
+ for s in 1...steps {
777
+ let t = CGFloat(s) / CGFloat(steps)
778
+ let p = CGPoint(x: a.x + (b.x - a.x) * t, y: a.y + (b.y - a.y) * t)
779
+ post(withClickState(CGEvent(mouseEventSource: eventSource, mouseType: dragType, mouseCursorPosition: p, mouseButton: button)))
780
+ usleep(8000)
781
+ }
782
+ }
783
+ usleep(40000)
784
+ post(withClickState(CGEvent(mouseEventSource: eventSource, mouseType: upType, mouseCursorPosition: points.last!, mouseButton: button)))
785
+ return [:]
786
+ }
787
+
788
+ // MARK: - Background input
789
+ //
790
+ // Delivers input to an app that stays in the background: its window is made
791
+ // key inside the app without activating it, and events are posted to the
792
+ // process with the window stamped on them. The real cursor never moves and the
793
+ // user's frontmost app keeps focus. This uses private SkyLight functions that
794
+ // Codex, Claude desktop, cua-driver, Peekaboo, and yabai also rely on; each is
795
+ // looked up at runtime, and when one is missing the caller falls back to the
796
+ // foreground route.
797
+
798
+ enum SkyLight {
799
+ typealias PostRecord = @convention(c) (UnsafeRawPointer, UnsafeRawPointer) -> Int32
800
+ typealias PostToPid = @convention(c) (pid_t, CGEvent) -> Void
801
+ typealias SetWindowLocation = @convention(c) (CGEvent, CGPoint) -> Void
802
+ typealias SetIntegerField = @convention(c) (CGEvent, UInt32, Int64) -> Void
803
+ typealias MainConnection = @convention(c) () -> UInt32
804
+ typealias WindowOwner = @convention(c) (UInt32, UInt32, UnsafeMutablePointer<UInt32>) -> Int32
805
+ typealias ConnectionPSN = @convention(c) (UInt32, UnsafeMutableRawPointer) -> Int32
806
+ typealias ProcessForPID = @convention(c) (pid_t, UnsafeMutableRawPointer) -> Int32
807
+
808
+ static let handle = dlopen("/System/Library/PrivateFrameworks/SkyLight.framework/SkyLight", RTLD_LAZY)
809
+
810
+ static func symbol<T>(_ name: String, _: T.Type) -> T? {
811
+ _ = handle
812
+ // RTLD_DEFAULT is ((void *)-2) and is not imported into Swift.
813
+ guard let pointer = dlsym(UnsafeMutableRawPointer(bitPattern: -2), name) else { return nil }
814
+ return unsafeBitCast(pointer, to: T.self)
815
+ }
816
+
817
+ static let postRecord = symbol("SLPSPostEventRecordTo", PostRecord.self)
818
+ static let postToPid = symbol("SLEventPostToPid", PostToPid.self)
819
+ static let setWindowLocation = symbol("CGEventSetWindowLocation", SetWindowLocation.self)
820
+ static let setIntegerField = symbol("CGEventSetIntegerValueField", SetIntegerField.self)
821
+ static let mainConnection = symbol("SLSMainConnectionID", MainConnection.self) ?? symbol("CGSMainConnectionID", MainConnection.self)
822
+ static let windowOwner = symbol("SLSGetWindowOwner", WindowOwner.self)
823
+ static let connectionPSN = symbol("SLSGetConnectionPSN", ConnectionPSN.self)
824
+ static let processForPID = symbol("GetProcessForPID", ProcessForPID.self)
825
+
826
+ // What the background route needs: records to make a window key, and a
827
+ // way to stamp window fields. Posting falls back to CGEventPostToPid.
828
+ static var available: Bool { postRecord != nil && setIntegerField != nil && (windowOwner != nil || processForPID != nil) }
829
+
830
+ static var capabilities: [String: Bool] {
831
+ [
832
+ "postEventRecord": postRecord != nil,
833
+ "postToPid": postToPid != nil,
834
+ "setWindowLocation": setWindowLocation != nil,
835
+ "setIntegerField": setIntegerField != nil,
836
+ "processSerialNumber": windowOwner != nil || processForPID != nil,
837
+ ]
838
+ }
839
+
840
+ // The 8-byte ProcessSerialNumber of the process that owns `windowId`.
841
+ static func psn(pid: pid_t, windowId: UInt32) -> [UInt8]? {
842
+ var psn = [UInt8](repeating: 0, count: 8)
843
+ if let owner = windowOwner, let toPSN = connectionPSN, let main = mainConnection {
844
+ var cid: UInt32 = 0
845
+ if owner(main(), windowId, &cid) == 0, cid != 0,
846
+ psn.withUnsafeMutableBytes({ toPSN(cid, $0.baseAddress!) }) == 0 {
847
+ return psn
848
+ }
849
+ }
850
+ if let forPid = processForPID, psn.withUnsafeMutableBytes({ forPid(pid, $0.baseAddress!) }) == 0 {
851
+ return psn
852
+ }
853
+ return nil
854
+ }
855
+
856
+ // 248-byte event records. Layout from yabai and cua-driver (MIT).
857
+ static func focusRecord(_ windowId: UInt32, focus: Bool) -> [UInt8] {
858
+ var record = [UInt8](repeating: 0, count: 0xF8)
859
+ record[0x04] = 0xF8
860
+ record[0x08] = 0x0D
861
+ withUnsafeBytes(of: windowId.littleEndian) { record.replaceSubrange(0x3C..<0x40, with: $0) }
862
+ record[0x8A] = focus ? 0x01 : 0x02
863
+ return record
864
+ }
865
+
866
+ static func makeKeyRecord(_ windowId: UInt32, kind: UInt8) -> [UInt8] {
867
+ var record = [UInt8](repeating: 0, count: 0xF8)
868
+ record[0x04] = 0xF8
869
+ record[0x08] = kind
870
+ record[0x3A] = 0x10
871
+ for i in 0x20..<0x30 { record[i] = 0xFF }
872
+ withUnsafeBytes(of: windowId.littleEndian) { record.replaceSubrange(0x3C..<0x40, with: $0) }
873
+ return record
874
+ }
875
+
876
+ @discardableResult
877
+ static func send(_ record: [UInt8], to psn: [UInt8]) -> Bool {
878
+ guard let post = postRecord else { return false }
879
+ return psn.withUnsafeBytes { p in record.withUnsafeBytes { r in post(p.baseAddress!, r.baseAddress!) } } == 0
880
+ }
881
+ }
882
+
883
+ func bundleUsesChromium(_ pid: pid_t) -> Bool {
884
+ guard let url = NSRunningApplication(processIdentifier: pid)?.bundleURL else { return false }
885
+ let frameworks = url.appendingPathComponent("Contents/Frameworks")
886
+ let names = (try? FileManager.default.contentsOfDirectory(atPath: frameworks.path)) ?? []
887
+ if names.contains(where: { $0.hasPrefix("Electron Framework") || $0.hasPrefix("Chromium Embedded Framework") || $0.contains("Chromium Framework") }) {
888
+ return true
889
+ }
890
+ let chromiumBrowsers: Set<String> = ["com.google.Chrome", "com.microsoft.edgemac", "com.brave.Browser", "company.thebrowser.Browser", "com.vivaldi.Vivaldi", "com.operasoftware.Opera"]
891
+ return chromiumBrowsers.contains(NSRunningApplication(processIdentifier: pid)?.bundleIdentifier ?? "")
892
+ }
893
+
894
+ // The CGWindowID of the app's key window as Accessibility reports it.
895
+ func keyWindowId(_ pid: pid_t) -> UInt32? {
896
+ let app = AXUIElementCreateApplication(pid)
897
+ guard let window: AXUIElement = axAttr(app, kAXFocusedWindowAttribute), let frame = axFrame(window),
898
+ let id = cgWindowID(pid: Int(pid), frame: frame) else { return nil }
899
+ return UInt32(id)
900
+ }
901
+
902
+ func cgWindowBounds(_ windowId: UInt32) -> CGRect? {
903
+ guard let info = CGWindowListCopyWindowInfo([.optionIncludingWindow], CGWindowID(windowId)) as? [[String: Any]],
904
+ let bounds = info.first?[kCGWindowBounds as String] as? [String: Any] else { return nil }
905
+ return CGRect(dictionaryRepresentation: bounds as CFDictionary)
906
+ }
907
+
908
+ final class BackgroundRoute {
909
+ let pid: pid_t
910
+ let windowId: UInt32
911
+ let origin: CGPoint
912
+ let chromium: Bool
913
+ let localLocation: Bool
914
+ let gesture = Int64.random(in: 1...Int64(Int32.max))
915
+
916
+ init(pid: pid_t, windowId: UInt32, origin: CGPoint, localLocation: Bool) {
917
+ self.pid = pid
918
+ self.windowId = windowId
919
+ self.origin = origin
920
+ self.chromium = bundleUsesChromium(pid)
921
+ self.localLocation = localLocation
922
+ }
923
+
924
+ private func stamp(_ e: CGEvent) {
925
+ guard let set = SkyLight.setIntegerField else { return }
926
+ set(e, 40, Int64(pid)) // kCGEventTargetUnixProcessID
927
+ set(e, 51, Int64(windowId)) // window number
928
+ set(e, 91, Int64(windowId)) // kCGMouseEventWindowUnderMousePointer
929
+ set(e, 92, Int64(windowId)) // ...ThatCanHandleThisEvent
930
+ set(e, 58, gesture) // groups one gesture's events
931
+ if let setLocation = SkyLight.setWindowLocation {
932
+ let p = e.location
933
+ setLocation(e, localLocation ? CGPoint(x: p.x - origin.x, y: p.y - origin.y) : p)
934
+ }
935
+ }
936
+
937
+ // Chromium and Electron want the SkyLight route; AppKit takes the public
938
+ // one. Never both: AppKit controls would see each click twice.
939
+ func postMouse(_ e: CGEvent) {
940
+ stamp(e)
941
+ if chromium, let post = SkyLight.postToPid { post(pid, e) } else { e.postToPid(pid) }
942
+ }
943
+
944
+ func postKey(_ e: CGEvent) {
945
+ if let post = SkyLight.postToPid { post(pid, e) } else { e.postToPid(pid) }
946
+ }
947
+
948
+ // Scroll wheel events reach background AppKit views only through both
949
+ // routes (cua-driver does the same); a doubled scroll tick is harmless.
950
+ func postScroll(_ e: CGEvent) {
951
+ stamp(e)
952
+ e.postToPid(pid)
953
+ if let post = SkyLight.postToPid { post(pid, e) }
954
+ }
955
+
956
+ // Chromium ignores a background click until it has seen user activation;
957
+ // an off-screen click satisfies that without touching anything.
958
+ func primeChromiumClick() {
959
+ guard chromium else { return }
960
+ let off = CGPoint(x: -1, y: -1)
961
+ postMouse(CGEvent(mouseEventSource: eventSource, mouseType: .leftMouseDown, mouseCursorPosition: off, mouseButton: .left)!)
962
+ usleep(10000)
963
+ postMouse(CGEvent(mouseEventSource: eventSource, mouseType: .leftMouseUp, mouseCursorPosition: off, mouseButton: .left)!)
964
+ usleep(100000)
965
+ }
966
+ }
967
+
968
+ var backgroundRoute: BackgroundRoute?
969
+
970
+ // Makes `windowId` the key window inside its app without activating the app.
971
+ // Sends nothing to any other process. Returns whether AX confirms it.
972
+ func makeWindowKey(pid: pid_t, windowId: UInt32, settle: UInt32 = 120_000) -> Bool {
973
+ guard let psn = SkyLight.psn(pid: pid, windowId: windowId) else { return false }
974
+ if let current = keyWindowId(pid), current != windowId {
975
+ SkyLight.send(SkyLight.focusRecord(current, focus: false), to: psn)
976
+ usleep(settle)
977
+ }
978
+ SkyLight.send(SkyLight.focusRecord(windowId, focus: true), to: psn)
979
+ usleep(settle)
980
+ SkyLight.send(SkyLight.makeKeyRecord(windowId, kind: 0x01), to: psn)
981
+ SkyLight.send(SkyLight.makeKeyRecord(windowId, kind: 0x02), to: psn)
982
+ for _ in 0..<10 {
983
+ if keyWindowId(pid) == windowId { return true }
984
+ usleep(40000)
985
+ }
986
+ return false
987
+ }
988
+
989
+ enum InputKind { case keys, mouse }
990
+
991
+ // True when keyboard focus is in a small uncapturable window inside
992
+ // `windowId`, such as Finder's rename field editor. Keys already go there, and
993
+ // making the parent key again would end the edit. A background app reports no
994
+ // focused window, so look for a focused child instead.
995
+ func focusIsInside(pid: pid_t, windowId: UInt32) -> Bool {
996
+ guard let bounds = cgWindowBounds(windowId)?.insetBy(dx: -2, dy: -2) else { return false }
997
+ for w in axWindows(Int(pid)) {
998
+ guard let frame = axFrame(w), frame.height < 120, bounds.contains(frame),
999
+ cgWindowID(pid: Int(pid), frame: frame) == nil else { continue }
1000
+ // Finder's editor "window" is the focused text field itself.
1001
+ let children: [AXUIElement] = axAttr(w, kAXChildrenAttribute) ?? []
1002
+ if ([w] + children).contains(where: { (axAttr($0, kAXFocusedAttribute) as Bool?) == true }) { return true }
1003
+ }
1004
+ return false
1005
+ }
1006
+
1007
+ // Runs `body` with input routed to the background app named in params, and
1008
+ // restores the user's frontmost app if the target took focus anyway.
1009
+ func withInputRoute(_ params: [String: Any], _ kind: InputKind, _ body: () throws -> [String: Any]) throws -> [String: Any] {
1010
+ guard params.string("mode") == "background" else { return try body() }
1011
+ guard SkyLight.available else {
1012
+ throw HelperError("Background input is unavailable on this macOS (private SkyLight functions missing).", code: "background_unavailable")
1013
+ }
1014
+ let pid = pid_t(try params.requireInt("pid"))
1015
+ let windowId = UInt32(try params.requireInt("windowId"))
1016
+ guard let bounds = cgWindowBounds(windowId) else { throw HelperError("Window \(windowId) is gone", code: "no_window") }
1017
+ let front = NSWorkspace.shared.frontmostApplication
1018
+ var keyed = true
1019
+ if kind == .keys {
1020
+ keyed = focusIsInside(pid: pid, windowId: windowId) || makeWindowKey(pid: pid, windowId: windowId)
1021
+ if !keyed {
1022
+ throw HelperError("Could not make the window key in the background; use foreground mode for this app.", code: "background_unconfirmed")
1023
+ }
1024
+ } else if let psn = SkyLight.psn(pid: pid, windowId: windowId) {
1025
+ // Target-only focus record so AppKit routes the click to this window.
1026
+ SkyLight.send(SkyLight.focusRecord(windowId, focus: true), to: psn)
1027
+ usleep(50000)
1028
+ }
1029
+ let route = BackgroundRoute(pid: pid, windowId: windowId, origin: bounds.origin, localLocation: params.string("windowLocation") != "screen")
1030
+ backgroundRoute = route
1031
+ defer { backgroundRoute = nil }
1032
+ var result = try body()
1033
+ usleep(60000)
1034
+ var restored = false
1035
+ if let front = front, front.processIdentifier != pid,
1036
+ NSWorkspace.shared.frontmostApplication?.processIdentifier == pid {
1037
+ // The target took focus anyway: give it back to the user's app.
1038
+ AXUIElementSetAttributeValue(AXUIElementCreateApplication(front.processIdentifier), kAXFrontmostAttribute as CFString, kCFBooleanTrue)
1039
+ restored = true
1040
+ }
1041
+ result["route"] = route.chromium ? "background-skylight" : "background"
1042
+ result["focusRestored"] = restored
1043
+ return result
1044
+ }
1045
+
1046
+ // MARK: - Agent cursor
1047
+ //
1048
+ // A drawn pointer that shows where the agent acts. It sits just above the
1049
+ // target window (not above everything), ignores the mouse, and never moves
1050
+ // the real cursor.
1051
+
1052
+ final class AgentCursor {
1053
+ static let shared = AgentCursor()
1054
+ var enabled = true
1055
+ private var window: NSWindow?
1056
+ private var hideTimer: Timer?
1057
+ private let size = CGSize(width: 36, height: 36)
1058
+
1059
+ private func makeWindow() -> NSWindow {
1060
+ let w = NSWindow(contentRect: NSRect(origin: .zero, size: size), styleMask: .borderless, backing: .buffered, defer: false)
1061
+ w.isOpaque = false
1062
+ w.backgroundColor = .clear
1063
+ w.hasShadow = false
1064
+ w.ignoresMouseEvents = true
1065
+ w.level = .normal
1066
+ w.collectionBehavior = [.canJoinAllSpaces, .fullScreenAuxiliary, .stationary, .ignoresCycle]
1067
+ w.sharingType = .readOnly
1068
+ w.title = "glovebox cursor"
1069
+ let view = NSImageView(frame: NSRect(origin: .zero, size: size))
1070
+ let config = NSImage.SymbolConfiguration(pointSize: 22, weight: .regular).applying(.init(paletteColors: [.white, .systemPurple]))
1071
+ view.image = NSImage(systemSymbolName: "cursorarrow", accessibilityDescription: "agent cursor")?.withSymbolConfiguration(config)
1072
+ view.imageAlignment = .alignTopLeft
1073
+ w.contentView = view
1074
+ return w
1075
+ }
1076
+
1077
+ // Moves the cursor to a screen point (top-left origin) above `windowId`.
1078
+ // Blocks until the move animation finishes so the action lines up with it.
1079
+ func move(to point: CGPoint, above windowId: Int?) {
1080
+ guard enabled else { return }
1081
+ var from = NSPoint.zero
1082
+ var target = NSPoint.zero
1083
+ var duration: TimeInterval = 0
1084
+ DispatchQueue.main.sync {
1085
+ let w = self.window ?? self.makeWindow()
1086
+ self.window = w
1087
+ guard let primary = NSScreen.screens.first else { return }
1088
+ // AppKit frames are bottom-left origin; the arrow tip is the frame's top-left.
1089
+ target = NSPoint(x: point.x - 4, y: primary.frame.height - point.y - self.size.height + 3)
1090
+ from = w.isVisible ? w.frame.origin : target
1091
+ let distance = hypot(target.x - from.x, target.y - from.y)
1092
+ duration = w.isVisible ? min(0.45, 0.12 + Double(distance) / 2400) : 0
1093
+ if let id = windowId { w.order(.above, relativeTo: id) } else { w.orderFrontRegardless() }
1094
+ if duration == 0 { w.setFrameOrigin(target) }
1095
+ self.hideTimer?.invalidate()
1096
+ self.hideTimer = Timer.scheduledTimer(withTimeInterval: 4, repeats: false) { _ in self.hide() }
1097
+ }
1098
+ guard duration > 0 else { return }
1099
+ // Stepped by hand: an animator() frame move left the window where it was.
1100
+ let steps = max(1, Int(duration * 60))
1101
+ for step in 1...steps {
1102
+ let t = Double(step) / Double(steps)
1103
+ let eased = t * t * (3 - 2 * t)
1104
+ let next = NSPoint(x: from.x + (target.x - from.x) * eased, y: from.y + (target.y - from.y) * eased)
1105
+ DispatchQueue.main.sync { self.window?.setFrameOrigin(next) }
1106
+ usleep(useconds_t(duration / Double(steps) * 1_000_000))
1107
+ }
1108
+ }
1109
+
1110
+ func hide() {
1111
+ let work: () -> Void = { self.window?.orderOut(nil) }
1112
+ if Thread.isMainThread { work() } else { DispatchQueue.main.async(execute: work) }
1113
+ }
1114
+ }
1115
+
1116
+ func cursorCommand(_ params: [String: Any]) throws -> [String: Any] {
1117
+ if let enabled = params.bool("enabled") {
1118
+ AgentCursor.shared.enabled = enabled
1119
+ if !enabled { AgentCursor.shared.hide() }
1120
+ }
1121
+ if params.bool("hide") == true { AgentCursor.shared.hide() }
1122
+ if let x = params.double("x"), let y = params.double("y") {
1123
+ AgentCursor.shared.move(to: CGPoint(x: x, y: y), above: params.int("windowId"))
1124
+ }
1125
+ return ["enabled": AgentCursor.shared.enabled]
1126
+ }
1127
+
1128
+ // MARK: - Screenshots
1129
+
1130
+ func encode(_ image: CGImage, format: String, quality: Double) throws -> Data {
1131
+ let data = NSMutableData()
1132
+ let type = format == "jpeg" ? UTType.jpeg.identifier : UTType.png.identifier
1133
+ guard let dest = CGImageDestinationCreateWithData(data, type as CFString, 1, nil) else { throw HelperError("Cannot encode image") }
1134
+ let options: [CFString: Any] = format == "jpeg" ? [kCGImageDestinationLossyCompressionQuality: quality] : [:]
1135
+ CGImageDestinationAddImage(dest, image, options as CFDictionary)
1136
+ guard CGImageDestinationFinalize(dest) else { throw HelperError("Cannot encode image") }
1137
+ return data as Data
1138
+ }
1139
+
1140
+ func screenshot(_ params: [String: Any]) throws -> [String: Any] {
1141
+ let windowId = try params.requireInt("windowId")
1142
+ let width = try params.requireInt("width")
1143
+ let height = try params.requireInt("height")
1144
+ let format = params.string("format") ?? "png"
1145
+ let quality = params.double("quality") ?? 0.8
1146
+
1147
+ // CGPreflightScreenCaptureAccess can report false for a granted ad-hoc
1148
+ // signed app, so ScreenCaptureKit itself is the source of truth.
1149
+ let done = DispatchSemaphore(value: 0)
1150
+ var captured: CGImage?
1151
+ var failure: Error?
1152
+ Task {
1153
+ do {
1154
+ let content = try await SCShareableContent.excludingDesktopWindows(false, onScreenWindowsOnly: false)
1155
+ guard let window = content.windows.first(where: { Int($0.windowID) == windowId }) else {
1156
+ throw HelperError("Window \(windowId) is not capturable (closed, or on another Space?)", code: "no_window")
1157
+ }
1158
+ let filter = SCContentFilter(desktopIndependentWindow: window)
1159
+ let config = SCStreamConfiguration()
1160
+ config.width = width
1161
+ config.height = height
1162
+ config.showsCursor = false
1163
+ config.scalesToFit = true
1164
+ if #available(macOS 14.0, *) { config.ignoreShadowsSingleWindow = true }
1165
+ captured = try await SCScreenshotManager.captureImage(contentFilter: filter, configuration: config)
1166
+ } catch {
1167
+ failure = error
1168
+ }
1169
+ done.signal()
1170
+ }
1171
+ if done.wait(timeout: .now() + 15) == .timedOut { throw HelperError("Screenshot timed out") }
1172
+ if let error = failure as? HelperError { throw error }
1173
+ if let error = failure {
1174
+ if !screenCaptureAllowed() {
1175
+ throw HelperError("Screen Recording permission is missing for Glovebox Helper. Run `glovebox doctor`.", code: "permission")
1176
+ }
1177
+ throw HelperError("Screenshot failed: \(error.localizedDescription)")
1178
+ }
1179
+ guard let image = captured else { throw HelperError("Screenshot returned nothing") }
1180
+ let data = try encode(image, format: format, quality: quality)
1181
+ return ["data": data.base64EncodedString(), "width": image.width, "height": image.height, "mimeType": format == "jpeg" ? "image/jpeg" : "image/png"]
1182
+ }
1183
+
1184
+ // True when ScreenCaptureKit will hand us window content.
1185
+ func screenCaptureAllowed() -> Bool {
1186
+ if CGPreflightScreenCaptureAccess() { return true }
1187
+ let done = DispatchSemaphore(value: 0)
1188
+ var ok = false
1189
+ Task {
1190
+ ok = (try? await SCShareableContent.excludingDesktopWindows(false, onScreenWindowsOnly: true)) != nil
1191
+ done.signal()
1192
+ }
1193
+ return done.wait(timeout: .now() + 5) == .success && ok
1194
+ }
1195
+
1196
+ // MARK: - Stopping scripts
1197
+ //
1198
+ // Two ways for the user to stop a running script, both broadcast as an
1199
+ // "escape" event that clients turn into an abort:
1200
+ // - a physical Escape press (synthetic events we post carry syntheticMarker
1201
+ // and are ignored);
1202
+ // - "Stop Running Scripts" in the menu-bar item shown while agents are connected.
1203
+
1204
+ var watchTap: CFMachPort?
1205
+ var keyMonitor: Any?
1206
+ var lastEscape = Date.distantPast
1207
+
1208
+ func userPressedEscape() {
1209
+ // The tap and the monitor can both see one press.
1210
+ if Date().timeIntervalSince(lastEscape) < 0.2 { return }
1211
+ lastEscape = Date()
1212
+ emit(["event": "escape"])
1213
+ }
1214
+
1215
+ // A global key monitor. macOS delivers key events to it only with Input
1216
+ // Monitoring granted; Accessibility alone installs it but it stays silent.
1217
+ func startEscapeMonitor() {
1218
+ guard keyMonitor == nil, AXIsProcessTrusted() else { return }
1219
+ let install = {
1220
+ keyMonitor = NSEvent.addGlobalMonitorForEvents(matching: .keyDown) { event in
1221
+ guard event.keyCode == 53 else { return }
1222
+ if let cg = event.cgEvent, cg.getIntegerValueField(.eventSourceUserData) == syntheticMarker { return }
1223
+ userPressedEscape()
1224
+ }
1225
+ }
1226
+ if Thread.isMainThread { install() } else { DispatchQueue.main.sync(execute: install) }
1227
+ }
1228
+
1229
+ // An event tap also sees keys sent to apps that bypass the monitor, but
1230
+ // macOS only feeds it with Input Monitoring; without it the tap is silent.
1231
+ func startEscapeTap() {
1232
+ guard watchTap == nil, CGPreflightListenEventAccess() else { return }
1233
+ let mask = CGEventMask(1 << CGEventType.keyDown.rawValue)
1234
+ let callback: CGEventTapCallBack = { _, type, event, _ in
1235
+ if type == .tapDisabledByTimeout || type == .tapDisabledByUserInput {
1236
+ if let tap = watchTap { CGEvent.tapEnable(tap: tap, enable: true) }
1237
+ return Unmanaged.passUnretained(event)
1238
+ }
1239
+ if type == .keyDown,
1240
+ event.getIntegerValueField(.keyboardEventKeycode) == 53,
1241
+ event.getIntegerValueField(.eventSourceUserData) != syntheticMarker {
1242
+ userPressedEscape()
1243
+ }
1244
+ return Unmanaged.passUnretained(event)
1245
+ }
1246
+ guard let tap = CGEvent.tapCreate(tap: .cgSessionEventTap, place: .headInsertEventTap, options: .listenOnly,
1247
+ eventsOfInterest: mask, callback: callback, userInfo: nil) else { return }
1248
+ watchTap = tap
1249
+ let source = CFMachPortCreateRunLoopSource(nil, tap, 0)
1250
+ CFRunLoopAddSource(CFRunLoopGetMain(), source, .commonModes)
1251
+ CGEvent.tapEnable(tap: tap, enable: true)
1252
+ }
1253
+
1254
+ @discardableResult
1255
+ func startEscapeWatcher() -> Bool {
1256
+ startEscapeMonitor()
1257
+ startEscapeTap()
1258
+ return keyMonitor != nil || watchTap != nil
1259
+ }
1260
+
1261
+ func escapeWatcherActive() -> Bool {
1262
+ startEscapeWatcher()
1263
+ }
1264
+
1265
+ // MARK: - Menu-bar item
1266
+
1267
+ final class StatusMenu: NSObject, NSMenuDelegate {
1268
+ var item: NSStatusItem?
1269
+
1270
+ // Called on the main thread whenever clients connect or disconnect.
1271
+ func refresh() {
1272
+ let connections = Connection.all()
1273
+ if connections.isEmpty {
1274
+ if let item = item { NSStatusBar.system.removeStatusItem(item) }
1275
+ item = nil
1276
+ return
1277
+ }
1278
+ if item == nil {
1279
+ let created = NSStatusBar.system.statusItem(withLength: NSStatusItem.variableLength)
1280
+ if let image = NSImage(systemSymbolName: "hand.raised.fill", accessibilityDescription: "glovebox") {
1281
+ image.isTemplate = true
1282
+ created.button?.image = image
1283
+ } else {
1284
+ created.button?.title = "glovebox"
1285
+ }
1286
+ let menu = NSMenu()
1287
+ menu.delegate = self
1288
+ created.menu = menu
1289
+ item = created
1290
+ }
1291
+ item?.button?.toolTip = "glovebox: \(connections.count) agent session(s) connected"
1292
+ }
1293
+
1294
+ func menuNeedsUpdate(_ menu: NSMenu) {
1295
+ menu.removeAllItems()
1296
+ let connections = Connection.all()
1297
+ menu.addItem(withTitle: "glovebox — \(connections.count) connected", action: nil, keyEquivalent: "")
1298
+ for c in connections {
1299
+ let apps = c.approved.sorted().joined(separator: ", ")
1300
+ menu.addItem(withTitle: " \(c.client)\(apps.isEmpty ? "" : ": \(apps)")", action: nil, keyEquivalent: "")
1301
+ }
1302
+ menu.addItem(.separator())
1303
+ let stop = NSMenuItem(title: "Stop Running Scripts", action: #selector(stopScripts), keyEquivalent: "")
1304
+ stop.target = self
1305
+ menu.addItem(stop)
1306
+ let revoke = NSMenuItem(title: "Revoke Session Approvals", action: #selector(revokeApprovals), keyEquivalent: "")
1307
+ revoke.target = self
1308
+ menu.addItem(revoke)
1309
+ let cursor = NSMenuItem(title: "Hide Agent Cursor", action: #selector(toggleCursor), keyEquivalent: "")
1310
+ cursor.target = self
1311
+ cursor.state = AgentCursor.shared.enabled ? .off : .on
1312
+ menu.addItem(cursor)
1313
+ let quit = NSMenuItem(title: "Quit Glovebox Helper", action: #selector(quitHelper), keyEquivalent: "")
1314
+ quit.target = self
1315
+ menu.addItem(quit)
1316
+ }
1317
+
1318
+ @objc func stopScripts() { emit(["event": "escape"]) }
1319
+
1320
+ @objc func toggleCursor() {
1321
+ AgentCursor.shared.enabled.toggle()
1322
+ if !AgentCursor.shared.enabled { AgentCursor.shared.hide() }
1323
+ }
1324
+
1325
+ @objc func revokeApprovals() {
1326
+ work.async { for c in Connection.all() { c.approved.removeAll() } }
1327
+ }
1328
+
1329
+ @objc func quitHelper() {
1330
+ unlink(socketPath)
1331
+ exit(0)
1332
+ }
1333
+ }
1334
+
1335
+ let statusMenu = StatusMenu()
1336
+
1337
+ func refreshStatusMenu() {
1338
+ DispatchQueue.main.async { statusMenu.refresh() }
1339
+ }
1340
+
1341
+ // MARK: - Approval policy
1342
+ //
1343
+ // The approval gate lives here, at the process that holds the Accessibility
1344
+ // grant, so a client cannot skip it by talking to the socket directly.
1345
+
1346
+ let gloveboxHome: String = ProcessInfo.processInfo.environment["GLOVEBOX_HOME"] ?? (NSHomeDirectory() + "/.glovebox")
1347
+ let configPath = gloveboxHome + "/config.json"
1348
+
1349
+ // Apps that can run arbitrary commands, hold secrets, or change system
1350
+ // security. These are never offered in a dialog; only an explicit `allow`
1351
+ // entry in config.json unlocks one.
1352
+ let blockedByDefault: Set<String> = [
1353
+ "com.apple.terminal", "com.googlecode.iterm2", "com.mitchellh.ghostty", "dev.warp.warp-stable",
1354
+ "com.github.wez.wezterm", "org.alacritty", "net.kovidgoyal.kitty", "org.tabby", "co.zeit.hyper",
1355
+ "com.apple.systempreferences", "com.apple.keychainaccess", "com.apple.passwords",
1356
+ "com.1password.1password", "com.agilebits.onepassword7", "com.bitwarden.desktop", "com.lastpass.lastpassmacdesktop",
1357
+ "com.apple.scripteditor2", "com.apple.automator", "io.github.vpatel9202.glovebox.helper",
1358
+ ]
1359
+
1360
+ func readConfig() -> [String: Any] {
1361
+ guard let data = FileManager.default.contents(atPath: configPath),
1362
+ let obj = try? JSONSerialization.jsonObject(with: data) as? [String: Any] else { return [:] }
1363
+ return obj
1364
+ }
1365
+
1366
+ func matches(_ list: [String], _ bundleId: String, _ name: String) -> Bool {
1367
+ let id = bundleId.lowercased(), n = name.lowercased()
1368
+ return list.contains { let e = $0.lowercased(); return e == id || e == n }
1369
+ }
1370
+
1371
+ func persistAllow(_ bundleId: String) {
1372
+ var config = readConfig()
1373
+ var allow = (config["allow"] as? [String]) ?? []
1374
+ if !allow.contains(where: { $0.lowercased() == bundleId.lowercased() }) { allow.append(bundleId) }
1375
+ config["allow"] = allow
1376
+ try? FileManager.default.createDirectory(atPath: gloveboxHome, withIntermediateDirectories: true, attributes: [.posixPermissions: 0o700])
1377
+ if let data = try? JSONSerialization.data(withJSONObject: config, options: [.prettyPrinted, .sortedKeys]) {
1378
+ FileManager.default.createFile(atPath: configPath, contents: data, attributes: [.posixPermissions: 0o600])
1379
+ }
1380
+ }
1381
+
1382
+ enum Choice { case session, always, deny }
1383
+
1384
+ func askUser(client: String, app: String, bundleId: String) -> Choice {
1385
+ var choice = Choice.deny
1386
+ DispatchQueue.main.sync {
1387
+ let alert = NSAlert()
1388
+ alert.messageText = "Allow \(client) to control \(app)?"
1389
+ alert.informativeText = "An agent using glovebox wants to click, type, and read the screen in \(app) (\(bundleId)). Every action is logged in \(gloveboxHome)/actions.log. Press Escape any time to stop a running script."
1390
+ alert.alertStyle = .warning
1391
+ alert.addButton(withTitle: "Allow for This Session")
1392
+ alert.addButton(withTitle: "Always Allow")
1393
+ alert.addButton(withTitle: "Deny")
1394
+ NSApp.activate(ignoringOtherApps: true)
1395
+ alert.window.level = .floating
1396
+ switch alert.runModal() {
1397
+ case .alertFirstButtonReturn: choice = .session
1398
+ case .alertSecondButtonReturn: choice = .always
1399
+ default: choice = .deny
1400
+ }
1401
+ }
1402
+ return choice
1403
+ }
1404
+
1405
+ func authorize(_ connection: Connection, bundleId: String, name: String) throws {
1406
+ if connection.approved.contains(bundleId) { return }
1407
+ let config = readConfig()
1408
+ let allow = (config["allow"] as? [String]) ?? []
1409
+ let deny = (config["deny"] as? [String]) ?? []
1410
+ let mode = (ProcessInfo.processInfo.environment["GLOVEBOX_APPROVAL"] ?? config["approval"] as? String) ?? "ask"
1411
+ if matches(deny, bundleId, name) {
1412
+ throw HelperError("\(name) is on the deny list in \(configPath).", code: "denied")
1413
+ }
1414
+ if matches(allow, bundleId, name) {
1415
+ connection.approved.insert(bundleId)
1416
+ return
1417
+ }
1418
+ if blockedByDefault.contains(bundleId.lowercased()) {
1419
+ throw HelperError("\(name) is blocked by default because it can run commands or holds secrets. To allow it anyway, run: glovebox allow \(bundleId)", code: "blocked")
1420
+ }
1421
+ switch mode {
1422
+ case "allow-all":
1423
+ connection.approved.insert(bundleId)
1424
+ case "allowlist":
1425
+ throw HelperError("\(name) is not in the allowlist. Run: glovebox allow \"\(name)\"", code: "not_allowed")
1426
+ default:
1427
+ switch askUser(client: connection.client, app: name, bundleId: bundleId) {
1428
+ case .session: connection.approved.insert(bundleId)
1429
+ case .always: connection.approved.insert(bundleId); persistAllow(bundleId)
1430
+ case .deny: throw HelperError("The user denied access to \(name).", code: "denied")
1431
+ }
1432
+ }
1433
+ }
1434
+
1435
+ // Methods that only read the app list or the helper's own state.
1436
+ let ungatedMethods: Set<String> = ["hello", "ping", "permissions", "listApps", "resolveApp", "forget", "cursor"]
1437
+
1438
+ func gate(_ connection: Connection, _ method: String, _ params: [String: Any]) throws {
1439
+ if ungatedMethods.contains(method) { return }
1440
+ if method == "launch" {
1441
+ let bundleId = params.string("bundleId") ?? params.string("path").flatMap { Bundle(path: $0)?.bundleIdentifier } ?? ""
1442
+ let name = params.string("name") ?? bundleId
1443
+ if bundleId.isEmpty { throw HelperError("launch needs a bundleId", code: "bad_request") }
1444
+ try authorize(connection, bundleId: bundleId, name: name)
1445
+ return
1446
+ }
1447
+ guard let pid = params.int("pid") else { throw HelperError("\(method) needs a pid", code: "bad_request") }
1448
+ let app = try runningApp(pid)
1449
+ guard let bundleId = app.bundleIdentifier else { throw HelperError("App has no bundle identifier", code: "denied") }
1450
+ try authorize(connection, bundleId: bundleId, name: app.localizedName ?? bundleId)
1451
+ }
1452
+
1453
+ // MARK: - Dispatch
1454
+
1455
+ func permissions(_ params: [String: Any]) -> [String: Any] {
1456
+ let prompt = params.bool("prompt") ?? false
1457
+ let options = [kAXTrustedCheckOptionPrompt.takeUnretainedValue() as String: prompt] as CFDictionary
1458
+ let ax = AXIsProcessTrustedWithOptions(options)
1459
+ var input = CGPreflightListenEventAccess()
1460
+ if prompt && params.bool("inputMonitoring") == true && !input { input = CGRequestListenEventAccess() }
1461
+ var screen = screenCaptureAllowed()
1462
+ if prompt && !screen { screen = CGRequestScreenCaptureAccess() }
1463
+ return ["accessibility": ax, "screenRecording": screen, "escapeWatcher": escapeWatcherActive(), "inputMonitoring": input]
1464
+ }
1465
+
1466
+ func handle(_ connection: Connection, _ method: String, _ params: [String: Any]) throws -> Any {
1467
+ try gate(connection, method, params)
1468
+ switch method {
1469
+ case "hello":
1470
+ connection.client = params.string("client") ?? connection.client
1471
+ return ["version": helperVersion, "platform": "macos", "pid": Int(getpid())]
1472
+ case "ping": return ["version": helperVersion, "platform": "macos", "pid": Int(getpid()), "background": SkyLight.capabilities]
1473
+ case "permissions": return permissions(params)
1474
+ case "listApps": return listApps()
1475
+ case "resolveApp": return try resolveApp(params.string("query") ?? "")
1476
+ case "launch": return try launch(params)
1477
+ case "activate": try activate(try params.requireInt("pid")); return [:]
1478
+ case "isActive": return ["active": try runningApp(try params.requireInt("pid")).isActive]
1479
+ case "windows": return windowsInfo(try params.requireInt("pid"))
1480
+ case "axTree": return try axTree(params)
1481
+ case "hitTest": return try hitTest(params)
1482
+ case "elementFrame": return try elementFrame(params)
1483
+ case "performAction": return try performAction(params)
1484
+ case "actionNames": return try actionNames(params)
1485
+ case "setValue": return try setValue(params)
1486
+ case "selectText": return try selectText(params)
1487
+ case "focusElement":
1488
+ var p = params
1489
+ p["attribute"] = kAXFocusedAttribute
1490
+ p["value"] = true
1491
+ return try setValue(p)
1492
+ case "keys": return try withInputRoute(params, .keys) { try withKeyTarget(params) { try pressKeys(params) } }
1493
+ case "type": return try withInputRoute(params, .keys) { try withKeyTarget(params) { try typeText(params) } }
1494
+ case "click": return try withInputRoute(params, .mouse) { try click(params) }
1495
+ case "cursor": return try cursorCommand(params)
1496
+ case "makeKey":
1497
+ let ok = makeWindowKey(pid: pid_t(try params.requireInt("pid")), windowId: UInt32(try params.requireInt("windowId")))
1498
+ return ["keyed": ok]
1499
+ case "move": return try moveMouse(params)
1500
+ case "scroll": return try withInputRoute(params, .mouse) { try scroll(params) }
1501
+ case "drag": return try withInputRoute(params, .mouse) { try drag(params) }
1502
+ case "screenshot": return try screenshot(params)
1503
+ case "forget": tables[try params.requireInt("pid")] = nil; return [:]
1504
+ default: throw HelperError("Unknown method \(method)", code: "bad_request")
1505
+ }
1506
+ }
1507
+
1508
+ // MARK: - Socket server
1509
+
1510
+ // All GUI work runs on one queue so actions from different clients never interleave.
1511
+ let work = DispatchQueue(label: "glovebox.work")
1512
+
1513
+ final class Connection {
1514
+ static var lock = NSLock()
1515
+ static var open: [ObjectIdentifier: Connection] = [:]
1516
+ static var lastClosed = Date()
1517
+
1518
+ static func all() -> [Connection] {
1519
+ lock.lock(); defer { lock.unlock() }
1520
+ return Array(open.values)
1521
+ }
1522
+
1523
+ let fd: Int32
1524
+ let writeLock = NSLock()
1525
+ var client = "An agent"
1526
+ private let approvedLock = NSLock()
1527
+ private var approvedApps = Set<String>()
1528
+ // Read from the menu on the main thread and written on the work queue.
1529
+ var approved: Set<String> {
1530
+ get { approvedLock.lock(); defer { approvedLock.unlock() }; return approvedApps }
1531
+ set { approvedLock.lock(); approvedApps = newValue; approvedLock.unlock() }
1532
+ }
1533
+
1534
+ init(fd: Int32) { self.fd = fd }
1535
+
1536
+ func send(_ object: [String: Any]) {
1537
+ guard var data = try? JSONSerialization.data(withJSONObject: object, options: []) else { return }
1538
+ data.append(0x0A)
1539
+ writeLock.lock(); defer { writeLock.unlock() }
1540
+ data.withUnsafeBytes { raw in
1541
+ var offset = 0
1542
+ while offset < raw.count {
1543
+ let n = Darwin.write(fd, raw.baseAddress! + offset, raw.count - offset)
1544
+ if n <= 0 { return }
1545
+ offset += n
1546
+ }
1547
+ }
1548
+ }
1549
+
1550
+ func run() {
1551
+ Connection.lock.lock(); Connection.open[ObjectIdentifier(self)] = self; Connection.lock.unlock()
1552
+ refreshStatusMenu()
1553
+ var buffer = Data()
1554
+ var chunk = [UInt8](repeating: 0, count: 65536)
1555
+ while true {
1556
+ let n = read(fd, &chunk, chunk.count)
1557
+ if n <= 0 { break }
1558
+ buffer.append(chunk, count: n)
1559
+ while let newline = buffer.firstIndex(of: 0x0A) {
1560
+ let line = buffer.subdata(in: buffer.startIndex..<newline)
1561
+ buffer.removeSubrange(buffer.startIndex...newline)
1562
+ guard !line.isEmpty,
1563
+ let obj = try? JSONSerialization.jsonObject(with: line) as? [String: Any],
1564
+ let method = obj["method"] as? String else { continue }
1565
+ let id = obj["id"] ?? NSNull()
1566
+ let params = (obj["params"] as? [String: Any]) ?? [:]
1567
+ work.async {
1568
+ do {
1569
+ let result = try handle(self, method, params)
1570
+ self.send(["id": id, "result": result])
1571
+ } catch let error as HelperError {
1572
+ self.send(["id": id, "error": ["message": error.message, "code": error.code]])
1573
+ } catch {
1574
+ self.send(["id": id, "error": ["message": "\(error)", "code": "failed"]])
1575
+ }
1576
+ }
1577
+ }
1578
+ }
1579
+ close(fd)
1580
+ Connection.lock.lock()
1581
+ Connection.open[ObjectIdentifier(self)] = nil
1582
+ Connection.lastClosed = Date()
1583
+ Connection.lock.unlock()
1584
+ refreshStatusMenu()
1585
+ }
1586
+ }
1587
+
1588
+ func socketAddress(_ path: String) -> sockaddr_un {
1589
+ var addr = sockaddr_un()
1590
+ addr.sun_family = sa_family_t(AF_UNIX)
1591
+ let bytes = Array(path.utf8)
1592
+ withUnsafeMutableBytes(of: &addr.sun_path) { raw in
1593
+ for (i, b) in bytes.prefix(raw.count - 1).enumerated() { raw[i] = b }
1594
+ }
1595
+ return addr
1596
+ }
1597
+
1598
+ func listen(at path: String) -> Int32 {
1599
+ // A live helper already owns the socket: let it keep serving.
1600
+ let probe = socket(AF_UNIX, SOCK_STREAM, 0)
1601
+ var addr = socketAddress(path)
1602
+ let alive = withUnsafePointer(to: &addr) {
1603
+ $0.withMemoryRebound(to: sockaddr.self, capacity: 1) { connect(probe, $0, socklen_t(MemoryLayout<sockaddr_un>.size)) }
1604
+ } == 0
1605
+ close(probe)
1606
+ if alive {
1607
+ FileHandle.standardError.write("glovebox-helper: already running\n".data(using: .utf8)!)
1608
+ exit(0)
1609
+ }
1610
+ unlink(path)
1611
+ let fd = socket(AF_UNIX, SOCK_STREAM, 0)
1612
+ let bound = withUnsafePointer(to: &addr) {
1613
+ $0.withMemoryRebound(to: sockaddr.self, capacity: 1) { bind(fd, $0, socklen_t(MemoryLayout<sockaddr_un>.size)) }
1614
+ }
1615
+ if bound != 0 || Darwin.listen(fd, 16) != 0 {
1616
+ FileHandle.standardError.write("glovebox-helper: cannot listen on \(path)\n".data(using: .utf8)!)
1617
+ exit(1)
1618
+ }
1619
+ chmod(path, 0o600)
1620
+ return fd
1621
+ }
1622
+
1623
+ func acceptLoop(_ server: Int32) {
1624
+ while true {
1625
+ let fd = accept(server, nil, nil)
1626
+ if fd < 0 { continue }
1627
+ // Only the user who owns this helper may drive it.
1628
+ var uid: uid_t = 0
1629
+ var gid: gid_t = 0
1630
+ if getpeereid(fd, &uid, &gid) != 0 || uid != getuid() {
1631
+ close(fd)
1632
+ continue
1633
+ }
1634
+ var on: Int32 = 1
1635
+ setsockopt(fd, SOL_SOCKET, SO_NOSIGPIPE, &on, socklen_t(MemoryLayout<Int32>.size))
1636
+ let connection = Connection(fd: fd)
1637
+ Thread.detachNewThread { connection.run() }
1638
+ }
1639
+ }
1640
+
1641
+ let socketPath = ProcessInfo.processInfo.environment["GLOVEBOX_SOCKET"] ?? (gloveboxHome + "/helper.sock")
1642
+ let idleQuitSeconds: TimeInterval = 30 * 60
1643
+
1644
+ try? FileManager.default.createDirectory(atPath: gloveboxHome, withIntermediateDirectories: true, attributes: [.posixPermissions: 0o700])
1645
+ _ = NSApplication.shared
1646
+ NSApp.setActivationPolicy(.accessory)
1647
+ signal(SIGPIPE, SIG_IGN)
1648
+ if CommandLine.arguments.contains("--request-permissions") {
1649
+ _ = permissions(["prompt": true])
1650
+ }
1651
+ _ = startEscapeWatcher()
1652
+ let server = listen(at: socketPath)
1653
+ Thread.detachNewThread { acceptLoop(server) }
1654
+ // Quit after a long stretch with no clients; the next client relaunches us.
1655
+ Timer.scheduledTimer(withTimeInterval: 60, repeats: true) { _ in
1656
+ Connection.lock.lock()
1657
+ let idle = Connection.open.isEmpty && Date().timeIntervalSince(Connection.lastClosed) > idleQuitSeconds
1658
+ Connection.lock.unlock()
1659
+ if idle { unlink(socketPath); exit(0) }
1660
+ }
1661
+ NSApp.run()