@forsion/tangu-computer-use 0.5.8

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (120) hide show
  1. package/CHANGELOG.md +223 -0
  2. package/LICENSE +26 -0
  3. package/LICENSE.upstream +21 -0
  4. package/README.md +169 -0
  5. package/UPSTREAM.md +277 -0
  6. package/check.mjs +369 -0
  7. package/icon.png +0 -0
  8. package/install.sh +112 -0
  9. package/main.js +541 -0
  10. package/manifest.json +47 -0
  11. package/native/linux/bridge-rs/Cargo.lock +1204 -0
  12. package/native/linux/bridge-rs/Cargo.toml +18 -0
  13. package/native/linux/bridge-rs/src/atspi.rs +640 -0
  14. package/native/linux/bridge-rs/src/error.rs +65 -0
  15. package/native/linux/bridge-rs/src/lib.rs +11 -0
  16. package/native/linux/bridge-rs/src/main.rs +1074 -0
  17. package/native/linux/bridge-rs/src/protocol.rs +49 -0
  18. package/native/linux/bridge-rs/src/state.rs +293 -0
  19. package/native/linux/bridge-rs/src/wayland.rs +89 -0
  20. package/native/linux/bridge-rs/src/x11.rs +909 -0
  21. package/native/linux/bridge-rs/tests/protocol_tests.rs +73 -0
  22. package/native/macos/agent_cursor.swift +184 -0
  23. package/native/macos/agent_cursor_motion.swift +252 -0
  24. package/native/macos/agent_cursor_tests.swift +128 -0
  25. package/native/macos/agent_highlight.swift +229 -0
  26. package/native/macos/agent_highlight_tests.swift +99 -0
  27. package/native/macos/bridge.swift +3852 -0
  28. package/native/macos/foreground_activity.swift +57 -0
  29. package/native/macos/foreground_activity_tests.swift +31 -0
  30. package/native/macos/live_stream.swift +289 -0
  31. package/native/windows/bridge-rs/Cargo.lock +396 -0
  32. package/native/windows/bridge-rs/Cargo.toml +30 -0
  33. package/native/windows/bridge-rs/src/capture.rs +518 -0
  34. package/native/windows/bridge-rs/src/error.rs +81 -0
  35. package/native/windows/bridge-rs/src/input.rs +501 -0
  36. package/native/windows/bridge-rs/src/lib.rs +16 -0
  37. package/native/windows/bridge-rs/src/main.rs +1683 -0
  38. package/native/windows/bridge-rs/src/protocol.rs +53 -0
  39. package/native/windows/bridge-rs/src/refs.rs +237 -0
  40. package/native/windows/bridge-rs/src/state.rs +55 -0
  41. package/native/windows/bridge-rs/src/uia.rs +1252 -0
  42. package/native/windows/bridge-rs/src/window.rs +718 -0
  43. package/native/windows/bridge-rs/tests/protocol_tests.rs +252 -0
  44. package/native/windows/bridge-rs/tests/refs_tests.rs +86 -0
  45. package/native/windows/bridge-rs/tests/state_tests.rs +33 -0
  46. package/package.json +74 -0
  47. package/prebuilt/linux/arm64/linux-bridge +0 -0
  48. package/prebuilt/linux/x64/linux-bridge +0 -0
  49. package/prebuilt/macos/arm64/bridge +0 -0
  50. package/prebuilt/macos/arm64/tangu-computer-use.app.json +10 -0
  51. package/prebuilt/macos/arm64/tangu-computer-use.app.zip +0 -0
  52. package/prebuilt/macos/x64/bridge +0 -0
  53. package/prebuilt/macos/x64/tangu-computer-use.app.json +10 -0
  54. package/prebuilt/macos/x64/tangu-computer-use.app.zip +0 -0
  55. package/prebuilt/windows/windows-bridge.exe +0 -0
  56. package/scripts/blind-click.check.mjs +128 -0
  57. package/scripts/build-native.mjs +302 -0
  58. package/scripts/build.mjs +84 -0
  59. package/scripts/calc-fixture.mjs +78 -0
  60. package/scripts/cli-exit.check.mjs +49 -0
  61. package/scripts/helper-path.check.mjs +82 -0
  62. package/scripts/helper-refresh.check.mjs +62 -0
  63. package/scripts/helper-signal.check.mjs +59 -0
  64. package/scripts/highlight.check.mjs +32 -0
  65. package/scripts/keychain-free-install.check.mjs +140 -0
  66. package/scripts/live-view.check.mjs +124 -0
  67. package/scripts/macos-bundle.d.mts +1 -0
  68. package/scripts/macos-bundle.mjs +119 -0
  69. package/scripts/make-signing-cert.sh +57 -0
  70. package/scripts/mini-foreground.check.mjs +11 -0
  71. package/scripts/no-foreground.check.mjs +81 -0
  72. package/scripts/overlay-visible.check.mjs +221 -0
  73. package/scripts/package-macos-app.mjs +39 -0
  74. package/scripts/permissions.check.mjs +199 -0
  75. package/scripts/platform-contract.check.mjs +30 -0
  76. package/scripts/setup-helper.mjs +154 -0
  77. package/scripts/tangu-computer-use.entitlements +10 -0
  78. package/scripts/verify-macos-bundles.mjs +25 -0
  79. package/scripts/verify-package.mjs +74 -0
  80. package/skills/computer-use/SKILL.md +110 -0
  81. package/src/foregroundNote.ts +43 -0
  82. package/src/helperState.ts +31 -0
  83. package/src/index.ts +63 -0
  84. package/src/onboarding.ts +198 -0
  85. package/src/pi-compat.ts +49 -0
  86. package/src/settings.ts +11 -0
  87. package/src/setup.ts +97 -0
  88. package/src/tools.ts +289 -0
  89. package/src/vendor/actions.ts +130 -0
  90. package/src/vendor/bridge.ts +2405 -0
  91. package/src/vendor/cdp.ts +658 -0
  92. package/src/vendor/config.ts +113 -0
  93. package/src/vendor/contract.ts +104 -0
  94. package/src/vendor/note.ts +195 -0
  95. package/src/vendor/outline.ts +651 -0
  96. package/src/vendor/output.ts +134 -0
  97. package/src/vendor/permissions.ts +111 -0
  98. package/src/vendor/platform/architecture.ts +23 -0
  99. package/src/vendor/platform/coerce.ts +16 -0
  100. package/src/vendor/platform/index.ts +59 -0
  101. package/src/vendor/platform/linux/backend.ts +186 -0
  102. package/src/vendor/platform/linux/helper.ts +238 -0
  103. package/src/vendor/platform/macos/backend.ts +131 -0
  104. package/src/vendor/platform/macos/browser.ts +110 -0
  105. package/src/vendor/platform/macos/helper-path.d.mts +9 -0
  106. package/src/vendor/platform/macos/helper-path.mjs +34 -0
  107. package/src/vendor/platform/macos/helper.ts +291 -0
  108. package/src/vendor/platform/macos/permissions.ts +146 -0
  109. package/src/vendor/platform/types.ts +222 -0
  110. package/src/vendor/platform/windows/backend.ts +142 -0
  111. package/src/vendor/platform/windows/helper.ts +140 -0
  112. package/src/vendor/root-selection.ts +25 -0
  113. package/src/vendor/runtime.ts +129 -0
  114. package/src/vendor/state.ts +146 -0
  115. package/src/vendor/view.ts +147 -0
  116. package/tangu-plugins/computer-use/dist/foregroundNote.js +19 -0
  117. package/tangu-plugins/computer-use/dist/index.js +5672 -0
  118. package/tangu-plugins/computer-use/tangu-plugin.json +9 -0
  119. package/tsconfig.json +22 -0
  120. package/types/tangu-agent.d.ts +142 -0
@@ -0,0 +1,3852 @@
1
+ import Foundation
2
+ import AppKit
3
+ import ApplicationServices
4
+ import Darwin
5
+ import Vision
6
+ import ImageIO
7
+ import ScreenCaptureKit
8
+
9
+ struct BridgeFailure: Error {
10
+ let message: String
11
+ let code: String
12
+ }
13
+
14
+ final class AXRefStore {
15
+ struct Snapshot {
16
+ let role: String
17
+ let identifier: String
18
+ let label: String
19
+ let rect: CGRect
20
+ }
21
+
22
+ private var nextId: UInt64 = 0
23
+ private var windows: [String: AXUIElement] = [:]
24
+ private var elements: [String: AXUIElement] = [:]
25
+ private var snapshots: [String: Snapshot] = [:]
26
+ private let lock = NSLock()
27
+
28
+ func storeWindow(_ window: AXUIElement) -> String {
29
+ lock.lock()
30
+ defer { lock.unlock() }
31
+ for (ref, existing) in windows {
32
+ if CFEqual(existing, window) {
33
+ return ref
34
+ }
35
+ }
36
+ nextId += 1
37
+ let ref = "w\(nextId)"
38
+ windows[ref] = window
39
+ return ref
40
+ }
41
+
42
+ func storeElement(_ element: AXUIElement, snapshot: Snapshot? = nil) -> String {
43
+ lock.lock()
44
+ defer { lock.unlock() }
45
+ nextId += 1
46
+ let ref = "e\(nextId)"
47
+ elements[ref] = element
48
+ snapshots[ref] = snapshot
49
+ return ref
50
+ }
51
+
52
+ func window(for ref: String) -> AXUIElement? {
53
+ lock.lock()
54
+ defer { lock.unlock() }
55
+ return windows[ref]
56
+ }
57
+
58
+ func element(for ref: String) -> AXUIElement? {
59
+ lock.lock()
60
+ defer { lock.unlock() }
61
+ return elements[ref]
62
+ }
63
+
64
+ func snapshot(for ref: String) -> Snapshot? {
65
+ lock.lock()
66
+ defer { lock.unlock() }
67
+ return snapshots[ref]
68
+ }
69
+ }
70
+
71
+ private struct CGWindowCandidate {
72
+ let windowId: UInt32
73
+ let title: String
74
+ let bounds: CGRect
75
+ let isOnscreen: Bool
76
+ let layer: Int
77
+ let zOrder: Int
78
+ }
79
+
80
+ private struct CGWindowOwnerSummary {
81
+ let pid: Int32
82
+ let name: String
83
+ }
84
+
85
+ private struct AXDescendant {
86
+ let element: AXUIElement
87
+ let depth: Int
88
+ let insideWebArea: Bool
89
+ let axVisible: Bool
90
+ }
91
+
92
+ private struct WindowPairing {
93
+ let candidate: CGWindowCandidate?
94
+ let score: Double
95
+ let confidence: String
96
+ }
97
+
98
+ private struct CapturedWindowImage {
99
+ let image: CGImage
100
+ let windowId: UInt32
101
+ let frame: CGRect
102
+ }
103
+
104
+ private struct LookRecord {
105
+ let lookId: String
106
+ let windowId: UInt32
107
+ let windowFrame: CGRect
108
+ let imageWidth: Int
109
+ let imageHeight: Int
110
+ let hasImage: Bool
111
+ }
112
+
113
+ private struct RootAXEvent {
114
+ let sequence: UInt64
115
+ let timestamp: TimeInterval
116
+ let notification: String
117
+ let element: AXUIElement?
118
+ }
119
+
120
+ private final class RootAXObserverState {
121
+ let pid: Int32
122
+ let observer: AXObserver
123
+ let change = NSCondition()
124
+ var changeGeneration: UInt64 = 0
125
+ var events: [RootAXEvent] = []
126
+ var nextSequence: UInt64 = 1
127
+ var lastUsed: TimeInterval = Date().timeIntervalSince1970
128
+
129
+ init(pid: Int32, observer: AXObserver) {
130
+ self.pid = pid
131
+ self.observer = observer
132
+ }
133
+ }
134
+
135
+ private struct OCRBox {
136
+ let string: String
137
+ let confidence: Double
138
+ let rect: CGRect
139
+ }
140
+
141
+ final class LookNode {
142
+ let element: AXUIElement?
143
+ let ref: String
144
+ let role: String
145
+ let subrole: String
146
+ let identifier: String
147
+ let title: String
148
+ let description: String
149
+ let value: String
150
+ let actions: [String]
151
+ let canPress: Bool
152
+ let canFocus: Bool
153
+ let canSetValue: Bool
154
+ let canScroll: Bool
155
+ let canIncrement: Bool
156
+ let canDecrement: Bool
157
+ let isTextInput: Bool
158
+ let rect: CGRect
159
+ let focused: Bool
160
+ var offscreen: Bool
161
+ var pictureOnly: Bool
162
+ var truncated: Bool
163
+ var scrollExtent: [String: Int]?
164
+ var text: [[String: Any]]
165
+ var children: [LookNode]
166
+
167
+ init(element: AXUIElement?, ref: String, role: String, subrole: String, identifier: String, title: String, description: String, value: String, actions: [String], canPress: Bool, canFocus: Bool, canSetValue: Bool, canScroll: Bool, canIncrement: Bool, canDecrement: Bool, isTextInput: Bool, rect: CGRect, focused: Bool = false, offscreen: Bool = false, pictureOnly: Bool = false) {
168
+ self.element = element
169
+ self.ref = ref
170
+ self.role = role
171
+ self.subrole = subrole
172
+ self.identifier = identifier
173
+ self.title = title
174
+ self.description = description
175
+ self.value = value
176
+ self.actions = actions
177
+ self.canPress = canPress
178
+ self.canFocus = canFocus
179
+ self.canSetValue = canSetValue
180
+ self.canScroll = canScroll
181
+ self.canIncrement = canIncrement
182
+ self.canDecrement = canDecrement
183
+ self.isTextInput = isTextInput
184
+ self.rect = rect
185
+ self.focused = focused
186
+ self.offscreen = offscreen
187
+ self.pictureOnly = pictureOnly
188
+ self.truncated = false
189
+ self.text = []
190
+ self.children = []
191
+ }
192
+
193
+ func payload() -> [String: Any] {
194
+ var output: [String: Any] = [
195
+ "ref": ref,
196
+ "role": role,
197
+ "subrole": subrole,
198
+ "identifier": identifier,
199
+ "title": title,
200
+ "description": description,
201
+ "value": value,
202
+ "actions": actions,
203
+ "canPress": canPress,
204
+ "canFocus": canFocus,
205
+ "canSetValue": canSetValue,
206
+ "canScroll": canScroll,
207
+ "canIncrement": canIncrement,
208
+ "canDecrement": canDecrement,
209
+ "isTextInput": isTextInput,
210
+ "rect": ["x": rect.origin.x, "y": rect.origin.y, "w": rect.width, "h": rect.height],
211
+ "children": children.map { $0.payload() },
212
+ ]
213
+ if focused { output["focused"] = true }
214
+ if offscreen { output["offscreen"] = true }
215
+ if pictureOnly { output["pictureOnly"] = true }
216
+ if truncated { output["truncated"] = true }
217
+ if let scrollExtent { output["scrollExtent"] = scrollExtent }
218
+ if !text.isEmpty { output["text"] = text }
219
+ return output
220
+ }
221
+
222
+ }
223
+
224
+ final class Box<T> {
225
+ var value: T
226
+ init(_ value: T) {
227
+ self.value = value
228
+ }
229
+ }
230
+
231
+ final class InputSuppressionGuard {
232
+ static let maxSuppressionSeconds: TimeInterval = 30
233
+
234
+ private let lock = NSLock()
235
+ private var eventTap: CFMachPort?
236
+ private var eventTapSource: CFRunLoopSource?
237
+ private var tapRunLoop: CFRunLoop?
238
+ private var tapThread: Thread?
239
+
240
+ func begin() throws {
241
+ lock.lock()
242
+ if eventTap != nil {
243
+ lock.unlock()
244
+ return
245
+ }
246
+ lock.unlock()
247
+
248
+ let eventTypes: [CGEventType] = [
249
+ .keyDown,
250
+ .keyUp,
251
+ .flagsChanged,
252
+ .leftMouseDown,
253
+ .leftMouseUp,
254
+ .rightMouseDown,
255
+ .rightMouseUp,
256
+ .otherMouseDown,
257
+ .otherMouseUp,
258
+ .mouseMoved,
259
+ .leftMouseDragged,
260
+ .rightMouseDragged,
261
+ .otherMouseDragged,
262
+ .scrollWheel,
263
+ .tabletPointer,
264
+ .tabletProximity,
265
+ ]
266
+ let mask = eventTypes.reduce(CGEventMask(0)) { partial, type in
267
+ partial | (CGEventMask(1) << CGEventMask(type.rawValue))
268
+ }
269
+
270
+ let callback: CGEventTapCallBack = { _proxy, type, event, userInfo in
271
+ guard let userInfo else { return Unmanaged.passUnretained(event) }
272
+ let inputGuard = Unmanaged<InputSuppressionGuard>.fromOpaque(userInfo).takeUnretainedValue()
273
+ if type == .tapDisabledByTimeout || type == .tapDisabledByUserInput {
274
+ inputGuard.reenableTap()
275
+ return Unmanaged.passUnretained(event)
276
+ }
277
+ return nil
278
+ }
279
+
280
+ guard let tap = CGEvent.tapCreate(
281
+ tap: .cgSessionEventTap,
282
+ place: .headInsertEventTap,
283
+ options: .defaultTap,
284
+ eventsOfInterest: mask,
285
+ callback: callback,
286
+ // passUnretained is only safe because Bridge owns this guard for the
287
+ // whole process lifetime; do not give it a shorter-lived owner.
288
+ userInfo: UnsafeMutableRawPointer(Unmanaged.passUnretained(self).toOpaque())
289
+ ) else {
290
+ throw BridgeFailure(message: "Failed to create input suppression event tap", code: "input_suppression_unavailable")
291
+ }
292
+
293
+ lock.lock()
294
+ eventTap = tap
295
+ lock.unlock()
296
+ let thread = Thread { [weak self] in
297
+ guard let self else { return }
298
+ let runLoop = CFRunLoopGetCurrent()
299
+ self.lock.lock()
300
+ self.tapRunLoop = runLoop
301
+ self.eventTapSource = CFMachPortCreateRunLoopSource(kCFAllocatorDefault, tap, 0)
302
+ if let source = self.eventTapSource {
303
+ CFRunLoopAddSource(runLoop, source, .commonModes)
304
+ }
305
+ // Watchdog: if end() never arrives (e.g. the parent process hangs
306
+ // mid-action), release the tap so the user is not locked out of
307
+ // keyboard and mouse input indefinitely.
308
+ let watchdog = CFRunLoopTimerCreateWithHandler(
309
+ kCFAllocatorDefault,
310
+ CFAbsoluteTimeGetCurrent() + InputSuppressionGuard.maxSuppressionSeconds,
311
+ 0,
312
+ 0,
313
+ 0
314
+ ) { [weak self] _ in
315
+ self?.end()
316
+ }
317
+ if let watchdog {
318
+ CFRunLoopAddTimer(runLoop, watchdog, .commonModes)
319
+ }
320
+ CGEvent.tapEnable(tap: tap, enable: true)
321
+ self.lock.unlock()
322
+ CFRunLoopRun()
323
+ }
324
+ thread.name = "tangu-computer-use-input-suppression"
325
+ lock.lock()
326
+ tapThread = thread
327
+ lock.unlock()
328
+ thread.start()
329
+
330
+ let deadline = Date().addingTimeInterval(1.0)
331
+ while tapRunLoop == nil && Date() < deadline {
332
+ Thread.sleep(forTimeInterval: 0.01)
333
+ }
334
+ if tapRunLoop == nil {
335
+ end()
336
+ throw BridgeFailure(message: "Timed out starting input suppression", code: "input_suppression_timeout")
337
+ }
338
+ }
339
+
340
+ func end() {
341
+ lock.lock()
342
+ let tap = eventTap
343
+ let source = eventTapSource
344
+ let runLoop = tapRunLoop
345
+ eventTap = nil
346
+ eventTapSource = nil
347
+ tapRunLoop = nil
348
+ tapThread = nil
349
+ lock.unlock()
350
+
351
+ if let tap {
352
+ CGEvent.tapEnable(tap: tap, enable: false)
353
+ }
354
+ if let source, let runLoop {
355
+ CFRunLoopRemoveSource(runLoop, source, .commonModes)
356
+ CFRunLoopStop(runLoop)
357
+ }
358
+ }
359
+
360
+ func reenableTap() {
361
+ lock.lock()
362
+ let tap = eventTap
363
+ lock.unlock()
364
+ if let tap {
365
+ CGEvent.tapEnable(tap: tap, enable: true)
366
+ }
367
+ }
368
+
369
+ }
370
+
371
+ final class Bridge {
372
+ private let protocolVersion = 12
373
+ private let refStore = AXRefStore()
374
+ private let inputSuppressionGuard = InputSuppressionGuard()
375
+ private let physicalInputLock = NSRecursiveLock()
376
+ private let supportsAgentCursor = CommandLine.arguments.contains("serve")
377
+ private let browserBundleIds: Set<String> = [
378
+ "com.apple.Safari", "com.google.Chrome", "org.chromium.Chromium", "company.thebrowser.Browser", "com.brave.Browser", "com.microsoft.edgemac", "com.vivaldi.Vivaldi", "net.imput.helium", "org.mozilla.firefox",
379
+ ]
380
+ private var enhancedAccessibilityPids = Set<Int32>()
381
+ private let enhancedAccessibilityLock = NSLock()
382
+ private var stdinBuffer = Data()
383
+ private var output = FileHandle.standardOutput
384
+ private var nextLookId: UInt64 = 0
385
+ private var lookRecords: [String: LookRecord] = [:]
386
+ private var lookRecordOrder: [String] = []
387
+ private let lookRecordLock = NSLock()
388
+ private let rootObserverLock = NSLock()
389
+ private var rootObservers: [Int32: RootAXObserverState] = [:]
390
+ private let maxRootObservers = 4
391
+ private let permissionCacheLock = NSLock()
392
+ private var grantedPermissionStatus: [String: Any]?
393
+ private let completedRequestLock = NSLock()
394
+ private var recentCompletedRequestIds: [String] = []
395
+ // Tangu 新增:最近一次被操控的窗口(供边缘光效跟随 + liveView 取画面)。serve 线程写、主线程读。
396
+ private let controlledLock = NSLock()
397
+ private var controlledWindowId: UInt32 = 0
398
+ private var controlledPid: Int32 = 0
399
+ private var controlledAt: Date?
400
+ private var highlightFollowTimer: Timer?
401
+
402
+ func run() {
403
+ if CommandLine.arguments.contains("serve") {
404
+ let socketPath = argumentValue("--socket") ?? defaultSocketPath()
405
+ Thread.detachNewThread { [self] in runServer(socketPath: socketPath) }
406
+ NSApp.run()
407
+ return
408
+ }
409
+ while true {
410
+ autoreleasepool {
411
+ let data = FileHandle.standardInput.availableData
412
+ if data.isEmpty {
413
+ exit(0)
414
+ }
415
+ stdinBuffer.append(data)
416
+ processBufferedInput()
417
+ }
418
+ }
419
+ }
420
+
421
+ private func argumentValue(_ name: String) -> String? {
422
+ guard let index = CommandLine.arguments.firstIndex(of: name), CommandLine.arguments.indices.contains(index + 1) else { return nil }
423
+ return CommandLine.arguments[index + 1]
424
+ }
425
+
426
+ private func defaultSocketPath() -> String {
427
+ let home = FileManager.default.homeDirectoryForCurrentUser.path
428
+ return "\(home)/Library/Caches/tangu-computer-use/bridge.sock"
429
+ }
430
+
431
+ private func runServer(socketPath: String) {
432
+ _ = signal(SIGPIPE, SIG_IGN)
433
+ try? FileManager.default.createDirectory(atPath: (socketPath as NSString).deletingLastPathComponent, withIntermediateDirectories: true)
434
+ // LaunchServices may race multiple `open -n` requests while the first
435
+ // daemon is still binding. Keep process ownership separate from socket
436
+ // cleanup so a late launcher can never unlink the live daemon's socket.
437
+ let lockPath = "\(socketPath).lock"
438
+ let lockFile = open(lockPath, O_CREAT | O_RDWR, S_IRUSR | S_IWUSR)
439
+ if lockFile < 0 || flock(lockFile, LOCK_EX | LOCK_NB) != 0 {
440
+ if lockFile >= 0 { close(lockFile) }
441
+ exit(0)
442
+ }
443
+ ForegroundActivity.shared.configure(socketPath: socketPath)
444
+ unlink(socketPath)
445
+ let server = socket(AF_UNIX, SOCK_STREAM, 0)
446
+ if server < 0 { close(lockFile); exit(1) }
447
+ var address = sockaddr_un()
448
+ address.sun_family = sa_family_t(AF_UNIX)
449
+ let sunPathCapacity = MemoryLayout.size(ofValue: address.sun_path)
450
+ let bytes = Array(socketPath.utf8.prefix(sunPathCapacity - 1))
451
+ withUnsafeMutablePointer(to: &address.sun_path) { pointer in
452
+ pointer.withMemoryRebound(to: CChar.self, capacity: sunPathCapacity) { dest in
453
+ for (index, byte) in bytes.enumerated() { dest[index] = CChar(bitPattern: byte) }
454
+ }
455
+ }
456
+ let bindStatus = withUnsafePointer(to: &address) { pointer in
457
+ pointer.withMemoryRebound(to: sockaddr.self, capacity: 1) { bind(server, $0, socklen_t(MemoryLayout<sockaddr_un>.size)) }
458
+ }
459
+ if bindStatus != 0 || listen(server, 8) != 0 { close(server); close(lockFile); exit(1) }
460
+ while true {
461
+ let client = accept(server, nil, nil)
462
+ if client < 0 { continue }
463
+ var noSigPipe: Int32 = 1
464
+ _ = setsockopt(client, SOL_SOCKET, SO_NOSIGPIPE, &noSigPipe, socklen_t(MemoryLayout.size(ofValue: noSigPipe)))
465
+ Thread.detachNewThread { [weak self] in self?.processClient(client) }
466
+ }
467
+ }
468
+
469
+ private func processClient(_ client: Int32) {
470
+ let clientInput = FileHandle(fileDescriptor: client, closeOnDealloc: true)
471
+ var buffer = Data()
472
+ let newline = Data([0x0A])
473
+ while true {
474
+ let data = clientInput.availableData
475
+ if data.isEmpty { break }
476
+ buffer.append(data)
477
+ while let range = buffer.range(of: newline) {
478
+ let lineData = buffer.subdata(in: 0..<range.lowerBound)
479
+ buffer.removeSubrange(0..<range.upperBound)
480
+ if let line = String(data: lineData, encoding: .utf8), !line.isEmpty { handleLine(line, to: client) }
481
+ }
482
+ }
483
+ clientInput.closeFile()
484
+ }
485
+
486
+ private func processBufferedInput() {
487
+ let newline = Data([0x0A])
488
+ while let range = stdinBuffer.range(of: newline) {
489
+ let lineData = stdinBuffer.subdata(in: 0..<range.lowerBound)
490
+ stdinBuffer.removeSubrange(0..<range.upperBound)
491
+
492
+ guard !lineData.isEmpty else { continue }
493
+ guard let line = String(data: lineData, encoding: .utf8) else { continue }
494
+ handleLine(line)
495
+ }
496
+ }
497
+
498
+ private func handleLine(_ line: String, to responseSocket: Int32? = nil) {
499
+ let trimmed = line.trimmingCharacters(in: .whitespacesAndNewlines)
500
+ guard !trimmed.isEmpty else { return }
501
+
502
+ let fallbackId = "invalid"
503
+ var completionId: String?
504
+ defer {
505
+ if responseSocket != nil, let completionId { recordCompletedRequest(completionId) }
506
+ }
507
+ do {
508
+ guard let jsonData = trimmed.data(using: .utf8) else {
509
+ throw BridgeFailure(message: "Input was not valid UTF-8", code: "invalid_request")
510
+ }
511
+ guard let object = try JSONSerialization.jsonObject(with: jsonData) as? [String: Any] else {
512
+ throw BridgeFailure(message: "Request must be a JSON object", code: "invalid_request")
513
+ }
514
+ let id = (object["id"] as? String) ?? fallbackId
515
+ completionId = id
516
+
517
+ do {
518
+ let result = try handleRequest(object)
519
+ send([
520
+ "id": id,
521
+ "ok": true,
522
+ "result": result,
523
+ ], to: responseSocket)
524
+ } catch let failure as BridgeFailure {
525
+ send([
526
+ "id": id,
527
+ "ok": false,
528
+ "error": [
529
+ "message": failure.message,
530
+ "code": failure.code,
531
+ ],
532
+ ], to: responseSocket)
533
+ } catch {
534
+ send([
535
+ "id": id,
536
+ "ok": false,
537
+ "error": [
538
+ "message": error.localizedDescription,
539
+ "code": "internal_error",
540
+ ],
541
+ ], to: responseSocket)
542
+ }
543
+ } catch let failure as BridgeFailure {
544
+ send([
545
+ "id": fallbackId,
546
+ "ok": false,
547
+ "error": [
548
+ "message": failure.message,
549
+ "code": failure.code,
550
+ ],
551
+ ], to: responseSocket)
552
+ } catch {
553
+ send([
554
+ "id": fallbackId,
555
+ "ok": false,
556
+ "error": [
557
+ "message": error.localizedDescription,
558
+ "code": "internal_error",
559
+ ],
560
+ ], to: responseSocket)
561
+ }
562
+ }
563
+
564
+ private func send(_ payload: [String: Any], to responseSocket: Int32? = nil) {
565
+ guard JSONSerialization.isValidJSONObject(payload),
566
+ let data = try? JSONSerialization.data(withJSONObject: payload),
567
+ let line = String(data: data, encoding: .utf8),
568
+ let out = (line + "\n").data(using: .utf8)
569
+ else {
570
+ return
571
+ }
572
+
573
+ guard let responseSocket else {
574
+ try? output.write(contentsOf: out)
575
+ return
576
+ }
577
+ out.withUnsafeBytes { raw in
578
+ guard let base = raw.baseAddress else { return }
579
+ var offset = 0
580
+ while offset < raw.count {
581
+ let sent = Darwin.send(responseSocket, base.advanced(by: offset), raw.count - offset, 0)
582
+ if sent <= 0 { return }
583
+ offset += sent
584
+ }
585
+ }
586
+ }
587
+
588
+ private func recordCompletedRequest(_ id: String) {
589
+ completedRequestLock.lock()
590
+ recentCompletedRequestIds.removeAll { $0 == id }
591
+ recentCompletedRequestIds.append(id)
592
+ if recentCompletedRequestIds.count > 32 {
593
+ recentCompletedRequestIds.removeFirst(recentCompletedRequestIds.count - 32)
594
+ }
595
+ completedRequestLock.unlock()
596
+ }
597
+
598
+ private func completedRequestIds() -> [String] {
599
+ completedRequestLock.lock()
600
+ defer { completedRequestLock.unlock() }
601
+ return recentCompletedRequestIds
602
+ }
603
+
604
+ // MARK: - Tangu 扩展:被操控窗口的边缘光效 + 实时画面(非上游)
605
+
606
+ /// 记下"现在正被操控的是哪个窗口",并按需点亮边缘光效。每次 act 调一次。
607
+ /// 记录与光效分开:即使用户关了光效,liveView 仍要知道该看哪个窗口。
608
+ private func noteControlledWindow(windowId: UInt32, pid: Int32, highlight: Bool) {
609
+ controlledLock.lock()
610
+ controlledWindowId = windowId
611
+ controlledPid = pid
612
+ controlledAt = Date()
613
+ controlledLock.unlock()
614
+ guard supportsAgentCursor, highlight else { return }
615
+ Task { @MainActor in AgentHighlight.shared.show(windowId: windowId) }
616
+ }
617
+
618
+ private func controlledSnapshot() -> (windowId: UInt32, pid: Int32, at: Date)? {
619
+ controlledLock.lock()
620
+ defer { controlledLock.unlock() }
621
+ guard controlledWindowId != 0, let at = controlledAt else { return nil }
622
+ return (controlledWindowId, controlledPid, at)
623
+ }
624
+
625
+ /// 最近被操控窗口的一帧画面。给 Forsion 桌面的"被操控窗口"视图轮询用(≤2fps)。
626
+ /// 只读:**绝不**启动 helper、绝不改焦点。没有正在被操控的窗口就 active=false。
627
+ /// 拿不到画面(如未授予录屏)也照实回 active=true + error,不编一张图。
628
+ private func liveView(_ request: [String: Any]) throws -> [String: Any] {
629
+ let activeWithinMs = optionalIntArg(request, "activeWithinMs") ?? 120_000
630
+ guard let controlled = controlledSnapshot() else { return ["active": false] }
631
+ let ageMs = Int(Date().timeIntervalSince(controlled.at) * 1000)
632
+ if ageMs > max(0, activeWithinMs) { return ["active": false] }
633
+
634
+ var payload: [String: Any] = ["active": true, "windowId": Int(controlled.windowId), "pid": Int(controlled.pid), "ageMs": ageMs]
635
+ if let app = NSRunningApplication(processIdentifier: controlled.pid) {
636
+ if let name = app.localizedName { payload["app"] = name }
637
+ if let bundleId = app.bundleIdentifier { payload["bundleId"] = bundleId }
638
+ }
639
+ if let window = windowElement(pid: controlled.pid, windowId: controlled.windowId),
640
+ let title = stringAttribute(window, attribute: kAXTitleAttribute as CFString) {
641
+ payload["title"] = title
642
+ }
643
+ guard request["image"] as? Bool ?? true else { return payload }
644
+ let maxDimension = optionalIntArg(request, "maxDimension") ?? 1280
645
+ let quality = (request["quality"] as? NSNumber)?.doubleValue ?? 0.6
646
+ // 先问常驻取景流要最新一帧(真·实时;见 live_stream.swift)。流还没起来 —— 首帧、刚换窗口 ——
647
+ // 退到**同样按窗口尺寸出图**的单帧兜底,所以第一帧照样立刻有画面、也照样没有留白。
648
+ // `source` 让调用方看得见走的哪条路。
649
+ if #available(macOS 14.0, *) {
650
+ let live = LiveWindowStream.shared
651
+ var source = "stream"
652
+ var image: CGImage?
653
+ if let fresh = live.latestFrame(windowId: controlled.windowId, maxDimension: maxDimension) {
654
+ payload["frameSeq"] = String(fresh.seq)
655
+ // 静止的窗口不会有新帧。调用方带着上次见过的帧号来,一样就直接说"没变" ——
656
+ // 否则同一张图会被每秒重编 8 次 JPEG + base64 + 过 socket + 换一次 <img src>,纯白烧。
657
+ if let seen = optionalStringArg(request, "sinceFrame"), seen == String(fresh.seq) {
658
+ payload["unchanged"] = true
659
+ payload["source"] = source
660
+ return payload
661
+ }
662
+ image = fresh.image
663
+ } else {
664
+ source = "snapshot"
665
+ image = live.captureOnce(windowId: controlled.windowId, maxDimension: maxDimension)
666
+ }
667
+ if let image {
668
+ guard let jpeg = jpegData(image: image, quality: quality) else {
669
+ payload["error"] = "encode_failed"
670
+ return payload
671
+ }
672
+ payload["width"] = image.width
673
+ payload["height"] = image.height
674
+ payload["jpegBase64"] = jpeg.base64EncodedString()
675
+ payload["source"] = source
676
+ return payload
677
+ }
678
+ }
679
+ // 最后一条老路(macOS <14,或上面两条都拿不到):上游的 captureWindow。它按整块屏幕出图,
680
+ // 窗口缩在左上角、四周留白 —— 有画面总比没有强,但打上 legacy 标记别让人以为这是正常尺寸。
681
+ do {
682
+ let captured = try captureWindow(windowId: controlled.windowId)
683
+ let image = downscaledImage(captured.image, maxDimension: maxDimension) ?? captured.image
684
+ guard let jpeg = jpegData(image: image, quality: quality) else {
685
+ payload["error"] = "encode_failed"
686
+ return payload
687
+ }
688
+ payload["width"] = image.width
689
+ payload["height"] = image.height
690
+ payload["jpegBase64"] = jpeg.base64EncodedString()
691
+ payload["source"] = "legacy"
692
+ } catch let failure as BridgeFailure {
693
+ payload["error"] = failure.code
694
+ } catch {
695
+ payload["error"] = "capture_failed"
696
+ }
697
+ return payload
698
+ }
699
+
700
+ private func handleRequest(_ request: [String: Any]) throws -> Any {
701
+ let cmd = try stringArg(request, "cmd")
702
+
703
+ switch cmd {
704
+ case "diagnostics":
705
+ return diagnostics()
706
+ case "permissionStatus":
707
+ return permissionStatus()
708
+ case "checkPermissions":
709
+ return checkPermissions(fresh: boolArg(request, "fresh") ?? false)
710
+ case "registerPermissions":
711
+ return try registerPermissions(request)
712
+ case "openPermissionPane":
713
+ return try openPermissionPane(request)
714
+ case "shutdown":
715
+ // Reply first, then exit: the caller relaunches the helper to get a
716
+ // process with a fresh TCC client (grant answers are cached per
717
+ // process, so a helper that saw "denied" keeps answering "denied"
718
+ // after the user grants — only a new process re-queries tccd).
719
+ // Background queue: the serve loop occupies the main thread, so a
720
+ // main-queue timer would never fire.
721
+ DispatchQueue.global().asyncAfter(deadline: .now() + 0.2) { exit(0) }
722
+ return ["shuttingDown": true]
723
+ case "listApps":
724
+ return listApps()
725
+ case "listWindows":
726
+ return try listWindows(pid: Int32(try intArg(request, "pid")))
727
+ case "listRoots":
728
+ return try listRoots(pid: optionalIntArg(request, "pid").map { Int32($0) }, title: optionalStringArg(request, "title"))
729
+ case "getFrontmost":
730
+ return try getFrontmost()
731
+ case "getUserContext":
732
+ return try getUserContext()
733
+ case "beginInputSuppression":
734
+ return try beginInputSuppression()
735
+ case "endInputSuppression":
736
+ return endInputSuppression()
737
+ case "restoreUserFocus":
738
+ return try restoreUserFocus(request)
739
+ case "focusWindow":
740
+ return try focusWindow(request)
741
+ case "setWindowFrame":
742
+ return try setWindowFrame(request)
743
+ case "look":
744
+ return try look(request)
745
+ case "act":
746
+ return try act(request)
747
+ case "actBatch":
748
+ return try actBatch(request)
749
+ case "liveView":
750
+ return try liveView(request)
751
+ case "hitTest":
752
+ return try hitTest(request)
753
+ case "axWaitFor":
754
+ return try axWaitFor(request)
755
+ case "focusedElement":
756
+ return try focusedElement(request)
757
+ case "axReadText":
758
+ return try axReadText(request)
759
+ case "getMousePosition":
760
+ return getMousePosition()
761
+ default:
762
+ throw BridgeFailure(message: "Unknown command '\(cmd)'", code: "unknown_command")
763
+ }
764
+ }
765
+
766
+ private func stringArg(_ request: [String: Any], _ key: String) throws -> String {
767
+ if let value = request[key] as? String {
768
+ return value
769
+ }
770
+ throw BridgeFailure(message: "Missing string argument '\(key)'", code: "invalid_args")
771
+ }
772
+
773
+ private func optionalStringArg(_ request: [String: Any], _ key: String) -> String? {
774
+ if let value = request[key] as? String {
775
+ return value
776
+ }
777
+ return nil
778
+ }
779
+
780
+ private func intArg(_ request: [String: Any], _ key: String) throws -> Int {
781
+ if let value = request[key] as? Int {
782
+ return value
783
+ }
784
+ if let value = request[key] as? NSNumber {
785
+ return value.intValue
786
+ }
787
+ if let value = request[key] as? Double {
788
+ return Int(value)
789
+ }
790
+ throw BridgeFailure(message: "Missing integer argument '\(key)'", code: "invalid_args")
791
+ }
792
+
793
+ private func optionalIntArg(_ request: [String: Any], _ key: String) -> Int? {
794
+ if let value = request[key] as? Int {
795
+ return value
796
+ }
797
+ if let value = request[key] as? NSNumber {
798
+ return value.intValue
799
+ }
800
+ if let value = request[key] as? Double {
801
+ return Int(value)
802
+ }
803
+ return nil
804
+ }
805
+
806
+ private func boolArg(_ request: [String: Any], _ key: String) -> Bool? {
807
+ if let value = request[key] as? Bool { return value }
808
+ if let value = request[key] as? NSNumber { return value.boolValue }
809
+ return nil
810
+ }
811
+
812
+ private func doubleArg(_ request: [String: Any], _ key: String) throws -> Double {
813
+ if let value = request[key] as? Double {
814
+ return value
815
+ }
816
+ if let value = request[key] as? NSNumber {
817
+ return value.doubleValue
818
+ }
819
+ if let value = request[key] as? Int {
820
+ return Double(value)
821
+ }
822
+ throw BridgeFailure(message: "Missing numeric argument '\(key)'", code: "invalid_args")
823
+ }
824
+
825
+ private func processPath(pid: pid_t) -> String? {
826
+ var buffer = [CChar](repeating: 0, count: 4096)
827
+ let length = proc_pidpath(pid, &buffer, UInt32(buffer.count))
828
+ return length > 0 ? String(cString: buffer) : nil
829
+ }
830
+
831
+ private func processName(pid: pid_t) -> String? {
832
+ processPath(pid: pid).map { URL(fileURLWithPath: $0).deletingPathExtension().lastPathComponent }
833
+ }
834
+
835
+ private func diagnostics() -> [String: Any] {
836
+ // Cheap booleans only — diagnostics doubles as the daemon liveness
837
+ // probe (1s client timeout), so it must not run the ScreenCaptureKit
838
+ // capturable check (up to 3s when ungranted). Permission truth comes
839
+ // from checkPermissions.
840
+ let permissions: [String: Any] = [
841
+ "accessibility": AXIsProcessTrusted(),
842
+ "screenRecording": {
843
+ if #available(macOS 10.15, *) { return CGPreflightScreenCaptureAccess() }
844
+ return true
845
+ }(),
846
+ ]
847
+ #if arch(arm64)
848
+ let arch = "arm64"
849
+ #elseif arch(x86_64)
850
+ let arch = "x86_64"
851
+ #else
852
+ let arch = "unknown"
853
+ #endif
854
+ let parentPid = Int32(getppid())
855
+ let parentApp = NSRunningApplication(processIdentifier: parentPid)
856
+ let parentPath = processPath(pid: parentPid)
857
+ var output: [String: Any] = [
858
+ "protocolVersion": protocolVersion,
859
+ "architectureVersion": 1,
860
+ "invariants": ["state-scoped-observations", "bounded-observation-history", "multi-root-forest", "progressive-disclosure", "atomic-physical-input", "concurrent-requests", "transactional-batching"],
861
+ "pid": Int32(getpid()),
862
+ "parentPid": parentPid,
863
+ "executablePath": CommandLine.arguments.first ?? "",
864
+ "macOS": ProcessInfo.processInfo.operatingSystemVersionString,
865
+ "arch": arch,
866
+ "accessibility": permissions["accessibility"] ?? false,
867
+ "screenRecording": permissions["screenRecording"] ?? false,
868
+ "recentCompletedRequestIds": completedRequestIds(),
869
+ ]
870
+ if let parentPath {
871
+ output["parentPath"] = parentPath
872
+ }
873
+ if let parentAppName = parentApp?.localizedName ?? parentPath.map({ URL(fileURLWithPath: $0).lastPathComponent }) {
874
+ output["parentAppName"] = parentAppName
875
+ }
876
+ if let parentBundleId = parentApp?.bundleIdentifier {
877
+ output["parentBundleId"] = parentBundleId
878
+ }
879
+ return output
880
+ }
881
+
882
+ /// Live Screen Recording probe. `CGPreflightScreenCaptureAccess()`
883
+ /// answers from a per-process cache that goes stale after `tccutil
884
+ /// reset` or a Settings toggle; a ScreenCaptureKit content fetch only
885
+ /// succeeds when THIS process can genuinely capture right now. When the
886
+ /// two disagree, the preflight boolean is the one lying.
887
+ private func screenRecordingCapturable() -> Bool {
888
+ if #available(macOS 14.0, *) {
889
+ let sema = DispatchSemaphore(value: 0)
890
+ let capturable = Box<Bool>(false)
891
+ SCShareableContent.getExcludingDesktopWindows(false, onScreenWindowsOnly: false) { shareable, error in
892
+ if let shareable = shareable {
893
+ capturable.value = !shareable.displays.isEmpty
894
+ }
895
+ sema.signal()
896
+ }
897
+ guard sema.wait(timeout: .now() + .seconds(5)) == .success else { return false }
898
+ return capturable.value
899
+ }
900
+ if #available(macOS 10.15, *) {
901
+ return CGPreflightScreenCaptureAccess()
902
+ }
903
+ return true
904
+ }
905
+
906
+ /// Which TCC identity the permission booleans reflect. macOS attributes
907
+ /// grants to the *responsible process* (the LaunchServices launching
908
+ /// app), so:
909
+ /// - "helper-app": running from the installed bundle, launched via
910
+ /// LaunchServices — grants belong to the canonical helper identity.
911
+ /// - "caller": anything else (dev binary under a terminal, etc.) —
912
+ /// the booleans reflect whatever app spawned us, NOT the canonical
913
+ /// helper. The extension surfaces this instead of guessing.
914
+ private func permissionSource() -> [String: Any] {
915
+ let parentPid = Int32(getppid())
916
+ let executable = CommandLine.arguments.first ?? ""
917
+ var source: [String: Any] = [
918
+ "pid": Int(getpid()),
919
+ "parentPid": Int(parentPid),
920
+ "executablePath": executable,
921
+ "macOS": ProcessInfo.processInfo.operatingSystemVersionString,
922
+ ]
923
+ if let parentPath = processPath(pid: parentPid) {
924
+ source["parentPath"] = parentPath
925
+ }
926
+ if let parentBundleId = NSRunningApplication(processIdentifier: parentPid)?.bundleIdentifier {
927
+ source["parentBundleId"] = parentBundleId
928
+ }
929
+ let attribution: String
930
+ if executable.contains("/tangu-computer-use.app/Contents/MacOS/"), parentPid == 1 {
931
+ // Non-spoofable signals only: installed-bundle executable path +
932
+ // launchd parent (`open` handed us to LaunchServices). A dev
933
+ // binary or a directly-spawned copy fails closed to "caller".
934
+ attribution = "helper-app"
935
+ } else {
936
+ attribution = "caller"
937
+ }
938
+ source["attribution"] = attribution
939
+ return source
940
+ }
941
+
942
+ /// Passive onboarding status only. Preflight is a hint, not capture proof;
943
+ /// never call ScreenCaptureKit or prompt APIs here, even after a grant.
944
+ private func permissionStatus() -> [String: Any] {
945
+ var result: [String: Any] = [
946
+ "accessibility": AXIsProcessTrusted(),
947
+ "screenRecordingPreflight": CGPreflightScreenCaptureAccess(),
948
+ "source": permissionSource(),
949
+ "settingsFrontmost": NSWorkspace.shared.frontmostApplication?.bundleIdentifier == "com.apple.systempreferences",
950
+ ]
951
+ if let bounds = systemSettingsWindowBounds() {
952
+ // Quartz global coordinates: top-left origin, y increases downwards.
953
+ result["settingsWindow"] = ["x": bounds.minX, "y": bounds.minY, "width": bounds.width, "height": bounds.height]
954
+ }
955
+ return result
956
+ }
957
+
958
+ /// Window metadata remains available before AX / Screen Recording grants.
959
+ /// Do not filter by window title (redacted without Screen Recording), query
960
+ /// AX, or use the capture-backed windowInfo/currentWindowBounds helpers.
961
+ private func systemSettingsWindowBounds() -> CGRect? {
962
+ guard let settings = NSWorkspace.shared.runningApplications.first(where: { $0.bundleIdentifier == "com.apple.systempreferences" }),
963
+ let entries = CGWindowListCopyWindowInfo([.optionOnScreenOnly, .excludeDesktopElements], kCGNullWindowID) as? [[String: Any]]
964
+ else { return nil }
965
+ return entries.compactMap { entry -> CGRect? in
966
+ guard (entry[kCGWindowOwnerPID as String] as? NSNumber)?.int32Value == settings.processIdentifier,
967
+ (entry[kCGWindowLayer as String] as? NSNumber)?.intValue == 0,
968
+ (entry[kCGWindowIsOnscreen as String] as? NSNumber)?.boolValue == true,
969
+ let dictionary = entry[kCGWindowBounds as String] as? [String: Any],
970
+ let bounds = CGRect(dictionaryRepresentation: dictionary as CFDictionary),
971
+ bounds.origin.x.isFinite, bounds.origin.y.isFinite,
972
+ bounds.width.isFinite, bounds.height.isFinite,
973
+ bounds.width > 0, bounds.height > 0
974
+ else { return nil }
975
+ return bounds
976
+ }.max(by: { $0.width * $0.height < $1.width * $1.height })
977
+ }
978
+
979
+ private func checkPermissions(fresh: Bool = false) -> [String: Any] {
980
+ permissionCacheLock.lock()
981
+ defer { permissionCacheLock.unlock() }
982
+ // Serialize probes so an older in-flight success cannot repopulate the
983
+ // cache after a fresh check detects revocation. A failed fresh check must
984
+ // leave no old success behind. This does not reset macOS's own TCC cache.
985
+ if fresh { grantedPermissionStatus = nil }
986
+ if let cached = grantedPermissionStatus {
987
+ return cached
988
+ }
989
+ let accessibility = AXIsProcessTrusted()
990
+ let screenRecordingPreflight: Bool
991
+ if #available(macOS 10.15, *) {
992
+ screenRecordingPreflight = CGPreflightScreenCaptureAccess()
993
+ } else {
994
+ screenRecordingPreflight = true
995
+ }
996
+ let capturable = screenRecordingCapturable()
997
+ let result: [String: Any] = [
998
+ "accessibility": accessibility,
999
+ // The live probe is authoritative; the preflight boolean is kept
1000
+ // for diagnostics (a true/false split identifies a stale cache or
1001
+ // a grant belonging to a different responsible process).
1002
+ "screenRecording": capturable,
1003
+ "screenRecordingPreflight": screenRecordingPreflight,
1004
+ "screenRecordingCapturable": capturable,
1005
+ "source": permissionSource(),
1006
+ ]
1007
+ // A successful TCC grant is process-stable in practice. Cache only the
1008
+ // positive result so missing grants are always rechecked after the user
1009
+ // enables them, while fresh agent processes avoid repeating a multi-second
1010
+ // ScreenCaptureKit probe against the same long-lived helper daemon.
1011
+ if accessibility && capturable {
1012
+ grantedPermissionStatus = result
1013
+ }
1014
+ return result
1015
+ }
1016
+
1017
+ /// Register this process's identity with TCC for both grants so the app
1018
+ /// appears in the Settings panes BEFORE the user is sent there. The AX
1019
+ /// request registers (and prompts for) Accessibility; on recent macOS an
1020
+ /// app only appears under Screen Recording after a real ScreenCaptureKit
1021
+ /// attempt, which the capturable probe performs.
1022
+ private func registerPermissions(_ request: [String: Any]) throws -> [String: Any] {
1023
+ // User-initiated, single-permission requests must not touch the other
1024
+ // permission or run a capturable probe. Omitted kind preserves legacy setup.
1025
+ if let rawKind = request["kind"] {
1026
+ guard let kind = rawKind as? String, kind == "accessibility" || kind == "screenRecording" else {
1027
+ throw BridgeFailure(message: "Permission kind must be 'accessibility' or 'screenRecording'", code: "invalid_args")
1028
+ }
1029
+ if kind == "accessibility" {
1030
+ let options = [kAXTrustedCheckOptionPrompt.takeUnretainedValue() as String: true] as CFDictionary
1031
+ return ["accessibility": AXIsProcessTrustedWithOptions(options)]
1032
+ }
1033
+ // Request result only; checkPermissions(fresh: true) verifies usability.
1034
+ return ["screenRecording": CGRequestScreenCaptureAccess()]
1035
+ }
1036
+ let options = [kAXTrustedCheckOptionPrompt.takeUnretainedValue() as String: true] as CFDictionary
1037
+ let accessibility = AXIsProcessTrustedWithOptions(options)
1038
+ if #available(macOS 10.15, *) {
1039
+ _ = CGRequestScreenCaptureAccess()
1040
+ }
1041
+ let capturable = screenRecordingCapturable()
1042
+ return [
1043
+ "accessibility": accessibility,
1044
+ "screenRecording": capturable,
1045
+ "screenRecordingCapturable": capturable,
1046
+ ]
1047
+ }
1048
+
1049
+ private func openPermissionPane(_ request: [String: Any]) throws -> [String: Any] {
1050
+ let kind = try stringArg(request, "kind")
1051
+ let urlString: String
1052
+ switch kind {
1053
+ case "accessibility":
1054
+ urlString = "x-apple.systempreferences:com.apple.preference.security?Privacy_Accessibility"
1055
+ case "screenRecording", "screenrecording":
1056
+ urlString = "x-apple.systempreferences:com.apple.preference.security?Privacy_ScreenCapture"
1057
+ default:
1058
+ throw BridgeFailure(message: "Unknown permission pane '\(kind)'", code: "invalid_args")
1059
+ }
1060
+
1061
+ guard let url = URL(string: urlString) else {
1062
+ throw BridgeFailure(message: "Invalid permission pane URL", code: "internal_error")
1063
+ }
1064
+ let opened = NSWorkspace.shared.open(url)
1065
+ return ["opened": opened]
1066
+ }
1067
+
1068
+ private func pidIsAlive(_ pid: pid_t) -> Bool {
1069
+ pid > 0 && kill(pid, 0) == 0
1070
+ }
1071
+
1072
+ private func listApps(cgEntries: [[String: Any]]? = nil) -> [[String: Any]] {
1073
+ let frontmostPid = NSWorkspace.shared.frontmostApplication?.processIdentifier
1074
+ let apps = NSWorkspace.shared.runningApplications.filter { app in
1075
+ // Computer-use targets are windows, not Dock-visible applications. Some
1076
+ // benchmark/test apps and utility-style apps expose perfectly valid AX
1077
+ // windows while using an accessory/prohibited activation policy, so do not
1078
+ // gate discovery on `.regular` here. `listWindows` and the higher-level
1079
+ // window collector filter to actual controllable windows.
1080
+ app.processIdentifier != getpid() && pidIsAlive(app.processIdentifier)
1081
+ }
1082
+ var seen = Set<Int32>()
1083
+ var output = apps.map { app in
1084
+ seen.insert(app.processIdentifier)
1085
+ var data: [String: Any] = [
1086
+ "appName": app.localizedName ?? processName(pid: app.processIdentifier) ?? "Unknown App",
1087
+ "pid": Int(app.processIdentifier),
1088
+ "isFrontmost": app.processIdentifier == frontmostPid,
1089
+ ]
1090
+ if let bundleId = app.bundleIdentifier {
1091
+ data["bundleId"] = bundleId
1092
+ }
1093
+ return data
1094
+ }
1095
+
1096
+ // NSWorkspace can miss apps launched from ad-hoc bundles or test harnesses
1097
+ // even when their windows are visible and AX-controllable. Add CGWindow
1098
+ // owners as acquisition candidates so callers can still resolve by pid/title
1099
+ // and then build the normal AX scene through listWindows(pid:).
1100
+ for owner in cgWindowOwners(entries: cgEntries) where owner.pid != getpid() && !seen.contains(owner.pid) && pidIsAlive(owner.pid) {
1101
+ seen.insert(owner.pid)
1102
+ output.append([
1103
+ "appName": owner.name,
1104
+ "pid": Int(owner.pid),
1105
+ "isFrontmost": owner.pid == frontmostPid,
1106
+ ])
1107
+ }
1108
+ return output
1109
+ }
1110
+
1111
+ private func getFrontmost() throws -> [String: Any] {
1112
+ guard let app = NSWorkspace.shared.frontmostApplication else {
1113
+ throw BridgeFailure(message: "No frontmost app available", code: "frontmost_unavailable")
1114
+ }
1115
+ let pid = app.processIdentifier
1116
+ let windows = try listWindows(pid: pid)
1117
+
1118
+ var result: [String: Any] = [
1119
+ "appName": app.localizedName ?? "Unknown App",
1120
+ "pid": Int(pid),
1121
+ ]
1122
+ if let bundleId = app.bundleIdentifier {
1123
+ result["bundleId"] = bundleId
1124
+ }
1125
+
1126
+ if let chosen = windows.sorted(by: { scoreWindow($0) > scoreWindow($1) }).first {
1127
+ result["windowTitle"] = (chosen["title"] as? String) ?? ""
1128
+ if let windowId = chosen["windowId"] {
1129
+ result["windowId"] = windowId
1130
+ }
1131
+ if let windowRef = chosen["windowRef"] as? String {
1132
+ result["windowRef"] = windowRef
1133
+ }
1134
+ }
1135
+ return result
1136
+ }
1137
+
1138
+ private func getUserContext() throws -> [String: Any] {
1139
+ guard let app = NSWorkspace.shared.frontmostApplication else {
1140
+ throw BridgeFailure(message: "No frontmost app available", code: "frontmost_unavailable")
1141
+ }
1142
+ let pid = app.processIdentifier
1143
+ ensureEnhancedAccessibility(pid: pid)
1144
+ let appElement = AXUIElementCreateApplication(pid)
1145
+ let focusedWindow = copyAttribute(appElement, attribute: kAXFocusedWindowAttribute as CFString).flatMap(asAXElement)
1146
+ let focusedElement = copyAttribute(appElement, attribute: kAXFocusedUIElementAttribute as CFString).flatMap(asAXElement)
1147
+ var result: [String: Any] = [
1148
+ "appName": app.localizedName ?? "Unknown App",
1149
+ "pid": Int(pid),
1150
+ ]
1151
+ if let bundleId = app.bundleIdentifier {
1152
+ result["bundleId"] = bundleId
1153
+ }
1154
+ if let window = focusedWindow {
1155
+ result["window"] = [
1156
+ "title": stringAttribute(window, attribute: kAXTitleAttribute as CFString) ?? "",
1157
+ "role": stringAttribute(window, attribute: kAXRoleAttribute as CFString) ?? "",
1158
+ "subrole": stringAttribute(window, attribute: kAXSubroleAttribute as CFString) ?? "",
1159
+ ]
1160
+ }
1161
+ if let element = focusedElement {
1162
+ let focusedRole = stringAttribute(element, attribute: kAXRoleAttribute as CFString) ?? ""
1163
+ let focusedSubrole = stringAttribute(element, attribute: kAXSubroleAttribute as CFString) ?? ""
1164
+ result["focusedElement"] = [
1165
+ "role": focusedRole,
1166
+ "subrole": focusedSubrole,
1167
+ "title": stringAttribute(element, attribute: kAXTitleAttribute as CFString) ?? "",
1168
+ "description": stringAttribute(element, attribute: kAXDescriptionAttribute as CFString) ?? "",
1169
+ "value": displayValue(element, role: focusedRole, subrole: focusedSubrole),
1170
+ ]
1171
+ }
1172
+ return result
1173
+ }
1174
+
1175
+ private func beginInputSuppression() throws -> [String: Any] {
1176
+ try inputSuppressionGuard.begin()
1177
+ return ["active": true]
1178
+ }
1179
+
1180
+ private func endInputSuppression() -> [String: Any] {
1181
+ inputSuppressionGuard.end()
1182
+ return ["active": false]
1183
+ }
1184
+
1185
+ private func restoreUserFocus(_ request: [String: Any]) throws -> [String: Any] {
1186
+ let pid = Int32(try intArg(request, "pid"))
1187
+ let targetTitle = optionalStringArg(request, "windowTitle")?.trimmingCharacters(in: .whitespacesAndNewlines) ?? ""
1188
+ guard let app = NSRunningApplication(processIdentifier: pid) else {
1189
+ throw BridgeFailure(message: "App with pid \(pid) is no longer running", code: "app_not_found")
1190
+ }
1191
+
1192
+ let appRestored = app.activate()
1193
+ var restoredWindowTitle = ""
1194
+ var windowRestored = false
1195
+
1196
+ if !targetTitle.isEmpty {
1197
+ let appElement = AXUIElementCreateApplication(pid)
1198
+ let windows = axElementArray(appElement, attribute: kAXWindowsAttribute as CFString)
1199
+ let normalizedTarget = targetTitle.lowercased()
1200
+ if let match = windows.first(where: {
1201
+ (stringAttribute($0, attribute: kAXTitleAttribute as CFString) ?? "")
1202
+ .trimmingCharacters(in: .whitespacesAndNewlines)
1203
+ .lowercased() == normalizedTarget
1204
+ }) {
1205
+ restoredWindowTitle = stringAttribute(match, attribute: kAXTitleAttribute as CFString) ?? ""
1206
+ let setMainStatus = AXUIElementSetAttributeValue(match, kAXMainAttribute as CFString, kCFBooleanTrue)
1207
+ let setFocusedStatus = AXUIElementSetAttributeValue(match, kAXFocusedAttribute as CFString, kCFBooleanTrue)
1208
+ let raiseStatus = AXUIElementPerformAction(match, kAXRaiseAction as CFString)
1209
+ windowRestored = setMainStatus == .success || setFocusedStatus == .success || raiseStatus == .success
1210
+ }
1211
+ }
1212
+
1213
+ return [
1214
+ "restored": appRestored || windowRestored,
1215
+ "appRestored": appRestored,
1216
+ "windowRestored": windowRestored,
1217
+ "appName": app.localizedName ?? "Unknown App",
1218
+ "windowTitle": restoredWindowTitle,
1219
+ ]
1220
+ }
1221
+
1222
+ private func setWindowFrame(_ request: [String: Any]) throws -> [String: Any] {
1223
+ let pid = Int32(try intArg(request, "pid"))
1224
+ let windowId = optionalIntArg(request, "windowId").map { UInt32($0) }
1225
+ let windowRef = optionalStringArg(request, "windowRef")
1226
+ guard let window = windowElement(pid: pid, windowId: windowId, windowRef: windowRef) else {
1227
+ return ["ok": false, "reason": "window_not_found"]
1228
+ }
1229
+ let x = try doubleArg(request, "x")
1230
+ let y = try doubleArg(request, "y")
1231
+ let width = max(100.0, try doubleArg(request, "width"))
1232
+ let height = max(80.0, try doubleArg(request, "height"))
1233
+ var origin = CGPoint(x: x, y: y)
1234
+ var size = CGSize(width: width, height: height)
1235
+ guard let originValue = AXValueCreate(.cgPoint, &origin), let sizeValue = AXValueCreate(.cgSize, &size) else {
1236
+ throw BridgeFailure(message: "Failed to create AX frame values", code: "frame_value_failed")
1237
+ }
1238
+ let positionStatus = AXUIElementSetAttributeValue(window, kAXPositionAttribute as CFString, originValue)
1239
+ let sizeStatus = AXUIElementSetAttributeValue(window, kAXSizeAttribute as CFString, sizeValue)
1240
+ let frame = frameForWindow(window)
1241
+ return [
1242
+ "ok": positionStatus == .success || sizeStatus == .success,
1243
+ "positionStatus": Int(positionStatus.rawValue),
1244
+ "sizeStatus": Int(sizeStatus.rawValue),
1245
+ "framePoints": ["x": frame.origin.x, "y": frame.origin.y, "w": frame.width, "h": frame.height],
1246
+ ]
1247
+ }
1248
+
1249
+ private func focusWindow(_ request: [String: Any]) throws -> [String: Any] {
1250
+ let pid = Int32(try intArg(request, "pid"))
1251
+ let windowId = optionalIntArg(request, "windowId").map { UInt32($0) }
1252
+ let windowRef = optionalStringArg(request, "windowRef")
1253
+ guard let window = windowElement(pid: pid, windowId: windowId, windowRef: windowRef) else {
1254
+ return ["focused": false, "reason": "window_not_found"]
1255
+ }
1256
+
1257
+ let appElement = AXUIElementCreateApplication(pid)
1258
+ if let focusedWindow = copyAttribute(appElement, attribute: kAXFocusedWindowAttribute as CFString).flatMap(asAXElement),
1259
+ sameElement(focusedWindow, window)
1260
+ {
1261
+ return ["focused": true, "alreadyFocused": true]
1262
+ }
1263
+
1264
+ let setMainStatus = AXUIElementSetAttributeValue(window, kAXMainAttribute as CFString, kCFBooleanTrue)
1265
+ let setFocusedStatus = AXUIElementSetAttributeValue(window, kAXFocusedAttribute as CFString, kCFBooleanTrue)
1266
+ let raiseStatus = AXUIElementPerformAction(window, kAXRaiseAction as CFString)
1267
+ let focused = setMainStatus == .success || setFocusedStatus == .success || raiseStatus == .success
1268
+ var result: [String: Any] = [
1269
+ "focused": focused,
1270
+ "setMain": setMainStatus == .success,
1271
+ "setFocused": setFocusedStatus == .success,
1272
+ "raised": raiseStatus == .success,
1273
+ ]
1274
+ if !focused {
1275
+ result["reason"] = "focus_failed"
1276
+ }
1277
+ return result
1278
+ }
1279
+
1280
+ private func scoreWindow(_ window: [String: Any]) -> Int {
1281
+ var score = 0
1282
+ if (window["isFocused"] as? Bool) == true { score += 100 }
1283
+ if (window["isMain"] as? Bool) == true { score += 80 }
1284
+ if (window["isMinimized"] as? Bool) == false { score += 40 }
1285
+ if (window["isOnscreen"] as? Bool) == true { score += 20 }
1286
+ if window["windowId"] != nil { score += 10 }
1287
+ return score
1288
+ }
1289
+
1290
+ private func rootKind(role: String, subrole: String) -> String {
1291
+ if role == "AXSheet" { return "sheet" }
1292
+ if subrole.localizedCaseInsensitiveContains("popover") { return "popover" }
1293
+ if subrole.localizedCaseInsensitiveContains("dialog") || role == "AXDialog" { return "dialog" }
1294
+ return "window"
1295
+ }
1296
+
1297
+ private func isDialogLikeRoot(role: String, subrole: String) -> Bool {
1298
+ let text = "\(role) \(subrole)"
1299
+ return text.range(of: "dialog", options: [.caseInsensitive]) != nil
1300
+ || text.range(of: "modal", options: [.caseInsensitive]) != nil
1301
+ || text.range(of: "sheet", options: [.caseInsensitive]) != nil
1302
+ }
1303
+
1304
+ private func rootMetadata(pairing: WindowPairing, sheetCount: Int) -> [String: Any] {
1305
+ ["pairing": ["confidence": pairing.confidence, "score": pairing.score], "sheetCount": sheetCount]
1306
+ }
1307
+
1308
+ private func broadRootCandidateApps(entries: [[String: Any]]) -> [[String: Any]] {
1309
+ cgBroadRootOwners(entries: entries).compactMap { owner in
1310
+ guard owner.pid != getpid(), pidIsAlive(owner.pid) else { return nil }
1311
+ var app: [String: Any] = ["appName": owner.name, "pid": Int(owner.pid)]
1312
+ if let bundleId = NSRunningApplication(processIdentifier: owner.pid)?.bundleIdentifier {
1313
+ app["bundleId"] = bundleId
1314
+ }
1315
+ return app
1316
+ }
1317
+ }
1318
+
1319
+ private func listRoots(pid: Int32?, title: String? = nil) throws -> [String: Any] {
1320
+ let requestedTitle = title?.trimmingCharacters(in: .whitespacesAndNewlines).lowercased() ?? ""
1321
+ let entries = allCGWindowEntries()
1322
+ let isBroadDiscovery = pid == nil && requestedTitle.isEmpty
1323
+ let apps: [[String: Any]]
1324
+ if let pid {
1325
+ apps = [["pid": Int(pid)]]
1326
+ } else if !requestedTitle.isEmpty {
1327
+ let matchingPids = Set(entries.compactMap { entry -> Int32? in
1328
+ let candidate = ((entry[kCGWindowName as String] as? String) ?? "").trimmingCharacters(in: .whitespacesAndNewlines).lowercased()
1329
+ guard candidate == requestedTitle || candidate.contains(requestedTitle) else { return nil }
1330
+ return (entry[kCGWindowOwnerPID as String] as? NSNumber)?.int32Value
1331
+ })
1332
+ apps = listApps(cgEntries: entries).filter { app in
1333
+ guard let rawPid = app["pid"] as? Int else { return false }
1334
+ return matchingPids.contains(Int32(rawPid))
1335
+ }
1336
+ } else {
1337
+ apps = broadRootCandidateApps(entries: entries)
1338
+ }
1339
+ var roots: [[String: Any]] = []
1340
+ for app in apps {
1341
+ guard let rawPid = app["pid"] as? Int else { continue }
1342
+ let appPid = Int32(rawPid)
1343
+ let appName = app["appName"] as? String ?? processName(pid: appPid) ?? "Unknown App"
1344
+ let bundleId = app["bundleId"] as? String
1345
+ for var root in (try? listWindows(pid: appPid, cgEntries: entries, messagingTimeout: isBroadDiscovery ? 0.25 : 1.0)) ?? [] {
1346
+ root["pid"] = rawPid
1347
+ root["appName"] = appName
1348
+ if let bundleId { root["bundleId"] = bundleId }
1349
+ roots.append(root)
1350
+ }
1351
+ let popupCandidates = cgPopupMenuCandidates(pid: appPid, entries: entries)
1352
+ let menuElements = popupCandidates.isEmpty ? [] : openMenuElements(pid: appPid, messagingTimeout: isBroadDiscovery ? 0.25 : 1.0)
1353
+ for (index, candidate) in popupCandidates.enumerated() {
1354
+ let menuElement = index < menuElements.count ? menuElements[index] : nil
1355
+ let menuRef = menuElement.map { refStore.storeWindow($0) } ?? "cgmenu:\(candidate.windowId)"
1356
+ var menu: [String: Any] = [
1357
+ "kind": "menu",
1358
+ "rootRef": menuRef,
1359
+ "windowRef": menuRef,
1360
+ "windowId": Int(candidate.windowId),
1361
+ "zOrder": candidate.zOrder,
1362
+ "title": menuElement.flatMap { stringAttribute($0, attribute: kAXTitleAttribute as CFString) } ?? candidate.title,
1363
+ "role": "AXMenu",
1364
+ "subrole": "",
1365
+ "isModal": false,
1366
+ "framePoints": ["x": candidate.bounds.origin.x, "y": candidate.bounds.origin.y, "w": candidate.bounds.width, "h": candidate.bounds.height],
1367
+ "scaleFactor": displayScaleFactor(for: candidate.bounds),
1368
+ "isMinimized": false,
1369
+ "isOnscreen": candidate.isOnscreen,
1370
+ "isMain": false,
1371
+ "isFocused": true,
1372
+ "metadata": ["pairing": ["confidence": menuElement == nil ? "low" : "high", "score": menuElement == nil ? 0 : 100], "sheetCount": 0],
1373
+ "pid": rawPid,
1374
+ "appName": appName,
1375
+ ]
1376
+ if let bundleId { menu["bundleId"] = bundleId }
1377
+ roots.append(menu)
1378
+ }
1379
+ }
1380
+ roots.sort { (($0["zOrder"] as? Int) ?? Int.max) < (($1["zOrder"] as? Int) ?? Int.max) }
1381
+ return ["roots": roots]
1382
+ }
1383
+
1384
+ private func listWindows(pid: Int32, cgEntries: [[String: Any]]? = nil, messagingTimeout: Float = 1.0) throws -> [[String: Any]] {
1385
+ ensureEnhancedAccessibility(pid: pid)
1386
+ let appElement = AXUIElementCreateApplication(pid)
1387
+ AXUIElementSetMessagingTimeout(appElement, messagingTimeout)
1388
+ let windows = Array(axElementArray(appElement, attribute: kAXWindowsAttribute as CFString).prefix(128))
1389
+ let candidates = cgWindowCandidates(pid: pid, entries: cgEntries)
1390
+ let pairings = windowPairings(windows: windows, candidates: candidates)
1391
+
1392
+ var output: [[String: Any]] = []
1393
+ for (zIndex, window) in windows.enumerated() {
1394
+ let axTitle = stringAttribute(window, attribute: kAXTitleAttribute as CFString) ?? ""
1395
+ let axRole = stringAttribute(window, attribute: kAXRoleAttribute as CFString) ?? ""
1396
+ let axSubrole = stringAttribute(window, attribute: kAXSubroleAttribute as CFString) ?? ""
1397
+ let axFrame = frameForWindow(window)
1398
+ let pairing = pairings[ObjectIdentifier(window)] ?? WindowPairing(candidate: nil, score: -Double.greatestFiniteMagnitude, confidence: "low")
1399
+ let candidate = pairing.candidate
1400
+
1401
+ let effectiveFrame = axFrame.width > 1 && axFrame.height > 1 ? axFrame : (candidate?.bounds ?? axFrame)
1402
+ if effectiveFrame.width < 100 || effectiveFrame.height < 80 { continue }
1403
+ let hasUsableAXFrame = axFrame.width > 1 && axFrame.height > 1
1404
+ let title = hasUsableAXFrame && !axTitle.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty ? axTitle : (candidate?.title.isEmpty == false ? candidate!.title : axTitle)
1405
+ let windowRef = refStore.storeWindow(window)
1406
+ let isMinimized = boolAttribute(window, attribute: kAXMinimizedAttribute as CFString) ?? false
1407
+ let isMain = boolAttribute(window, attribute: kAXMainAttribute as CFString) ?? false
1408
+ let isFocused = boolAttribute(window, attribute: kAXFocusedAttribute as CFString) ?? false
1409
+ let sheetCount = sheetElements(of: window).count
1410
+ let isModal = (boolAttribute(window, attribute: "AXModal" as CFString) ?? false) || sheetCount > 0 || isDialogLikeRoot(role: axRole, subrole: axSubrole)
1411
+ let scale = displayScaleFactor(for: effectiveFrame)
1412
+
1413
+ var item: [String: Any] = [
1414
+ "kind": rootKind(role: axRole, subrole: axSubrole),
1415
+ "rootRef": windowRef,
1416
+ "windowRef": windowRef,
1417
+ "zOrder": candidate?.zOrder ?? zIndex,
1418
+ "title": title,
1419
+ "role": axRole,
1420
+ "subrole": axSubrole,
1421
+ "isModal": isModal,
1422
+ "framePoints": [
1423
+ "x": effectiveFrame.origin.x,
1424
+ "y": effectiveFrame.origin.y,
1425
+ "w": effectiveFrame.size.width,
1426
+ "h": effectiveFrame.size.height,
1427
+ ],
1428
+ "scaleFactor": scale,
1429
+ "isMinimized": isMinimized,
1430
+ "isOnscreen": candidate?.isOnscreen ?? !isMinimized,
1431
+ "isMain": isMain,
1432
+ "isFocused": isFocused,
1433
+ "metadata": rootMetadata(pairing: pairing, sheetCount: sheetCount),
1434
+ ]
1435
+ if let candidate {
1436
+ item["windowId"] = Int(candidate.windowId)
1437
+ }
1438
+ output.append(item)
1439
+
1440
+ for sheet in sheetElements(of: window) {
1441
+ let sheetRef = refStore.storeWindow(sheet)
1442
+ let sheetFrame = frameForWindow(sheet)
1443
+ let sheetCandidate = bestCandidate(for: sheet, candidates: candidates)
1444
+ var sheetItem: [String: Any] = [
1445
+ "kind": "sheet",
1446
+ "rootRef": sheetRef,
1447
+ "windowRef": sheetRef,
1448
+ "zOrder": sheetCandidate?.zOrder ?? candidate?.zOrder ?? zIndex,
1449
+ "title": stringAttribute(sheet, attribute: kAXTitleAttribute as CFString) ?? title,
1450
+ "role": stringAttribute(sheet, attribute: kAXRoleAttribute as CFString) ?? "AXSheet",
1451
+ "subrole": stringAttribute(sheet, attribute: kAXSubroleAttribute as CFString) ?? "",
1452
+ "isModal": true,
1453
+ "framePoints": ["x": sheetFrame.origin.x, "y": sheetFrame.origin.y, "w": sheetFrame.width, "h": sheetFrame.height],
1454
+ "scaleFactor": displayScaleFactor(for: sheetFrame),
1455
+ "isMinimized": false,
1456
+ "isOnscreen": sheetCandidate?.isOnscreen ?? candidate?.isOnscreen ?? !isMinimized,
1457
+ "isMain": false,
1458
+ "isFocused": isFocused,
1459
+ "metadata": ["pairing": ["confidence": sheetCandidate == nil ? pairing.confidence : "high", "score": sheetCandidate == nil ? pairing.score : 100], "sheetCount": 0],
1460
+ ]
1461
+ if let sheetCandidate { sheetItem["windowId"] = Int(sheetCandidate.windowId) }
1462
+ output.append(sheetItem)
1463
+ }
1464
+ }
1465
+ return output
1466
+ }
1467
+
1468
+ private func look(_ request: [String: Any]) throws -> [String: Any] {
1469
+ let windowId = optionalIntArg(request, "windowId").map { UInt32($0) }
1470
+ let windowRef = optionalStringArg(request, "windowRef")
1471
+ let maxDimension = optionalIntArg(request, "maxDimension").map { max(1, $0) }
1472
+ let readText = optionalStringArg(request, "readText") ?? "auto"
1473
+ let baseLookId = optionalStringArg(request, "baseLookId")
1474
+ let includeImage = boolArg(request, "includeImage") ?? true
1475
+ guard readText == "auto" || readText == "always" || readText == "never" else {
1476
+ throw BridgeFailure(message: "readText must be auto, always, or never", code: "invalid_args")
1477
+ }
1478
+
1479
+ let requestedRoot = windowRef.flatMap { refStore.window(for: $0) }
1480
+ let requestedRole = requestedRoot.flatMap { stringAttribute($0, attribute: kAXRoleAttribute as CFString) } ?? ""
1481
+ let isMenuRoot = requestedRole == "AXMenu" || (windowRef?.hasPrefix("cgmenu:") == true)
1482
+ let captureStart = Date()
1483
+ let shouldCapture = !isMenuRoot && (includeImage || readText == "always")
1484
+ let capture = try shouldCapture ? windowId.map { try captureWindow(windowId: $0) } : nil
1485
+ let captureMs = capture.map { _ in elapsedMs(captureStart) } ?? 0
1486
+
1487
+ let pid: Int32
1488
+ if let windowId, let ownerPid = pidForWindowId(windowId) {
1489
+ pid = ownerPid
1490
+ } else if let requestedRoot, let owner = pidForElement(requestedRoot) {
1491
+ pid = owner
1492
+ } else {
1493
+ throw BridgeFailure(message: "Root is not owned by a running app", code: "root_not_found")
1494
+ }
1495
+ ensureEnhancedAccessibility(pid: pid)
1496
+ if let windowRef, windowRef.hasPrefix("cgmenu:"), refStore.window(for: windowRef) == nil {
1497
+ let frame = windowId.flatMap { windowInfo(windowId: $0)?.bounds } ?? CGRect(x: 0, y: 0, width: 1, height: 1)
1498
+ let lookId = freshLookId()
1499
+ storeLookRecord(LookRecord(lookId: lookId, windowId: windowId ?? 0, windowFrame: frame, imageWidth: max(1, Int(frame.width)), imageHeight: max(1, Int(frame.height)), hasImage: false))
1500
+ let outline = LookNode(element: nil, ref: windowRef, role: "AXMenu", subrole: "", identifier: "", title: "Menu", description: "", value: "", actions: [], canPress: false, canFocus: false, canSetValue: false, canScroll: false, canIncrement: false, canDecrement: false, isTextInput: false, rect: CGRect(x: 0, y: 0, width: max(1, frame.width), height: max(1, frame.height)), pictureOnly: true)
1501
+ return [
1502
+ "lookId": lookId,
1503
+ "capturedAt": captureStart.timeIntervalSince1970,
1504
+ "window": ["windowId": Int(windowId ?? 0), "rootRef": windowRef, "kind": "menu", "framePoints": ["x": frame.origin.x, "y": frame.origin.y, "w": frame.width, "h": frame.height], "scaleFactor": displayScaleFactor(for: frame), "isModal": false, "metadata": ["pairing": ["confidence": "low", "score": 0], "sheetCount": 0], "role": "AXMenu", "subrole": ""],
1505
+ "outline": outline.payload(),
1506
+ "timings": ["captureMs": 0, "describeMs": 0, "readTextMs": 0],
1507
+ ]
1508
+ }
1509
+ guard let window = windowElement(pid: pid, windowId: windowId, windowRef: windowRef) else {
1510
+ throw BridgeFailure(message: "Root is not available through Accessibility", code: "root_not_found")
1511
+ }
1512
+ let rootElement: AXUIElement
1513
+ if let scopeRef = optionalStringArg(request, "scopeRef") {
1514
+ guard let scoped = refStore.element(for: scopeRef), isElement(scoped, descendantOf: window) else {
1515
+ throw BridgeFailure(message: "Scope ref is stale or outside the target root", code: "element_ref_invalid")
1516
+ }
1517
+ rootElement = scoped
1518
+ } else {
1519
+ rootElement = window
1520
+ }
1521
+
1522
+ let rootFrame = frameForWindow(window)
1523
+ let imageWidth: Int
1524
+ let imageHeight: Int
1525
+ let imagePayload: [String: Any]?
1526
+ let transform: (CGRect) -> CGRect
1527
+ if let capture {
1528
+ let outputImage = downscaledImage(capture.image, maxDimension: maxDimension) ?? capture.image
1529
+ imageWidth = outputImage.width
1530
+ imageHeight = outputImage.height
1531
+ transform = rectTransform(windowFrame: capture.frame, imageWidth: outputImage.width, imageHeight: outputImage.height)
1532
+ if includeImage {
1533
+ guard let jpeg = jpegData(image: outputImage, quality: 0.8) else {
1534
+ throw BridgeFailure(message: "Failed to encode look image as JPEG", code: "encoding_failed")
1535
+ }
1536
+ imagePayload = ["jpegBase64": jpeg.base64EncodedString(), "width": outputImage.width, "height": outputImage.height]
1537
+ } else {
1538
+ imagePayload = nil
1539
+ }
1540
+ } else {
1541
+ imageWidth = max(1, Int(rootFrame.width))
1542
+ imageHeight = max(1, Int(rootFrame.height))
1543
+ imagePayload = nil
1544
+ transform = rectTransform(windowFrame: rootFrame, imageWidth: imageWidth, imageHeight: imageHeight)
1545
+ }
1546
+ let describeStart = Date()
1547
+ let outline = buildLookOutline(root: rootElement, transform: transform)
1548
+ let describeMs = elapsedMs(describeStart)
1549
+
1550
+ var readTextMs = 0
1551
+ var readTextExecuted = false
1552
+ if let capture, readText == "always" {
1553
+ readTextExecuted = true
1554
+ let textStart = Date()
1555
+ let boxes = try recognizeText(in: capture.image, outputWidth: imageWidth, outputHeight: imageHeight)
1556
+ attachOCR(boxes, to: outline)
1557
+ readTextMs = elapsedMs(textStart)
1558
+ }
1559
+
1560
+ let lookId = freshLookId()
1561
+ let baseRecord = baseLookId.flatMap { lookRecord(for: $0) }
1562
+ storeLookRecord(LookRecord(
1563
+ lookId: lookId,
1564
+ windowId: windowId ?? baseRecord?.windowId ?? 0,
1565
+ windowFrame: baseRecord?.windowFrame ?? capture?.frame ?? rootFrame,
1566
+ imageWidth: baseRecord?.imageWidth ?? imageWidth,
1567
+ imageHeight: baseRecord?.imageHeight ?? imageHeight,
1568
+ hasImage: baseRecord?.hasImage ?? (capture != nil)
1569
+ ))
1570
+ let scale = (capture?.frame.width ?? rootFrame.width) > 0 ? Double(imageWidth) / (capture?.frame.width ?? rootFrame.width) : displayScaleFactor(for: rootFrame)
1571
+ let pairing = pairingForWindow(window, pid: pid)
1572
+ let role = stringAttribute(window, attribute: kAXRoleAttribute as CFString) ?? ""
1573
+ let subrole = stringAttribute(window, attribute: kAXSubroleAttribute as CFString) ?? ""
1574
+ let sheetCount = sheetElements(of: window).count
1575
+ var response: [String: Any] = [
1576
+ "lookId": lookId,
1577
+ "capturedAt": captureStart.timeIntervalSince1970,
1578
+ "window": [
1579
+ "windowId": Int(windowId ?? 0),
1580
+ "rootRef": windowRef ?? refStore.storeWindow(window),
1581
+ "kind": rootKind(role: role, subrole: subrole),
1582
+ "framePoints": ["x": (capture?.frame ?? rootFrame).origin.x, "y": (capture?.frame ?? rootFrame).origin.y, "w": (capture?.frame ?? rootFrame).width, "h": (capture?.frame ?? rootFrame).height],
1583
+ "scaleFactor": scale,
1584
+ "isModal": (boolAttribute(window, attribute: "AXModal" as CFString) ?? false) || sheetCount > 0 || isDialogLikeRoot(role: role, subrole: subrole),
1585
+ "metadata": rootMetadata(pairing: pairing, sheetCount: sheetCount),
1586
+ "role": role,
1587
+ "subrole": subrole,
1588
+ ],
1589
+ "outline": outline.payload(),
1590
+ "timings": ["captureMs": captureMs, "describeMs": describeMs, "readTextMs": readTextMs],
1591
+ "readText": ["requested": readText, "executed": readTextExecuted],
1592
+ ]
1593
+ if let imagePayload { response["image"] = imagePayload }
1594
+ return response
1595
+ }
1596
+
1597
+ private func elapsedMs(_ start: Date) -> Int {
1598
+ max(0, Int(Date().timeIntervalSince(start) * 1000.0))
1599
+ }
1600
+
1601
+ private func storeLookRecord(_ record: LookRecord) {
1602
+ lookRecordLock.lock()
1603
+ lookRecords[record.lookId] = record
1604
+ lookRecordOrder.append(record.lookId)
1605
+ while lookRecordOrder.count > 8 {
1606
+ let oldest = lookRecordOrder.removeFirst()
1607
+ lookRecords.removeValue(forKey: oldest)
1608
+ }
1609
+ lookRecordLock.unlock()
1610
+ // Tangu:每次观察也算"agent 正在看这个窗口",liveView 据此选画面(唯一的 look 收口点,
1611
+ // 比在 look() 的各个分支里各挂一次可靠)。**不点光效** —— 光效只表示"正在动手"。
1612
+ if record.windowId != 0 {
1613
+ controlledLock.lock()
1614
+ controlledWindowId = record.windowId
1615
+ controlledPid = pidForWindowId(record.windowId) ?? controlledPid
1616
+ controlledAt = Date()
1617
+ controlledLock.unlock()
1618
+ }
1619
+ }
1620
+
1621
+ private func freshLookId() -> String {
1622
+ lookRecordLock.lock()
1623
+ defer { lookRecordLock.unlock() }
1624
+ nextLookId += 1
1625
+ return "look_\(nextLookId)"
1626
+ }
1627
+
1628
+ private func lookRecord(for lookId: String) -> LookRecord? {
1629
+ lookRecordLock.lock()
1630
+ defer { lookRecordLock.unlock() }
1631
+ return lookRecords[lookId]
1632
+ }
1633
+
1634
+ private func rectTransform(windowFrame: CGRect, imageWidth: Int, imageHeight: Int) -> (CGRect) -> CGRect {
1635
+ let sx = windowFrame.width > 0 ? Double(imageWidth) / windowFrame.width : 1.0
1636
+ let sy = windowFrame.height > 0 ? Double(imageHeight) / windowFrame.height : 1.0
1637
+ return { frame in
1638
+ let x = (frame.origin.x - windowFrame.origin.x) * sx
1639
+ let y = (frame.origin.y - windowFrame.origin.y) * sy
1640
+ let w = frame.width * sx
1641
+ let h = frame.height * sy
1642
+ return self.clampRect(CGRect(x: x, y: y, width: w, height: h), width: imageWidth, height: imageHeight)
1643
+ }
1644
+ }
1645
+
1646
+ private func clampRect(_ rect: CGRect, width: Int, height: Int) -> CGRect {
1647
+ let maxX = Double(width)
1648
+ let maxY = Double(height)
1649
+ let x1 = min(max(rect.minX, 0), maxX)
1650
+ let y1 = min(max(rect.minY, 0), maxY)
1651
+ let x2 = min(max(rect.maxX, 0), maxX)
1652
+ let y2 = min(max(rect.maxY, 0), maxY)
1653
+ return CGRect(x: x1, y: y1, width: max(0, x2 - x1), height: max(0, y2 - y1))
1654
+ }
1655
+
1656
+ private func buildLookOutline(root: AXUIElement, transform: @escaping (CGRect) -> CGRect) -> LookNode {
1657
+ let rootNode = lookNode(element: root, transform: transform, offscreen: false)
1658
+ let nodeLimit = 2000
1659
+ // Apps with slow AX servers (e.g. Outlook) can take >30s to describe; the
1660
+ // client aborts at 33s, so stop walking well before that and return a
1661
+ // truncated outline instead.
1662
+ let deadline = Date().addingTimeInterval(20.0)
1663
+ var walked = 1
1664
+ var seen = Set<ObjectIdentifier>([ObjectIdentifier(root)])
1665
+ var queue: [(AXUIElement, LookNode)] = [(root, rootNode)]
1666
+ var index = 0
1667
+ while index < queue.count {
1668
+ let (element, node) = queue[index]
1669
+ index += 1
1670
+ let children = axElementArray(element, attribute: kAXChildrenAttribute as CFString)
1671
+ if children.isEmpty { continue }
1672
+ if walked >= nodeLimit || Date() >= deadline {
1673
+ node.truncated = true
1674
+ continue
1675
+ }
1676
+ let visibleByKind = visibleChildrenByKind(element)
1677
+ for child in children {
1678
+ if walked >= nodeLimit || Date() >= deadline {
1679
+ node.truncated = true
1680
+ break
1681
+ }
1682
+ let identity = ObjectIdentifier(child)
1683
+ if seen.contains(identity) { continue }
1684
+ seen.insert(identity)
1685
+ let role = stringAttribute(child, attribute: kAXRoleAttribute as CFString) ?? ""
1686
+ let offscreen = childOffscreen(child, role: role, visibleByKind: visibleByKind)
1687
+ let childNode = lookNode(element: child, transform: transform, offscreen: offscreen)
1688
+ node.children.append(childNode)
1689
+ queue.append((child, childNode))
1690
+ walked += 1
1691
+ }
1692
+ }
1693
+ return rootNode
1694
+ }
1695
+
1696
+ private func lookNode(element: AXUIElement, transform: (CGRect) -> CGRect, offscreen: Bool) -> LookNode {
1697
+ let role = stringAttribute(element, attribute: kAXRoleAttribute as CFString) ?? ""
1698
+ let subrole = stringAttribute(element, attribute: kAXSubroleAttribute as CFString) ?? ""
1699
+ let actions = actionNames(element)
1700
+ var valueSettable = DarwinBoolean(false)
1701
+ let valueStatus = AXUIElementIsAttributeSettable(element, kAXValueAttribute as CFString, &valueSettable)
1702
+ var focusedSettable = DarwinBoolean(false)
1703
+ let focusedStatus = AXUIElementIsAttributeSettable(element, kAXFocusedAttribute as CFString, &focusedSettable)
1704
+ let textRoles: Set<String> = ["AXTextField", "AXTextArea", "AXTextView", "AXSearchField", "AXComboBox", "AXEditableText", "AXSecureTextField"]
1705
+ let title = stringAttribute(element, attribute: kAXTitleAttribute as CFString) ?? ""
1706
+ let description = stringAttribute(element, attribute: kAXDescriptionAttribute as CFString) ?? ""
1707
+ let value = displayValue(element, role: role, subrole: subrole)
1708
+ let screenRect = frameForElement(element) ?? .zero
1709
+ let node = LookNode(
1710
+ element: element,
1711
+ ref: refStore.storeElement(element, snapshot: AXRefStore.Snapshot(
1712
+ role: role,
1713
+ identifier: stringAttribute(element, attribute: "AXIdentifier" as CFString) ?? "",
1714
+ label: normalizedLabel([title, description, value].joined(separator: " ")),
1715
+ rect: screenRect
1716
+ )),
1717
+ role: role,
1718
+ subrole: subrole,
1719
+ identifier: stringAttribute(element, attribute: "AXIdentifier" as CFString) ?? "",
1720
+ title: title,
1721
+ description: description,
1722
+ value: value,
1723
+ actions: actions,
1724
+ canPress: actions.contains(kAXPressAction as String),
1725
+ canFocus: focusedStatus == .success && focusedSettable.boolValue,
1726
+ canSetValue: valueStatus == .success && valueSettable.boolValue,
1727
+ canScroll: supportsAnyScrollAction(element),
1728
+ canIncrement: actions.contains(kAXIncrementAction as String),
1729
+ canDecrement: actions.contains(kAXDecrementAction as String),
1730
+ isTextInput: textRoles.contains(role),
1731
+ rect: transform(screenRect),
1732
+ focused: boolAttribute(element, attribute: kAXFocusedAttribute as CFString) == true,
1733
+ offscreen: offscreen
1734
+ )
1735
+ if node.canScroll {
1736
+ let rows = axElementArray(element, attribute: kAXRowsAttribute as CFString)
1737
+ let visibleRows = axElementArrayIfPresent(element, attribute: kAXVisibleRowsAttribute as CFString)
1738
+ if !rows.isEmpty, let visibleRows {
1739
+ node.scrollExtent = ["seen": visibleRows.count, "total": rows.count]
1740
+ }
1741
+ }
1742
+ return node
1743
+ }
1744
+
1745
+ private func visibleChildrenByKind(_ element: AXUIElement) -> [String: [AXUIElement]?] {
1746
+ [
1747
+ "AXRow": axElementArrayIfPresent(element, attribute: kAXVisibleRowsAttribute as CFString),
1748
+ "AXColumn": axElementArrayIfPresent(element, attribute: kAXVisibleColumnsAttribute as CFString),
1749
+ "AXCell": axElementArrayIfPresent(element, attribute: kAXVisibleCellsAttribute as CFString),
1750
+ "*": axElementArrayIfPresent(element, attribute: kAXVisibleChildrenAttribute as CFString),
1751
+ ]
1752
+ }
1753
+
1754
+ private func childOffscreen(_ child: AXUIElement, role: String, visibleByKind: [String: [AXUIElement]?]) -> Bool {
1755
+ let key = role == "AXRow" || role == "AXColumn" || role == "AXCell" ? role : "*"
1756
+ guard let visible = visibleByKind[key] ?? nil else { return false }
1757
+ return !visible.contains { sameElement($0, child) }
1758
+ }
1759
+
1760
+ private func recognizeText(in image: CGImage, outputWidth: Int, outputHeight: Int) throws -> [OCRBox] {
1761
+ let semaphore = DispatchSemaphore(value: 0)
1762
+ let recognized = Box<[OCRBox]>([])
1763
+ let recognizedError = Box<Error?>(nil)
1764
+ let request = VNRecognizeTextRequest { request, error in
1765
+ defer { semaphore.signal() }
1766
+ if let error {
1767
+ recognizedError.value = error
1768
+ return
1769
+ }
1770
+ let observations = (request.results as? [VNRecognizedTextObservation]) ?? []
1771
+ recognized.value = observations.compactMap { observation in
1772
+ guard let candidate = observation.topCandidates(1).first else { return nil }
1773
+ let box = observation.boundingBox
1774
+ let x = box.origin.x * Double(outputWidth)
1775
+ let y = (1.0 - box.origin.y - box.height) * Double(outputHeight)
1776
+ let w = box.width * Double(outputWidth)
1777
+ let h = box.height * Double(outputHeight)
1778
+ return OCRBox(string: candidate.string, confidence: Double(candidate.confidence), rect: CGRect(x: x, y: y, width: w, height: h))
1779
+ }
1780
+ }
1781
+ request.recognitionLevel = .accurate
1782
+ request.usesLanguageCorrection = false
1783
+ try VNImageRequestHandler(cgImage: image, options: [:]).perform([request])
1784
+ if semaphore.wait(timeout: .now() + .seconds(8)) == .timedOut {
1785
+ throw BridgeFailure(message: "Text recognition timed out", code: "text_recognition_timeout")
1786
+ }
1787
+ if let error = recognizedError.value {
1788
+ throw BridgeFailure(message: "Text recognition failed: \(error.localizedDescription)", code: "text_recognition_failed")
1789
+ }
1790
+ return recognized.value
1791
+ }
1792
+
1793
+ private func attachOCR(_ boxes: [OCRBox], to root: LookNode) {
1794
+ var pictureOnlyIndex = 0
1795
+ for box in boxes {
1796
+ if ocrBoxDuplicatesAXLabel(box, in: root) { continue }
1797
+ let center = CGPoint(x: box.rect.midX, y: box.rect.midY)
1798
+ if let node = deepestNode(containing: center, in: root) {
1799
+ node.text.append(["string": box.string, "confidence": box.confidence, "rect": ["x": box.rect.origin.x, "y": box.rect.origin.y, "w": box.rect.width, "h": box.rect.height]])
1800
+ } else {
1801
+ pictureOnlyIndex += 1
1802
+ let parent = deepestContainer(containing: center, in: root) ?? root
1803
+ parent.children.append(LookNode(element: nil, ref: "pic_\(pictureOnlyIndex)", role: "AXImage", subrole: "", identifier: "", title: box.string, description: "", value: "", actions: [], canPress: false, canFocus: false, canSetValue: false, canScroll: false, canIncrement: false, canDecrement: false, isTextInput: false, rect: box.rect, pictureOnly: true))
1804
+ }
1805
+ }
1806
+ }
1807
+
1808
+ private func ocrBoxDuplicatesAXLabel(_ box: OCRBox, in root: LookNode) -> Bool {
1809
+ let boxLabel = normalizedLabel(box.string)
1810
+ if boxLabel.isEmpty { return true }
1811
+ var queue = [root]
1812
+ var index = 0
1813
+ while index < queue.count {
1814
+ let node = queue[index]
1815
+ index += 1
1816
+ if !node.pictureOnly, node.rect.intersects(box.rect) {
1817
+ let fields = [node.title, node.value, node.description]
1818
+ if fields.contains(where: { normalizedLabel($0) == boxLabel }) { return true }
1819
+ }
1820
+ queue.append(contentsOf: node.children)
1821
+ }
1822
+ return false
1823
+ }
1824
+
1825
+ private func deepestNode(containing point: CGPoint, in root: LookNode) -> LookNode? {
1826
+ guard root.rect.contains(point), !root.pictureOnly else { return nil }
1827
+ for child in root.children.reversed() {
1828
+ if let match = deepestNode(containing: point, in: child) { return match }
1829
+ }
1830
+ return root
1831
+ }
1832
+
1833
+ private func deepestContainer(containing point: CGPoint, in root: LookNode) -> LookNode? {
1834
+ guard root.rect.contains(point) else { return nil }
1835
+ for child in root.children.reversed() {
1836
+ if let match = deepestContainer(containing: point, in: child) { return match }
1837
+ }
1838
+ return root
1839
+ }
1840
+
1841
+ private func lookPoint(record: LookRecord, x: Double, y: Double) -> CGPoint {
1842
+ let relX = min(max(x / max(1.0, Double(record.imageWidth)), 0), 1)
1843
+ let relY = min(max(y / max(1.0, Double(record.imageHeight)), 0), 1)
1844
+ return CGPoint(x: record.windowFrame.origin.x + record.windowFrame.width * relX, y: record.windowFrame.origin.y + record.windowFrame.height * relY)
1845
+ }
1846
+
1847
+ private func payloadNode(element: AXUIElement) -> [String: Any] {
1848
+ let node = lookNode(element: element, transform: { $0 }, offscreen: false)
1849
+ var payload = node.payload()
1850
+ payload["children"] = []
1851
+ return payload
1852
+ }
1853
+
1854
+ private func normalizedLabel(_ value: String) -> String {
1855
+ value.lowercased().components(separatedBy: CharacterSet.whitespacesAndNewlines).filter { !$0.isEmpty }.joined(separator: " ")
1856
+ }
1857
+
1858
+ private func ensureRootObserver(pid: Int32) -> Bool {
1859
+ rootObserverLock.lock()
1860
+ if let existing = rootObservers[pid] {
1861
+ existing.lastUsed = Date().timeIntervalSince1970
1862
+ rootObserverLock.unlock()
1863
+ return true
1864
+ }
1865
+ rootObserverLock.unlock()
1866
+
1867
+ let appElement = AXUIElementCreateApplication(pid)
1868
+ AXUIElementSetMessagingTimeout(appElement, 0.25)
1869
+ var observer: AXObserver?
1870
+ let createStatus = AXObserverCreate(pid, { observer, element, notification, refcon in
1871
+ guard let refcon else { return }
1872
+ let bridge = Unmanaged<Bridge>.fromOpaque(refcon).takeUnretainedValue()
1873
+ bridge.recordRootAXEvent(observer: observer, notification: notification as String, element: element)
1874
+ }, &observer)
1875
+ guard createStatus == .success, let observer else { return false }
1876
+
1877
+ let state = RootAXObserverState(pid: pid, observer: observer)
1878
+ let context = UnsafeMutableRawPointer(Unmanaged.passUnretained(self).toOpaque())
1879
+ let notifications: [CFString] = [
1880
+ "AXWindowCreated" as CFString,
1881
+ "AXSheetCreated" as CFString,
1882
+ "AXMenuOpened" as CFString,
1883
+ "AXMenuClosed" as CFString,
1884
+ "AXUIElementDestroyed" as CFString,
1885
+ "AXFocusedWindowChanged" as CFString,
1886
+ kAXValueChangedNotification as CFString,
1887
+ kAXTitleChangedNotification as CFString,
1888
+ kAXSelectedChildrenChangedNotification as CFString,
1889
+ kAXLayoutChangedNotification as CFString,
1890
+ ]
1891
+ var registered = false
1892
+ let observedElements = [appElement] + axElementArray(appElement, attribute: kAXWindowsAttribute as CFString)
1893
+ for observed in observedElements {
1894
+ for notification in notifications {
1895
+ let status = AXObserverAddNotification(observer, observed, notification, context)
1896
+ if status == .success || status == .notificationAlreadyRegistered { registered = true }
1897
+ }
1898
+ }
1899
+ guard registered else { return false }
1900
+
1901
+ rootObserverLock.lock()
1902
+ if rootObservers.count >= maxRootObservers,
1903
+ let evict = rootObservers.values.min(by: { $0.lastUsed < $1.lastUsed })?.pid
1904
+ {
1905
+ rootObservers.removeValue(forKey: evict)
1906
+ }
1907
+ rootObservers[pid] = state
1908
+ rootObserverLock.unlock()
1909
+
1910
+ let source = AXObserverGetRunLoopSource(observer)
1911
+ Thread.detachNewThread {
1912
+ // Keep each AXObserver on its own run loop so its callbacks never compete
1913
+ // with AppKit rendering on the helper's main thread.
1914
+ CFRunLoopAddSource(CFRunLoopGetCurrent(), source, .commonModes)
1915
+ CFRunLoopRun()
1916
+ }
1917
+ return true
1918
+ }
1919
+
1920
+ private func recordRootAXEvent(observer: AXObserver, notification: String, element: AXUIElement) {
1921
+ rootObserverLock.lock()
1922
+ let state = rootObservers.values.first { CFEqual($0.observer, observer) }
1923
+ if let state {
1924
+ state.events.append(RootAXEvent(sequence: state.nextSequence, timestamp: Date().timeIntervalSince1970, notification: notification, element: element))
1925
+ state.nextSequence += 1
1926
+ if state.events.count > 64 { state.events.removeFirst(state.events.count - 64) }
1927
+ }
1928
+ rootObserverLock.unlock()
1929
+ if let state {
1930
+ state.change.lock()
1931
+ state.changeGeneration += 1
1932
+ state.change.broadcast()
1933
+ state.change.unlock()
1934
+ }
1935
+ }
1936
+
1937
+ private func rootChangeGeneration(pid: Int32) -> UInt64 {
1938
+ rootObserverLock.lock()
1939
+ let state = rootObservers[pid]
1940
+ rootObserverLock.unlock()
1941
+ guard let state else { return 0 }
1942
+ state.change.lock()
1943
+ defer { state.change.unlock() }
1944
+ return state.changeGeneration
1945
+ }
1946
+
1947
+ private func waitForRootChange(pid: Int32, since generation: UInt64, until deadline: Date) {
1948
+ rootObserverLock.lock()
1949
+ let state = rootObservers[pid]
1950
+ rootObserverLock.unlock()
1951
+ guard let state else {
1952
+ Thread.sleep(forTimeInterval: min(0.2, max(0, deadline.timeIntervalSinceNow)))
1953
+ return
1954
+ }
1955
+ state.change.lock()
1956
+ if state.changeGeneration == generation {
1957
+ _ = state.change.wait(until: min(deadline, Date().addingTimeInterval(0.2)))
1958
+ }
1959
+ state.change.unlock()
1960
+ }
1961
+
1962
+ private func rootEventCursor(pid: Int32) -> UInt64 {
1963
+ rootObserverLock.lock()
1964
+ defer { rootObserverLock.unlock() }
1965
+ return rootObservers[pid]?.nextSequence ?? 1
1966
+ }
1967
+
1968
+ private func rootEvents(pid: Int32, since cursor: UInt64) -> [RootAXEvent] {
1969
+ rootObserverLock.lock()
1970
+ defer { rootObserverLock.unlock() }
1971
+ guard let events = rootObservers[pid]?.events else { return [] }
1972
+ return events.filter { $0.sequence >= cursor }
1973
+ }
1974
+
1975
+ // Onscreen CGWindowList id set for one pid: ~1-2ms per call, so it can be
1976
+ // polled tightly where a full AX enumeration cannot.
1977
+ private func cgRootSignature(pid: Int32) -> Set<UInt32> {
1978
+ guard let entries = CGWindowListCopyWindowInfo([.optionOnScreenOnly], kCGNullWindowID) as? [[String: Any]] else { return [] }
1979
+ var ids = Set<UInt32>()
1980
+ for entry in entries {
1981
+ guard let owner = (entry[kCGWindowOwnerPID as String] as? NSNumber)?.int32Value, owner == pid else { continue }
1982
+ guard let windowId = (entry[kCGWindowNumber as String] as? NSNumber)?.uint32Value else { continue }
1983
+ ids.insert(windowId)
1984
+ }
1985
+ return ids
1986
+ }
1987
+
1988
+ private func refindElement(ref: String, pid: Int32, windowId: UInt32) -> AXUIElement? {
1989
+ guard let snapshot = refStore.snapshot(for: ref),
1990
+ let window = windowElement(pid: pid, windowId: windowId)
1991
+ else { return nil }
1992
+ let targetCenter = CGPoint(x: snapshot.rect.midX, y: snapshot.rect.midY)
1993
+ let candidates = collectDescendants(startingAt: window, maxDepth: 8).filter { candidate in
1994
+ let role = stringAttribute(candidate, attribute: kAXRoleAttribute as CFString) ?? ""
1995
+ guard role == snapshot.role else { return false }
1996
+ let identifier = stringAttribute(candidate, attribute: "AXIdentifier" as CFString) ?? ""
1997
+ if !snapshot.identifier.isEmpty { return identifier == snapshot.identifier }
1998
+ let subrole = stringAttribute(candidate, attribute: kAXSubroleAttribute as CFString) ?? ""
1999
+ let title = stringAttribute(candidate, attribute: kAXTitleAttribute as CFString) ?? ""
2000
+ let description = stringAttribute(candidate, attribute: kAXDescriptionAttribute as CFString) ?? ""
2001
+ let value = displayValue(candidate, role: role, subrole: subrole)
2002
+ return normalizedLabel([title, description, value].joined(separator: " ")) == snapshot.label
2003
+ }
2004
+ return candidates.min { left, right in
2005
+ let leftFrame = frameForElement(left) ?? .zero
2006
+ let rightFrame = frameForElement(right) ?? .zero
2007
+ let leftDistance = hypot(leftFrame.midX - targetCenter.x, leftFrame.midY - targetCenter.y)
2008
+ let rightDistance = hypot(rightFrame.midX - targetCenter.x, rightFrame.midY - targetCenter.y)
2009
+ return leftDistance < rightDistance
2010
+ }
2011
+ }
2012
+
2013
+ private func hitTest(_ request: [String: Any]) throws -> [String: Any] {
2014
+ let lookId = try stringArg(request, "lookId")
2015
+ guard let record = lookRecord(for: lookId) else {
2016
+ throw BridgeFailure(message: "Look id '\(lookId)' is no longer available", code: "stale_look")
2017
+ }
2018
+ let point = lookPoint(record: record, x: try doubleArg(request, "x"), y: try doubleArg(request, "y"))
2019
+ guard let element = hitTestElement(at: point) else {
2020
+ throw BridgeFailure(message: "No element at point", code: "hit_test_failed")
2021
+ }
2022
+ return payloadNode(element: element)
2023
+ }
2024
+
2025
+ private func act(_ request: [String: Any]) throws -> [String: Any] {
2026
+ let lookId = try stringArg(request, "lookId")
2027
+ guard let record = lookRecord(for: lookId) else {
2028
+ throw BridgeFailure(message: "Look id '\(lookId)' is no longer available", code: "stale_look")
2029
+ }
2030
+ let pid = Int32(try intArg(request, "pid"))
2031
+ let action = try stringArg(request, "action")
2032
+ let target = request["target"] as? [String: Any] ?? [:]
2033
+ let params = request["params"] as? [String: Any] ?? [:]
2034
+ let policy = optionalStringArg(request, "policy") ?? "default"
2035
+ // Tangu:登记"正在操控这个窗口"(供 liveView),并按设置点亮边缘光效。放在最前面 —— 后面任何
2036
+ // 一步抛错都不该让用户失去"agent 正在动这个窗口"的可见信号。
2037
+ // 复用 cursorOverlay 这一个开关(vendor 的 act 请求里本就带着它),不为光效另开配置键:
2038
+ // 二者是同一件事——"把 agent 的动作画出来给人看"。要拆开时再加键不迟。
2039
+ noteControlledWindow(windowId: record.windowId, pid: pid, highlight: request["cursorOverlay"] as? Bool ?? true)
2040
+ let deferRootDelta = boolArg(request, "deferRootDelta") ?? false
2041
+ let delivery = policy == "background" ? "pid" : ((params["delivery"] as? String) == "pid" ? "pid" : "hid")
2042
+ var holdsPhysicalInput = false
2043
+ func acquirePhysicalInputIfNeeded() {
2044
+ if delivery == "hid" && !holdsPhysicalInput {
2045
+ physicalInputLock.lock(); ForegroundActivity.shared.enter()
2046
+ holdsPhysicalInput = true
2047
+ }
2048
+ }
2049
+ defer { if holdsPhysicalInput { ForegroundActivity.shared.leave(); physicalInputLock.unlock() } }
2050
+ var performed: [String: Any] = ["delivery": delivery]
2051
+ var element: AXUIElement?
2052
+ var rawPoint: CGPoint?
2053
+ var preflightCapsUnknown = false
2054
+ let eventsLive = !deferRootDelta && ensureRootObserver(pid: pid)
2055
+ let eventCursor = eventsLive ? rootEventCursor(pid: pid) : 0
2056
+ let beforeRootSnapshot = deferRootDelta ? [:] : rootMetadataSnapshot(pid: pid)
2057
+ let beforeCgSignature = deferRootDelta ? [] : cgRootSignature(pid: pid)
2058
+ let beforeFrontmostPid = deferRootDelta ? nil : NSWorkspace.shared.frontmostApplication?.processIdentifier
2059
+ let beforeSheetCount = windowElement(pid: pid, windowId: record.windowId).map { sheetElements(of: $0).count } ?? 0
2060
+ let beforeFocusedWindow = focusedWindowSummary(pid: pid)
2061
+ let beforeValue: String?
2062
+ let beforeSelected: String?
2063
+ func finish(_ response: [String: Any]) -> [String: Any] {
2064
+ if deferRootDelta { return response }
2065
+ return attachRootDelta(to: response, before: beforeRootSnapshot, beforeFrontmostPid: beforeFrontmostPid, pid: pid, eventsLive: eventsLive, eventCursor: eventCursor, beforeCgSignature: beforeCgSignature)
2066
+ }
2067
+
2068
+ if let ref = target["ref"] as? String {
2069
+ var refound = false
2070
+ let cached = refStore.element(for: ref)
2071
+ let cachedIsLive = cached.map {
2072
+ stringAttribute($0, attribute: kAXRoleAttribute as CFString) != nil && frameForElement($0) != nil
2073
+ } ?? false
2074
+ let resolved: AXUIElement?
2075
+ if cachedIsLive {
2076
+ resolved = cached
2077
+ } else {
2078
+ refound = true
2079
+ resolved = refindElement(ref: ref, pid: pid, windowId: record.windowId)
2080
+ }
2081
+ guard let stored = resolved else {
2082
+ throw BridgeFailure(message: "Element reference is stale", code: "stale_ref")
2083
+ }
2084
+ if refound { performed["refound"] = true }
2085
+ element = stored
2086
+ beforeValue = stringAttribute(stored, attribute: kAXValueAttribute as CFString)
2087
+ beforeSelected = stringAttribute(stored, attribute: kAXSelectedTextAttribute as CFString)
2088
+ } else if let xNumber = target["x"] as? NSNumber, let yNumber = target["y"] as? NSNumber {
2089
+ guard record.hasImage else {
2090
+ throw BridgeFailure(message: "Coordinate targeting is unavailable for this outline-only root", code: "coordinate_unavailable_for_root")
2091
+ }
2092
+ rawPoint = lookPoint(record: record, x: xNumber.doubleValue, y: yNumber.doubleValue)
2093
+ beforeValue = nil
2094
+ beforeSelected = nil
2095
+ } else {
2096
+ throw BridgeFailure(message: "act target must include ref or x/y", code: "invalid_args")
2097
+ }
2098
+
2099
+ func coordinatePoint() throws -> CGPoint {
2100
+ if let element, let frame = frameForElement(element) {
2101
+ return CGPoint(x: frame.midX, y: frame.midY)
2102
+ }
2103
+ if let rawPoint { return rawPoint }
2104
+ throw BridgeFailure(message: "No coordinate grounding is available", code: "coordinate_unavailable")
2105
+ }
2106
+
2107
+ /// `viaAx = true`:这一下是用 AX 直接按压完成的,压根没发物理事件 —— 无论 delivery 名义上
2108
+ /// 是什么,系统指针都没动过,所以虚拟光标该画。否则前台策略下用 AX 打成的那一下毫无反馈。
2109
+ func animateCursor(at point: CGPoint, viaAx: Bool = false) {
2110
+ guard supportsAgentCursor,
2111
+ (request["cursorOverlay"] as? Bool ?? true),
2112
+ viaAx || (delivery == "pid" && policy != "ax_only"),
2113
+ ["press", "click", "moveMouse", "scroll", "drag"].contains(action)
2114
+ else { return }
2115
+ Task { @MainActor in AgentCursor.shared.animate(to: point, above: record.windowId) }
2116
+ }
2117
+
2118
+ /// Tangu 新增:**纯坐标点击先做一次 AX 命中测试**。命中可按压元素就直接 AXPress ——
2119
+ /// 不抢前台、不动真鼠标、画虚拟光标。命不中再照上游回落物理事件。
2120
+ ///
2121
+ /// 为什么需要它:物理事件那条后台路(CGEventPostToPid)对非 key 窗口的 App 会被直接丢掉,
2122
+ /// 所以上游对坐标点击一律直接判 `needsForeground`(actions.ts)走 HID 前台。于是"看着截图
2123
+ /// 盲点"必然抢前台 —— 这是用户实报的症结。AX 按压是唯一能在后台真正落下去的点击。
2124
+ ///
2125
+ /// ⚠️命中测试的根**必须是目标 App**,不能用 systemWide:目标窗口在后台时,那个屏幕坐标上压着的
2126
+ /// 是别人的窗口,systemWide 会把别人窗口里的控件返回给我们 —— 那一下就点到别的 App 上了。
2127
+ /// 再加一道窗口子树校验:同一个 App 开多个窗口时,同样会点错窗口。
2128
+ func backgroundPressAtPoint(_ point: CGPoint) -> Bool {
2129
+ guard action == "press" || action == "click" else { return false }
2130
+ // 右键/中键/双击的语义 AXPress 表达不了,老实走物理事件。
2131
+ guard (params["button"] as? String ?? "left") == "left",
2132
+ ((params["clickCount"] as? NSNumber)?.intValue ?? 1) <= 1
2133
+ else { return false }
2134
+ var hitRef: AXUIElement?
2135
+ guard AXUIElementCopyElementAtPosition(AXUIElementCreateApplication(pid), Float(point.x), Float(point.y), &hitRef) == .success,
2136
+ let hit = hitRef
2137
+ else { return false }
2138
+ var hitPid: pid_t = 0
2139
+ guard AXUIElementGetPid(hit, &hitPid) == .success, hitPid == pid else { return false }
2140
+ guard let windowRoot = windowElement(pid: pid, windowId: record.windowId),
2141
+ sameElement(hit, windowRoot) || isElement(hit, descendantOf: windowRoot)
2142
+ else { return false }
2143
+ // 与 ref 路径同一条规则:网页内容与文本框要的是真实指针焦点,AXPress 打不动。
2144
+ let hitRole = stringAttribute(hit, attribute: kAXRoleAttribute as CFString) ?? ""
2145
+ let textRoles: Set<String> = ["AXTextField", "AXTextArea", "AXTextView", "AXSearchField", "AXComboBox", "AXEditableText", "AXSecureTextField"]
2146
+ guard !hasAncestorRole(hit, role: "AXWebArea"), !textRoles.contains(hitRole) else { return false }
2147
+ guard supportsAction(hit, action: kAXPressAction as CFString) else { return false }
2148
+ guard AXUIElementPerformAction(hit, kAXPressAction as CFString) == .success else { return false }
2149
+ performed["grounding"] = "hit_test"
2150
+ performed["delivery"] = "ax"
2151
+ performed["hitRole"] = hitRole
2152
+ // 坐标点击没有 before/after 值可比,和后台物理事件一样无从自证 —— 如实说要调用方自己查。
2153
+ performed["verification"] = "caller_required"
2154
+ animateCursor(at: point, viaAx: true)
2155
+ return true
2156
+ }
2157
+
2158
+ func focusTargetForPhysicalInput() {
2159
+ guard delivery == "hid" else { return }
2160
+ if let app = NSRunningApplication(processIdentifier: pid), !app.isActive {
2161
+ ForegroundActivity.shared.physicalInput()
2162
+ performed["activated"] = app.activate()
2163
+ }
2164
+ if let window = windowElement(pid: pid, windowId: record.windowId) {
2165
+ _ = AXUIElementSetAttributeValue(window, kAXMainAttribute as CFString, kCFBooleanTrue)
2166
+ _ = AXUIElementSetAttributeValue(window, kAXFocusedAttribute as CFString, kCFBooleanTrue)
2167
+ performed["raised"] = AXUIElementPerformAction(window, kAXRaiseAction as CFString) == .success
2168
+ }
2169
+ usleep(20_000)
2170
+ }
2171
+
2172
+ func preflight(_ point: CGPoint) throws {
2173
+ guard let element else { preflightCapsUnknown = true; return }
2174
+ for attempt in 0..<4 {
2175
+ guard let hit = hitTestElement(at: point) else { preflightCapsUnknown = true; return }
2176
+ if sameElement(hit, element) || isElement(hit, descendantOf: element) || isElement(element, descendantOf: hit) { return }
2177
+ let role = stringAttribute(hit, attribute: kAXRoleAttribute as CFString) ?? ""
2178
+ if role == "AXWindow" || role == "AXApplication" { preflightCapsUnknown = true; return }
2179
+ if delivery == "hid" && attempt < 3 {
2180
+ focusTargetForPhysicalInput()
2181
+ usleep(20_000)
2182
+ continue
2183
+ }
2184
+ throw BridgeFailure(message: "Target is occluded by \(payloadNode(element: hit))", code: "occluded_target")
2185
+ }
2186
+ }
2187
+
2188
+ func executeCoordinates(_ point: CGPoint) throws {
2189
+ guard element != nil || record.hasImage else {
2190
+ throw BridgeFailure(message: "Coordinate grounding is unavailable for this outline-only root", code: "coordinate_unavailable_for_root")
2191
+ }
2192
+ // 纯坐标点击先试后台 AX 按压。带 element 的路径不试:那边刚判过 AXPress 支不支持,
2193
+ // 走到这里就是已经失败过一次,再试一遍只是重复同一个失败。
2194
+ if element == nil, backgroundPressAtPoint(point) { return }
2195
+ performed["grounding"] = "coordinates"
2196
+ if delivery == "pid" { performed["verification"] = "caller_required" }
2197
+ acquirePhysicalInputIfNeeded()
2198
+ focusTargetForPhysicalInput()
2199
+ if delivery == "hid" { try preflight(point) }
2200
+ switch action {
2201
+ case "press", "click":
2202
+ animateCursor(at: point)
2203
+ try postMouseClick(at: point, pid: pid, button: mouseButton(params["button"] as? String ?? "left"), clickCount: max(1, min(3, (params["clickCount"] as? NSNumber)?.intValue ?? 1)), delivery: delivery)
2204
+ case "moveMouse":
2205
+ animateCursor(at: point)
2206
+ try postMouseMove(to: point, pid: pid, delivery: delivery)
2207
+ case "scroll":
2208
+ animateCursor(at: point)
2209
+ try postScrollWheel(at: point, deltaX: (params["scrollX"] as? NSNumber)?.intValue ?? 0, deltaY: (params["scrollY"] as? NSNumber)?.intValue ?? 0, pid: pid, delivery: delivery)
2210
+ case "drag":
2211
+ guard let rawPath = params["path"] as? [[String: Any]], rawPath.count >= 2 else {
2212
+ throw BridgeFailure(message: "drag requires path", code: "invalid_args")
2213
+ }
2214
+ let points = try rawPath.map { raw -> CGPoint in
2215
+ guard let x = (raw["x"] as? NSNumber)?.doubleValue, let y = (raw["y"] as? NSNumber)?.doubleValue else {
2216
+ throw BridgeFailure(message: "drag path entries require x and y", code: "invalid_args")
2217
+ }
2218
+ return lookPoint(record: record, x: x, y: y)
2219
+ }
2220
+ animateCursor(at: point)
2221
+ try postMouseDrag(points: points, pid: pid, delivery: delivery)
2222
+ default:
2223
+ throw BridgeFailure(message: "Action \(action) cannot use coordinate grounding", code: "invalid_args")
2224
+ }
2225
+ }
2226
+
2227
+ func refreshElement() -> AXUIElement? {
2228
+ guard let ref = target["ref"] as? String,
2229
+ let refreshed = refindElement(ref: ref, pid: pid, windowId: record.windowId)
2230
+ else { return nil }
2231
+ element = refreshed
2232
+ performed["refound"] = true
2233
+ return refreshed
2234
+ }
2235
+
2236
+ if let element, action == "press" || action == "click" {
2237
+ let elementRole = stringAttribute(element, attribute: kAXRoleAttribute as CFString) ?? ""
2238
+ let textRoles: Set<String> = ["AXTextField", "AXTextArea", "AXTextView", "AXSearchField", "AXComboBox", "AXEditableText", "AXSecureTextField"]
2239
+ let requiresPointerFocus = hasAncestorRole(element, role: "AXWebArea") || textRoles.contains(elementRole)
2240
+ if requiresPointerFocus && policy != "ax_only" {
2241
+ if policy == "foreground" {
2242
+ try executeCoordinates(coordinatePoint())
2243
+ } else {
2244
+ throw BridgeFailure(message: "Web content requires pointer input", code: "foreground_required")
2245
+ }
2246
+ } else if supportsAction(element, action: kAXPressAction as CFString) {
2247
+ let cursorPoint = try? coordinatePoint()
2248
+ var status = AXUIElementPerformAction(element, kAXPressAction as CFString)
2249
+ if status != .success, let refreshed = refreshElement(), supportsAction(refreshed, action: kAXPressAction as CFString) {
2250
+ status = AXUIElementPerformAction(refreshed, kAXPressAction as CFString)
2251
+ }
2252
+ if status == .success {
2253
+ performed["grounding"] = "description"
2254
+ performed["delivery"] = "ax"
2255
+ if let cursorPoint { animateCursor(at: cursorPoint, viaAx: true) }
2256
+ } else {
2257
+ try executeCoordinates(coordinatePoint())
2258
+ }
2259
+ } else {
2260
+ try executeCoordinates(coordinatePoint())
2261
+ }
2262
+ } else if let element, action == "setText" {
2263
+ let text = params["text"] as? String ?? ""
2264
+ if hasAncestorRole(element, role: "AXWebArea") {
2265
+ acquirePhysicalInputIfNeeded()
2266
+ focusTargetForPhysicalInput()
2267
+ _ = AXUIElementSetAttributeValue(element, kAXFocusedAttribute as CFString, kCFBooleanTrue)
2268
+ let currentValue = stringAttribute(element, attribute: kAXValueAttribute as CFString) ?? ""
2269
+ var range = CFRange(location: 0, length: (currentValue as NSString).length)
2270
+ let selected = AXValueCreate(.cfRange, &range).map {
2271
+ AXUIElementSetAttributeValue(element, kAXSelectedTextRangeAttribute as CFString, $0) == .success
2272
+ } ?? false
2273
+ if !selected {
2274
+ try postKeyPress(keys: ["cmd", "a"], pid: pid, delivery: delivery)
2275
+ usleep(20_000)
2276
+ }
2277
+ try postAtomicUnicodeText(text, pid: pid, delivery: delivery)
2278
+ usleep(40_000)
2279
+ let verificationElement = refreshElement() ?? element
2280
+ let value = stringAttribute(verificationElement, attribute: kAXValueAttribute as CFString) ?? ""
2281
+ performed["grounding"] = "keyboard-events"
2282
+ performed["delivery"] = delivery
2283
+ performed["selectionGrounding"] = selected ? "ax" : "keyboard"
2284
+ return finish(["outcome": value == text ? "worked" : "didnt", "performed": performed, "evidence": ["value": value]])
2285
+ }
2286
+ var targetElement = element
2287
+ var status = AXUIElementSetAttributeValue(targetElement, kAXValueAttribute as CFString, text as CFTypeRef)
2288
+ if status != .success, let refreshed = refreshElement() {
2289
+ targetElement = refreshed
2290
+ status = AXUIElementSetAttributeValue(targetElement, kAXValueAttribute as CFString, text as CFTypeRef)
2291
+ }
2292
+ if status == .success {
2293
+ performed["grounding"] = "description"
2294
+ performed["delivery"] = "ax"
2295
+ let value = stringAttribute(targetElement, attribute: kAXValueAttribute as CFString) ?? ""
2296
+ if value != text && policy != "foreground" {
2297
+ throw BridgeFailure(message: "The background accessibility value write was accepted but did not take effect", code: "foreground_required")
2298
+ }
2299
+ return finish(["outcome": value == text ? "worked" : "didnt", "performed": performed, "evidence": ["value": value]])
2300
+ }
2301
+ try executeCoordinates(coordinatePoint())
2302
+ } else if action == "typeText" {
2303
+ let preserveFocus = params["preserveFocus"] as? Bool ?? false
2304
+ if let element {
2305
+ let focused = AXUIElementSetAttributeValue(element, kAXFocusedAttribute as CFString, kCFBooleanTrue)
2306
+ if focused == .success { performed["focused"] = true }
2307
+ }
2308
+ acquirePhysicalInputIfNeeded()
2309
+ if delivery == "hid" && !preserveFocus { focusTargetForPhysicalInput() }
2310
+ let text = params["text"] as? String ?? ""
2311
+ try postUnicodeText(text, pid: pid, delivery: delivery)
2312
+ performed["grounding"] = "coordinates"
2313
+ if let element, !text.isEmpty {
2314
+ usleep(30_000)
2315
+ let afterValue = stringAttribute(element, attribute: kAXValueAttribute as CFString) ?? ""
2316
+ let changed = afterValue != (beforeValue ?? "")
2317
+ return finish(["outcome": changed ? "worked" : "didnt", "performed": performed, "evidence": ["value": afterValue, "valueChanged": changed]])
2318
+ }
2319
+ } else if action == "keypress" {
2320
+ let preserveFocus = params["preserveFocus"] as? Bool ?? false
2321
+ guard let keys = params["keys"] as? [String], !keys.isEmpty else {
2322
+ throw BridgeFailure(message: "keypress requires keys", code: "invalid_args")
2323
+ }
2324
+ if let element {
2325
+ let focused = AXUIElementSetAttributeValue(element, kAXFocusedAttribute as CFString, kCFBooleanTrue)
2326
+ if focused == .success { performed["focused"] = true }
2327
+ let normalizedKeys = keys.map { $0.trimmingCharacters(in: .whitespacesAndNewlines).lowercased() }
2328
+ if normalizedKeys.count == 2,
2329
+ normalizedKeys.last == "a",
2330
+ ["cmd", "command", "meta"].contains(normalizedKeys.first ?? ""),
2331
+ let value = stringAttribute(element, attribute: kAXValueAttribute as CFString)
2332
+ {
2333
+ var range = CFRange(location: 0, length: (value as NSString).length)
2334
+ if let selection = AXValueCreate(.cfRange, &range),
2335
+ AXUIElementSetAttributeValue(element, kAXSelectedTextRangeAttribute as CFString, selection) == .success
2336
+ {
2337
+ performed["selectionGrounding"] = "ax"
2338
+ }
2339
+ }
2340
+ }
2341
+ acquirePhysicalInputIfNeeded()
2342
+ if delivery == "hid" && !preserveFocus { focusTargetForPhysicalInput() }
2343
+ try postKeyPress(keys: keys, pid: pid, delivery: delivery)
2344
+ performed["grounding"] = "coordinates"
2345
+ } else if let element, action == "scroll" {
2346
+ let cursorPoint = try? coordinatePoint()
2347
+ let before = scrollPositionSignature(element)
2348
+ let result = performScrollActionOrAncestor(startingAt: element, targetPid: pid, scrollX: (params["scrollX"] as? NSNumber)?.intValue ?? 0, scrollY: (params["scrollY"] as? NSNumber)?.intValue ?? 0, steps: 1)
2349
+ if (result["scrolled"] as? Bool) == true {
2350
+ performed["grounding"] = "description"
2351
+ performed["delivery"] = "ax"
2352
+ if let cursorPoint { animateCursor(at: cursorPoint) }
2353
+ let after = scrollPositionSignature(element)
2354
+ return finish(["outcome": before != after ? "worked" : "unknown", "performed": performed])
2355
+ }
2356
+ try executeCoordinates(coordinatePoint())
2357
+ } else {
2358
+ try executeCoordinates(coordinatePoint())
2359
+ }
2360
+
2361
+ let afterSheetCount = windowElement(pid: pid, windowId: record.windowId).map { sheetElements(of: $0).count } ?? beforeSheetCount
2362
+ let windowChanged = beforeFocusedWindow != focusedWindowSummary(pid: pid) || beforeSheetCount != afterSheetCount
2363
+ var outcome = preflightCapsUnknown ? "unknown" : "unknown"
2364
+ var evidence: [String: Any] = [:]
2365
+ if let element, action == "press" || action == "click" {
2366
+ let afterValue = stringAttribute(element, attribute: kAXValueAttribute as CFString)
2367
+ let afterSelected = stringAttribute(element, attribute: kAXSelectedTextAttribute as CFString)
2368
+ if beforeValue != afterValue || beforeSelected != afterSelected || windowChanged {
2369
+ outcome = "worked"
2370
+ }
2371
+ } else if windowChanged {
2372
+ outcome = "worked"
2373
+ }
2374
+ if windowChanged { evidence["windowChanged"] = true }
2375
+ var response: [String: Any] = ["outcome": outcome, "performed": performed]
2376
+ if !evidence.isEmpty { response["evidence"] = evidence }
2377
+ return finish(response)
2378
+ }
2379
+
2380
+ private func actBatch(_ request: [String: Any]) throws -> [String: Any] {
2381
+ guard let actions = request["actions"] as? [[String: Any]], !actions.isEmpty, actions.count <= 20 else {
2382
+ throw BridgeFailure(message: "actBatch requires 1...20 actions", code: "invalid_args")
2383
+ }
2384
+ let pid = Int32(try intArg(actions[0], "pid"))
2385
+ let lookId = try stringArg(actions[0], "lookId")
2386
+ guard actions.allSatisfy({ ($0["pid"] as? NSNumber)?.int32Value == pid }) else {
2387
+ throw BridgeFailure(message: "actBatch actions must target one pid", code: "invalid_args")
2388
+ }
2389
+ guard actions.allSatisfy({ ($0["lookId"] as? String) == lookId }) else {
2390
+ throw BridgeFailure(message: "actBatch actions must belong to one look", code: "invalid_args")
2391
+ }
2392
+ let eventsLive = ensureRootObserver(pid: pid)
2393
+ let eventCursor = eventsLive ? rootEventCursor(pid: pid) : 0
2394
+ let beforeRootSnapshot = rootMetadataSnapshot(pid: pid)
2395
+ let beforeCgSignature = cgRootSignature(pid: pid)
2396
+ let beforeFrontmostPid = NSWorkspace.shared.frontmostApplication?.processIdentifier
2397
+ let mayUsePhysicalInput = actions.contains { action in
2398
+ (optionalStringArg(action, "policy") ?? "default") != "ax_only"
2399
+ }
2400
+ if mayUsePhysicalInput { physicalInputLock.lock(); ForegroundActivity.shared.enter() }
2401
+ defer { if mayUsePhysicalInput { ForegroundActivity.shared.leave(); physicalInputLock.unlock() } }
2402
+
2403
+ var steps: [[String: Any]] = []
2404
+ var stoppedAt: Int?
2405
+ for (index, action) in actions.enumerated() {
2406
+ var deferred = action
2407
+ deferred["deferRootDelta"] = true
2408
+ do {
2409
+ let step = try act(deferred)
2410
+ steps.append(step)
2411
+ if (step["outcome"] as? String) == "didnt" { stoppedAt = index; break }
2412
+ } catch let failure as BridgeFailure {
2413
+ steps.append(["outcome": "didnt", "error": ["code": failure.code, "message": failure.message]])
2414
+ stoppedAt = index
2415
+ break
2416
+ }
2417
+ }
2418
+ let outcomes = steps.compactMap { $0["outcome"] as? String }
2419
+ let outcome = outcomes.contains("didnt") ? "didnt" : (outcomes.contains("unknown") ? "unknown" : "worked")
2420
+ var response: [String: Any] = ["outcome": outcome, "performed": ["transaction": true, "actionCount": steps.count], "steps": steps]
2421
+ if let stoppedAt { response["stoppedAt"] = stoppedAt }
2422
+ return attachRootDelta(to: response, before: beforeRootSnapshot, beforeFrontmostPid: beforeFrontmostPid, pid: pid, eventsLive: eventsLive, eventCursor: eventCursor, beforeCgSignature: beforeCgSignature)
2423
+ }
2424
+
2425
+ private func rootIdentity(_ root: [String: Any]) -> String {
2426
+ if let windowId = root["windowId"] as? Int, windowId > 0 { return "window:\(windowId)" }
2427
+ // AXUIElement CFEqual/CFHash are not stable after re-enumeration for all
2428
+ // transient roots; use the metadata tuple that comes from the cheap pass.
2429
+ let kind = root["kind"] as? String ?? "window"
2430
+ let title = root["title"] as? String ?? ""
2431
+ let role = root["role"] as? String ?? ""
2432
+ let frame = root["framePoints"] as? [String: Any] ?? [:]
2433
+ let x = Int((frame["x"] as? NSNumber)?.doubleValue ?? 0)
2434
+ let y = Int((frame["y"] as? NSNumber)?.doubleValue ?? 0)
2435
+ let w = Int((frame["w"] as? NSNumber)?.doubleValue ?? 0)
2436
+ let h = Int((frame["h"] as? NSNumber)?.doubleValue ?? 0)
2437
+ return "meta:\(kind):\(role):\(title):\(x),\(y),\(w),\(h)"
2438
+ }
2439
+
2440
+ private func rootMetadataSnapshot(pid: Int32) -> [String: [String: Any]] {
2441
+ let roots = ((try? listRoots(pid: pid)["roots"] as? [[String: Any]]) ?? [])
2442
+ return Dictionary(uniqueKeysWithValues: roots.map { (rootIdentity($0), $0) })
2443
+ }
2444
+
2445
+ private func rootDelta(before: [String: [String: Any]], beforeFrontmostPid: pid_t?, pid: Int32) -> [[String: Any]] {
2446
+ let after = rootMetadataSnapshot(pid: pid)
2447
+ var delta: [[String: Any]] = []
2448
+ for (key, root) in after where before[key] == nil {
2449
+ delta.append(rootDeltaItem(change: "appeared", root: root, pid: pid))
2450
+ }
2451
+ for (key, root) in before where after[key] == nil {
2452
+ delta.append(rootDeltaItem(change: "closed", root: root, pid: pid))
2453
+ }
2454
+ for (key, root) in after {
2455
+ if (root["isFocused"] as? Bool) == true && (before[key]?["isFocused"] as? Bool) != true {
2456
+ delta.append(rootDeltaItem(change: "focused", root: root, pid: pid))
2457
+ }
2458
+ }
2459
+ if let beforeFrontmostPid, beforeFrontmostPid != NSWorkspace.shared.frontmostApplication?.processIdentifier {
2460
+ if let frontmost = NSWorkspace.shared.frontmostApplication {
2461
+ delta.append(["change": "focused", "kind": "app", "title": frontmost.localizedName ?? processName(pid: frontmost.processIdentifier) ?? "Unknown App", "pid": Int(frontmost.processIdentifier)])
2462
+ }
2463
+ }
2464
+ return delta
2465
+ }
2466
+
2467
+ private func rootDeltaItem(change: String, root: [String: Any], pid: Int32) -> [String: Any] {
2468
+ var item: [String: Any] = ["change": change, "kind": root["kind"] as? String ?? "window", "title": root["title"] as? String ?? "", "pid": root["pid"] as? Int ?? Int(pid)]
2469
+ if let isModal = root["isModal"] as? Bool { item["isModal"] = isModal }
2470
+ if let metadata = root["metadata"] as? [String: Any] { item["metadata"] = metadata }
2471
+ if let ref = root["rootRef"] as? String ?? root["windowRef"] as? String { item["ref"] = ref }
2472
+ return item
2473
+ }
2474
+
2475
+ // The AX snapshot diff is authoritative: macOS emits no AXObserver
2476
+ // notification at all when a sheet appears (verified on macOS 26), so
2477
+ // events can only accelerate the decision, never make it. A cheap
2478
+ // CGWindowList id-set poll detects real-window appearance/closure early;
2479
+ // the AX diff runs once at the first signal or at timeout.
2480
+ private func attachRootDelta(to response: [String: Any], before: [String: [String: Any]], beforeFrontmostPid: pid_t?, pid: Int32, eventsLive: Bool, eventCursor: UInt64, beforeCgSignature: Set<UInt32>) -> [String: Any] {
2481
+ var output = response
2482
+ var performed = output["performed"] as? [String: Any] ?? [:]
2483
+
2484
+ // AXUIElementDestroyed is deliberately not a signal: it fires for every
2485
+ // rebuilt list row; a genuinely closed root also leaves the CG set.
2486
+ let signalNotifications: Set<String> = ["AXWindowCreated", "AXSheetCreated", "AXMenuOpened", "AXMenuClosed", "AXFocusedWindowChanged"]
2487
+ var source = "snapshot"
2488
+ let deadline = Date().addingTimeInterval(0.40)
2489
+ while Date() < deadline {
2490
+ if cgRootSignature(pid: pid) != beforeCgSignature { source = "cg-poll"; break }
2491
+ if let beforeFrontmostPid, NSWorkspace.shared.frontmostApplication?.processIdentifier != beforeFrontmostPid { source = "cg-poll"; break }
2492
+ if eventsLive && rootEvents(pid: pid, since: eventCursor).contains(where: { signalNotifications.contains($0.notification) }) { source = "events"; break }
2493
+ usleep(30_000)
2494
+ }
2495
+
2496
+ var delta = rootDelta(before: before, beforeFrontmostPid: beforeFrontmostPid, pid: pid)
2497
+ if delta.isEmpty && source != "snapshot" {
2498
+ // A signal fired but the AX tree can lag the CG window; give it a
2499
+ // bounded moment to catch up.
2500
+ for _ in 0..<3 where delta.isEmpty {
2501
+ usleep(80_000)
2502
+ delta = rootDelta(before: before, beforeFrontmostPid: beforeFrontmostPid, pid: pid)
2503
+ }
2504
+ }
2505
+ performed["deltaSource"] = source
2506
+ output["performed"] = performed
2507
+ if !delta.isEmpty { output["rootDelta"] = delta }
2508
+ return output
2509
+ }
2510
+
2511
+ private func focusedWindowSummary(pid: Int32) -> String {
2512
+ let app = AXUIElementCreateApplication(pid)
2513
+ guard let element = copyAttribute(app, attribute: kAXFocusedWindowAttribute as CFString).flatMap(asAXElement) else { return "none" }
2514
+ let role = stringAttribute(element, attribute: kAXRoleAttribute as CFString) ?? ""
2515
+ let title = stringAttribute(element, attribute: kAXTitleAttribute as CFString) ?? ""
2516
+ return "\(role):\(title)"
2517
+ }
2518
+
2519
+ private func scrollPositionSignature(_ element: AXUIElement) -> String {
2520
+ let names: [CFString] = ["AXVerticalScrollBar" as CFString, "AXHorizontalScrollBar" as CFString, kAXValueAttribute as CFString]
2521
+ return names.map { String(describing: copyAttribute(element, attribute: $0) ?? "" as CFTypeRef) }.joined(separator: "|")
2522
+ }
2523
+
2524
+ private func axWaitFor(_ request: [String: Any]) throws -> [String: Any] {
2525
+ let pid = Int32(try intArg(request, "pid"))
2526
+ ensureEnhancedAccessibility(pid: pid)
2527
+ let windowId = optionalIntArg(request, "windowId").map { UInt32($0) }
2528
+ let windowRef = optionalStringArg(request, "windowRef")
2529
+ let role = optionalStringArg(request, "role")?.trimmingCharacters(in: .whitespacesAndNewlines)
2530
+ let text = optionalStringArg(request, "text")?.trimmingCharacters(in: .whitespacesAndNewlines).lowercased()
2531
+ let expectedValue = optionalStringArg(request, "value")?.trimmingCharacters(in: .whitespacesAndNewlines).lowercased()
2532
+ let waitForGone = boolArg(request, "gone") ?? false
2533
+ let scopeExact = boolArg(request, "scopeExact") ?? false
2534
+ let timeoutMs = max(100, min(60_000, optionalIntArg(request, "timeoutMs") ?? 10_000))
2535
+ let deadline = Date().addingTimeInterval(Double(timeoutMs) / 1000.0)
2536
+ guard role?.isEmpty == false || text?.isEmpty == false || expectedValue?.isEmpty == false else {
2537
+ throw BridgeFailure(message: "axWaitFor requires role, text, or value", code: "invalid_args")
2538
+ }
2539
+ guard let window = windowElement(pid: pid, windowId: windowId, windowRef: windowRef) else {
2540
+ return ["found": false, "reason": "window_not_found"]
2541
+ }
2542
+ let rootElement: AXUIElement
2543
+ if let scopeRef = optionalStringArg(request, "scopeRef") {
2544
+ guard let scoped = refStore.element(for: scopeRef), isElement(scoped, descendantOf: window) else {
2545
+ throw BridgeFailure(message: "Condition scope ref is stale or outside the target root", code: "element_ref_invalid")
2546
+ }
2547
+ rootElement = scoped
2548
+ } else {
2549
+ rootElement = window
2550
+ }
2551
+ _ = ensureRootObserver(pid: pid)
2552
+
2553
+ func matches(_ element: AXUIElement) -> Bool {
2554
+ let candidateRole = self.stringAttribute(element, attribute: kAXRoleAttribute as CFString) ?? ""
2555
+ if let role, !role.isEmpty, candidateRole != role { return false }
2556
+ if let text, !text.isEmpty {
2557
+ let subrole = self.stringAttribute(element, attribute: kAXSubroleAttribute as CFString) ?? ""
2558
+ let haystack = [
2559
+ self.stringAttribute(element, attribute: kAXTitleAttribute as CFString) ?? "",
2560
+ self.stringAttribute(element, attribute: kAXDescriptionAttribute as CFString) ?? "",
2561
+ self.displayValue(element, role: candidateRole, subrole: subrole),
2562
+ ].joined(separator: "\n").lowercased()
2563
+ if !haystack.contains(text) { return false }
2564
+ }
2565
+ if let expectedValue, !expectedValue.isEmpty {
2566
+ let subrole = self.stringAttribute(element, attribute: kAXSubroleAttribute as CFString) ?? ""
2567
+ if self.displayValue(element, role: candidateRole, subrole: subrole).trimmingCharacters(in: .whitespacesAndNewlines).lowercased() != expectedValue { return false }
2568
+ }
2569
+ return true
2570
+ }
2571
+
2572
+ var lastCount = 0
2573
+ repeat {
2574
+ let changeGeneration = rootChangeGeneration(pid: pid)
2575
+ let collected = collectDescendantsWithContext(startingAt: rootElement, maxDepth: 12, maxNodes: 2000)
2576
+ let descendants = scopeExact ? Array(collected.prefix(1)) : collected
2577
+ lastCount = descendants.count
2578
+ if let match = descendants.first(where: { matches($0.element) }) {
2579
+ if waitForGone {
2580
+ waitForRootChange(pid: pid, since: changeGeneration, until: deadline)
2581
+ continue
2582
+ }
2583
+ let candidateRole = self.stringAttribute(match.element, attribute: kAXRoleAttribute as CFString) ?? ""
2584
+ let isBrowser = isBrowser(pid: pid)
2585
+ let containsWebArea = descendants.contains { self.stringAttribute($0.element, attribute: kAXRoleAttribute as CFString) == "AXWebArea" }
2586
+ return [
2587
+ "found": true,
2588
+ "target": self.elementPayload(
2589
+ element: match.element,
2590
+ key: "target",
2591
+ source: self.axSource(role: candidateRole, insideWebArea: match.insideWebArea, isBrowser: isBrowser, containsWebArea: containsWebArea)
2592
+ ),
2593
+ "nodeCount": lastCount,
2594
+ ]
2595
+ }
2596
+ if waitForGone {
2597
+ return ["found": true, "gone": true, "nodeCount": lastCount]
2598
+ }
2599
+ waitForRootChange(pid: pid, since: changeGeneration, until: deadline)
2600
+ } while Date() < deadline
2601
+
2602
+ return ["found": false, "timedOut": true, "nodeCount": lastCount]
2603
+ }
2604
+
2605
+ private func hitTestElement(at point: CGPoint) -> AXUIElement? {
2606
+ let systemWide = AXUIElementCreateSystemWide()
2607
+ var hitElement: AXUIElement?
2608
+ let status = AXUIElementCopyElementAtPosition(systemWide, Float(point.x), Float(point.y), &hitElement)
2609
+ guard status == .success, let hitElement else { return nil }
2610
+ return hitElement
2611
+ }
2612
+
2613
+ private let axScrollDownAction = "AXScrollDown" as CFString
2614
+ private let axScrollUpAction = "AXScrollUp" as CFString
2615
+ private let axScrollLeftAction = "AXScrollLeft" as CFString
2616
+ private let axScrollRightAction = "AXScrollRight" as CFString
2617
+
2618
+ private func scrollActionNames(scrollX: Int, scrollY: Int) -> [CFString] {
2619
+ var actions: [CFString] = []
2620
+ if scrollY > 0 { actions.append(axScrollDownAction) }
2621
+ if scrollY < 0 { actions.append(axScrollUpAction) }
2622
+ if scrollX > 0 { actions.append(axScrollRightAction) }
2623
+ if scrollX < 0 { actions.append(axScrollLeftAction) }
2624
+ return actions
2625
+ }
2626
+
2627
+ private func supportsAnyScrollAction(_ element: AXUIElement) -> Bool {
2628
+ let actions = Set(actionNames(element))
2629
+ return actions.contains(axScrollDownAction as String) || actions.contains(axScrollUpAction as String) || actions.contains(axScrollLeftAction as String) || actions.contains(axScrollRightAction as String)
2630
+ }
2631
+
2632
+ private func performScrollActionOrAncestor(startingAt element: AXUIElement, targetPid: Int32, scrollX: Int, scrollY: Int, steps: Int) -> [String: Any] {
2633
+ let actions = scrollActionNames(scrollX: scrollX, scrollY: scrollY)
2634
+ guard !actions.isEmpty else { return ["scrolled": false, "reason": "zero_delta"] }
2635
+ var current: AXUIElement? = element
2636
+ var depth = 0
2637
+
2638
+ while let candidate = current, depth < 10 {
2639
+ if let pid = pidForElement(candidate), pid != targetPid {
2640
+ return ["scrolled": false, "reason": "pid_mismatch", "ownerPid": Int(pid)]
2641
+ }
2642
+ var didScroll = false
2643
+ for _ in 0..<steps {
2644
+ for action in actions where supportsAction(candidate, action: action) {
2645
+ let status = AXUIElementPerformAction(candidate, action)
2646
+ if status == .success { didScroll = true }
2647
+ }
2648
+ }
2649
+ if didScroll { return ["scrolled": true] }
2650
+ current = parentElement(candidate)
2651
+ depth += 1
2652
+ }
2653
+
2654
+ return ["scrolled": false, "reason": "no_scroll_action"]
2655
+ }
2656
+
2657
+ private func performActionOrAncestor(startingAt element: AXUIElement, action: CFString, targetPid: Int32) -> [String: Any] {
2658
+ var current: AXUIElement? = element
2659
+ var depth = 0
2660
+
2661
+ while let candidate = current, depth < 10 {
2662
+ if let pid = pidForElement(candidate), pid != targetPid {
2663
+ return ["performed": false, "reason": "pid_mismatch", "ownerPid": Int(pid)]
2664
+ }
2665
+
2666
+ if supportsAction(candidate, action: action) {
2667
+ let actionStatus = AXUIElementPerformAction(candidate, action)
2668
+ if actionStatus == .success {
2669
+ return ["performed": true]
2670
+ }
2671
+ }
2672
+
2673
+ current = parentElement(candidate)
2674
+ depth += 1
2675
+ }
2676
+
2677
+ return ["performed": false, "reason": "no_matching_action"]
2678
+ }
2679
+
2680
+ private func windowElement(pid: Int32, windowId: UInt32?, windowRef: String? = nil) -> AXUIElement? {
2681
+ if let windowRef, let stored = refStore.window(for: windowRef) {
2682
+ AXUIElementSetMessagingTimeout(stored, 1.0)
2683
+ var ownerPid: pid_t = 0
2684
+ if AXUIElementGetPid(stored, &ownerPid) == .success, ownerPid == pid {
2685
+ return stored
2686
+ }
2687
+ }
2688
+
2689
+ let appElement = AXUIElementCreateApplication(pid)
2690
+ AXUIElementSetMessagingTimeout(appElement, 1.0)
2691
+ let windows = Array(axElementArray(appElement, attribute: kAXWindowsAttribute as CFString).prefix(128))
2692
+ guard !windows.isEmpty else { return nil }
2693
+ guard let windowId else {
2694
+ return windows.first
2695
+ }
2696
+ let candidates = cgWindowCandidates(pid: pid)
2697
+ let pairings = windowPairings(windows: windows, candidates: candidates)
2698
+ for window in windows {
2699
+ if pairings[ObjectIdentifier(window)]?.candidate?.windowId == windowId {
2700
+ return window
2701
+ }
2702
+ for sheet in sheetElements(of: window) {
2703
+ if bestCandidate(for: sheet, candidates: candidates)?.windowId == windowId {
2704
+ return sheet
2705
+ }
2706
+ }
2707
+ }
2708
+ return nil
2709
+ }
2710
+
2711
+ private func findDescendant(startingAt root: AXUIElement, maxDepth: Int, predicate: (AXUIElement) -> Bool) -> AXUIElement? {
2712
+ collectDescendants(startingAt: root, maxDepth: maxDepth).first(where: predicate)
2713
+ }
2714
+
2715
+ private func ensureEnhancedAccessibility(pid: Int32) {
2716
+ enhancedAccessibilityLock.lock()
2717
+ let inserted = enhancedAccessibilityPids.insert(pid).inserted
2718
+ enhancedAccessibilityLock.unlock()
2719
+ if !inserted { return }
2720
+ let appElement = AXUIElementCreateApplication(pid)
2721
+ AXUIElementSetMessagingTimeout(appElement, 0.25)
2722
+ let enhancedStatus = AXUIElementSetAttributeValue(appElement, "AXEnhancedUserInterface" as CFString, kCFBooleanTrue)
2723
+ let manualStatus = AXUIElementSetAttributeValue(appElement, "AXManualAccessibility" as CFString, kCFBooleanTrue)
2724
+ // Chromium-family apps often materialize web-content AX asynchronously
2725
+ // after these toggles. Pay a small one-time settle cost per pid so the
2726
+ // first tree walk is less likely to see browser chrome only.
2727
+ if isBrowser(pid: pid) && (enhancedStatus == .success || manualStatus == .success) {
2728
+ Thread.sleep(forTimeInterval: 0.35)
2729
+ }
2730
+ }
2731
+
2732
+ private func isBrowser(pid: Int32) -> Bool {
2733
+ let app = NSRunningApplication(processIdentifier: pid)
2734
+ if browserBundleIds.contains(app?.bundleIdentifier ?? "") { return true }
2735
+ let name = (app?.localizedName ?? processName(pid: pid) ?? "").lowercased()
2736
+ return ["chrome", "chromium", "brave", "edge", "vivaldi", "opera", "firefox", "helium"].contains { name.contains($0) }
2737
+ }
2738
+
2739
+ private func collectDescendants(startingAt root: AXUIElement, maxDepth: Int, maxNodes: Int = 5000) -> [AXUIElement] {
2740
+ collectDescendantsWithContext(startingAt: root, maxDepth: maxDepth, maxNodes: maxNodes).map(\.element)
2741
+ }
2742
+
2743
+ private func collectDescendantsWithContext(startingAt root: AXUIElement, maxDepth: Int, maxNodes: Int = 5000) -> [AXDescendant] {
2744
+ let nodeLimit = max(1, maxNodes)
2745
+ var queue: [(AXUIElement, Int, Bool, Bool)] = [(root, 0, false, true)]
2746
+ var seen = Set<ObjectIdentifier>()
2747
+ var index = 0
2748
+ var output: [AXDescendant] = []
2749
+ while index < queue.count && output.count < nodeLimit {
2750
+ let (element, depth, parentInsideWebArea, inheritedVisible) = queue[index]
2751
+ index += 1
2752
+ let identity = ObjectIdentifier(element)
2753
+ if seen.contains(identity) { continue }
2754
+ seen.insert(identity)
2755
+ let role = stringAttribute(element, attribute: kAXRoleAttribute as CFString) ?? ""
2756
+ let insideWebArea = parentInsideWebArea || role == "AXWebArea"
2757
+ output.append(AXDescendant(element: element, depth: depth, insideWebArea: insideWebArea, axVisible: inheritedVisible))
2758
+ if depth >= maxDepth { continue }
2759
+ let children = axElementArray(element, attribute: kAXChildrenAttribute as CFString)
2760
+ let visibleChildren = visibleAXChildren(element)
2761
+ for child in children {
2762
+ if queue.count >= nodeLimit { break }
2763
+ let childVisible = inheritedVisible && (visibleChildren.map { set in set.contains { self.sameElement($0, child) } } ?? true)
2764
+ queue.append((child, depth + 1, insideWebArea, childVisible))
2765
+ }
2766
+ }
2767
+ return output
2768
+ }
2769
+
2770
+ private func visibleAXChildren(_ element: AXUIElement) -> [AXUIElement]? {
2771
+ let attributes: [CFString] = [
2772
+ kAXVisibleChildrenAttribute as CFString,
2773
+ kAXVisibleRowsAttribute as CFString,
2774
+ kAXVisibleColumnsAttribute as CFString,
2775
+ kAXVisibleCellsAttribute as CFString,
2776
+ ]
2777
+ let visible = attributes.flatMap { axElementArray(element, attribute: $0) }
2778
+ return visible.isEmpty ? nil : visible
2779
+ }
2780
+
2781
+ private func insideWebAreaMap(_ descendants: [AXDescendant]) -> [ObjectIdentifier: Bool] {
2782
+ var output: [ObjectIdentifier: Bool] = [:]
2783
+ for descendant in descendants {
2784
+ let key = ObjectIdentifier(descendant.element)
2785
+ output[key] = (output[key] ?? false) || descendant.insideWebArea
2786
+ }
2787
+ return output
2788
+ }
2789
+
2790
+ private func axSource(role: String, insideWebArea: Bool, isBrowser: Bool, containsWebArea: Bool) -> String {
2791
+ if insideWebArea || role == "AXWebArea" { return "web_content_ax" }
2792
+ if isBrowser || containsWebArea { return "browser_chrome_ax" }
2793
+ return "desktop_ax"
2794
+ }
2795
+
2796
+ private func frameForElement(_ element: AXUIElement) -> CGRect? {
2797
+ let origin = pointAttribute(element, attribute: kAXPositionAttribute as CFString)
2798
+ let size = sizeAttribute(element, attribute: kAXSizeAttribute as CFString)
2799
+ guard let origin, let size, size.width > 0, size.height > 0 else { return nil }
2800
+ return CGRect(origin: origin, size: size)
2801
+ }
2802
+
2803
+ private func elementPayload(element: AXUIElement, key: String, score: Double? = nil, source: String? = nil, axVisible: Bool = true) -> [String: Any] {
2804
+ let role = stringAttribute(element, attribute: kAXRoleAttribute as CFString) ?? ""
2805
+ let subrole = stringAttribute(element, attribute: kAXSubroleAttribute as CFString) ?? ""
2806
+ let title = stringAttribute(element, attribute: kAXTitleAttribute as CFString) ?? ""
2807
+ let description = stringAttribute(element, attribute: kAXDescriptionAttribute as CFString) ?? ""
2808
+ let identifier = stringAttribute(element, attribute: "AXIdentifier" as CFString) ?? ""
2809
+ let value = displayValue(element, role: role, subrole: subrole)
2810
+ let frame = frameForElement(element)
2811
+ let parentFrame = copyAttribute(element, attribute: kAXParentAttribute as CFString).flatMap(asAXElement).flatMap(frameForElement)
2812
+ let centerX = frame.map { $0.midX } ?? 0
2813
+ let centerY = frame.map { $0.midY } ?? 0
2814
+ var valueSettable = DarwinBoolean(false)
2815
+ let valueStatus = AXUIElementIsAttributeSettable(element, kAXValueAttribute as CFString, &valueSettable)
2816
+ var focusedSettable = DarwinBoolean(false)
2817
+ let focusedStatus = AXUIElementIsAttributeSettable(element, kAXFocusedAttribute as CFString, &focusedSettable)
2818
+ let actions = actionNames(element)
2819
+ let canSetValue = valueStatus == .success && valueSettable.boolValue
2820
+ let textRoles: Set<String> = [
2821
+ "AXTextField", "AXTextArea", "AXTextView", "AXSearchField", "AXComboBox", "AXEditableText", "AXSecureTextField",
2822
+ ]
2823
+ var payload: [String: Any] = [
2824
+ key: true,
2825
+ "elementRef": refStore.storeElement(element),
2826
+ "role": role,
2827
+ "subrole": subrole,
2828
+ "title": title,
2829
+ "description": description,
2830
+ "identifier": identifier,
2831
+ "value": value,
2832
+ "actions": actions,
2833
+ "isTextInput": textRoles.contains(role),
2834
+ "canSetValue": canSetValue,
2835
+ "canFocus": focusedStatus == .success && focusedSettable.boolValue,
2836
+ "canPress": actions.contains(kAXPressAction as String),
2837
+ "canScroll": supportsAnyScrollAction(element),
2838
+ "canIncrement": actions.contains(kAXIncrementAction as String),
2839
+ "canDecrement": actions.contains(kAXDecrementAction as String),
2840
+ "axVisible": axVisible,
2841
+ "x": centerX,
2842
+ "y": centerY,
2843
+ ]
2844
+ if let frame {
2845
+ payload["frame"] = ["x": frame.origin.x, "y": frame.origin.y, "w": frame.width, "h": frame.height]
2846
+ }
2847
+ if let parentFrame {
2848
+ payload["parentFrame"] = ["x": parentFrame.origin.x, "y": parentFrame.origin.y, "w": parentFrame.width, "h": parentFrame.height]
2849
+ }
2850
+ if let score {
2851
+ payload["score"] = score
2852
+ }
2853
+ if let source {
2854
+ payload["source"] = source
2855
+ }
2856
+ return payload
2857
+ }
2858
+
2859
+ private func pidForElement(_ element: AXUIElement) -> Int32? {
2860
+ var pid: pid_t = 0
2861
+ let status = AXUIElementGetPid(element, &pid)
2862
+ guard status == .success else { return nil }
2863
+ return Int32(pid)
2864
+ }
2865
+
2866
+ private func parentElement(_ element: AXUIElement) -> AXUIElement? {
2867
+ guard let value = copyAttribute(element, attribute: kAXParentAttribute as CFString) else {
2868
+ return nil
2869
+ }
2870
+ return asAXElement(value)
2871
+ }
2872
+
2873
+ private func sameElement(_ lhs: AXUIElement, _ rhs: AXUIElement) -> Bool {
2874
+ CFEqual(lhs as CFTypeRef, rhs as CFTypeRef)
2875
+ }
2876
+
2877
+ private func isElement(_ element: AXUIElement, descendantOf ancestor: AXUIElement) -> Bool {
2878
+ var current: AXUIElement? = element
2879
+ var depth = 0
2880
+ while let candidate = current, depth < 20 {
2881
+ if sameElement(candidate, ancestor) {
2882
+ return true
2883
+ }
2884
+ current = parentElement(candidate)
2885
+ depth += 1
2886
+ }
2887
+ return false
2888
+ }
2889
+
2890
+ private func hasAncestorRole(_ element: AXUIElement, role: String) -> Bool {
2891
+ var current: AXUIElement? = element
2892
+ var depth = 0
2893
+ while let candidate = current, depth < 30 {
2894
+ if stringAttribute(candidate, attribute: kAXRoleAttribute as CFString) == role { return true }
2895
+ current = parentElement(candidate)
2896
+ depth += 1
2897
+ }
2898
+ return false
2899
+ }
2900
+
2901
+ private func actionNames(_ element: AXUIElement) -> [String] {
2902
+ var actionsValue: CFArray?
2903
+ let status = AXUIElementCopyActionNames(element, &actionsValue)
2904
+ guard status == .success else { return [] }
2905
+ guard let actionsArray = actionsValue as? [AnyObject] else { return [] }
2906
+ return actionsArray.compactMap { $0 as? String }
2907
+ }
2908
+
2909
+ private func supportsAction(_ element: AXUIElement, action: CFString) -> Bool {
2910
+ actionNames(element).contains(action as String)
2911
+ }
2912
+
2913
+ private func focusedElement(_ request: [String: Any]) throws -> [String: Any] {
2914
+ let pid = Int32(try intArg(request, "pid"))
2915
+ let windowId = optionalIntArg(request, "windowId").map { UInt32($0) }
2916
+ let windowRef = optionalStringArg(request, "windowRef")
2917
+ let app = AXUIElementCreateApplication(pid)
2918
+ guard let focusedValue = copyAttribute(app, attribute: kAXFocusedUIElementAttribute as CFString),
2919
+ let element = asAXElement(focusedValue)
2920
+ else {
2921
+ return ["exists": false]
2922
+ }
2923
+ if windowId != nil || windowRef != nil {
2924
+ guard let window = windowElement(pid: pid, windowId: windowId, windowRef: windowRef) else {
2925
+ return ["exists": false, "reason": "window_not_found"]
2926
+ }
2927
+ guard isElement(element, descendantOf: window) else {
2928
+ return ["exists": false, "reason": "focused_element_outside_window"]
2929
+ }
2930
+ }
2931
+
2932
+ let role = stringAttribute(element, attribute: kAXRoleAttribute as CFString) ?? ""
2933
+ let subrole = stringAttribute(element, attribute: kAXSubroleAttribute as CFString) ?? ""
2934
+ let secure = role == "AXSecureTextField" || subrole == "AXSecureTextField"
2935
+
2936
+ var settable = DarwinBoolean(false)
2937
+ let settableStatus = AXUIElementIsAttributeSettable(element, kAXValueAttribute as CFString, &settable)
2938
+ let canSetValue = settableStatus == .success && settable.boolValue
2939
+
2940
+ let textRoles: Set<String> = [
2941
+ "AXTextField",
2942
+ "AXTextArea",
2943
+ "AXTextView",
2944
+ "AXSearchField",
2945
+ "AXComboBox",
2946
+ "AXEditableText",
2947
+ "AXSecureTextField",
2948
+ ]
2949
+
2950
+ let isTextInput = textRoles.contains(role) || canSetValue
2951
+ let elementRef = refStore.storeElement(element)
2952
+
2953
+ return [
2954
+ "exists": true,
2955
+ "elementRef": elementRef,
2956
+ "role": role,
2957
+ "subrole": subrole,
2958
+ "isTextInput": isTextInput,
2959
+ "isSecure": secure,
2960
+ "canSetValue": canSetValue,
2961
+ ]
2962
+ }
2963
+
2964
+ private func axReadText(_ request: [String: Any]) throws -> [String: Any] {
2965
+ let elementRef = try stringArg(request, "elementRef")
2966
+ let offset = max(0, optionalIntArg(request, "offset") ?? 0)
2967
+ let limit = max(1, min(100_000, optionalIntArg(request, "limit") ?? 4_000))
2968
+ guard let element = refStore.element(for: elementRef) else {
2969
+ throw BridgeFailure(message: "Element reference is no longer valid", code: "element_ref_invalid")
2970
+ }
2971
+ let role = stringAttribute(element, attribute: kAXRoleAttribute as CFString) ?? ""
2972
+ let subrole = stringAttribute(element, attribute: kAXSubroleAttribute as CFString) ?? ""
2973
+ guard !isSecureTextElement(role: role, subrole: subrole) else {
2974
+ throw BridgeFailure(message: "Refers to a secure text field; refusing to read its value", code: "secure_text_unreadable")
2975
+ }
2976
+ guard let value = stringAttribute(element, attribute: kAXValueAttribute as CFString) else {
2977
+ throw BridgeFailure(message: "Element has no readable AXValue. Call snapshot/screenshot and choose a text-bearing ref.", code: "text_unavailable")
2978
+ }
2979
+ let characters = Array(value)
2980
+ if offset >= characters.count {
2981
+ return ["text": "", "offset": offset, "limit": limit, "totalChars": characters.count, "hasMore": false]
2982
+ }
2983
+ let end = min(characters.count, offset + limit)
2984
+ return [
2985
+ "text": String(characters[offset..<end]),
2986
+ "offset": offset,
2987
+ "limit": limit,
2988
+ "totalChars": characters.count,
2989
+ "hasMore": end < characters.count,
2990
+ ]
2991
+ }
2992
+
2993
+ private func getMousePosition() -> [String: Any] {
2994
+ let position = NSEvent.mouseLocation
2995
+ return ["x": position.x, "y": position.y]
2996
+ }
2997
+
2998
+ private func copyAttribute(_ element: AXUIElement, attribute: CFString) -> AnyObject? {
2999
+ var value: AnyObject?
3000
+ let status = AXUIElementCopyAttributeValue(element, attribute, &value)
3001
+ guard status == .success else { return nil }
3002
+ return value
3003
+ }
3004
+
3005
+ private func boolAttribute(_ element: AXUIElement, attribute: CFString) -> Bool? {
3006
+ guard let value = copyAttribute(element, attribute: attribute) else { return nil }
3007
+ if let boolValue = value as? Bool {
3008
+ return boolValue
3009
+ }
3010
+ if let number = value as? NSNumber {
3011
+ return number.boolValue
3012
+ }
3013
+ return nil
3014
+ }
3015
+
3016
+ private func stringAttribute(_ element: AXUIElement, attribute: CFString) -> String? {
3017
+ copyAttribute(element, attribute: attribute) as? String
3018
+ }
3019
+
3020
+ // Secure fields can expose plaintext through AX value in non-native apps,
3021
+ // and serialized values flow into the model conversation. Never emit them.
3022
+ private func isSecureTextElement(role: String, subrole: String) -> Bool {
3023
+ role == "AXSecureTextField" || subrole == "AXSecureTextField"
3024
+ }
3025
+
3026
+ private func displayValue(_ element: AXUIElement, role: String, subrole: String) -> String {
3027
+ if isSecureTextElement(role: role, subrole: subrole) { return "" }
3028
+ return stringAttribute(element, attribute: kAXValueAttribute as CFString) ?? ""
3029
+ }
3030
+
3031
+ // kAXSheetsAttribute is unsupported (-25205) on recent macOS; sheets are
3032
+ // exposed only as AXSheet-role children. Merge both sources so sheet
3033
+ // discovery works across versions.
3034
+ private func sheetElements(of window: AXUIElement) -> [AXUIElement] {
3035
+ var sheets = axElementArray(window, attribute: "AXSheets" as CFString)
3036
+ for child in axElementArray(window, attribute: kAXChildrenAttribute as CFString) {
3037
+ guard (stringAttribute(child, attribute: kAXRoleAttribute as CFString) ?? "") == "AXSheet" else { continue }
3038
+ if !sheets.contains(where: { CFEqual($0, child) }) { sheets.append(child) }
3039
+ }
3040
+ return sheets
3041
+ }
3042
+
3043
+ private func axElementArray(_ element: AXUIElement, attribute: CFString) -> [AXUIElement] {
3044
+ guard let value = copyAttribute(element, attribute: attribute) else { return [] }
3045
+ if let array = value as? [AXUIElement] {
3046
+ return array
3047
+ }
3048
+ if let anyArray = value as? [AnyObject] {
3049
+ return anyArray.compactMap(asAXElement)
3050
+ }
3051
+ return []
3052
+ }
3053
+
3054
+ private func axElementArrayIfPresent(_ element: AXUIElement, attribute: CFString) -> [AXUIElement]? {
3055
+ var value: CFTypeRef?
3056
+ let status = AXUIElementCopyAttributeValue(element, attribute, &value)
3057
+ guard status == .success, let value else { return nil }
3058
+ if let array = value as? [AXUIElement] {
3059
+ return array
3060
+ }
3061
+ if let anyArray = value as? [AnyObject] {
3062
+ return anyArray.compactMap(asAXElement)
3063
+ }
3064
+ return []
3065
+ }
3066
+
3067
+ private func asAXElement(_ value: AnyObject) -> AXUIElement? {
3068
+ let cfValue = value as CFTypeRef
3069
+ guard CFGetTypeID(cfValue) == AXUIElementGetTypeID() else { return nil }
3070
+ return unsafeBitCast(cfValue, to: AXUIElement.self)
3071
+ }
3072
+
3073
+ private func pointAttribute(_ element: AXUIElement, attribute: CFString) -> CGPoint? {
3074
+ guard let value = copyAttribute(element, attribute: attribute) else { return nil }
3075
+ let cfValue = value as CFTypeRef
3076
+ guard CFGetTypeID(cfValue) == AXValueGetTypeID() else { return nil }
3077
+ let axValue = unsafeBitCast(cfValue, to: AXValue.self)
3078
+ guard AXValueGetType(axValue) == .cgPoint else { return nil }
3079
+ var point = CGPoint.zero
3080
+ guard AXValueGetValue(axValue, .cgPoint, &point) else { return nil }
3081
+ return point
3082
+ }
3083
+
3084
+ private func sizeAttribute(_ element: AXUIElement, attribute: CFString) -> CGSize? {
3085
+ guard let value = copyAttribute(element, attribute: attribute) else { return nil }
3086
+ let cfValue = value as CFTypeRef
3087
+ guard CFGetTypeID(cfValue) == AXValueGetTypeID() else { return nil }
3088
+ let axValue = unsafeBitCast(cfValue, to: AXValue.self)
3089
+ guard AXValueGetType(axValue) == .cgSize else { return nil }
3090
+ var size = CGSize.zero
3091
+ guard AXValueGetValue(axValue, .cgSize, &size) else { return nil }
3092
+ return size
3093
+ }
3094
+
3095
+ private func frameForWindow(_ window: AXUIElement) -> CGRect {
3096
+ let origin = pointAttribute(window, attribute: kAXPositionAttribute as CFString) ?? .zero
3097
+ let size = sizeAttribute(window, attribute: kAXSizeAttribute as CFString) ?? .zero
3098
+ return CGRect(origin: origin, size: size)
3099
+ }
3100
+
3101
+ private func allCGWindowEntries() -> [[String: Any]] {
3102
+ (CGWindowListCopyWindowInfo([.optionAll, .excludeDesktopElements], kCGNullWindowID) as? [[String: Any]]) ?? []
3103
+ }
3104
+
3105
+ private func cgWindowOwners(entries suppliedEntries: [[String: Any]]? = nil) -> [CGWindowOwnerSummary] {
3106
+ let entries = suppliedEntries ?? allCGWindowEntries()
3107
+ var seen = Set<Int32>()
3108
+ var owners: [CGWindowOwnerSummary] = []
3109
+ for entry in entries {
3110
+ guard let ownerPid = (entry[kCGWindowOwnerPID as String] as? NSNumber)?.int32Value else { continue }
3111
+ let layer = (entry[kCGWindowLayer as String] as? NSNumber)?.intValue ?? 0
3112
+ if layer != 0 || seen.contains(ownerPid) { continue }
3113
+ guard let boundsDict = entry[kCGWindowBounds as String] as? [String: Any],
3114
+ let bounds = CGRect(dictionaryRepresentation: boundsDict as CFDictionary),
3115
+ bounds.width >= 100,
3116
+ bounds.height >= 80
3117
+ else {
3118
+ continue
3119
+ }
3120
+ let ownerName = (entry[kCGWindowOwnerName as String] as? String) ?? processName(pid: ownerPid) ?? "Unknown App"
3121
+ seen.insert(ownerPid)
3122
+ owners.append(CGWindowOwnerSummary(pid: ownerPid, name: ownerName))
3123
+ }
3124
+ return owners
3125
+ }
3126
+
3127
+ private func pidForWindowId(_ windowId: UInt32) -> Int32? {
3128
+ windowInfo(windowId: windowId)?.pid
3129
+ }
3130
+
3131
+ private func cgWindowCandidates(pid: Int32, entries suppliedEntries: [[String: Any]]? = nil) -> [CGWindowCandidate] {
3132
+ let entries = suppliedEntries ?? allCGWindowEntries()
3133
+ var candidates: [CGWindowCandidate] = []
3134
+ for (zOrder, entry) in entries.enumerated() {
3135
+ guard let ownerPid = (entry[kCGWindowOwnerPID as String] as? NSNumber)?.int32Value,
3136
+ ownerPid == pid
3137
+ else {
3138
+ continue
3139
+ }
3140
+ let layer = (entry[kCGWindowLayer as String] as? NSNumber)?.intValue ?? 0
3141
+ if layer != 0 { continue }
3142
+
3143
+ guard let windowNumber = (entry[kCGWindowNumber as String] as? NSNumber)?.uint32Value else {
3144
+ continue
3145
+ }
3146
+ guard let boundsDict = entry[kCGWindowBounds as String] as? [String: Any],
3147
+ let bounds = CGRect(dictionaryRepresentation: boundsDict as CFDictionary)
3148
+ else {
3149
+ continue
3150
+ }
3151
+
3152
+ let title = (entry[kCGWindowName as String] as? String) ?? ""
3153
+ let isOnscreen = (entry[kCGWindowIsOnscreen as String] as? NSNumber)?.boolValue ?? true
3154
+ candidates.append(
3155
+ CGWindowCandidate(
3156
+ windowId: windowNumber,
3157
+ title: title,
3158
+ bounds: bounds,
3159
+ isOnscreen: isOnscreen,
3160
+ layer: layer,
3161
+ zOrder: zOrder
3162
+ )
3163
+ )
3164
+ if candidates.count == 128 { break }
3165
+ }
3166
+ return candidates
3167
+ }
3168
+
3169
+ private func cgBroadRootOwners(entries: [[String: Any]]) -> [CGWindowOwnerSummary] {
3170
+ let popupLevel = Int(CGWindowLevelForKey(.popUpMenuWindow))
3171
+ var seen = Set<Int32>()
3172
+ return entries.compactMap { entry -> CGWindowOwnerSummary? in
3173
+ let layer = (entry[kCGWindowLayer as String] as? NSNumber)?.intValue ?? 0
3174
+ guard layer == 0 || layer == popupLevel,
3175
+ let pid = (entry[kCGWindowOwnerPID as String] as? NSNumber)?.int32Value
3176
+ else { return nil }
3177
+ if layer == popupLevel {
3178
+ guard (entry[kCGWindowIsOnscreen as String] as? NSNumber)?.boolValue == true else { return nil }
3179
+ } else {
3180
+ guard let boundsDict = entry[kCGWindowBounds as String] as? [String: Any],
3181
+ let bounds = CGRect(dictionaryRepresentation: boundsDict as CFDictionary),
3182
+ bounds.width >= 100,
3183
+ bounds.height >= 80
3184
+ else { return nil }
3185
+ }
3186
+ guard seen.insert(pid).inserted else { return nil }
3187
+ let name = (entry[kCGWindowOwnerName as String] as? String) ?? processName(pid: pid) ?? "Unknown App"
3188
+ return CGWindowOwnerSummary(pid: pid, name: name)
3189
+ }
3190
+ }
3191
+
3192
+ private func cgPopupMenuCandidates(pid: Int32?, entries: [[String: Any]]) -> [CGWindowCandidate] {
3193
+ let popupLevel = Int(CGWindowLevelForKey(.popUpMenuWindow))
3194
+ var candidates: [CGWindowCandidate] = []
3195
+ for (zOrder, entry) in entries.enumerated() {
3196
+ guard let ownerPid = (entry[kCGWindowOwnerPID as String] as? NSNumber)?.int32Value else { continue }
3197
+ if let pid, ownerPid != pid { continue }
3198
+ let layer = (entry[kCGWindowLayer as String] as? NSNumber)?.intValue ?? 0
3199
+ if layer != popupLevel { continue }
3200
+ guard let windowNumber = (entry[kCGWindowNumber as String] as? NSNumber)?.uint32Value,
3201
+ let boundsDict = entry[kCGWindowBounds as String] as? [String: Any],
3202
+ let bounds = CGRect(dictionaryRepresentation: boundsDict as CFDictionary)
3203
+ else { continue }
3204
+ let title = (entry[kCGWindowName as String] as? String) ?? ""
3205
+ let isOnscreen = (entry[kCGWindowIsOnscreen as String] as? NSNumber)?.boolValue ?? false
3206
+ guard isOnscreen else { continue }
3207
+ candidates.append(CGWindowCandidate(windowId: windowNumber, title: title, bounds: bounds, isOnscreen: isOnscreen, layer: layer, zOrder: zOrder))
3208
+ if candidates.count == 128 { break }
3209
+ }
3210
+ return candidates
3211
+ }
3212
+
3213
+ private func openMenuElements(pid: Int32, messagingTimeout: Float = 1.0) -> [AXUIElement] {
3214
+ let app = AXUIElementCreateApplication(pid)
3215
+ AXUIElementSetMessagingTimeout(app, messagingTimeout)
3216
+ let descendants = collectDescendants(startingAt: app, maxDepth: 6)
3217
+ var menus = descendants.filter { (stringAttribute($0, attribute: kAXRoleAttribute as CFString) ?? "") == "AXMenu" }
3218
+ if menus.isEmpty,
3219
+ let focused = copyAttribute(app, attribute: kAXFocusedUIElementAttribute as CFString).flatMap(asAXElement)
3220
+ {
3221
+ var current: AXUIElement? = focused
3222
+ while let element = current {
3223
+ if (stringAttribute(element, attribute: kAXRoleAttribute as CFString) ?? "") == "AXMenu" {
3224
+ menus.append(element)
3225
+ break
3226
+ }
3227
+ current = copyAttribute(element, attribute: kAXParentAttribute as CFString).flatMap(asAXElement)
3228
+ }
3229
+ }
3230
+ return menus
3231
+ }
3232
+
3233
+ private func bestCandidate(for element: AXUIElement, candidates: [CGWindowCandidate]) -> CGWindowCandidate? {
3234
+ let title = stringAttribute(element, attribute: kAXTitleAttribute as CFString) ?? ""
3235
+ let frame = frameForWindow(element)
3236
+ return candidates.max { left, right in
3237
+ windowPairScore(frame: frame, title: title, candidate: left) < windowPairScore(frame: frame, title: title, candidate: right)
3238
+ }
3239
+ }
3240
+
3241
+ private func pairingForWindow(_ window: AXUIElement, pid: Int32) -> WindowPairing {
3242
+ windowPairings(windows: [window], candidates: cgWindowCandidates(pid: pid))[ObjectIdentifier(window)] ?? WindowPairing(candidate: nil, score: -Double.greatestFiniteMagnitude, confidence: "low")
3243
+ }
3244
+
3245
+ private func windowPairings(windows: [AXUIElement], candidates: [CGWindowCandidate]) -> [ObjectIdentifier: WindowPairing] {
3246
+ var pairs: [(window: AXUIElement, candidate: CGWindowCandidate, score: Double)] = []
3247
+ for window in windows {
3248
+ let title = stringAttribute(window, attribute: kAXTitleAttribute as CFString) ?? ""
3249
+ let frame = frameForWindow(window)
3250
+ for candidate in candidates {
3251
+ pairs.append((window, candidate, windowPairScore(frame: frame, title: title, candidate: candidate)))
3252
+ }
3253
+ }
3254
+ pairs.sort { $0.score > $1.score }
3255
+ var output: [ObjectIdentifier: WindowPairing] = [:]
3256
+ var usedWindows = Set<ObjectIdentifier>()
3257
+ var usedCandidateIds = Set<UInt32>()
3258
+ for pair in pairs {
3259
+ let key = ObjectIdentifier(pair.window)
3260
+ if usedWindows.contains(key) || usedCandidateIds.contains(pair.candidate.windowId) { continue }
3261
+ usedWindows.insert(key)
3262
+ usedCandidateIds.insert(pair.candidate.windowId)
3263
+ let frame = frameForWindow(pair.window)
3264
+ let title = stringAttribute(pair.window, attribute: kAXTitleAttribute as CFString) ?? ""
3265
+ output[key] = WindowPairing(candidate: pair.score >= 0 ? pair.candidate : nil, score: pair.score, confidence: pairingConfidence(frame: frame, title: title, candidate: pair.candidate, score: pair.score))
3266
+ }
3267
+ for window in windows {
3268
+ let key = ObjectIdentifier(window)
3269
+ if output[key] == nil {
3270
+ output[key] = WindowPairing(candidate: nil, score: -Double.greatestFiniteMagnitude, confidence: "low")
3271
+ }
3272
+ }
3273
+ return output
3274
+ }
3275
+
3276
+ private func windowPairScore(frame: CGRect, title: String, candidate: CGWindowCandidate) -> Double {
3277
+ var score = 0.0
3278
+ let normalizedTitle = title.trimmingCharacters(in: .whitespacesAndNewlines).lowercased()
3279
+ let candidateTitle = candidate.title.trimmingCharacters(in: .whitespacesAndNewlines).lowercased()
3280
+ if !normalizedTitle.isEmpty && !candidateTitle.isEmpty {
3281
+ if normalizedTitle == candidateTitle {
3282
+ score += 100
3283
+ } else if normalizedTitle.contains(candidateTitle) || candidateTitle.contains(normalizedTitle) {
3284
+ score += 50
3285
+ }
3286
+ }
3287
+ if frame.width > 1 && frame.height > 1 {
3288
+ let dx = abs(candidate.bounds.origin.x - frame.origin.x)
3289
+ let dy = abs(candidate.bounds.origin.y - frame.origin.y)
3290
+ let dw = abs(candidate.bounds.size.width - frame.size.width)
3291
+ let dh = abs(candidate.bounds.size.height - frame.size.height)
3292
+ score -= Double(dx + dy + dw + dh) / 20.0
3293
+ }
3294
+ if candidate.isOnscreen { score += 10 }
3295
+ return score
3296
+ }
3297
+
3298
+ private func pairingConfidence(frame: CGRect, title: String, candidate: CGWindowCandidate, score: Double) -> String {
3299
+ guard frame.width > 1 && frame.height > 1 else { return "low" }
3300
+ let normalizedTitle = title.trimmingCharacters(in: .whitespacesAndNewlines).lowercased()
3301
+ let candidateTitle = candidate.title.trimmingCharacters(in: .whitespacesAndNewlines).lowercased()
3302
+ let titleEqual = !normalizedTitle.isEmpty && !candidateTitle.isEmpty && normalizedTitle == candidateTitle
3303
+ let geometryExact = abs(frame.origin.x - candidate.bounds.origin.x) <= 2 && abs(frame.origin.y - candidate.bounds.origin.y) <= 2 && abs(frame.width - candidate.bounds.width) <= 2 && abs(frame.height - candidate.bounds.height) <= 2
3304
+ if titleEqual && geometryExact { return "exact" }
3305
+ if score >= 50 { return "high" }
3306
+ return "low"
3307
+ }
3308
+
3309
+ private func displayScaleFactor(for frame: CGRect) -> Double {
3310
+ var displayCount: UInt32 = 0
3311
+ guard CGGetOnlineDisplayList(0, nil, &displayCount) == .success, displayCount > 0 else {
3312
+ return Double(NSScreen.main?.backingScaleFactor ?? 1.0)
3313
+ }
3314
+
3315
+ var displays = Array(repeating: CGDirectDisplayID(), count: Int(displayCount))
3316
+ guard CGGetOnlineDisplayList(displayCount, &displays, &displayCount) == .success else {
3317
+ return Double(NSScreen.main?.backingScaleFactor ?? 1.0)
3318
+ }
3319
+
3320
+ var chosenDisplay: CGDirectDisplayID?
3321
+ var chosenArea: CGFloat = -1
3322
+ for display in displays {
3323
+ let bounds = CGDisplayBounds(display)
3324
+ let overlap = bounds.intersection(frame)
3325
+ let area = overlap.isNull ? 0 : overlap.width * overlap.height
3326
+ if area > chosenArea {
3327
+ chosenArea = area
3328
+ chosenDisplay = display
3329
+ }
3330
+ }
3331
+
3332
+ guard let display = chosenDisplay, let mode = CGDisplayCopyDisplayMode(display) else {
3333
+ return Double(NSScreen.main?.backingScaleFactor ?? 1.0)
3334
+ }
3335
+
3336
+ let width = Double(mode.width)
3337
+ guard width > 0 else { return 1.0 }
3338
+ let scale = Double(mode.pixelWidth) / width
3339
+ return scale > 0 ? scale : 1.0
3340
+ }
3341
+
3342
+ private func captureWindow(windowId: UInt32) throws -> CapturedWindowImage {
3343
+ if #available(macOS 14.0, *) {
3344
+ let semaphore = DispatchSemaphore(value: 0)
3345
+ let capturedImage = Box<CGImage?>(nil)
3346
+ let capturedError = Box<Error?>(nil)
3347
+
3348
+ let task = Task {
3349
+ defer { semaphore.signal() }
3350
+ do {
3351
+ if Task.isCancelled {
3352
+ return
3353
+ }
3354
+ let shareable = try await SCShareableContent.excludingDesktopWindows(false, onScreenWindowsOnly: false)
3355
+ guard let window = shareable.windows.first(where: { $0.windowID == windowId }) else {
3356
+ throw BridgeFailure(message: "Window \(windowId) is not available for capture", code: "window_not_found")
3357
+ }
3358
+
3359
+ let filter = SCContentFilter(desktopIndependentWindow: window)
3360
+ let config = SCStreamConfiguration()
3361
+ // Avoid ScreenCaptureKit's default 1920x1080 canvas for window captures.
3362
+ let scale = displayScaleFactor(for: window.frame)
3363
+ config.width = max(1, Int((window.frame.width * scale).rounded()))
3364
+ config.height = max(1, Int((window.frame.height * scale).rounded()))
3365
+ config.showsCursor = false
3366
+ config.ignoreShadowsSingleWindow = true
3367
+
3368
+ let image = try await SCScreenshotManager.captureImage(contentFilter: filter, configuration: config)
3369
+ capturedImage.value = image
3370
+ } catch {
3371
+ capturedError.value = error
3372
+ }
3373
+ }
3374
+
3375
+ if semaphore.wait(timeout: .now() + .seconds(8)) == .timedOut {
3376
+ task.cancel()
3377
+ if let payload = try cgWindowScreenshotFallback(windowId: windowId) {
3378
+ return payload
3379
+ }
3380
+ throw BridgeFailure(message: "Capture timed out while capturing window \(windowId)", code: "capture_timeout")
3381
+ }
3382
+
3383
+ if let error = capturedError.value {
3384
+ if let payload = try cgWindowScreenshotFallback(windowId: windowId) {
3385
+ return payload
3386
+ }
3387
+ if let failure = error as? BridgeFailure {
3388
+ throw failure
3389
+ }
3390
+ throw BridgeFailure(message: "Capture failed: \(error.localizedDescription)", code: "capture_failed")
3391
+ }
3392
+
3393
+ guard let image = capturedImage.value else {
3394
+ if let payload = try cgWindowScreenshotFallback(windowId: windowId) {
3395
+ return payload
3396
+ }
3397
+ throw BridgeFailure(message: "Capture failed", code: "capture_failed")
3398
+ }
3399
+
3400
+ return CapturedWindowImage(image: image, windowId: windowId, frame: currentWindowBounds(windowId: windowId) ?? CGRect(x: 0, y: 0, width: image.width, height: image.height))
3401
+ }
3402
+ if let payload = try cgWindowScreenshotFallback(windowId: windowId) {
3403
+ return payload
3404
+ }
3405
+ throw BridgeFailure(message: "Capture failed", code: "capture_failed")
3406
+ }
3407
+
3408
+ private func jpegData(image: CGImage, quality: Double) -> Data? {
3409
+ let data = NSMutableData()
3410
+ guard let destination = CGImageDestinationCreateWithData(data, "public.jpeg" as CFString, 1, nil) else { return nil }
3411
+ CGImageDestinationAddImage(destination, image, [kCGImageDestinationLossyCompressionQuality: quality] as CFDictionary)
3412
+ guard CGImageDestinationFinalize(destination) else { return nil }
3413
+ return data as Data
3414
+ }
3415
+
3416
+ private func downscaledImage(_ image: CGImage, maxDimension: Int?) -> CGImage? {
3417
+ guard let maxDimension, max(image.width, image.height) > maxDimension else { return nil }
3418
+ let scale = Double(maxDimension) / Double(max(image.width, image.height))
3419
+ let width = max(1, Int(Double(image.width) * scale))
3420
+ let height = max(1, Int(Double(image.height) * scale))
3421
+ guard let context = CGContext(
3422
+ data: nil,
3423
+ width: width,
3424
+ height: height,
3425
+ bitsPerComponent: image.bitsPerComponent,
3426
+ bytesPerRow: 0,
3427
+ space: image.colorSpace ?? CGColorSpaceCreateDeviceRGB(),
3428
+ bitmapInfo: image.bitmapInfo.rawValue
3429
+ ) else { return nil }
3430
+ context.interpolationQuality = .medium
3431
+ context.draw(image, in: CGRect(x: 0, y: 0, width: width, height: height))
3432
+ return context.makeImage()
3433
+ }
3434
+
3435
+ private func cgWindowScreenshotFallback(windowId: UInt32) throws -> CapturedWindowImage? {
3436
+ if let payload = try systemScreenshotWindow(windowId: windowId) {
3437
+ return payload
3438
+ }
3439
+ return nil
3440
+ }
3441
+
3442
+ private func systemScreenshotWindow(windowId: UInt32) throws -> CapturedWindowImage? {
3443
+ let tempUrl = FileManager.default.temporaryDirectory.appendingPathComponent("pi-cu-\(UUID().uuidString).png")
3444
+ defer { try? FileManager.default.removeItem(at: tempUrl) }
3445
+ // Owner-only perms in case TMPDIR ever resolves to a shared directory.
3446
+ FileManager.default.createFile(atPath: tempUrl.path, contents: nil, attributes: [.posixPermissions: 0o600])
3447
+
3448
+ let process = Process()
3449
+ process.executableURL = URL(fileURLWithPath: "/usr/sbin/screencapture")
3450
+ process.arguments = ["-x", "-l", String(windowId), tempUrl.path]
3451
+ try process.run()
3452
+ let deadline = Date().addingTimeInterval(5)
3453
+ while process.isRunning && Date() < deadline {
3454
+ Thread.sleep(forTimeInterval: 0.05)
3455
+ }
3456
+ if process.isRunning {
3457
+ process.terminate()
3458
+ Thread.sleep(forTimeInterval: 0.1)
3459
+ if process.isRunning { process.interrupt() }
3460
+ return nil
3461
+ }
3462
+ guard process.terminationStatus == 0 else { return nil }
3463
+ guard let data = try? Data(contentsOf: tempUrl), !data.isEmpty else { return nil }
3464
+ guard let imageRep = NSBitmapImageRep(data: data), let cgImage = imageRep.cgImage else { return nil }
3465
+ return CapturedWindowImage(image: cgImage, windowId: windowId, frame: currentWindowBounds(windowId: windowId) ?? CGRect(x: 0, y: 0, width: cgImage.width, height: cgImage.height))
3466
+ }
3467
+
3468
+ private func currentWindowBounds(windowId: UInt32) -> CGRect? {
3469
+ if #available(macOS 14.0, *), let scBounds = currentWindowBoundsViaScreenCaptureKit(windowId: windowId) {
3470
+ return scBounds
3471
+ }
3472
+ return windowInfo(windowId: windowId)?.bounds
3473
+ }
3474
+
3475
+ private func windowInfo(windowId: UInt32) -> (pid: Int32, bounds: CGRect)? {
3476
+ func matchingEntry(_ entries: [[String: Any]]?) -> [String: Any]? {
3477
+ entries?.first {
3478
+ ($0[kCGWindowNumber as String] as? NSNumber)?.uint32Value == windowId
3479
+ }
3480
+ }
3481
+
3482
+ let requestedIds = [NSNumber(value: windowId)] as CFArray
3483
+ let targetedEntries = CGWindowListCreateDescriptionFromArray(requestedIds) as? [[String: Any]]
3484
+ let entry = matchingEntry(targetedEntries) ?? matchingEntry(
3485
+ CGWindowListCopyWindowInfo([.optionAll], kCGNullWindowID) as? [[String: Any]]
3486
+ )
3487
+ guard let entry,
3488
+ let pid = (entry[kCGWindowOwnerPID as String] as? NSNumber)?.int32Value,
3489
+ let boundsDict = entry[kCGWindowBounds as String] as? [String: Any],
3490
+ let bounds = CGRect(dictionaryRepresentation: boundsDict as CFDictionary)
3491
+ else {
3492
+ return nil
3493
+ }
3494
+ return (pid, bounds)
3495
+ }
3496
+
3497
+ @available(macOS 14.0, *)
3498
+ private func currentWindowBoundsViaScreenCaptureKit(windowId: UInt32) -> CGRect? {
3499
+ let semaphore = DispatchSemaphore(value: 0)
3500
+ let output = Box<CGRect?>(nil)
3501
+
3502
+ let task = Task {
3503
+ defer { semaphore.signal() }
3504
+ do {
3505
+ if Task.isCancelled {
3506
+ return
3507
+ }
3508
+ let shareable = try await SCShareableContent.excludingDesktopWindows(false, onScreenWindowsOnly: false)
3509
+ if let window = shareable.windows.first(where: { $0.windowID == windowId }) {
3510
+ output.value = window.frame
3511
+ }
3512
+ } catch {
3513
+ output.value = nil
3514
+ }
3515
+ }
3516
+
3517
+ if semaphore.wait(timeout: .now() + .seconds(2)) == .timedOut {
3518
+ task.cancel()
3519
+ return nil
3520
+ }
3521
+ return output.value
3522
+ }
3523
+
3524
+ private func eventDelivery(_ request: [String: Any]) -> String {
3525
+ optionalStringArg(request, "delivery") == "pid" ? "pid" : "hid"
3526
+ }
3527
+
3528
+ private func postEvent(_ event: CGEvent, pid: Int32, delivery: String = "hid") {
3529
+ if delivery == "pid" {
3530
+ event.postToPid(pid)
3531
+ return
3532
+ }
3533
+ ForegroundActivity.shared.physicalInput()
3534
+ // Post as a real foreground HID event. AppKit views with mouseDown handlers
3535
+ // can ignore pid-targeted CGEvents even though postToPid reports success.
3536
+ // Keep the target app frontmost so the HID event is delivered to the intended
3537
+ // window, then post at the session event tap.
3538
+ if let app = NSRunningApplication(processIdentifier: pid), !app.isActive {
3539
+ if #available(macOS 14.0, *) {
3540
+ _ = app.activate()
3541
+ } else {
3542
+ _ = app.activate(options: [.activateIgnoringOtherApps])
3543
+ }
3544
+ usleep(20_000)
3545
+ }
3546
+ event.post(tap: .cghidEventTap)
3547
+ }
3548
+
3549
+ private func postMouseMove(to point: CGPoint, pid: Int32, delivery: String = "hid") throws {
3550
+ if delivery == "hid" { physicalInputLock.lock(); ForegroundActivity.shared.enter() }
3551
+ defer { if delivery == "hid" { ForegroundActivity.shared.leave(); physicalInputLock.unlock() } }
3552
+ guard let move = CGEvent(mouseEventSource: nil, mouseType: .mouseMoved, mouseCursorPosition: point, mouseButton: .left) else {
3553
+ throw BridgeFailure(message: "Failed to create mouse move event", code: "input_failed")
3554
+ }
3555
+ postEvent(move, pid: pid, delivery: delivery)
3556
+ }
3557
+
3558
+ private func mouseButton(_ name: String) -> CGMouseButton {
3559
+ switch name.trimmingCharacters(in: .whitespacesAndNewlines).lowercased() {
3560
+ case "right":
3561
+ return .right
3562
+ case "middle", "center":
3563
+ return .center
3564
+ default:
3565
+ return .left
3566
+ }
3567
+ }
3568
+
3569
+ private func mouseDownType(for button: CGMouseButton) -> CGEventType {
3570
+ switch button {
3571
+ case .right:
3572
+ return .rightMouseDown
3573
+ case .center:
3574
+ return .otherMouseDown
3575
+ default:
3576
+ return .leftMouseDown
3577
+ }
3578
+ }
3579
+
3580
+ private func mouseUpType(for button: CGMouseButton) -> CGEventType {
3581
+ switch button {
3582
+ case .right:
3583
+ return .rightMouseUp
3584
+ case .center:
3585
+ return .otherMouseUp
3586
+ default:
3587
+ return .leftMouseUp
3588
+ }
3589
+ }
3590
+
3591
+ private func mouseDraggedType(for button: CGMouseButton) -> CGEventType {
3592
+ switch button {
3593
+ case .right:
3594
+ return .rightMouseDragged
3595
+ case .center:
3596
+ return .otherMouseDragged
3597
+ default:
3598
+ return .leftMouseDragged
3599
+ }
3600
+ }
3601
+
3602
+ private func postMouseClick(at point: CGPoint, pid: Int32, button: CGMouseButton = .left, clickCount: Int = 1, delivery: String = "hid") throws {
3603
+ if delivery == "hid" { physicalInputLock.lock(); ForegroundActivity.shared.enter() }
3604
+ defer { if delivery == "hid" { ForegroundActivity.shared.leave(); physicalInputLock.unlock() } }
3605
+ // Tangu:HID 点击会把**用户的实物指针**挪到目标处并留在那里 —— 前台被抢已经够烦,
3606
+ // 鼠标还被丢在别处就更糟。点完把指针还回原位。
3607
+ // 后台(pid)路径不需要:那条本来就不动系统指针(只画 AgentCursor 虚拟光标)。
3608
+ // 守卫:只有指针仍停在我们投递的位置(= 这期间用户没自己动鼠标)才还原,绝不跟用户抢。
3609
+ let savedCursor: CGPoint? = delivery == "hid" ? CGEvent(source: nil)?.location : nil
3610
+ defer {
3611
+ // defer 里不能 return,只能靠嵌套 if
3612
+ if let savedCursor, let now = CGEvent(source: nil)?.location,
3613
+ abs(now.x - point.x) < 2, abs(now.y - point.y) < 2 {
3614
+ CGWarpMouseCursorPosition(savedCursor)
3615
+ // warp 之后必须重新关联,否则系统会短暂忽略真实鼠标移动(指针像卡住)。
3616
+ CGAssociateMouseAndMouseCursorPosition(1)
3617
+ }
3618
+ }
3619
+ try postMouseMove(to: point, pid: pid, delivery: delivery)
3620
+ for index in 1...max(1, clickCount) {
3621
+ guard let down = CGEvent(mouseEventSource: nil, mouseType: mouseDownType(for: button), mouseCursorPosition: point, mouseButton: button),
3622
+ let up = CGEvent(mouseEventSource: nil, mouseType: mouseUpType(for: button), mouseCursorPosition: point, mouseButton: button)
3623
+ else {
3624
+ throw BridgeFailure(message: "Failed to create mouse click event", code: "input_failed")
3625
+ }
3626
+ down.setIntegerValueField(.mouseEventClickState, value: Int64(index))
3627
+ up.setIntegerValueField(.mouseEventClickState, value: Int64(index))
3628
+ postEvent(down, pid: pid, delivery: delivery)
3629
+ usleep(12_000)
3630
+ postEvent(up, pid: pid, delivery: delivery)
3631
+ if index < clickCount {
3632
+ usleep(70_000)
3633
+ }
3634
+ }
3635
+ }
3636
+
3637
+ private func postMouseDrag(points: [CGPoint], pid: Int32, delivery: String = "hid") throws {
3638
+ if delivery == "hid" { physicalInputLock.lock(); ForegroundActivity.shared.enter() }
3639
+ defer { if delivery == "hid" { ForegroundActivity.shared.leave(); physicalInputLock.unlock() } }
3640
+ guard points.count >= 2, let first = points.first else {
3641
+ throw BridgeFailure(message: "Drag requires at least two points", code: "invalid_args")
3642
+ }
3643
+ try postMouseMove(to: first, pid: pid, delivery: delivery)
3644
+ guard let down = CGEvent(mouseEventSource: nil, mouseType: .leftMouseDown, mouseCursorPosition: first, mouseButton: .left) else {
3645
+ throw BridgeFailure(message: "Failed to create mouse down event", code: "input_failed")
3646
+ }
3647
+ postEvent(down, pid: pid, delivery: delivery)
3648
+ usleep(12_000)
3649
+
3650
+ for point in points.dropFirst() {
3651
+ guard let drag = CGEvent(mouseEventSource: nil, mouseType: mouseDraggedType(for: .left), mouseCursorPosition: point, mouseButton: .left) else {
3652
+ throw BridgeFailure(message: "Failed to create mouse drag event", code: "input_failed")
3653
+ }
3654
+ postEvent(drag, pid: pid, delivery: delivery)
3655
+ usleep(8_000)
3656
+ }
3657
+
3658
+ guard let last = points.last,
3659
+ let up = CGEvent(mouseEventSource: nil, mouseType: .leftMouseUp, mouseCursorPosition: last, mouseButton: .left)
3660
+ else {
3661
+ throw BridgeFailure(message: "Failed to create mouse up event", code: "input_failed")
3662
+ }
3663
+ postEvent(up, pid: pid, delivery: delivery)
3664
+ }
3665
+
3666
+ private func postScrollWheel(at point: CGPoint, deltaX: Int, deltaY: Int, pid: Int32, delivery: String = "hid") throws {
3667
+ if delivery == "hid" { physicalInputLock.lock(); ForegroundActivity.shared.enter() }
3668
+ defer { if delivery == "hid" { ForegroundActivity.shared.leave(); physicalInputLock.unlock() } }
3669
+ try postMouseMove(to: point, pid: pid, delivery: delivery)
3670
+ guard let event = CGEvent(
3671
+ scrollWheelEvent2Source: nil,
3672
+ units: .pixel,
3673
+ wheelCount: 2,
3674
+ wheel1: Int32(-deltaY),
3675
+ wheel2: Int32(deltaX),
3676
+ wheel3: 0
3677
+ ) else {
3678
+ throw BridgeFailure(message: "Failed to create scroll event", code: "input_failed")
3679
+ }
3680
+ event.location = point
3681
+ postEvent(event, pid: pid, delivery: delivery)
3682
+ }
3683
+
3684
+ private func modifierFlag(_ key: String) -> CGEventFlags? {
3685
+ switch key.trimmingCharacters(in: .whitespacesAndNewlines).lowercased() {
3686
+ case "cmd", "command", "meta":
3687
+ return .maskCommand
3688
+ case "ctrl", "control":
3689
+ return .maskControl
3690
+ case "shift":
3691
+ return .maskShift
3692
+ case "option", "alt":
3693
+ return .maskAlternate
3694
+ default:
3695
+ return nil
3696
+ }
3697
+ }
3698
+
3699
+ private func keyCode(_ key: String) -> CGKeyCode? {
3700
+ let normalized = key.trimmingCharacters(in: .whitespacesAndNewlines).lowercased()
3701
+ let table: [String: CGKeyCode] = [
3702
+ "a": 0, "s": 1, "d": 2, "f": 3, "h": 4, "g": 5, "z": 6, "x": 7, "c": 8, "v": 9, "b": 11,
3703
+ "q": 12, "w": 13, "e": 14, "r": 15, "y": 16, "t": 17, "1": 18, "2": 19, "3": 20, "4": 21,
3704
+ "6": 22, "5": 23, "=": 24, "+": 24, "9": 25, "7": 26, "-": 27, "8": 28, "0": 29,
3705
+ "]": 30, "o": 31, "u": 32, "[": 33, "i": 34, "p": 35, "return": 36, "enter": 36,
3706
+ "l": 37, "j": 38, "'": 39, "\"": 39, "k": 40, ";": 41, "\\": 42, ",": 43, "/": 44,
3707
+ "n": 45, "m": 46, ".": 47, "tab": 48, "space": 49, " ": 49, "`": 50, "~": 50,
3708
+ "backspace": 51, "delete": 51, "del": 51, "esc": 53, "escape": 53,
3709
+ "f1": 122, "f2": 120, "f3": 99, "f4": 118, "f5": 96, "f6": 97, "f7": 98, "f8": 100,
3710
+ "f9": 101, "f10": 109, "f11": 103, "f12": 111,
3711
+ "home": 115, "pageup": 116, "page_up": 116, "page down": 121, "pagedown": 121, "page_down": 121,
3712
+ "forwarddelete": 117, "forward_delete": 117, "end": 119,
3713
+ "left": 123, "arrowleft": 123, "arrow_left": 123,
3714
+ "right": 124, "arrowright": 124, "arrow_right": 124,
3715
+ "down": 125, "arrowdown": 125, "arrow_down": 125,
3716
+ "up": 126, "arrowup": 126, "arrow_up": 126,
3717
+ ]
3718
+ return table[normalized]
3719
+ }
3720
+
3721
+ private func keyChord(_ keys: [String]) -> (flags: CGEventFlags, key: String)? {
3722
+ guard keys.count >= 2 else { return nil }
3723
+ var flags = CGEventFlags()
3724
+ for key in keys.dropLast() {
3725
+ guard let flag = modifierFlag(key) else {
3726
+ return nil
3727
+ }
3728
+ flags.insert(flag)
3729
+ }
3730
+ return (flags, keys.last ?? "")
3731
+ }
3732
+
3733
+ private func postKeyPress(keys: [String], pid: Int32, delivery: String = "hid") throws {
3734
+ if delivery == "hid" { physicalInputLock.lock(); ForegroundActivity.shared.enter() }
3735
+ defer { if delivery == "hid" { ForegroundActivity.shared.leave(); physicalInputLock.unlock() } }
3736
+ if let chord = keyChord(keys) {
3737
+ try postKey(chord.key, flags: chord.flags, pid: pid, delivery: delivery)
3738
+ return
3739
+ }
3740
+
3741
+ for key in keys {
3742
+ let parts = key
3743
+ .split(separator: "+")
3744
+ .map { String($0).trimmingCharacters(in: .whitespacesAndNewlines) }
3745
+ .filter { !$0.isEmpty }
3746
+ if let chord = keyChord(parts) {
3747
+ try postKey(chord.key, flags: chord.flags, pid: pid, delivery: delivery)
3748
+ } else {
3749
+ try postKey(key, flags: [], pid: pid, delivery: delivery)
3750
+ }
3751
+ }
3752
+ }
3753
+
3754
+ private func postKey(_ key: String, flags: CGEventFlags, pid: Int32, delivery: String = "hid") throws {
3755
+ if delivery == "hid" { physicalInputLock.lock(); ForegroundActivity.shared.enter() }
3756
+ defer { if delivery == "hid" { ForegroundActivity.shared.leave(); physicalInputLock.unlock() } }
3757
+ guard let code = keyCode(key) else {
3758
+ if key.count == 1 {
3759
+ try postUnicodeText(key, pid: pid, delivery: delivery)
3760
+ return
3761
+ }
3762
+ throw BridgeFailure(message: "Unsupported key '\(key)'", code: "invalid_args")
3763
+ }
3764
+ guard let down = CGEvent(keyboardEventSource: nil, virtualKey: code, keyDown: true),
3765
+ let up = CGEvent(keyboardEventSource: nil, virtualKey: code, keyDown: false)
3766
+ else {
3767
+ throw BridgeFailure(message: "Failed to create key event", code: "input_failed")
3768
+ }
3769
+ down.flags = flags
3770
+ up.flags = flags
3771
+ postEvent(down, pid: pid, delivery: delivery)
3772
+ postEvent(up, pid: pid, delivery: delivery)
3773
+ usleep(8_000)
3774
+ }
3775
+
3776
+ private func postUnicodeText(_ text: String, pid: Int32, delivery: String = "hid") throws {
3777
+ if delivery == "hid" { physicalInputLock.lock(); ForegroundActivity.shared.enter() }
3778
+ defer { if delivery == "hid" { ForegroundActivity.shared.leave(); physicalInputLock.unlock() } }
3779
+ for scalar in text.unicodeScalars {
3780
+ let char = String(scalar)
3781
+ if let stroke = physicalKeyStroke(for: char) {
3782
+ try postKey(stroke.key, flags: stroke.flags, pid: pid, delivery: delivery)
3783
+ continue
3784
+ }
3785
+ guard let down = CGEvent(keyboardEventSource: nil, virtualKey: 0, keyDown: true),
3786
+ let up = CGEvent(keyboardEventSource: nil, virtualKey: 0, keyDown: false)
3787
+ else {
3788
+ throw BridgeFailure(message: "Failed to create unicode key event", code: "input_failed")
3789
+ }
3790
+ setUnicodeString(event: down, text: char)
3791
+ setUnicodeString(event: up, text: char)
3792
+ postEvent(down, pid: pid, delivery: delivery)
3793
+ postEvent(up, pid: pid, delivery: delivery)
3794
+ usleep(8_000)
3795
+ }
3796
+ }
3797
+
3798
+ private func postAtomicUnicodeText(_ text: String, pid: Int32, delivery: String = "hid") throws {
3799
+ if delivery == "hid" { physicalInputLock.lock(); ForegroundActivity.shared.enter() }
3800
+ defer { if delivery == "hid" { ForegroundActivity.shared.leave(); physicalInputLock.unlock() } }
3801
+ guard let down = CGEvent(keyboardEventSource: nil, virtualKey: 0, keyDown: true),
3802
+ let up = CGEvent(keyboardEventSource: nil, virtualKey: 0, keyDown: false)
3803
+ else { throw BridgeFailure(message: "Failed to create unicode text event", code: "input_failed") }
3804
+ setUnicodeString(event: down, text: text)
3805
+ setUnicodeString(event: up, text: text)
3806
+ postEvent(down, pid: pid, delivery: delivery)
3807
+ usleep(8_000)
3808
+ postEvent(up, pid: pid, delivery: delivery)
3809
+ }
3810
+
3811
+ /// Prefer physical key codes for characters represented by the US layout.
3812
+ /// AppKit field editors can observe synthetic Unicode events without applying
3813
+ /// them to the backing value, while normal key codes follow the same input
3814
+ /// path as a user keystroke. Unicode synthesis remains the fallback for text
3815
+ /// that has no direct key representation.
3816
+ private func physicalKeyStroke(for character: String) -> (key: String, flags: CGEventFlags)? {
3817
+ guard character.count == 1 else { return nil }
3818
+ if character >= "a" && character <= "z" { return (character, []) }
3819
+ if character >= "A" && character <= "Z" { return (character.lowercased(), [.maskShift]) }
3820
+ if character >= "0" && character <= "9" { return (character, []) }
3821
+ switch character {
3822
+ case " ": return ("space", [])
3823
+ case ".", ",", "/", "-", "=", ";", "'", "[", "]", "\\", "`": return (character, [])
3824
+ case "_": return ("-", [.maskShift])
3825
+ case "+": return ("=", [.maskShift])
3826
+ case ":": return (";", [.maskShift])
3827
+ case "\"": return ("'", [.maskShift])
3828
+ case "?": return ("/", [.maskShift])
3829
+ case "<": return (",", [.maskShift])
3830
+ case ">": return (".", [.maskShift])
3831
+ default: return nil
3832
+ }
3833
+ }
3834
+
3835
+ private func setUnicodeString(event: CGEvent, text: String) {
3836
+ var utf16 = Array(text.utf16)
3837
+ utf16.withUnsafeMutableBufferPointer { buffer in
3838
+ guard let base = buffer.baseAddress else { return }
3839
+ event.keyboardSetUnicodeString(stringLength: buffer.count, unicodeString: base)
3840
+ }
3841
+ }
3842
+
3843
+ }
3844
+
3845
+ @main
3846
+ struct PiComputerUseHelper {
3847
+ static func main() {
3848
+ _ = NSApplication.shared
3849
+ NSApp.setActivationPolicy(CommandLine.arguments.contains("serve") ? .accessory : .prohibited)
3850
+ Bridge().run()
3851
+ }
3852
+ }