@forsion/tangu-computer-use 0.5.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +223 -0
- package/LICENSE +26 -0
- package/LICENSE.upstream +21 -0
- package/README.md +169 -0
- package/UPSTREAM.md +277 -0
- package/check.mjs +369 -0
- package/icon.png +0 -0
- package/install.sh +112 -0
- package/main.js +541 -0
- package/manifest.json +47 -0
- package/native/linux/bridge-rs/Cargo.lock +1204 -0
- package/native/linux/bridge-rs/Cargo.toml +18 -0
- package/native/linux/bridge-rs/src/atspi.rs +640 -0
- package/native/linux/bridge-rs/src/error.rs +65 -0
- package/native/linux/bridge-rs/src/lib.rs +11 -0
- package/native/linux/bridge-rs/src/main.rs +1074 -0
- package/native/linux/bridge-rs/src/protocol.rs +49 -0
- package/native/linux/bridge-rs/src/state.rs +293 -0
- package/native/linux/bridge-rs/src/wayland.rs +89 -0
- package/native/linux/bridge-rs/src/x11.rs +909 -0
- package/native/linux/bridge-rs/tests/protocol_tests.rs +73 -0
- package/native/macos/agent_cursor.swift +184 -0
- package/native/macos/agent_cursor_motion.swift +252 -0
- package/native/macos/agent_cursor_tests.swift +128 -0
- package/native/macos/agent_highlight.swift +229 -0
- package/native/macos/agent_highlight_tests.swift +99 -0
- package/native/macos/bridge.swift +3852 -0
- package/native/macos/foreground_activity.swift +57 -0
- package/native/macos/foreground_activity_tests.swift +31 -0
- package/native/macos/live_stream.swift +289 -0
- package/native/windows/bridge-rs/Cargo.lock +396 -0
- package/native/windows/bridge-rs/Cargo.toml +30 -0
- package/native/windows/bridge-rs/src/capture.rs +518 -0
- package/native/windows/bridge-rs/src/error.rs +81 -0
- package/native/windows/bridge-rs/src/input.rs +501 -0
- package/native/windows/bridge-rs/src/lib.rs +16 -0
- package/native/windows/bridge-rs/src/main.rs +1683 -0
- package/native/windows/bridge-rs/src/protocol.rs +53 -0
- package/native/windows/bridge-rs/src/refs.rs +237 -0
- package/native/windows/bridge-rs/src/state.rs +55 -0
- package/native/windows/bridge-rs/src/uia.rs +1252 -0
- package/native/windows/bridge-rs/src/window.rs +718 -0
- package/native/windows/bridge-rs/tests/protocol_tests.rs +252 -0
- package/native/windows/bridge-rs/tests/refs_tests.rs +86 -0
- package/native/windows/bridge-rs/tests/state_tests.rs +33 -0
- package/package.json +74 -0
- package/prebuilt/linux/arm64/linux-bridge +0 -0
- package/prebuilt/linux/x64/linux-bridge +0 -0
- package/prebuilt/macos/arm64/bridge +0 -0
- package/prebuilt/macos/arm64/tangu-computer-use.app.json +10 -0
- package/prebuilt/macos/arm64/tangu-computer-use.app.zip +0 -0
- package/prebuilt/macos/x64/bridge +0 -0
- package/prebuilt/macos/x64/tangu-computer-use.app.json +10 -0
- package/prebuilt/macos/x64/tangu-computer-use.app.zip +0 -0
- package/prebuilt/windows/windows-bridge.exe +0 -0
- package/scripts/blind-click.check.mjs +128 -0
- package/scripts/build-native.mjs +302 -0
- package/scripts/build.mjs +84 -0
- package/scripts/calc-fixture.mjs +78 -0
- package/scripts/cli-exit.check.mjs +49 -0
- package/scripts/helper-path.check.mjs +82 -0
- package/scripts/helper-refresh.check.mjs +62 -0
- package/scripts/helper-signal.check.mjs +59 -0
- package/scripts/highlight.check.mjs +32 -0
- package/scripts/keychain-free-install.check.mjs +140 -0
- package/scripts/live-view.check.mjs +124 -0
- package/scripts/macos-bundle.d.mts +1 -0
- package/scripts/macos-bundle.mjs +119 -0
- package/scripts/make-signing-cert.sh +57 -0
- package/scripts/mini-foreground.check.mjs +11 -0
- package/scripts/no-foreground.check.mjs +81 -0
- package/scripts/overlay-visible.check.mjs +221 -0
- package/scripts/package-macos-app.mjs +39 -0
- package/scripts/permissions.check.mjs +199 -0
- package/scripts/platform-contract.check.mjs +30 -0
- package/scripts/setup-helper.mjs +154 -0
- package/scripts/tangu-computer-use.entitlements +10 -0
- package/scripts/verify-macos-bundles.mjs +25 -0
- package/scripts/verify-package.mjs +74 -0
- package/skills/computer-use/SKILL.md +110 -0
- package/src/foregroundNote.ts +43 -0
- package/src/helperState.ts +31 -0
- package/src/index.ts +63 -0
- package/src/onboarding.ts +198 -0
- package/src/pi-compat.ts +49 -0
- package/src/settings.ts +11 -0
- package/src/setup.ts +97 -0
- package/src/tools.ts +289 -0
- package/src/vendor/actions.ts +130 -0
- package/src/vendor/bridge.ts +2405 -0
- package/src/vendor/cdp.ts +658 -0
- package/src/vendor/config.ts +113 -0
- package/src/vendor/contract.ts +104 -0
- package/src/vendor/note.ts +195 -0
- package/src/vendor/outline.ts +651 -0
- package/src/vendor/output.ts +134 -0
- package/src/vendor/permissions.ts +111 -0
- package/src/vendor/platform/architecture.ts +23 -0
- package/src/vendor/platform/coerce.ts +16 -0
- package/src/vendor/platform/index.ts +59 -0
- package/src/vendor/platform/linux/backend.ts +186 -0
- package/src/vendor/platform/linux/helper.ts +238 -0
- package/src/vendor/platform/macos/backend.ts +131 -0
- package/src/vendor/platform/macos/browser.ts +110 -0
- package/src/vendor/platform/macos/helper-path.d.mts +9 -0
- package/src/vendor/platform/macos/helper-path.mjs +34 -0
- package/src/vendor/platform/macos/helper.ts +291 -0
- package/src/vendor/platform/macos/permissions.ts +146 -0
- package/src/vendor/platform/types.ts +222 -0
- package/src/vendor/platform/windows/backend.ts +142 -0
- package/src/vendor/platform/windows/helper.ts +140 -0
- package/src/vendor/root-selection.ts +25 -0
- package/src/vendor/runtime.ts +129 -0
- package/src/vendor/state.ts +146 -0
- package/src/vendor/view.ts +147 -0
- package/tangu-plugins/computer-use/dist/foregroundNote.js +19 -0
- package/tangu-plugins/computer-use/dist/index.js +5672 -0
- package/tangu-plugins/computer-use/tangu-plugin.json +9 -0
- package/tsconfig.json +22 -0
- package/types/tangu-agent.d.ts +142 -0
|
@@ -0,0 +1,3852 @@
|
|
|
1
|
+
import Foundation
|
|
2
|
+
import AppKit
|
|
3
|
+
import ApplicationServices
|
|
4
|
+
import Darwin
|
|
5
|
+
import Vision
|
|
6
|
+
import ImageIO
|
|
7
|
+
import ScreenCaptureKit
|
|
8
|
+
|
|
9
|
+
struct BridgeFailure: Error {
|
|
10
|
+
let message: String
|
|
11
|
+
let code: String
|
|
12
|
+
}
|
|
13
|
+
|
|
14
|
+
final class AXRefStore {
|
|
15
|
+
struct Snapshot {
|
|
16
|
+
let role: String
|
|
17
|
+
let identifier: String
|
|
18
|
+
let label: String
|
|
19
|
+
let rect: CGRect
|
|
20
|
+
}
|
|
21
|
+
|
|
22
|
+
private var nextId: UInt64 = 0
|
|
23
|
+
private var windows: [String: AXUIElement] = [:]
|
|
24
|
+
private var elements: [String: AXUIElement] = [:]
|
|
25
|
+
private var snapshots: [String: Snapshot] = [:]
|
|
26
|
+
private let lock = NSLock()
|
|
27
|
+
|
|
28
|
+
func storeWindow(_ window: AXUIElement) -> String {
|
|
29
|
+
lock.lock()
|
|
30
|
+
defer { lock.unlock() }
|
|
31
|
+
for (ref, existing) in windows {
|
|
32
|
+
if CFEqual(existing, window) {
|
|
33
|
+
return ref
|
|
34
|
+
}
|
|
35
|
+
}
|
|
36
|
+
nextId += 1
|
|
37
|
+
let ref = "w\(nextId)"
|
|
38
|
+
windows[ref] = window
|
|
39
|
+
return ref
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
func storeElement(_ element: AXUIElement, snapshot: Snapshot? = nil) -> String {
|
|
43
|
+
lock.lock()
|
|
44
|
+
defer { lock.unlock() }
|
|
45
|
+
nextId += 1
|
|
46
|
+
let ref = "e\(nextId)"
|
|
47
|
+
elements[ref] = element
|
|
48
|
+
snapshots[ref] = snapshot
|
|
49
|
+
return ref
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
func window(for ref: String) -> AXUIElement? {
|
|
53
|
+
lock.lock()
|
|
54
|
+
defer { lock.unlock() }
|
|
55
|
+
return windows[ref]
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
func element(for ref: String) -> AXUIElement? {
|
|
59
|
+
lock.lock()
|
|
60
|
+
defer { lock.unlock() }
|
|
61
|
+
return elements[ref]
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
func snapshot(for ref: String) -> Snapshot? {
|
|
65
|
+
lock.lock()
|
|
66
|
+
defer { lock.unlock() }
|
|
67
|
+
return snapshots[ref]
|
|
68
|
+
}
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
private struct CGWindowCandidate {
|
|
72
|
+
let windowId: UInt32
|
|
73
|
+
let title: String
|
|
74
|
+
let bounds: CGRect
|
|
75
|
+
let isOnscreen: Bool
|
|
76
|
+
let layer: Int
|
|
77
|
+
let zOrder: Int
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
private struct CGWindowOwnerSummary {
|
|
81
|
+
let pid: Int32
|
|
82
|
+
let name: String
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
private struct AXDescendant {
|
|
86
|
+
let element: AXUIElement
|
|
87
|
+
let depth: Int
|
|
88
|
+
let insideWebArea: Bool
|
|
89
|
+
let axVisible: Bool
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
private struct WindowPairing {
|
|
93
|
+
let candidate: CGWindowCandidate?
|
|
94
|
+
let score: Double
|
|
95
|
+
let confidence: String
|
|
96
|
+
}
|
|
97
|
+
|
|
98
|
+
private struct CapturedWindowImage {
|
|
99
|
+
let image: CGImage
|
|
100
|
+
let windowId: UInt32
|
|
101
|
+
let frame: CGRect
|
|
102
|
+
}
|
|
103
|
+
|
|
104
|
+
private struct LookRecord {
|
|
105
|
+
let lookId: String
|
|
106
|
+
let windowId: UInt32
|
|
107
|
+
let windowFrame: CGRect
|
|
108
|
+
let imageWidth: Int
|
|
109
|
+
let imageHeight: Int
|
|
110
|
+
let hasImage: Bool
|
|
111
|
+
}
|
|
112
|
+
|
|
113
|
+
private struct RootAXEvent {
|
|
114
|
+
let sequence: UInt64
|
|
115
|
+
let timestamp: TimeInterval
|
|
116
|
+
let notification: String
|
|
117
|
+
let element: AXUIElement?
|
|
118
|
+
}
|
|
119
|
+
|
|
120
|
+
private final class RootAXObserverState {
|
|
121
|
+
let pid: Int32
|
|
122
|
+
let observer: AXObserver
|
|
123
|
+
let change = NSCondition()
|
|
124
|
+
var changeGeneration: UInt64 = 0
|
|
125
|
+
var events: [RootAXEvent] = []
|
|
126
|
+
var nextSequence: UInt64 = 1
|
|
127
|
+
var lastUsed: TimeInterval = Date().timeIntervalSince1970
|
|
128
|
+
|
|
129
|
+
init(pid: Int32, observer: AXObserver) {
|
|
130
|
+
self.pid = pid
|
|
131
|
+
self.observer = observer
|
|
132
|
+
}
|
|
133
|
+
}
|
|
134
|
+
|
|
135
|
+
private struct OCRBox {
|
|
136
|
+
let string: String
|
|
137
|
+
let confidence: Double
|
|
138
|
+
let rect: CGRect
|
|
139
|
+
}
|
|
140
|
+
|
|
141
|
+
final class LookNode {
|
|
142
|
+
let element: AXUIElement?
|
|
143
|
+
let ref: String
|
|
144
|
+
let role: String
|
|
145
|
+
let subrole: String
|
|
146
|
+
let identifier: String
|
|
147
|
+
let title: String
|
|
148
|
+
let description: String
|
|
149
|
+
let value: String
|
|
150
|
+
let actions: [String]
|
|
151
|
+
let canPress: Bool
|
|
152
|
+
let canFocus: Bool
|
|
153
|
+
let canSetValue: Bool
|
|
154
|
+
let canScroll: Bool
|
|
155
|
+
let canIncrement: Bool
|
|
156
|
+
let canDecrement: Bool
|
|
157
|
+
let isTextInput: Bool
|
|
158
|
+
let rect: CGRect
|
|
159
|
+
let focused: Bool
|
|
160
|
+
var offscreen: Bool
|
|
161
|
+
var pictureOnly: Bool
|
|
162
|
+
var truncated: Bool
|
|
163
|
+
var scrollExtent: [String: Int]?
|
|
164
|
+
var text: [[String: Any]]
|
|
165
|
+
var children: [LookNode]
|
|
166
|
+
|
|
167
|
+
init(element: AXUIElement?, ref: String, role: String, subrole: String, identifier: String, title: String, description: String, value: String, actions: [String], canPress: Bool, canFocus: Bool, canSetValue: Bool, canScroll: Bool, canIncrement: Bool, canDecrement: Bool, isTextInput: Bool, rect: CGRect, focused: Bool = false, offscreen: Bool = false, pictureOnly: Bool = false) {
|
|
168
|
+
self.element = element
|
|
169
|
+
self.ref = ref
|
|
170
|
+
self.role = role
|
|
171
|
+
self.subrole = subrole
|
|
172
|
+
self.identifier = identifier
|
|
173
|
+
self.title = title
|
|
174
|
+
self.description = description
|
|
175
|
+
self.value = value
|
|
176
|
+
self.actions = actions
|
|
177
|
+
self.canPress = canPress
|
|
178
|
+
self.canFocus = canFocus
|
|
179
|
+
self.canSetValue = canSetValue
|
|
180
|
+
self.canScroll = canScroll
|
|
181
|
+
self.canIncrement = canIncrement
|
|
182
|
+
self.canDecrement = canDecrement
|
|
183
|
+
self.isTextInput = isTextInput
|
|
184
|
+
self.rect = rect
|
|
185
|
+
self.focused = focused
|
|
186
|
+
self.offscreen = offscreen
|
|
187
|
+
self.pictureOnly = pictureOnly
|
|
188
|
+
self.truncated = false
|
|
189
|
+
self.text = []
|
|
190
|
+
self.children = []
|
|
191
|
+
}
|
|
192
|
+
|
|
193
|
+
func payload() -> [String: Any] {
|
|
194
|
+
var output: [String: Any] = [
|
|
195
|
+
"ref": ref,
|
|
196
|
+
"role": role,
|
|
197
|
+
"subrole": subrole,
|
|
198
|
+
"identifier": identifier,
|
|
199
|
+
"title": title,
|
|
200
|
+
"description": description,
|
|
201
|
+
"value": value,
|
|
202
|
+
"actions": actions,
|
|
203
|
+
"canPress": canPress,
|
|
204
|
+
"canFocus": canFocus,
|
|
205
|
+
"canSetValue": canSetValue,
|
|
206
|
+
"canScroll": canScroll,
|
|
207
|
+
"canIncrement": canIncrement,
|
|
208
|
+
"canDecrement": canDecrement,
|
|
209
|
+
"isTextInput": isTextInput,
|
|
210
|
+
"rect": ["x": rect.origin.x, "y": rect.origin.y, "w": rect.width, "h": rect.height],
|
|
211
|
+
"children": children.map { $0.payload() },
|
|
212
|
+
]
|
|
213
|
+
if focused { output["focused"] = true }
|
|
214
|
+
if offscreen { output["offscreen"] = true }
|
|
215
|
+
if pictureOnly { output["pictureOnly"] = true }
|
|
216
|
+
if truncated { output["truncated"] = true }
|
|
217
|
+
if let scrollExtent { output["scrollExtent"] = scrollExtent }
|
|
218
|
+
if !text.isEmpty { output["text"] = text }
|
|
219
|
+
return output
|
|
220
|
+
}
|
|
221
|
+
|
|
222
|
+
}
|
|
223
|
+
|
|
224
|
+
final class Box<T> {
|
|
225
|
+
var value: T
|
|
226
|
+
init(_ value: T) {
|
|
227
|
+
self.value = value
|
|
228
|
+
}
|
|
229
|
+
}
|
|
230
|
+
|
|
231
|
+
final class InputSuppressionGuard {
|
|
232
|
+
static let maxSuppressionSeconds: TimeInterval = 30
|
|
233
|
+
|
|
234
|
+
private let lock = NSLock()
|
|
235
|
+
private var eventTap: CFMachPort?
|
|
236
|
+
private var eventTapSource: CFRunLoopSource?
|
|
237
|
+
private var tapRunLoop: CFRunLoop?
|
|
238
|
+
private var tapThread: Thread?
|
|
239
|
+
|
|
240
|
+
func begin() throws {
|
|
241
|
+
lock.lock()
|
|
242
|
+
if eventTap != nil {
|
|
243
|
+
lock.unlock()
|
|
244
|
+
return
|
|
245
|
+
}
|
|
246
|
+
lock.unlock()
|
|
247
|
+
|
|
248
|
+
let eventTypes: [CGEventType] = [
|
|
249
|
+
.keyDown,
|
|
250
|
+
.keyUp,
|
|
251
|
+
.flagsChanged,
|
|
252
|
+
.leftMouseDown,
|
|
253
|
+
.leftMouseUp,
|
|
254
|
+
.rightMouseDown,
|
|
255
|
+
.rightMouseUp,
|
|
256
|
+
.otherMouseDown,
|
|
257
|
+
.otherMouseUp,
|
|
258
|
+
.mouseMoved,
|
|
259
|
+
.leftMouseDragged,
|
|
260
|
+
.rightMouseDragged,
|
|
261
|
+
.otherMouseDragged,
|
|
262
|
+
.scrollWheel,
|
|
263
|
+
.tabletPointer,
|
|
264
|
+
.tabletProximity,
|
|
265
|
+
]
|
|
266
|
+
let mask = eventTypes.reduce(CGEventMask(0)) { partial, type in
|
|
267
|
+
partial | (CGEventMask(1) << CGEventMask(type.rawValue))
|
|
268
|
+
}
|
|
269
|
+
|
|
270
|
+
let callback: CGEventTapCallBack = { _proxy, type, event, userInfo in
|
|
271
|
+
guard let userInfo else { return Unmanaged.passUnretained(event) }
|
|
272
|
+
let inputGuard = Unmanaged<InputSuppressionGuard>.fromOpaque(userInfo).takeUnretainedValue()
|
|
273
|
+
if type == .tapDisabledByTimeout || type == .tapDisabledByUserInput {
|
|
274
|
+
inputGuard.reenableTap()
|
|
275
|
+
return Unmanaged.passUnretained(event)
|
|
276
|
+
}
|
|
277
|
+
return nil
|
|
278
|
+
}
|
|
279
|
+
|
|
280
|
+
guard let tap = CGEvent.tapCreate(
|
|
281
|
+
tap: .cgSessionEventTap,
|
|
282
|
+
place: .headInsertEventTap,
|
|
283
|
+
options: .defaultTap,
|
|
284
|
+
eventsOfInterest: mask,
|
|
285
|
+
callback: callback,
|
|
286
|
+
// passUnretained is only safe because Bridge owns this guard for the
|
|
287
|
+
// whole process lifetime; do not give it a shorter-lived owner.
|
|
288
|
+
userInfo: UnsafeMutableRawPointer(Unmanaged.passUnretained(self).toOpaque())
|
|
289
|
+
) else {
|
|
290
|
+
throw BridgeFailure(message: "Failed to create input suppression event tap", code: "input_suppression_unavailable")
|
|
291
|
+
}
|
|
292
|
+
|
|
293
|
+
lock.lock()
|
|
294
|
+
eventTap = tap
|
|
295
|
+
lock.unlock()
|
|
296
|
+
let thread = Thread { [weak self] in
|
|
297
|
+
guard let self else { return }
|
|
298
|
+
let runLoop = CFRunLoopGetCurrent()
|
|
299
|
+
self.lock.lock()
|
|
300
|
+
self.tapRunLoop = runLoop
|
|
301
|
+
self.eventTapSource = CFMachPortCreateRunLoopSource(kCFAllocatorDefault, tap, 0)
|
|
302
|
+
if let source = self.eventTapSource {
|
|
303
|
+
CFRunLoopAddSource(runLoop, source, .commonModes)
|
|
304
|
+
}
|
|
305
|
+
// Watchdog: if end() never arrives (e.g. the parent process hangs
|
|
306
|
+
// mid-action), release the tap so the user is not locked out of
|
|
307
|
+
// keyboard and mouse input indefinitely.
|
|
308
|
+
let watchdog = CFRunLoopTimerCreateWithHandler(
|
|
309
|
+
kCFAllocatorDefault,
|
|
310
|
+
CFAbsoluteTimeGetCurrent() + InputSuppressionGuard.maxSuppressionSeconds,
|
|
311
|
+
0,
|
|
312
|
+
0,
|
|
313
|
+
0
|
|
314
|
+
) { [weak self] _ in
|
|
315
|
+
self?.end()
|
|
316
|
+
}
|
|
317
|
+
if let watchdog {
|
|
318
|
+
CFRunLoopAddTimer(runLoop, watchdog, .commonModes)
|
|
319
|
+
}
|
|
320
|
+
CGEvent.tapEnable(tap: tap, enable: true)
|
|
321
|
+
self.lock.unlock()
|
|
322
|
+
CFRunLoopRun()
|
|
323
|
+
}
|
|
324
|
+
thread.name = "tangu-computer-use-input-suppression"
|
|
325
|
+
lock.lock()
|
|
326
|
+
tapThread = thread
|
|
327
|
+
lock.unlock()
|
|
328
|
+
thread.start()
|
|
329
|
+
|
|
330
|
+
let deadline = Date().addingTimeInterval(1.0)
|
|
331
|
+
while tapRunLoop == nil && Date() < deadline {
|
|
332
|
+
Thread.sleep(forTimeInterval: 0.01)
|
|
333
|
+
}
|
|
334
|
+
if tapRunLoop == nil {
|
|
335
|
+
end()
|
|
336
|
+
throw BridgeFailure(message: "Timed out starting input suppression", code: "input_suppression_timeout")
|
|
337
|
+
}
|
|
338
|
+
}
|
|
339
|
+
|
|
340
|
+
func end() {
|
|
341
|
+
lock.lock()
|
|
342
|
+
let tap = eventTap
|
|
343
|
+
let source = eventTapSource
|
|
344
|
+
let runLoop = tapRunLoop
|
|
345
|
+
eventTap = nil
|
|
346
|
+
eventTapSource = nil
|
|
347
|
+
tapRunLoop = nil
|
|
348
|
+
tapThread = nil
|
|
349
|
+
lock.unlock()
|
|
350
|
+
|
|
351
|
+
if let tap {
|
|
352
|
+
CGEvent.tapEnable(tap: tap, enable: false)
|
|
353
|
+
}
|
|
354
|
+
if let source, let runLoop {
|
|
355
|
+
CFRunLoopRemoveSource(runLoop, source, .commonModes)
|
|
356
|
+
CFRunLoopStop(runLoop)
|
|
357
|
+
}
|
|
358
|
+
}
|
|
359
|
+
|
|
360
|
+
func reenableTap() {
|
|
361
|
+
lock.lock()
|
|
362
|
+
let tap = eventTap
|
|
363
|
+
lock.unlock()
|
|
364
|
+
if let tap {
|
|
365
|
+
CGEvent.tapEnable(tap: tap, enable: true)
|
|
366
|
+
}
|
|
367
|
+
}
|
|
368
|
+
|
|
369
|
+
}
|
|
370
|
+
|
|
371
|
+
final class Bridge {
|
|
372
|
+
private let protocolVersion = 12
|
|
373
|
+
private let refStore = AXRefStore()
|
|
374
|
+
private let inputSuppressionGuard = InputSuppressionGuard()
|
|
375
|
+
private let physicalInputLock = NSRecursiveLock()
|
|
376
|
+
private let supportsAgentCursor = CommandLine.arguments.contains("serve")
|
|
377
|
+
private let browserBundleIds: Set<String> = [
|
|
378
|
+
"com.apple.Safari", "com.google.Chrome", "org.chromium.Chromium", "company.thebrowser.Browser", "com.brave.Browser", "com.microsoft.edgemac", "com.vivaldi.Vivaldi", "net.imput.helium", "org.mozilla.firefox",
|
|
379
|
+
]
|
|
380
|
+
private var enhancedAccessibilityPids = Set<Int32>()
|
|
381
|
+
private let enhancedAccessibilityLock = NSLock()
|
|
382
|
+
private var stdinBuffer = Data()
|
|
383
|
+
private var output = FileHandle.standardOutput
|
|
384
|
+
private var nextLookId: UInt64 = 0
|
|
385
|
+
private var lookRecords: [String: LookRecord] = [:]
|
|
386
|
+
private var lookRecordOrder: [String] = []
|
|
387
|
+
private let lookRecordLock = NSLock()
|
|
388
|
+
private let rootObserverLock = NSLock()
|
|
389
|
+
private var rootObservers: [Int32: RootAXObserverState] = [:]
|
|
390
|
+
private let maxRootObservers = 4
|
|
391
|
+
private let permissionCacheLock = NSLock()
|
|
392
|
+
private var grantedPermissionStatus: [String: Any]?
|
|
393
|
+
private let completedRequestLock = NSLock()
|
|
394
|
+
private var recentCompletedRequestIds: [String] = []
|
|
395
|
+
// Tangu 新增:最近一次被操控的窗口(供边缘光效跟随 + liveView 取画面)。serve 线程写、主线程读。
|
|
396
|
+
private let controlledLock = NSLock()
|
|
397
|
+
private var controlledWindowId: UInt32 = 0
|
|
398
|
+
private var controlledPid: Int32 = 0
|
|
399
|
+
private var controlledAt: Date?
|
|
400
|
+
private var highlightFollowTimer: Timer?
|
|
401
|
+
|
|
402
|
+
func run() {
|
|
403
|
+
if CommandLine.arguments.contains("serve") {
|
|
404
|
+
let socketPath = argumentValue("--socket") ?? defaultSocketPath()
|
|
405
|
+
Thread.detachNewThread { [self] in runServer(socketPath: socketPath) }
|
|
406
|
+
NSApp.run()
|
|
407
|
+
return
|
|
408
|
+
}
|
|
409
|
+
while true {
|
|
410
|
+
autoreleasepool {
|
|
411
|
+
let data = FileHandle.standardInput.availableData
|
|
412
|
+
if data.isEmpty {
|
|
413
|
+
exit(0)
|
|
414
|
+
}
|
|
415
|
+
stdinBuffer.append(data)
|
|
416
|
+
processBufferedInput()
|
|
417
|
+
}
|
|
418
|
+
}
|
|
419
|
+
}
|
|
420
|
+
|
|
421
|
+
private func argumentValue(_ name: String) -> String? {
|
|
422
|
+
guard let index = CommandLine.arguments.firstIndex(of: name), CommandLine.arguments.indices.contains(index + 1) else { return nil }
|
|
423
|
+
return CommandLine.arguments[index + 1]
|
|
424
|
+
}
|
|
425
|
+
|
|
426
|
+
private func defaultSocketPath() -> String {
|
|
427
|
+
let home = FileManager.default.homeDirectoryForCurrentUser.path
|
|
428
|
+
return "\(home)/Library/Caches/tangu-computer-use/bridge.sock"
|
|
429
|
+
}
|
|
430
|
+
|
|
431
|
+
private func runServer(socketPath: String) {
|
|
432
|
+
_ = signal(SIGPIPE, SIG_IGN)
|
|
433
|
+
try? FileManager.default.createDirectory(atPath: (socketPath as NSString).deletingLastPathComponent, withIntermediateDirectories: true)
|
|
434
|
+
// LaunchServices may race multiple `open -n` requests while the first
|
|
435
|
+
// daemon is still binding. Keep process ownership separate from socket
|
|
436
|
+
// cleanup so a late launcher can never unlink the live daemon's socket.
|
|
437
|
+
let lockPath = "\(socketPath).lock"
|
|
438
|
+
let lockFile = open(lockPath, O_CREAT | O_RDWR, S_IRUSR | S_IWUSR)
|
|
439
|
+
if lockFile < 0 || flock(lockFile, LOCK_EX | LOCK_NB) != 0 {
|
|
440
|
+
if lockFile >= 0 { close(lockFile) }
|
|
441
|
+
exit(0)
|
|
442
|
+
}
|
|
443
|
+
ForegroundActivity.shared.configure(socketPath: socketPath)
|
|
444
|
+
unlink(socketPath)
|
|
445
|
+
let server = socket(AF_UNIX, SOCK_STREAM, 0)
|
|
446
|
+
if server < 0 { close(lockFile); exit(1) }
|
|
447
|
+
var address = sockaddr_un()
|
|
448
|
+
address.sun_family = sa_family_t(AF_UNIX)
|
|
449
|
+
let sunPathCapacity = MemoryLayout.size(ofValue: address.sun_path)
|
|
450
|
+
let bytes = Array(socketPath.utf8.prefix(sunPathCapacity - 1))
|
|
451
|
+
withUnsafeMutablePointer(to: &address.sun_path) { pointer in
|
|
452
|
+
pointer.withMemoryRebound(to: CChar.self, capacity: sunPathCapacity) { dest in
|
|
453
|
+
for (index, byte) in bytes.enumerated() { dest[index] = CChar(bitPattern: byte) }
|
|
454
|
+
}
|
|
455
|
+
}
|
|
456
|
+
let bindStatus = withUnsafePointer(to: &address) { pointer in
|
|
457
|
+
pointer.withMemoryRebound(to: sockaddr.self, capacity: 1) { bind(server, $0, socklen_t(MemoryLayout<sockaddr_un>.size)) }
|
|
458
|
+
}
|
|
459
|
+
if bindStatus != 0 || listen(server, 8) != 0 { close(server); close(lockFile); exit(1) }
|
|
460
|
+
while true {
|
|
461
|
+
let client = accept(server, nil, nil)
|
|
462
|
+
if client < 0 { continue }
|
|
463
|
+
var noSigPipe: Int32 = 1
|
|
464
|
+
_ = setsockopt(client, SOL_SOCKET, SO_NOSIGPIPE, &noSigPipe, socklen_t(MemoryLayout.size(ofValue: noSigPipe)))
|
|
465
|
+
Thread.detachNewThread { [weak self] in self?.processClient(client) }
|
|
466
|
+
}
|
|
467
|
+
}
|
|
468
|
+
|
|
469
|
+
private func processClient(_ client: Int32) {
|
|
470
|
+
let clientInput = FileHandle(fileDescriptor: client, closeOnDealloc: true)
|
|
471
|
+
var buffer = Data()
|
|
472
|
+
let newline = Data([0x0A])
|
|
473
|
+
while true {
|
|
474
|
+
let data = clientInput.availableData
|
|
475
|
+
if data.isEmpty { break }
|
|
476
|
+
buffer.append(data)
|
|
477
|
+
while let range = buffer.range(of: newline) {
|
|
478
|
+
let lineData = buffer.subdata(in: 0..<range.lowerBound)
|
|
479
|
+
buffer.removeSubrange(0..<range.upperBound)
|
|
480
|
+
if let line = String(data: lineData, encoding: .utf8), !line.isEmpty { handleLine(line, to: client) }
|
|
481
|
+
}
|
|
482
|
+
}
|
|
483
|
+
clientInput.closeFile()
|
|
484
|
+
}
|
|
485
|
+
|
|
486
|
+
private func processBufferedInput() {
|
|
487
|
+
let newline = Data([0x0A])
|
|
488
|
+
while let range = stdinBuffer.range(of: newline) {
|
|
489
|
+
let lineData = stdinBuffer.subdata(in: 0..<range.lowerBound)
|
|
490
|
+
stdinBuffer.removeSubrange(0..<range.upperBound)
|
|
491
|
+
|
|
492
|
+
guard !lineData.isEmpty else { continue }
|
|
493
|
+
guard let line = String(data: lineData, encoding: .utf8) else { continue }
|
|
494
|
+
handleLine(line)
|
|
495
|
+
}
|
|
496
|
+
}
|
|
497
|
+
|
|
498
|
+
private func handleLine(_ line: String, to responseSocket: Int32? = nil) {
|
|
499
|
+
let trimmed = line.trimmingCharacters(in: .whitespacesAndNewlines)
|
|
500
|
+
guard !trimmed.isEmpty else { return }
|
|
501
|
+
|
|
502
|
+
let fallbackId = "invalid"
|
|
503
|
+
var completionId: String?
|
|
504
|
+
defer {
|
|
505
|
+
if responseSocket != nil, let completionId { recordCompletedRequest(completionId) }
|
|
506
|
+
}
|
|
507
|
+
do {
|
|
508
|
+
guard let jsonData = trimmed.data(using: .utf8) else {
|
|
509
|
+
throw BridgeFailure(message: "Input was not valid UTF-8", code: "invalid_request")
|
|
510
|
+
}
|
|
511
|
+
guard let object = try JSONSerialization.jsonObject(with: jsonData) as? [String: Any] else {
|
|
512
|
+
throw BridgeFailure(message: "Request must be a JSON object", code: "invalid_request")
|
|
513
|
+
}
|
|
514
|
+
let id = (object["id"] as? String) ?? fallbackId
|
|
515
|
+
completionId = id
|
|
516
|
+
|
|
517
|
+
do {
|
|
518
|
+
let result = try handleRequest(object)
|
|
519
|
+
send([
|
|
520
|
+
"id": id,
|
|
521
|
+
"ok": true,
|
|
522
|
+
"result": result,
|
|
523
|
+
], to: responseSocket)
|
|
524
|
+
} catch let failure as BridgeFailure {
|
|
525
|
+
send([
|
|
526
|
+
"id": id,
|
|
527
|
+
"ok": false,
|
|
528
|
+
"error": [
|
|
529
|
+
"message": failure.message,
|
|
530
|
+
"code": failure.code,
|
|
531
|
+
],
|
|
532
|
+
], to: responseSocket)
|
|
533
|
+
} catch {
|
|
534
|
+
send([
|
|
535
|
+
"id": id,
|
|
536
|
+
"ok": false,
|
|
537
|
+
"error": [
|
|
538
|
+
"message": error.localizedDescription,
|
|
539
|
+
"code": "internal_error",
|
|
540
|
+
],
|
|
541
|
+
], to: responseSocket)
|
|
542
|
+
}
|
|
543
|
+
} catch let failure as BridgeFailure {
|
|
544
|
+
send([
|
|
545
|
+
"id": fallbackId,
|
|
546
|
+
"ok": false,
|
|
547
|
+
"error": [
|
|
548
|
+
"message": failure.message,
|
|
549
|
+
"code": failure.code,
|
|
550
|
+
],
|
|
551
|
+
], to: responseSocket)
|
|
552
|
+
} catch {
|
|
553
|
+
send([
|
|
554
|
+
"id": fallbackId,
|
|
555
|
+
"ok": false,
|
|
556
|
+
"error": [
|
|
557
|
+
"message": error.localizedDescription,
|
|
558
|
+
"code": "internal_error",
|
|
559
|
+
],
|
|
560
|
+
], to: responseSocket)
|
|
561
|
+
}
|
|
562
|
+
}
|
|
563
|
+
|
|
564
|
+
private func send(_ payload: [String: Any], to responseSocket: Int32? = nil) {
|
|
565
|
+
guard JSONSerialization.isValidJSONObject(payload),
|
|
566
|
+
let data = try? JSONSerialization.data(withJSONObject: payload),
|
|
567
|
+
let line = String(data: data, encoding: .utf8),
|
|
568
|
+
let out = (line + "\n").data(using: .utf8)
|
|
569
|
+
else {
|
|
570
|
+
return
|
|
571
|
+
}
|
|
572
|
+
|
|
573
|
+
guard let responseSocket else {
|
|
574
|
+
try? output.write(contentsOf: out)
|
|
575
|
+
return
|
|
576
|
+
}
|
|
577
|
+
out.withUnsafeBytes { raw in
|
|
578
|
+
guard let base = raw.baseAddress else { return }
|
|
579
|
+
var offset = 0
|
|
580
|
+
while offset < raw.count {
|
|
581
|
+
let sent = Darwin.send(responseSocket, base.advanced(by: offset), raw.count - offset, 0)
|
|
582
|
+
if sent <= 0 { return }
|
|
583
|
+
offset += sent
|
|
584
|
+
}
|
|
585
|
+
}
|
|
586
|
+
}
|
|
587
|
+
|
|
588
|
+
private func recordCompletedRequest(_ id: String) {
|
|
589
|
+
completedRequestLock.lock()
|
|
590
|
+
recentCompletedRequestIds.removeAll { $0 == id }
|
|
591
|
+
recentCompletedRequestIds.append(id)
|
|
592
|
+
if recentCompletedRequestIds.count > 32 {
|
|
593
|
+
recentCompletedRequestIds.removeFirst(recentCompletedRequestIds.count - 32)
|
|
594
|
+
}
|
|
595
|
+
completedRequestLock.unlock()
|
|
596
|
+
}
|
|
597
|
+
|
|
598
|
+
private func completedRequestIds() -> [String] {
|
|
599
|
+
completedRequestLock.lock()
|
|
600
|
+
defer { completedRequestLock.unlock() }
|
|
601
|
+
return recentCompletedRequestIds
|
|
602
|
+
}
|
|
603
|
+
|
|
604
|
+
// MARK: - Tangu 扩展:被操控窗口的边缘光效 + 实时画面(非上游)
|
|
605
|
+
|
|
606
|
+
/// 记下"现在正被操控的是哪个窗口",并按需点亮边缘光效。每次 act 调一次。
|
|
607
|
+
/// 记录与光效分开:即使用户关了光效,liveView 仍要知道该看哪个窗口。
|
|
608
|
+
private func noteControlledWindow(windowId: UInt32, pid: Int32, highlight: Bool) {
|
|
609
|
+
controlledLock.lock()
|
|
610
|
+
controlledWindowId = windowId
|
|
611
|
+
controlledPid = pid
|
|
612
|
+
controlledAt = Date()
|
|
613
|
+
controlledLock.unlock()
|
|
614
|
+
guard supportsAgentCursor, highlight else { return }
|
|
615
|
+
Task { @MainActor in AgentHighlight.shared.show(windowId: windowId) }
|
|
616
|
+
}
|
|
617
|
+
|
|
618
|
+
private func controlledSnapshot() -> (windowId: UInt32, pid: Int32, at: Date)? {
|
|
619
|
+
controlledLock.lock()
|
|
620
|
+
defer { controlledLock.unlock() }
|
|
621
|
+
guard controlledWindowId != 0, let at = controlledAt else { return nil }
|
|
622
|
+
return (controlledWindowId, controlledPid, at)
|
|
623
|
+
}
|
|
624
|
+
|
|
625
|
+
/// 最近被操控窗口的一帧画面。给 Forsion 桌面的"被操控窗口"视图轮询用(≤2fps)。
|
|
626
|
+
/// 只读:**绝不**启动 helper、绝不改焦点。没有正在被操控的窗口就 active=false。
|
|
627
|
+
/// 拿不到画面(如未授予录屏)也照实回 active=true + error,不编一张图。
|
|
628
|
+
private func liveView(_ request: [String: Any]) throws -> [String: Any] {
|
|
629
|
+
let activeWithinMs = optionalIntArg(request, "activeWithinMs") ?? 120_000
|
|
630
|
+
guard let controlled = controlledSnapshot() else { return ["active": false] }
|
|
631
|
+
let ageMs = Int(Date().timeIntervalSince(controlled.at) * 1000)
|
|
632
|
+
if ageMs > max(0, activeWithinMs) { return ["active": false] }
|
|
633
|
+
|
|
634
|
+
var payload: [String: Any] = ["active": true, "windowId": Int(controlled.windowId), "pid": Int(controlled.pid), "ageMs": ageMs]
|
|
635
|
+
if let app = NSRunningApplication(processIdentifier: controlled.pid) {
|
|
636
|
+
if let name = app.localizedName { payload["app"] = name }
|
|
637
|
+
if let bundleId = app.bundleIdentifier { payload["bundleId"] = bundleId }
|
|
638
|
+
}
|
|
639
|
+
if let window = windowElement(pid: controlled.pid, windowId: controlled.windowId),
|
|
640
|
+
let title = stringAttribute(window, attribute: kAXTitleAttribute as CFString) {
|
|
641
|
+
payload["title"] = title
|
|
642
|
+
}
|
|
643
|
+
guard request["image"] as? Bool ?? true else { return payload }
|
|
644
|
+
let maxDimension = optionalIntArg(request, "maxDimension") ?? 1280
|
|
645
|
+
let quality = (request["quality"] as? NSNumber)?.doubleValue ?? 0.6
|
|
646
|
+
// 先问常驻取景流要最新一帧(真·实时;见 live_stream.swift)。流还没起来 —— 首帧、刚换窗口 ——
|
|
647
|
+
// 退到**同样按窗口尺寸出图**的单帧兜底,所以第一帧照样立刻有画面、也照样没有留白。
|
|
648
|
+
// `source` 让调用方看得见走的哪条路。
|
|
649
|
+
if #available(macOS 14.0, *) {
|
|
650
|
+
let live = LiveWindowStream.shared
|
|
651
|
+
var source = "stream"
|
|
652
|
+
var image: CGImage?
|
|
653
|
+
if let fresh = live.latestFrame(windowId: controlled.windowId, maxDimension: maxDimension) {
|
|
654
|
+
payload["frameSeq"] = String(fresh.seq)
|
|
655
|
+
// 静止的窗口不会有新帧。调用方带着上次见过的帧号来,一样就直接说"没变" ——
|
|
656
|
+
// 否则同一张图会被每秒重编 8 次 JPEG + base64 + 过 socket + 换一次 <img src>,纯白烧。
|
|
657
|
+
if let seen = optionalStringArg(request, "sinceFrame"), seen == String(fresh.seq) {
|
|
658
|
+
payload["unchanged"] = true
|
|
659
|
+
payload["source"] = source
|
|
660
|
+
return payload
|
|
661
|
+
}
|
|
662
|
+
image = fresh.image
|
|
663
|
+
} else {
|
|
664
|
+
source = "snapshot"
|
|
665
|
+
image = live.captureOnce(windowId: controlled.windowId, maxDimension: maxDimension)
|
|
666
|
+
}
|
|
667
|
+
if let image {
|
|
668
|
+
guard let jpeg = jpegData(image: image, quality: quality) else {
|
|
669
|
+
payload["error"] = "encode_failed"
|
|
670
|
+
return payload
|
|
671
|
+
}
|
|
672
|
+
payload["width"] = image.width
|
|
673
|
+
payload["height"] = image.height
|
|
674
|
+
payload["jpegBase64"] = jpeg.base64EncodedString()
|
|
675
|
+
payload["source"] = source
|
|
676
|
+
return payload
|
|
677
|
+
}
|
|
678
|
+
}
|
|
679
|
+
// 最后一条老路(macOS <14,或上面两条都拿不到):上游的 captureWindow。它按整块屏幕出图,
|
|
680
|
+
// 窗口缩在左上角、四周留白 —— 有画面总比没有强,但打上 legacy 标记别让人以为这是正常尺寸。
|
|
681
|
+
do {
|
|
682
|
+
let captured = try captureWindow(windowId: controlled.windowId)
|
|
683
|
+
let image = downscaledImage(captured.image, maxDimension: maxDimension) ?? captured.image
|
|
684
|
+
guard let jpeg = jpegData(image: image, quality: quality) else {
|
|
685
|
+
payload["error"] = "encode_failed"
|
|
686
|
+
return payload
|
|
687
|
+
}
|
|
688
|
+
payload["width"] = image.width
|
|
689
|
+
payload["height"] = image.height
|
|
690
|
+
payload["jpegBase64"] = jpeg.base64EncodedString()
|
|
691
|
+
payload["source"] = "legacy"
|
|
692
|
+
} catch let failure as BridgeFailure {
|
|
693
|
+
payload["error"] = failure.code
|
|
694
|
+
} catch {
|
|
695
|
+
payload["error"] = "capture_failed"
|
|
696
|
+
}
|
|
697
|
+
return payload
|
|
698
|
+
}
|
|
699
|
+
|
|
700
|
+
private func handleRequest(_ request: [String: Any]) throws -> Any {
|
|
701
|
+
let cmd = try stringArg(request, "cmd")
|
|
702
|
+
|
|
703
|
+
switch cmd {
|
|
704
|
+
case "diagnostics":
|
|
705
|
+
return diagnostics()
|
|
706
|
+
case "permissionStatus":
|
|
707
|
+
return permissionStatus()
|
|
708
|
+
case "checkPermissions":
|
|
709
|
+
return checkPermissions(fresh: boolArg(request, "fresh") ?? false)
|
|
710
|
+
case "registerPermissions":
|
|
711
|
+
return try registerPermissions(request)
|
|
712
|
+
case "openPermissionPane":
|
|
713
|
+
return try openPermissionPane(request)
|
|
714
|
+
case "shutdown":
|
|
715
|
+
// Reply first, then exit: the caller relaunches the helper to get a
|
|
716
|
+
// process with a fresh TCC client (grant answers are cached per
|
|
717
|
+
// process, so a helper that saw "denied" keeps answering "denied"
|
|
718
|
+
// after the user grants — only a new process re-queries tccd).
|
|
719
|
+
// Background queue: the serve loop occupies the main thread, so a
|
|
720
|
+
// main-queue timer would never fire.
|
|
721
|
+
DispatchQueue.global().asyncAfter(deadline: .now() + 0.2) { exit(0) }
|
|
722
|
+
return ["shuttingDown": true]
|
|
723
|
+
case "listApps":
|
|
724
|
+
return listApps()
|
|
725
|
+
case "listWindows":
|
|
726
|
+
return try listWindows(pid: Int32(try intArg(request, "pid")))
|
|
727
|
+
case "listRoots":
|
|
728
|
+
return try listRoots(pid: optionalIntArg(request, "pid").map { Int32($0) }, title: optionalStringArg(request, "title"))
|
|
729
|
+
case "getFrontmost":
|
|
730
|
+
return try getFrontmost()
|
|
731
|
+
case "getUserContext":
|
|
732
|
+
return try getUserContext()
|
|
733
|
+
case "beginInputSuppression":
|
|
734
|
+
return try beginInputSuppression()
|
|
735
|
+
case "endInputSuppression":
|
|
736
|
+
return endInputSuppression()
|
|
737
|
+
case "restoreUserFocus":
|
|
738
|
+
return try restoreUserFocus(request)
|
|
739
|
+
case "focusWindow":
|
|
740
|
+
return try focusWindow(request)
|
|
741
|
+
case "setWindowFrame":
|
|
742
|
+
return try setWindowFrame(request)
|
|
743
|
+
case "look":
|
|
744
|
+
return try look(request)
|
|
745
|
+
case "act":
|
|
746
|
+
return try act(request)
|
|
747
|
+
case "actBatch":
|
|
748
|
+
return try actBatch(request)
|
|
749
|
+
case "liveView":
|
|
750
|
+
return try liveView(request)
|
|
751
|
+
case "hitTest":
|
|
752
|
+
return try hitTest(request)
|
|
753
|
+
case "axWaitFor":
|
|
754
|
+
return try axWaitFor(request)
|
|
755
|
+
case "focusedElement":
|
|
756
|
+
return try focusedElement(request)
|
|
757
|
+
case "axReadText":
|
|
758
|
+
return try axReadText(request)
|
|
759
|
+
case "getMousePosition":
|
|
760
|
+
return getMousePosition()
|
|
761
|
+
default:
|
|
762
|
+
throw BridgeFailure(message: "Unknown command '\(cmd)'", code: "unknown_command")
|
|
763
|
+
}
|
|
764
|
+
}
|
|
765
|
+
|
|
766
|
+
private func stringArg(_ request: [String: Any], _ key: String) throws -> String {
|
|
767
|
+
if let value = request[key] as? String {
|
|
768
|
+
return value
|
|
769
|
+
}
|
|
770
|
+
throw BridgeFailure(message: "Missing string argument '\(key)'", code: "invalid_args")
|
|
771
|
+
}
|
|
772
|
+
|
|
773
|
+
private func optionalStringArg(_ request: [String: Any], _ key: String) -> String? {
|
|
774
|
+
if let value = request[key] as? String {
|
|
775
|
+
return value
|
|
776
|
+
}
|
|
777
|
+
return nil
|
|
778
|
+
}
|
|
779
|
+
|
|
780
|
+
private func intArg(_ request: [String: Any], _ key: String) throws -> Int {
|
|
781
|
+
if let value = request[key] as? Int {
|
|
782
|
+
return value
|
|
783
|
+
}
|
|
784
|
+
if let value = request[key] as? NSNumber {
|
|
785
|
+
return value.intValue
|
|
786
|
+
}
|
|
787
|
+
if let value = request[key] as? Double {
|
|
788
|
+
return Int(value)
|
|
789
|
+
}
|
|
790
|
+
throw BridgeFailure(message: "Missing integer argument '\(key)'", code: "invalid_args")
|
|
791
|
+
}
|
|
792
|
+
|
|
793
|
+
private func optionalIntArg(_ request: [String: Any], _ key: String) -> Int? {
|
|
794
|
+
if let value = request[key] as? Int {
|
|
795
|
+
return value
|
|
796
|
+
}
|
|
797
|
+
if let value = request[key] as? NSNumber {
|
|
798
|
+
return value.intValue
|
|
799
|
+
}
|
|
800
|
+
if let value = request[key] as? Double {
|
|
801
|
+
return Int(value)
|
|
802
|
+
}
|
|
803
|
+
return nil
|
|
804
|
+
}
|
|
805
|
+
|
|
806
|
+
private func boolArg(_ request: [String: Any], _ key: String) -> Bool? {
|
|
807
|
+
if let value = request[key] as? Bool { return value }
|
|
808
|
+
if let value = request[key] as? NSNumber { return value.boolValue }
|
|
809
|
+
return nil
|
|
810
|
+
}
|
|
811
|
+
|
|
812
|
+
private func doubleArg(_ request: [String: Any], _ key: String) throws -> Double {
|
|
813
|
+
if let value = request[key] as? Double {
|
|
814
|
+
return value
|
|
815
|
+
}
|
|
816
|
+
if let value = request[key] as? NSNumber {
|
|
817
|
+
return value.doubleValue
|
|
818
|
+
}
|
|
819
|
+
if let value = request[key] as? Int {
|
|
820
|
+
return Double(value)
|
|
821
|
+
}
|
|
822
|
+
throw BridgeFailure(message: "Missing numeric argument '\(key)'", code: "invalid_args")
|
|
823
|
+
}
|
|
824
|
+
|
|
825
|
+
private func processPath(pid: pid_t) -> String? {
|
|
826
|
+
var buffer = [CChar](repeating: 0, count: 4096)
|
|
827
|
+
let length = proc_pidpath(pid, &buffer, UInt32(buffer.count))
|
|
828
|
+
return length > 0 ? String(cString: buffer) : nil
|
|
829
|
+
}
|
|
830
|
+
|
|
831
|
+
private func processName(pid: pid_t) -> String? {
|
|
832
|
+
processPath(pid: pid).map { URL(fileURLWithPath: $0).deletingPathExtension().lastPathComponent }
|
|
833
|
+
}
|
|
834
|
+
|
|
835
|
+
private func diagnostics() -> [String: Any] {
|
|
836
|
+
// Cheap booleans only — diagnostics doubles as the daemon liveness
|
|
837
|
+
// probe (1s client timeout), so it must not run the ScreenCaptureKit
|
|
838
|
+
// capturable check (up to 3s when ungranted). Permission truth comes
|
|
839
|
+
// from checkPermissions.
|
|
840
|
+
let permissions: [String: Any] = [
|
|
841
|
+
"accessibility": AXIsProcessTrusted(),
|
|
842
|
+
"screenRecording": {
|
|
843
|
+
if #available(macOS 10.15, *) { return CGPreflightScreenCaptureAccess() }
|
|
844
|
+
return true
|
|
845
|
+
}(),
|
|
846
|
+
]
|
|
847
|
+
#if arch(arm64)
|
|
848
|
+
let arch = "arm64"
|
|
849
|
+
#elseif arch(x86_64)
|
|
850
|
+
let arch = "x86_64"
|
|
851
|
+
#else
|
|
852
|
+
let arch = "unknown"
|
|
853
|
+
#endif
|
|
854
|
+
let parentPid = Int32(getppid())
|
|
855
|
+
let parentApp = NSRunningApplication(processIdentifier: parentPid)
|
|
856
|
+
let parentPath = processPath(pid: parentPid)
|
|
857
|
+
var output: [String: Any] = [
|
|
858
|
+
"protocolVersion": protocolVersion,
|
|
859
|
+
"architectureVersion": 1,
|
|
860
|
+
"invariants": ["state-scoped-observations", "bounded-observation-history", "multi-root-forest", "progressive-disclosure", "atomic-physical-input", "concurrent-requests", "transactional-batching"],
|
|
861
|
+
"pid": Int32(getpid()),
|
|
862
|
+
"parentPid": parentPid,
|
|
863
|
+
"executablePath": CommandLine.arguments.first ?? "",
|
|
864
|
+
"macOS": ProcessInfo.processInfo.operatingSystemVersionString,
|
|
865
|
+
"arch": arch,
|
|
866
|
+
"accessibility": permissions["accessibility"] ?? false,
|
|
867
|
+
"screenRecording": permissions["screenRecording"] ?? false,
|
|
868
|
+
"recentCompletedRequestIds": completedRequestIds(),
|
|
869
|
+
]
|
|
870
|
+
if let parentPath {
|
|
871
|
+
output["parentPath"] = parentPath
|
|
872
|
+
}
|
|
873
|
+
if let parentAppName = parentApp?.localizedName ?? parentPath.map({ URL(fileURLWithPath: $0).lastPathComponent }) {
|
|
874
|
+
output["parentAppName"] = parentAppName
|
|
875
|
+
}
|
|
876
|
+
if let parentBundleId = parentApp?.bundleIdentifier {
|
|
877
|
+
output["parentBundleId"] = parentBundleId
|
|
878
|
+
}
|
|
879
|
+
return output
|
|
880
|
+
}
|
|
881
|
+
|
|
882
|
+
/// Live Screen Recording probe. `CGPreflightScreenCaptureAccess()`
|
|
883
|
+
/// answers from a per-process cache that goes stale after `tccutil
|
|
884
|
+
/// reset` or a Settings toggle; a ScreenCaptureKit content fetch only
|
|
885
|
+
/// succeeds when THIS process can genuinely capture right now. When the
|
|
886
|
+
/// two disagree, the preflight boolean is the one lying.
|
|
887
|
+
private func screenRecordingCapturable() -> Bool {
|
|
888
|
+
if #available(macOS 14.0, *) {
|
|
889
|
+
let sema = DispatchSemaphore(value: 0)
|
|
890
|
+
let capturable = Box<Bool>(false)
|
|
891
|
+
SCShareableContent.getExcludingDesktopWindows(false, onScreenWindowsOnly: false) { shareable, error in
|
|
892
|
+
if let shareable = shareable {
|
|
893
|
+
capturable.value = !shareable.displays.isEmpty
|
|
894
|
+
}
|
|
895
|
+
sema.signal()
|
|
896
|
+
}
|
|
897
|
+
guard sema.wait(timeout: .now() + .seconds(5)) == .success else { return false }
|
|
898
|
+
return capturable.value
|
|
899
|
+
}
|
|
900
|
+
if #available(macOS 10.15, *) {
|
|
901
|
+
return CGPreflightScreenCaptureAccess()
|
|
902
|
+
}
|
|
903
|
+
return true
|
|
904
|
+
}
|
|
905
|
+
|
|
906
|
+
/// Which TCC identity the permission booleans reflect. macOS attributes
|
|
907
|
+
/// grants to the *responsible process* (the LaunchServices launching
|
|
908
|
+
/// app), so:
|
|
909
|
+
/// - "helper-app": running from the installed bundle, launched via
|
|
910
|
+
/// LaunchServices — grants belong to the canonical helper identity.
|
|
911
|
+
/// - "caller": anything else (dev binary under a terminal, etc.) —
|
|
912
|
+
/// the booleans reflect whatever app spawned us, NOT the canonical
|
|
913
|
+
/// helper. The extension surfaces this instead of guessing.
|
|
914
|
+
private func permissionSource() -> [String: Any] {
|
|
915
|
+
let parentPid = Int32(getppid())
|
|
916
|
+
let executable = CommandLine.arguments.first ?? ""
|
|
917
|
+
var source: [String: Any] = [
|
|
918
|
+
"pid": Int(getpid()),
|
|
919
|
+
"parentPid": Int(parentPid),
|
|
920
|
+
"executablePath": executable,
|
|
921
|
+
"macOS": ProcessInfo.processInfo.operatingSystemVersionString,
|
|
922
|
+
]
|
|
923
|
+
if let parentPath = processPath(pid: parentPid) {
|
|
924
|
+
source["parentPath"] = parentPath
|
|
925
|
+
}
|
|
926
|
+
if let parentBundleId = NSRunningApplication(processIdentifier: parentPid)?.bundleIdentifier {
|
|
927
|
+
source["parentBundleId"] = parentBundleId
|
|
928
|
+
}
|
|
929
|
+
let attribution: String
|
|
930
|
+
if executable.contains("/tangu-computer-use.app/Contents/MacOS/"), parentPid == 1 {
|
|
931
|
+
// Non-spoofable signals only: installed-bundle executable path +
|
|
932
|
+
// launchd parent (`open` handed us to LaunchServices). A dev
|
|
933
|
+
// binary or a directly-spawned copy fails closed to "caller".
|
|
934
|
+
attribution = "helper-app"
|
|
935
|
+
} else {
|
|
936
|
+
attribution = "caller"
|
|
937
|
+
}
|
|
938
|
+
source["attribution"] = attribution
|
|
939
|
+
return source
|
|
940
|
+
}
|
|
941
|
+
|
|
942
|
+
/// Passive onboarding status only. Preflight is a hint, not capture proof;
|
|
943
|
+
/// never call ScreenCaptureKit or prompt APIs here, even after a grant.
|
|
944
|
+
private func permissionStatus() -> [String: Any] {
|
|
945
|
+
var result: [String: Any] = [
|
|
946
|
+
"accessibility": AXIsProcessTrusted(),
|
|
947
|
+
"screenRecordingPreflight": CGPreflightScreenCaptureAccess(),
|
|
948
|
+
"source": permissionSource(),
|
|
949
|
+
"settingsFrontmost": NSWorkspace.shared.frontmostApplication?.bundleIdentifier == "com.apple.systempreferences",
|
|
950
|
+
]
|
|
951
|
+
if let bounds = systemSettingsWindowBounds() {
|
|
952
|
+
// Quartz global coordinates: top-left origin, y increases downwards.
|
|
953
|
+
result["settingsWindow"] = ["x": bounds.minX, "y": bounds.minY, "width": bounds.width, "height": bounds.height]
|
|
954
|
+
}
|
|
955
|
+
return result
|
|
956
|
+
}
|
|
957
|
+
|
|
958
|
+
/// Window metadata remains available before AX / Screen Recording grants.
|
|
959
|
+
/// Do not filter by window title (redacted without Screen Recording), query
|
|
960
|
+
/// AX, or use the capture-backed windowInfo/currentWindowBounds helpers.
|
|
961
|
+
private func systemSettingsWindowBounds() -> CGRect? {
|
|
962
|
+
guard let settings = NSWorkspace.shared.runningApplications.first(where: { $0.bundleIdentifier == "com.apple.systempreferences" }),
|
|
963
|
+
let entries = CGWindowListCopyWindowInfo([.optionOnScreenOnly, .excludeDesktopElements], kCGNullWindowID) as? [[String: Any]]
|
|
964
|
+
else { return nil }
|
|
965
|
+
return entries.compactMap { entry -> CGRect? in
|
|
966
|
+
guard (entry[kCGWindowOwnerPID as String] as? NSNumber)?.int32Value == settings.processIdentifier,
|
|
967
|
+
(entry[kCGWindowLayer as String] as? NSNumber)?.intValue == 0,
|
|
968
|
+
(entry[kCGWindowIsOnscreen as String] as? NSNumber)?.boolValue == true,
|
|
969
|
+
let dictionary = entry[kCGWindowBounds as String] as? [String: Any],
|
|
970
|
+
let bounds = CGRect(dictionaryRepresentation: dictionary as CFDictionary),
|
|
971
|
+
bounds.origin.x.isFinite, bounds.origin.y.isFinite,
|
|
972
|
+
bounds.width.isFinite, bounds.height.isFinite,
|
|
973
|
+
bounds.width > 0, bounds.height > 0
|
|
974
|
+
else { return nil }
|
|
975
|
+
return bounds
|
|
976
|
+
}.max(by: { $0.width * $0.height < $1.width * $1.height })
|
|
977
|
+
}
|
|
978
|
+
|
|
979
|
+
private func checkPermissions(fresh: Bool = false) -> [String: Any] {
|
|
980
|
+
permissionCacheLock.lock()
|
|
981
|
+
defer { permissionCacheLock.unlock() }
|
|
982
|
+
// Serialize probes so an older in-flight success cannot repopulate the
|
|
983
|
+
// cache after a fresh check detects revocation. A failed fresh check must
|
|
984
|
+
// leave no old success behind. This does not reset macOS's own TCC cache.
|
|
985
|
+
if fresh { grantedPermissionStatus = nil }
|
|
986
|
+
if let cached = grantedPermissionStatus {
|
|
987
|
+
return cached
|
|
988
|
+
}
|
|
989
|
+
let accessibility = AXIsProcessTrusted()
|
|
990
|
+
let screenRecordingPreflight: Bool
|
|
991
|
+
if #available(macOS 10.15, *) {
|
|
992
|
+
screenRecordingPreflight = CGPreflightScreenCaptureAccess()
|
|
993
|
+
} else {
|
|
994
|
+
screenRecordingPreflight = true
|
|
995
|
+
}
|
|
996
|
+
let capturable = screenRecordingCapturable()
|
|
997
|
+
let result: [String: Any] = [
|
|
998
|
+
"accessibility": accessibility,
|
|
999
|
+
// The live probe is authoritative; the preflight boolean is kept
|
|
1000
|
+
// for diagnostics (a true/false split identifies a stale cache or
|
|
1001
|
+
// a grant belonging to a different responsible process).
|
|
1002
|
+
"screenRecording": capturable,
|
|
1003
|
+
"screenRecordingPreflight": screenRecordingPreflight,
|
|
1004
|
+
"screenRecordingCapturable": capturable,
|
|
1005
|
+
"source": permissionSource(),
|
|
1006
|
+
]
|
|
1007
|
+
// A successful TCC grant is process-stable in practice. Cache only the
|
|
1008
|
+
// positive result so missing grants are always rechecked after the user
|
|
1009
|
+
// enables them, while fresh agent processes avoid repeating a multi-second
|
|
1010
|
+
// ScreenCaptureKit probe against the same long-lived helper daemon.
|
|
1011
|
+
if accessibility && capturable {
|
|
1012
|
+
grantedPermissionStatus = result
|
|
1013
|
+
}
|
|
1014
|
+
return result
|
|
1015
|
+
}
|
|
1016
|
+
|
|
1017
|
+
/// Register this process's identity with TCC for both grants so the app
|
|
1018
|
+
/// appears in the Settings panes BEFORE the user is sent there. The AX
|
|
1019
|
+
/// request registers (and prompts for) Accessibility; on recent macOS an
|
|
1020
|
+
/// app only appears under Screen Recording after a real ScreenCaptureKit
|
|
1021
|
+
/// attempt, which the capturable probe performs.
|
|
1022
|
+
private func registerPermissions(_ request: [String: Any]) throws -> [String: Any] {
|
|
1023
|
+
// User-initiated, single-permission requests must not touch the other
|
|
1024
|
+
// permission or run a capturable probe. Omitted kind preserves legacy setup.
|
|
1025
|
+
if let rawKind = request["kind"] {
|
|
1026
|
+
guard let kind = rawKind as? String, kind == "accessibility" || kind == "screenRecording" else {
|
|
1027
|
+
throw BridgeFailure(message: "Permission kind must be 'accessibility' or 'screenRecording'", code: "invalid_args")
|
|
1028
|
+
}
|
|
1029
|
+
if kind == "accessibility" {
|
|
1030
|
+
let options = [kAXTrustedCheckOptionPrompt.takeUnretainedValue() as String: true] as CFDictionary
|
|
1031
|
+
return ["accessibility": AXIsProcessTrustedWithOptions(options)]
|
|
1032
|
+
}
|
|
1033
|
+
// Request result only; checkPermissions(fresh: true) verifies usability.
|
|
1034
|
+
return ["screenRecording": CGRequestScreenCaptureAccess()]
|
|
1035
|
+
}
|
|
1036
|
+
let options = [kAXTrustedCheckOptionPrompt.takeUnretainedValue() as String: true] as CFDictionary
|
|
1037
|
+
let accessibility = AXIsProcessTrustedWithOptions(options)
|
|
1038
|
+
if #available(macOS 10.15, *) {
|
|
1039
|
+
_ = CGRequestScreenCaptureAccess()
|
|
1040
|
+
}
|
|
1041
|
+
let capturable = screenRecordingCapturable()
|
|
1042
|
+
return [
|
|
1043
|
+
"accessibility": accessibility,
|
|
1044
|
+
"screenRecording": capturable,
|
|
1045
|
+
"screenRecordingCapturable": capturable,
|
|
1046
|
+
]
|
|
1047
|
+
}
|
|
1048
|
+
|
|
1049
|
+
private func openPermissionPane(_ request: [String: Any]) throws -> [String: Any] {
|
|
1050
|
+
let kind = try stringArg(request, "kind")
|
|
1051
|
+
let urlString: String
|
|
1052
|
+
switch kind {
|
|
1053
|
+
case "accessibility":
|
|
1054
|
+
urlString = "x-apple.systempreferences:com.apple.preference.security?Privacy_Accessibility"
|
|
1055
|
+
case "screenRecording", "screenrecording":
|
|
1056
|
+
urlString = "x-apple.systempreferences:com.apple.preference.security?Privacy_ScreenCapture"
|
|
1057
|
+
default:
|
|
1058
|
+
throw BridgeFailure(message: "Unknown permission pane '\(kind)'", code: "invalid_args")
|
|
1059
|
+
}
|
|
1060
|
+
|
|
1061
|
+
guard let url = URL(string: urlString) else {
|
|
1062
|
+
throw BridgeFailure(message: "Invalid permission pane URL", code: "internal_error")
|
|
1063
|
+
}
|
|
1064
|
+
let opened = NSWorkspace.shared.open(url)
|
|
1065
|
+
return ["opened": opened]
|
|
1066
|
+
}
|
|
1067
|
+
|
|
1068
|
+
private func pidIsAlive(_ pid: pid_t) -> Bool {
|
|
1069
|
+
pid > 0 && kill(pid, 0) == 0
|
|
1070
|
+
}
|
|
1071
|
+
|
|
1072
|
+
private func listApps(cgEntries: [[String: Any]]? = nil) -> [[String: Any]] {
|
|
1073
|
+
let frontmostPid = NSWorkspace.shared.frontmostApplication?.processIdentifier
|
|
1074
|
+
let apps = NSWorkspace.shared.runningApplications.filter { app in
|
|
1075
|
+
// Computer-use targets are windows, not Dock-visible applications. Some
|
|
1076
|
+
// benchmark/test apps and utility-style apps expose perfectly valid AX
|
|
1077
|
+
// windows while using an accessory/prohibited activation policy, so do not
|
|
1078
|
+
// gate discovery on `.regular` here. `listWindows` and the higher-level
|
|
1079
|
+
// window collector filter to actual controllable windows.
|
|
1080
|
+
app.processIdentifier != getpid() && pidIsAlive(app.processIdentifier)
|
|
1081
|
+
}
|
|
1082
|
+
var seen = Set<Int32>()
|
|
1083
|
+
var output = apps.map { app in
|
|
1084
|
+
seen.insert(app.processIdentifier)
|
|
1085
|
+
var data: [String: Any] = [
|
|
1086
|
+
"appName": app.localizedName ?? processName(pid: app.processIdentifier) ?? "Unknown App",
|
|
1087
|
+
"pid": Int(app.processIdentifier),
|
|
1088
|
+
"isFrontmost": app.processIdentifier == frontmostPid,
|
|
1089
|
+
]
|
|
1090
|
+
if let bundleId = app.bundleIdentifier {
|
|
1091
|
+
data["bundleId"] = bundleId
|
|
1092
|
+
}
|
|
1093
|
+
return data
|
|
1094
|
+
}
|
|
1095
|
+
|
|
1096
|
+
// NSWorkspace can miss apps launched from ad-hoc bundles or test harnesses
|
|
1097
|
+
// even when their windows are visible and AX-controllable. Add CGWindow
|
|
1098
|
+
// owners as acquisition candidates so callers can still resolve by pid/title
|
|
1099
|
+
// and then build the normal AX scene through listWindows(pid:).
|
|
1100
|
+
for owner in cgWindowOwners(entries: cgEntries) where owner.pid != getpid() && !seen.contains(owner.pid) && pidIsAlive(owner.pid) {
|
|
1101
|
+
seen.insert(owner.pid)
|
|
1102
|
+
output.append([
|
|
1103
|
+
"appName": owner.name,
|
|
1104
|
+
"pid": Int(owner.pid),
|
|
1105
|
+
"isFrontmost": owner.pid == frontmostPid,
|
|
1106
|
+
])
|
|
1107
|
+
}
|
|
1108
|
+
return output
|
|
1109
|
+
}
|
|
1110
|
+
|
|
1111
|
+
private func getFrontmost() throws -> [String: Any] {
|
|
1112
|
+
guard let app = NSWorkspace.shared.frontmostApplication else {
|
|
1113
|
+
throw BridgeFailure(message: "No frontmost app available", code: "frontmost_unavailable")
|
|
1114
|
+
}
|
|
1115
|
+
let pid = app.processIdentifier
|
|
1116
|
+
let windows = try listWindows(pid: pid)
|
|
1117
|
+
|
|
1118
|
+
var result: [String: Any] = [
|
|
1119
|
+
"appName": app.localizedName ?? "Unknown App",
|
|
1120
|
+
"pid": Int(pid),
|
|
1121
|
+
]
|
|
1122
|
+
if let bundleId = app.bundleIdentifier {
|
|
1123
|
+
result["bundleId"] = bundleId
|
|
1124
|
+
}
|
|
1125
|
+
|
|
1126
|
+
if let chosen = windows.sorted(by: { scoreWindow($0) > scoreWindow($1) }).first {
|
|
1127
|
+
result["windowTitle"] = (chosen["title"] as? String) ?? ""
|
|
1128
|
+
if let windowId = chosen["windowId"] {
|
|
1129
|
+
result["windowId"] = windowId
|
|
1130
|
+
}
|
|
1131
|
+
if let windowRef = chosen["windowRef"] as? String {
|
|
1132
|
+
result["windowRef"] = windowRef
|
|
1133
|
+
}
|
|
1134
|
+
}
|
|
1135
|
+
return result
|
|
1136
|
+
}
|
|
1137
|
+
|
|
1138
|
+
private func getUserContext() throws -> [String: Any] {
|
|
1139
|
+
guard let app = NSWorkspace.shared.frontmostApplication else {
|
|
1140
|
+
throw BridgeFailure(message: "No frontmost app available", code: "frontmost_unavailable")
|
|
1141
|
+
}
|
|
1142
|
+
let pid = app.processIdentifier
|
|
1143
|
+
ensureEnhancedAccessibility(pid: pid)
|
|
1144
|
+
let appElement = AXUIElementCreateApplication(pid)
|
|
1145
|
+
let focusedWindow = copyAttribute(appElement, attribute: kAXFocusedWindowAttribute as CFString).flatMap(asAXElement)
|
|
1146
|
+
let focusedElement = copyAttribute(appElement, attribute: kAXFocusedUIElementAttribute as CFString).flatMap(asAXElement)
|
|
1147
|
+
var result: [String: Any] = [
|
|
1148
|
+
"appName": app.localizedName ?? "Unknown App",
|
|
1149
|
+
"pid": Int(pid),
|
|
1150
|
+
]
|
|
1151
|
+
if let bundleId = app.bundleIdentifier {
|
|
1152
|
+
result["bundleId"] = bundleId
|
|
1153
|
+
}
|
|
1154
|
+
if let window = focusedWindow {
|
|
1155
|
+
result["window"] = [
|
|
1156
|
+
"title": stringAttribute(window, attribute: kAXTitleAttribute as CFString) ?? "",
|
|
1157
|
+
"role": stringAttribute(window, attribute: kAXRoleAttribute as CFString) ?? "",
|
|
1158
|
+
"subrole": stringAttribute(window, attribute: kAXSubroleAttribute as CFString) ?? "",
|
|
1159
|
+
]
|
|
1160
|
+
}
|
|
1161
|
+
if let element = focusedElement {
|
|
1162
|
+
let focusedRole = stringAttribute(element, attribute: kAXRoleAttribute as CFString) ?? ""
|
|
1163
|
+
let focusedSubrole = stringAttribute(element, attribute: kAXSubroleAttribute as CFString) ?? ""
|
|
1164
|
+
result["focusedElement"] = [
|
|
1165
|
+
"role": focusedRole,
|
|
1166
|
+
"subrole": focusedSubrole,
|
|
1167
|
+
"title": stringAttribute(element, attribute: kAXTitleAttribute as CFString) ?? "",
|
|
1168
|
+
"description": stringAttribute(element, attribute: kAXDescriptionAttribute as CFString) ?? "",
|
|
1169
|
+
"value": displayValue(element, role: focusedRole, subrole: focusedSubrole),
|
|
1170
|
+
]
|
|
1171
|
+
}
|
|
1172
|
+
return result
|
|
1173
|
+
}
|
|
1174
|
+
|
|
1175
|
+
private func beginInputSuppression() throws -> [String: Any] {
|
|
1176
|
+
try inputSuppressionGuard.begin()
|
|
1177
|
+
return ["active": true]
|
|
1178
|
+
}
|
|
1179
|
+
|
|
1180
|
+
private func endInputSuppression() -> [String: Any] {
|
|
1181
|
+
inputSuppressionGuard.end()
|
|
1182
|
+
return ["active": false]
|
|
1183
|
+
}
|
|
1184
|
+
|
|
1185
|
+
private func restoreUserFocus(_ request: [String: Any]) throws -> [String: Any] {
|
|
1186
|
+
let pid = Int32(try intArg(request, "pid"))
|
|
1187
|
+
let targetTitle = optionalStringArg(request, "windowTitle")?.trimmingCharacters(in: .whitespacesAndNewlines) ?? ""
|
|
1188
|
+
guard let app = NSRunningApplication(processIdentifier: pid) else {
|
|
1189
|
+
throw BridgeFailure(message: "App with pid \(pid) is no longer running", code: "app_not_found")
|
|
1190
|
+
}
|
|
1191
|
+
|
|
1192
|
+
let appRestored = app.activate()
|
|
1193
|
+
var restoredWindowTitle = ""
|
|
1194
|
+
var windowRestored = false
|
|
1195
|
+
|
|
1196
|
+
if !targetTitle.isEmpty {
|
|
1197
|
+
let appElement = AXUIElementCreateApplication(pid)
|
|
1198
|
+
let windows = axElementArray(appElement, attribute: kAXWindowsAttribute as CFString)
|
|
1199
|
+
let normalizedTarget = targetTitle.lowercased()
|
|
1200
|
+
if let match = windows.first(where: {
|
|
1201
|
+
(stringAttribute($0, attribute: kAXTitleAttribute as CFString) ?? "")
|
|
1202
|
+
.trimmingCharacters(in: .whitespacesAndNewlines)
|
|
1203
|
+
.lowercased() == normalizedTarget
|
|
1204
|
+
}) {
|
|
1205
|
+
restoredWindowTitle = stringAttribute(match, attribute: kAXTitleAttribute as CFString) ?? ""
|
|
1206
|
+
let setMainStatus = AXUIElementSetAttributeValue(match, kAXMainAttribute as CFString, kCFBooleanTrue)
|
|
1207
|
+
let setFocusedStatus = AXUIElementSetAttributeValue(match, kAXFocusedAttribute as CFString, kCFBooleanTrue)
|
|
1208
|
+
let raiseStatus = AXUIElementPerformAction(match, kAXRaiseAction as CFString)
|
|
1209
|
+
windowRestored = setMainStatus == .success || setFocusedStatus == .success || raiseStatus == .success
|
|
1210
|
+
}
|
|
1211
|
+
}
|
|
1212
|
+
|
|
1213
|
+
return [
|
|
1214
|
+
"restored": appRestored || windowRestored,
|
|
1215
|
+
"appRestored": appRestored,
|
|
1216
|
+
"windowRestored": windowRestored,
|
|
1217
|
+
"appName": app.localizedName ?? "Unknown App",
|
|
1218
|
+
"windowTitle": restoredWindowTitle,
|
|
1219
|
+
]
|
|
1220
|
+
}
|
|
1221
|
+
|
|
1222
|
+
private func setWindowFrame(_ request: [String: Any]) throws -> [String: Any] {
|
|
1223
|
+
let pid = Int32(try intArg(request, "pid"))
|
|
1224
|
+
let windowId = optionalIntArg(request, "windowId").map { UInt32($0) }
|
|
1225
|
+
let windowRef = optionalStringArg(request, "windowRef")
|
|
1226
|
+
guard let window = windowElement(pid: pid, windowId: windowId, windowRef: windowRef) else {
|
|
1227
|
+
return ["ok": false, "reason": "window_not_found"]
|
|
1228
|
+
}
|
|
1229
|
+
let x = try doubleArg(request, "x")
|
|
1230
|
+
let y = try doubleArg(request, "y")
|
|
1231
|
+
let width = max(100.0, try doubleArg(request, "width"))
|
|
1232
|
+
let height = max(80.0, try doubleArg(request, "height"))
|
|
1233
|
+
var origin = CGPoint(x: x, y: y)
|
|
1234
|
+
var size = CGSize(width: width, height: height)
|
|
1235
|
+
guard let originValue = AXValueCreate(.cgPoint, &origin), let sizeValue = AXValueCreate(.cgSize, &size) else {
|
|
1236
|
+
throw BridgeFailure(message: "Failed to create AX frame values", code: "frame_value_failed")
|
|
1237
|
+
}
|
|
1238
|
+
let positionStatus = AXUIElementSetAttributeValue(window, kAXPositionAttribute as CFString, originValue)
|
|
1239
|
+
let sizeStatus = AXUIElementSetAttributeValue(window, kAXSizeAttribute as CFString, sizeValue)
|
|
1240
|
+
let frame = frameForWindow(window)
|
|
1241
|
+
return [
|
|
1242
|
+
"ok": positionStatus == .success || sizeStatus == .success,
|
|
1243
|
+
"positionStatus": Int(positionStatus.rawValue),
|
|
1244
|
+
"sizeStatus": Int(sizeStatus.rawValue),
|
|
1245
|
+
"framePoints": ["x": frame.origin.x, "y": frame.origin.y, "w": frame.width, "h": frame.height],
|
|
1246
|
+
]
|
|
1247
|
+
}
|
|
1248
|
+
|
|
1249
|
+
private func focusWindow(_ request: [String: Any]) throws -> [String: Any] {
|
|
1250
|
+
let pid = Int32(try intArg(request, "pid"))
|
|
1251
|
+
let windowId = optionalIntArg(request, "windowId").map { UInt32($0) }
|
|
1252
|
+
let windowRef = optionalStringArg(request, "windowRef")
|
|
1253
|
+
guard let window = windowElement(pid: pid, windowId: windowId, windowRef: windowRef) else {
|
|
1254
|
+
return ["focused": false, "reason": "window_not_found"]
|
|
1255
|
+
}
|
|
1256
|
+
|
|
1257
|
+
let appElement = AXUIElementCreateApplication(pid)
|
|
1258
|
+
if let focusedWindow = copyAttribute(appElement, attribute: kAXFocusedWindowAttribute as CFString).flatMap(asAXElement),
|
|
1259
|
+
sameElement(focusedWindow, window)
|
|
1260
|
+
{
|
|
1261
|
+
return ["focused": true, "alreadyFocused": true]
|
|
1262
|
+
}
|
|
1263
|
+
|
|
1264
|
+
let setMainStatus = AXUIElementSetAttributeValue(window, kAXMainAttribute as CFString, kCFBooleanTrue)
|
|
1265
|
+
let setFocusedStatus = AXUIElementSetAttributeValue(window, kAXFocusedAttribute as CFString, kCFBooleanTrue)
|
|
1266
|
+
let raiseStatus = AXUIElementPerformAction(window, kAXRaiseAction as CFString)
|
|
1267
|
+
let focused = setMainStatus == .success || setFocusedStatus == .success || raiseStatus == .success
|
|
1268
|
+
var result: [String: Any] = [
|
|
1269
|
+
"focused": focused,
|
|
1270
|
+
"setMain": setMainStatus == .success,
|
|
1271
|
+
"setFocused": setFocusedStatus == .success,
|
|
1272
|
+
"raised": raiseStatus == .success,
|
|
1273
|
+
]
|
|
1274
|
+
if !focused {
|
|
1275
|
+
result["reason"] = "focus_failed"
|
|
1276
|
+
}
|
|
1277
|
+
return result
|
|
1278
|
+
}
|
|
1279
|
+
|
|
1280
|
+
private func scoreWindow(_ window: [String: Any]) -> Int {
|
|
1281
|
+
var score = 0
|
|
1282
|
+
if (window["isFocused"] as? Bool) == true { score += 100 }
|
|
1283
|
+
if (window["isMain"] as? Bool) == true { score += 80 }
|
|
1284
|
+
if (window["isMinimized"] as? Bool) == false { score += 40 }
|
|
1285
|
+
if (window["isOnscreen"] as? Bool) == true { score += 20 }
|
|
1286
|
+
if window["windowId"] != nil { score += 10 }
|
|
1287
|
+
return score
|
|
1288
|
+
}
|
|
1289
|
+
|
|
1290
|
+
private func rootKind(role: String, subrole: String) -> String {
|
|
1291
|
+
if role == "AXSheet" { return "sheet" }
|
|
1292
|
+
if subrole.localizedCaseInsensitiveContains("popover") { return "popover" }
|
|
1293
|
+
if subrole.localizedCaseInsensitiveContains("dialog") || role == "AXDialog" { return "dialog" }
|
|
1294
|
+
return "window"
|
|
1295
|
+
}
|
|
1296
|
+
|
|
1297
|
+
private func isDialogLikeRoot(role: String, subrole: String) -> Bool {
|
|
1298
|
+
let text = "\(role) \(subrole)"
|
|
1299
|
+
return text.range(of: "dialog", options: [.caseInsensitive]) != nil
|
|
1300
|
+
|| text.range(of: "modal", options: [.caseInsensitive]) != nil
|
|
1301
|
+
|| text.range(of: "sheet", options: [.caseInsensitive]) != nil
|
|
1302
|
+
}
|
|
1303
|
+
|
|
1304
|
+
private func rootMetadata(pairing: WindowPairing, sheetCount: Int) -> [String: Any] {
|
|
1305
|
+
["pairing": ["confidence": pairing.confidence, "score": pairing.score], "sheetCount": sheetCount]
|
|
1306
|
+
}
|
|
1307
|
+
|
|
1308
|
+
private func broadRootCandidateApps(entries: [[String: Any]]) -> [[String: Any]] {
|
|
1309
|
+
cgBroadRootOwners(entries: entries).compactMap { owner in
|
|
1310
|
+
guard owner.pid != getpid(), pidIsAlive(owner.pid) else { return nil }
|
|
1311
|
+
var app: [String: Any] = ["appName": owner.name, "pid": Int(owner.pid)]
|
|
1312
|
+
if let bundleId = NSRunningApplication(processIdentifier: owner.pid)?.bundleIdentifier {
|
|
1313
|
+
app["bundleId"] = bundleId
|
|
1314
|
+
}
|
|
1315
|
+
return app
|
|
1316
|
+
}
|
|
1317
|
+
}
|
|
1318
|
+
|
|
1319
|
+
private func listRoots(pid: Int32?, title: String? = nil) throws -> [String: Any] {
|
|
1320
|
+
let requestedTitle = title?.trimmingCharacters(in: .whitespacesAndNewlines).lowercased() ?? ""
|
|
1321
|
+
let entries = allCGWindowEntries()
|
|
1322
|
+
let isBroadDiscovery = pid == nil && requestedTitle.isEmpty
|
|
1323
|
+
let apps: [[String: Any]]
|
|
1324
|
+
if let pid {
|
|
1325
|
+
apps = [["pid": Int(pid)]]
|
|
1326
|
+
} else if !requestedTitle.isEmpty {
|
|
1327
|
+
let matchingPids = Set(entries.compactMap { entry -> Int32? in
|
|
1328
|
+
let candidate = ((entry[kCGWindowName as String] as? String) ?? "").trimmingCharacters(in: .whitespacesAndNewlines).lowercased()
|
|
1329
|
+
guard candidate == requestedTitle || candidate.contains(requestedTitle) else { return nil }
|
|
1330
|
+
return (entry[kCGWindowOwnerPID as String] as? NSNumber)?.int32Value
|
|
1331
|
+
})
|
|
1332
|
+
apps = listApps(cgEntries: entries).filter { app in
|
|
1333
|
+
guard let rawPid = app["pid"] as? Int else { return false }
|
|
1334
|
+
return matchingPids.contains(Int32(rawPid))
|
|
1335
|
+
}
|
|
1336
|
+
} else {
|
|
1337
|
+
apps = broadRootCandidateApps(entries: entries)
|
|
1338
|
+
}
|
|
1339
|
+
var roots: [[String: Any]] = []
|
|
1340
|
+
for app in apps {
|
|
1341
|
+
guard let rawPid = app["pid"] as? Int else { continue }
|
|
1342
|
+
let appPid = Int32(rawPid)
|
|
1343
|
+
let appName = app["appName"] as? String ?? processName(pid: appPid) ?? "Unknown App"
|
|
1344
|
+
let bundleId = app["bundleId"] as? String
|
|
1345
|
+
for var root in (try? listWindows(pid: appPid, cgEntries: entries, messagingTimeout: isBroadDiscovery ? 0.25 : 1.0)) ?? [] {
|
|
1346
|
+
root["pid"] = rawPid
|
|
1347
|
+
root["appName"] = appName
|
|
1348
|
+
if let bundleId { root["bundleId"] = bundleId }
|
|
1349
|
+
roots.append(root)
|
|
1350
|
+
}
|
|
1351
|
+
let popupCandidates = cgPopupMenuCandidates(pid: appPid, entries: entries)
|
|
1352
|
+
let menuElements = popupCandidates.isEmpty ? [] : openMenuElements(pid: appPid, messagingTimeout: isBroadDiscovery ? 0.25 : 1.0)
|
|
1353
|
+
for (index, candidate) in popupCandidates.enumerated() {
|
|
1354
|
+
let menuElement = index < menuElements.count ? menuElements[index] : nil
|
|
1355
|
+
let menuRef = menuElement.map { refStore.storeWindow($0) } ?? "cgmenu:\(candidate.windowId)"
|
|
1356
|
+
var menu: [String: Any] = [
|
|
1357
|
+
"kind": "menu",
|
|
1358
|
+
"rootRef": menuRef,
|
|
1359
|
+
"windowRef": menuRef,
|
|
1360
|
+
"windowId": Int(candidate.windowId),
|
|
1361
|
+
"zOrder": candidate.zOrder,
|
|
1362
|
+
"title": menuElement.flatMap { stringAttribute($0, attribute: kAXTitleAttribute as CFString) } ?? candidate.title,
|
|
1363
|
+
"role": "AXMenu",
|
|
1364
|
+
"subrole": "",
|
|
1365
|
+
"isModal": false,
|
|
1366
|
+
"framePoints": ["x": candidate.bounds.origin.x, "y": candidate.bounds.origin.y, "w": candidate.bounds.width, "h": candidate.bounds.height],
|
|
1367
|
+
"scaleFactor": displayScaleFactor(for: candidate.bounds),
|
|
1368
|
+
"isMinimized": false,
|
|
1369
|
+
"isOnscreen": candidate.isOnscreen,
|
|
1370
|
+
"isMain": false,
|
|
1371
|
+
"isFocused": true,
|
|
1372
|
+
"metadata": ["pairing": ["confidence": menuElement == nil ? "low" : "high", "score": menuElement == nil ? 0 : 100], "sheetCount": 0],
|
|
1373
|
+
"pid": rawPid,
|
|
1374
|
+
"appName": appName,
|
|
1375
|
+
]
|
|
1376
|
+
if let bundleId { menu["bundleId"] = bundleId }
|
|
1377
|
+
roots.append(menu)
|
|
1378
|
+
}
|
|
1379
|
+
}
|
|
1380
|
+
roots.sort { (($0["zOrder"] as? Int) ?? Int.max) < (($1["zOrder"] as? Int) ?? Int.max) }
|
|
1381
|
+
return ["roots": roots]
|
|
1382
|
+
}
|
|
1383
|
+
|
|
1384
|
+
private func listWindows(pid: Int32, cgEntries: [[String: Any]]? = nil, messagingTimeout: Float = 1.0) throws -> [[String: Any]] {
|
|
1385
|
+
ensureEnhancedAccessibility(pid: pid)
|
|
1386
|
+
let appElement = AXUIElementCreateApplication(pid)
|
|
1387
|
+
AXUIElementSetMessagingTimeout(appElement, messagingTimeout)
|
|
1388
|
+
let windows = Array(axElementArray(appElement, attribute: kAXWindowsAttribute as CFString).prefix(128))
|
|
1389
|
+
let candidates = cgWindowCandidates(pid: pid, entries: cgEntries)
|
|
1390
|
+
let pairings = windowPairings(windows: windows, candidates: candidates)
|
|
1391
|
+
|
|
1392
|
+
var output: [[String: Any]] = []
|
|
1393
|
+
for (zIndex, window) in windows.enumerated() {
|
|
1394
|
+
let axTitle = stringAttribute(window, attribute: kAXTitleAttribute as CFString) ?? ""
|
|
1395
|
+
let axRole = stringAttribute(window, attribute: kAXRoleAttribute as CFString) ?? ""
|
|
1396
|
+
let axSubrole = stringAttribute(window, attribute: kAXSubroleAttribute as CFString) ?? ""
|
|
1397
|
+
let axFrame = frameForWindow(window)
|
|
1398
|
+
let pairing = pairings[ObjectIdentifier(window)] ?? WindowPairing(candidate: nil, score: -Double.greatestFiniteMagnitude, confidence: "low")
|
|
1399
|
+
let candidate = pairing.candidate
|
|
1400
|
+
|
|
1401
|
+
let effectiveFrame = axFrame.width > 1 && axFrame.height > 1 ? axFrame : (candidate?.bounds ?? axFrame)
|
|
1402
|
+
if effectiveFrame.width < 100 || effectiveFrame.height < 80 { continue }
|
|
1403
|
+
let hasUsableAXFrame = axFrame.width > 1 && axFrame.height > 1
|
|
1404
|
+
let title = hasUsableAXFrame && !axTitle.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty ? axTitle : (candidate?.title.isEmpty == false ? candidate!.title : axTitle)
|
|
1405
|
+
let windowRef = refStore.storeWindow(window)
|
|
1406
|
+
let isMinimized = boolAttribute(window, attribute: kAXMinimizedAttribute as CFString) ?? false
|
|
1407
|
+
let isMain = boolAttribute(window, attribute: kAXMainAttribute as CFString) ?? false
|
|
1408
|
+
let isFocused = boolAttribute(window, attribute: kAXFocusedAttribute as CFString) ?? false
|
|
1409
|
+
let sheetCount = sheetElements(of: window).count
|
|
1410
|
+
let isModal = (boolAttribute(window, attribute: "AXModal" as CFString) ?? false) || sheetCount > 0 || isDialogLikeRoot(role: axRole, subrole: axSubrole)
|
|
1411
|
+
let scale = displayScaleFactor(for: effectiveFrame)
|
|
1412
|
+
|
|
1413
|
+
var item: [String: Any] = [
|
|
1414
|
+
"kind": rootKind(role: axRole, subrole: axSubrole),
|
|
1415
|
+
"rootRef": windowRef,
|
|
1416
|
+
"windowRef": windowRef,
|
|
1417
|
+
"zOrder": candidate?.zOrder ?? zIndex,
|
|
1418
|
+
"title": title,
|
|
1419
|
+
"role": axRole,
|
|
1420
|
+
"subrole": axSubrole,
|
|
1421
|
+
"isModal": isModal,
|
|
1422
|
+
"framePoints": [
|
|
1423
|
+
"x": effectiveFrame.origin.x,
|
|
1424
|
+
"y": effectiveFrame.origin.y,
|
|
1425
|
+
"w": effectiveFrame.size.width,
|
|
1426
|
+
"h": effectiveFrame.size.height,
|
|
1427
|
+
],
|
|
1428
|
+
"scaleFactor": scale,
|
|
1429
|
+
"isMinimized": isMinimized,
|
|
1430
|
+
"isOnscreen": candidate?.isOnscreen ?? !isMinimized,
|
|
1431
|
+
"isMain": isMain,
|
|
1432
|
+
"isFocused": isFocused,
|
|
1433
|
+
"metadata": rootMetadata(pairing: pairing, sheetCount: sheetCount),
|
|
1434
|
+
]
|
|
1435
|
+
if let candidate {
|
|
1436
|
+
item["windowId"] = Int(candidate.windowId)
|
|
1437
|
+
}
|
|
1438
|
+
output.append(item)
|
|
1439
|
+
|
|
1440
|
+
for sheet in sheetElements(of: window) {
|
|
1441
|
+
let sheetRef = refStore.storeWindow(sheet)
|
|
1442
|
+
let sheetFrame = frameForWindow(sheet)
|
|
1443
|
+
let sheetCandidate = bestCandidate(for: sheet, candidates: candidates)
|
|
1444
|
+
var sheetItem: [String: Any] = [
|
|
1445
|
+
"kind": "sheet",
|
|
1446
|
+
"rootRef": sheetRef,
|
|
1447
|
+
"windowRef": sheetRef,
|
|
1448
|
+
"zOrder": sheetCandidate?.zOrder ?? candidate?.zOrder ?? zIndex,
|
|
1449
|
+
"title": stringAttribute(sheet, attribute: kAXTitleAttribute as CFString) ?? title,
|
|
1450
|
+
"role": stringAttribute(sheet, attribute: kAXRoleAttribute as CFString) ?? "AXSheet",
|
|
1451
|
+
"subrole": stringAttribute(sheet, attribute: kAXSubroleAttribute as CFString) ?? "",
|
|
1452
|
+
"isModal": true,
|
|
1453
|
+
"framePoints": ["x": sheetFrame.origin.x, "y": sheetFrame.origin.y, "w": sheetFrame.width, "h": sheetFrame.height],
|
|
1454
|
+
"scaleFactor": displayScaleFactor(for: sheetFrame),
|
|
1455
|
+
"isMinimized": false,
|
|
1456
|
+
"isOnscreen": sheetCandidate?.isOnscreen ?? candidate?.isOnscreen ?? !isMinimized,
|
|
1457
|
+
"isMain": false,
|
|
1458
|
+
"isFocused": isFocused,
|
|
1459
|
+
"metadata": ["pairing": ["confidence": sheetCandidate == nil ? pairing.confidence : "high", "score": sheetCandidate == nil ? pairing.score : 100], "sheetCount": 0],
|
|
1460
|
+
]
|
|
1461
|
+
if let sheetCandidate { sheetItem["windowId"] = Int(sheetCandidate.windowId) }
|
|
1462
|
+
output.append(sheetItem)
|
|
1463
|
+
}
|
|
1464
|
+
}
|
|
1465
|
+
return output
|
|
1466
|
+
}
|
|
1467
|
+
|
|
1468
|
+
private func look(_ request: [String: Any]) throws -> [String: Any] {
|
|
1469
|
+
let windowId = optionalIntArg(request, "windowId").map { UInt32($0) }
|
|
1470
|
+
let windowRef = optionalStringArg(request, "windowRef")
|
|
1471
|
+
let maxDimension = optionalIntArg(request, "maxDimension").map { max(1, $0) }
|
|
1472
|
+
let readText = optionalStringArg(request, "readText") ?? "auto"
|
|
1473
|
+
let baseLookId = optionalStringArg(request, "baseLookId")
|
|
1474
|
+
let includeImage = boolArg(request, "includeImage") ?? true
|
|
1475
|
+
guard readText == "auto" || readText == "always" || readText == "never" else {
|
|
1476
|
+
throw BridgeFailure(message: "readText must be auto, always, or never", code: "invalid_args")
|
|
1477
|
+
}
|
|
1478
|
+
|
|
1479
|
+
let requestedRoot = windowRef.flatMap { refStore.window(for: $0) }
|
|
1480
|
+
let requestedRole = requestedRoot.flatMap { stringAttribute($0, attribute: kAXRoleAttribute as CFString) } ?? ""
|
|
1481
|
+
let isMenuRoot = requestedRole == "AXMenu" || (windowRef?.hasPrefix("cgmenu:") == true)
|
|
1482
|
+
let captureStart = Date()
|
|
1483
|
+
let shouldCapture = !isMenuRoot && (includeImage || readText == "always")
|
|
1484
|
+
let capture = try shouldCapture ? windowId.map { try captureWindow(windowId: $0) } : nil
|
|
1485
|
+
let captureMs = capture.map { _ in elapsedMs(captureStart) } ?? 0
|
|
1486
|
+
|
|
1487
|
+
let pid: Int32
|
|
1488
|
+
if let windowId, let ownerPid = pidForWindowId(windowId) {
|
|
1489
|
+
pid = ownerPid
|
|
1490
|
+
} else if let requestedRoot, let owner = pidForElement(requestedRoot) {
|
|
1491
|
+
pid = owner
|
|
1492
|
+
} else {
|
|
1493
|
+
throw BridgeFailure(message: "Root is not owned by a running app", code: "root_not_found")
|
|
1494
|
+
}
|
|
1495
|
+
ensureEnhancedAccessibility(pid: pid)
|
|
1496
|
+
if let windowRef, windowRef.hasPrefix("cgmenu:"), refStore.window(for: windowRef) == nil {
|
|
1497
|
+
let frame = windowId.flatMap { windowInfo(windowId: $0)?.bounds } ?? CGRect(x: 0, y: 0, width: 1, height: 1)
|
|
1498
|
+
let lookId = freshLookId()
|
|
1499
|
+
storeLookRecord(LookRecord(lookId: lookId, windowId: windowId ?? 0, windowFrame: frame, imageWidth: max(1, Int(frame.width)), imageHeight: max(1, Int(frame.height)), hasImage: false))
|
|
1500
|
+
let outline = LookNode(element: nil, ref: windowRef, role: "AXMenu", subrole: "", identifier: "", title: "Menu", description: "", value: "", actions: [], canPress: false, canFocus: false, canSetValue: false, canScroll: false, canIncrement: false, canDecrement: false, isTextInput: false, rect: CGRect(x: 0, y: 0, width: max(1, frame.width), height: max(1, frame.height)), pictureOnly: true)
|
|
1501
|
+
return [
|
|
1502
|
+
"lookId": lookId,
|
|
1503
|
+
"capturedAt": captureStart.timeIntervalSince1970,
|
|
1504
|
+
"window": ["windowId": Int(windowId ?? 0), "rootRef": windowRef, "kind": "menu", "framePoints": ["x": frame.origin.x, "y": frame.origin.y, "w": frame.width, "h": frame.height], "scaleFactor": displayScaleFactor(for: frame), "isModal": false, "metadata": ["pairing": ["confidence": "low", "score": 0], "sheetCount": 0], "role": "AXMenu", "subrole": ""],
|
|
1505
|
+
"outline": outline.payload(),
|
|
1506
|
+
"timings": ["captureMs": 0, "describeMs": 0, "readTextMs": 0],
|
|
1507
|
+
]
|
|
1508
|
+
}
|
|
1509
|
+
guard let window = windowElement(pid: pid, windowId: windowId, windowRef: windowRef) else {
|
|
1510
|
+
throw BridgeFailure(message: "Root is not available through Accessibility", code: "root_not_found")
|
|
1511
|
+
}
|
|
1512
|
+
let rootElement: AXUIElement
|
|
1513
|
+
if let scopeRef = optionalStringArg(request, "scopeRef") {
|
|
1514
|
+
guard let scoped = refStore.element(for: scopeRef), isElement(scoped, descendantOf: window) else {
|
|
1515
|
+
throw BridgeFailure(message: "Scope ref is stale or outside the target root", code: "element_ref_invalid")
|
|
1516
|
+
}
|
|
1517
|
+
rootElement = scoped
|
|
1518
|
+
} else {
|
|
1519
|
+
rootElement = window
|
|
1520
|
+
}
|
|
1521
|
+
|
|
1522
|
+
let rootFrame = frameForWindow(window)
|
|
1523
|
+
let imageWidth: Int
|
|
1524
|
+
let imageHeight: Int
|
|
1525
|
+
let imagePayload: [String: Any]?
|
|
1526
|
+
let transform: (CGRect) -> CGRect
|
|
1527
|
+
if let capture {
|
|
1528
|
+
let outputImage = downscaledImage(capture.image, maxDimension: maxDimension) ?? capture.image
|
|
1529
|
+
imageWidth = outputImage.width
|
|
1530
|
+
imageHeight = outputImage.height
|
|
1531
|
+
transform = rectTransform(windowFrame: capture.frame, imageWidth: outputImage.width, imageHeight: outputImage.height)
|
|
1532
|
+
if includeImage {
|
|
1533
|
+
guard let jpeg = jpegData(image: outputImage, quality: 0.8) else {
|
|
1534
|
+
throw BridgeFailure(message: "Failed to encode look image as JPEG", code: "encoding_failed")
|
|
1535
|
+
}
|
|
1536
|
+
imagePayload = ["jpegBase64": jpeg.base64EncodedString(), "width": outputImage.width, "height": outputImage.height]
|
|
1537
|
+
} else {
|
|
1538
|
+
imagePayload = nil
|
|
1539
|
+
}
|
|
1540
|
+
} else {
|
|
1541
|
+
imageWidth = max(1, Int(rootFrame.width))
|
|
1542
|
+
imageHeight = max(1, Int(rootFrame.height))
|
|
1543
|
+
imagePayload = nil
|
|
1544
|
+
transform = rectTransform(windowFrame: rootFrame, imageWidth: imageWidth, imageHeight: imageHeight)
|
|
1545
|
+
}
|
|
1546
|
+
let describeStart = Date()
|
|
1547
|
+
let outline = buildLookOutline(root: rootElement, transform: transform)
|
|
1548
|
+
let describeMs = elapsedMs(describeStart)
|
|
1549
|
+
|
|
1550
|
+
var readTextMs = 0
|
|
1551
|
+
var readTextExecuted = false
|
|
1552
|
+
if let capture, readText == "always" {
|
|
1553
|
+
readTextExecuted = true
|
|
1554
|
+
let textStart = Date()
|
|
1555
|
+
let boxes = try recognizeText(in: capture.image, outputWidth: imageWidth, outputHeight: imageHeight)
|
|
1556
|
+
attachOCR(boxes, to: outline)
|
|
1557
|
+
readTextMs = elapsedMs(textStart)
|
|
1558
|
+
}
|
|
1559
|
+
|
|
1560
|
+
let lookId = freshLookId()
|
|
1561
|
+
let baseRecord = baseLookId.flatMap { lookRecord(for: $0) }
|
|
1562
|
+
storeLookRecord(LookRecord(
|
|
1563
|
+
lookId: lookId,
|
|
1564
|
+
windowId: windowId ?? baseRecord?.windowId ?? 0,
|
|
1565
|
+
windowFrame: baseRecord?.windowFrame ?? capture?.frame ?? rootFrame,
|
|
1566
|
+
imageWidth: baseRecord?.imageWidth ?? imageWidth,
|
|
1567
|
+
imageHeight: baseRecord?.imageHeight ?? imageHeight,
|
|
1568
|
+
hasImage: baseRecord?.hasImage ?? (capture != nil)
|
|
1569
|
+
))
|
|
1570
|
+
let scale = (capture?.frame.width ?? rootFrame.width) > 0 ? Double(imageWidth) / (capture?.frame.width ?? rootFrame.width) : displayScaleFactor(for: rootFrame)
|
|
1571
|
+
let pairing = pairingForWindow(window, pid: pid)
|
|
1572
|
+
let role = stringAttribute(window, attribute: kAXRoleAttribute as CFString) ?? ""
|
|
1573
|
+
let subrole = stringAttribute(window, attribute: kAXSubroleAttribute as CFString) ?? ""
|
|
1574
|
+
let sheetCount = sheetElements(of: window).count
|
|
1575
|
+
var response: [String: Any] = [
|
|
1576
|
+
"lookId": lookId,
|
|
1577
|
+
"capturedAt": captureStart.timeIntervalSince1970,
|
|
1578
|
+
"window": [
|
|
1579
|
+
"windowId": Int(windowId ?? 0),
|
|
1580
|
+
"rootRef": windowRef ?? refStore.storeWindow(window),
|
|
1581
|
+
"kind": rootKind(role: role, subrole: subrole),
|
|
1582
|
+
"framePoints": ["x": (capture?.frame ?? rootFrame).origin.x, "y": (capture?.frame ?? rootFrame).origin.y, "w": (capture?.frame ?? rootFrame).width, "h": (capture?.frame ?? rootFrame).height],
|
|
1583
|
+
"scaleFactor": scale,
|
|
1584
|
+
"isModal": (boolAttribute(window, attribute: "AXModal" as CFString) ?? false) || sheetCount > 0 || isDialogLikeRoot(role: role, subrole: subrole),
|
|
1585
|
+
"metadata": rootMetadata(pairing: pairing, sheetCount: sheetCount),
|
|
1586
|
+
"role": role,
|
|
1587
|
+
"subrole": subrole,
|
|
1588
|
+
],
|
|
1589
|
+
"outline": outline.payload(),
|
|
1590
|
+
"timings": ["captureMs": captureMs, "describeMs": describeMs, "readTextMs": readTextMs],
|
|
1591
|
+
"readText": ["requested": readText, "executed": readTextExecuted],
|
|
1592
|
+
]
|
|
1593
|
+
if let imagePayload { response["image"] = imagePayload }
|
|
1594
|
+
return response
|
|
1595
|
+
}
|
|
1596
|
+
|
|
1597
|
+
private func elapsedMs(_ start: Date) -> Int {
|
|
1598
|
+
max(0, Int(Date().timeIntervalSince(start) * 1000.0))
|
|
1599
|
+
}
|
|
1600
|
+
|
|
1601
|
+
private func storeLookRecord(_ record: LookRecord) {
|
|
1602
|
+
lookRecordLock.lock()
|
|
1603
|
+
lookRecords[record.lookId] = record
|
|
1604
|
+
lookRecordOrder.append(record.lookId)
|
|
1605
|
+
while lookRecordOrder.count > 8 {
|
|
1606
|
+
let oldest = lookRecordOrder.removeFirst()
|
|
1607
|
+
lookRecords.removeValue(forKey: oldest)
|
|
1608
|
+
}
|
|
1609
|
+
lookRecordLock.unlock()
|
|
1610
|
+
// Tangu:每次观察也算"agent 正在看这个窗口",liveView 据此选画面(唯一的 look 收口点,
|
|
1611
|
+
// 比在 look() 的各个分支里各挂一次可靠)。**不点光效** —— 光效只表示"正在动手"。
|
|
1612
|
+
if record.windowId != 0 {
|
|
1613
|
+
controlledLock.lock()
|
|
1614
|
+
controlledWindowId = record.windowId
|
|
1615
|
+
controlledPid = pidForWindowId(record.windowId) ?? controlledPid
|
|
1616
|
+
controlledAt = Date()
|
|
1617
|
+
controlledLock.unlock()
|
|
1618
|
+
}
|
|
1619
|
+
}
|
|
1620
|
+
|
|
1621
|
+
private func freshLookId() -> String {
|
|
1622
|
+
lookRecordLock.lock()
|
|
1623
|
+
defer { lookRecordLock.unlock() }
|
|
1624
|
+
nextLookId += 1
|
|
1625
|
+
return "look_\(nextLookId)"
|
|
1626
|
+
}
|
|
1627
|
+
|
|
1628
|
+
private func lookRecord(for lookId: String) -> LookRecord? {
|
|
1629
|
+
lookRecordLock.lock()
|
|
1630
|
+
defer { lookRecordLock.unlock() }
|
|
1631
|
+
return lookRecords[lookId]
|
|
1632
|
+
}
|
|
1633
|
+
|
|
1634
|
+
private func rectTransform(windowFrame: CGRect, imageWidth: Int, imageHeight: Int) -> (CGRect) -> CGRect {
|
|
1635
|
+
let sx = windowFrame.width > 0 ? Double(imageWidth) / windowFrame.width : 1.0
|
|
1636
|
+
let sy = windowFrame.height > 0 ? Double(imageHeight) / windowFrame.height : 1.0
|
|
1637
|
+
return { frame in
|
|
1638
|
+
let x = (frame.origin.x - windowFrame.origin.x) * sx
|
|
1639
|
+
let y = (frame.origin.y - windowFrame.origin.y) * sy
|
|
1640
|
+
let w = frame.width * sx
|
|
1641
|
+
let h = frame.height * sy
|
|
1642
|
+
return self.clampRect(CGRect(x: x, y: y, width: w, height: h), width: imageWidth, height: imageHeight)
|
|
1643
|
+
}
|
|
1644
|
+
}
|
|
1645
|
+
|
|
1646
|
+
private func clampRect(_ rect: CGRect, width: Int, height: Int) -> CGRect {
|
|
1647
|
+
let maxX = Double(width)
|
|
1648
|
+
let maxY = Double(height)
|
|
1649
|
+
let x1 = min(max(rect.minX, 0), maxX)
|
|
1650
|
+
let y1 = min(max(rect.minY, 0), maxY)
|
|
1651
|
+
let x2 = min(max(rect.maxX, 0), maxX)
|
|
1652
|
+
let y2 = min(max(rect.maxY, 0), maxY)
|
|
1653
|
+
return CGRect(x: x1, y: y1, width: max(0, x2 - x1), height: max(0, y2 - y1))
|
|
1654
|
+
}
|
|
1655
|
+
|
|
1656
|
+
private func buildLookOutline(root: AXUIElement, transform: @escaping (CGRect) -> CGRect) -> LookNode {
|
|
1657
|
+
let rootNode = lookNode(element: root, transform: transform, offscreen: false)
|
|
1658
|
+
let nodeLimit = 2000
|
|
1659
|
+
// Apps with slow AX servers (e.g. Outlook) can take >30s to describe; the
|
|
1660
|
+
// client aborts at 33s, so stop walking well before that and return a
|
|
1661
|
+
// truncated outline instead.
|
|
1662
|
+
let deadline = Date().addingTimeInterval(20.0)
|
|
1663
|
+
var walked = 1
|
|
1664
|
+
var seen = Set<ObjectIdentifier>([ObjectIdentifier(root)])
|
|
1665
|
+
var queue: [(AXUIElement, LookNode)] = [(root, rootNode)]
|
|
1666
|
+
var index = 0
|
|
1667
|
+
while index < queue.count {
|
|
1668
|
+
let (element, node) = queue[index]
|
|
1669
|
+
index += 1
|
|
1670
|
+
let children = axElementArray(element, attribute: kAXChildrenAttribute as CFString)
|
|
1671
|
+
if children.isEmpty { continue }
|
|
1672
|
+
if walked >= nodeLimit || Date() >= deadline {
|
|
1673
|
+
node.truncated = true
|
|
1674
|
+
continue
|
|
1675
|
+
}
|
|
1676
|
+
let visibleByKind = visibleChildrenByKind(element)
|
|
1677
|
+
for child in children {
|
|
1678
|
+
if walked >= nodeLimit || Date() >= deadline {
|
|
1679
|
+
node.truncated = true
|
|
1680
|
+
break
|
|
1681
|
+
}
|
|
1682
|
+
let identity = ObjectIdentifier(child)
|
|
1683
|
+
if seen.contains(identity) { continue }
|
|
1684
|
+
seen.insert(identity)
|
|
1685
|
+
let role = stringAttribute(child, attribute: kAXRoleAttribute as CFString) ?? ""
|
|
1686
|
+
let offscreen = childOffscreen(child, role: role, visibleByKind: visibleByKind)
|
|
1687
|
+
let childNode = lookNode(element: child, transform: transform, offscreen: offscreen)
|
|
1688
|
+
node.children.append(childNode)
|
|
1689
|
+
queue.append((child, childNode))
|
|
1690
|
+
walked += 1
|
|
1691
|
+
}
|
|
1692
|
+
}
|
|
1693
|
+
return rootNode
|
|
1694
|
+
}
|
|
1695
|
+
|
|
1696
|
+
private func lookNode(element: AXUIElement, transform: (CGRect) -> CGRect, offscreen: Bool) -> LookNode {
|
|
1697
|
+
let role = stringAttribute(element, attribute: kAXRoleAttribute as CFString) ?? ""
|
|
1698
|
+
let subrole = stringAttribute(element, attribute: kAXSubroleAttribute as CFString) ?? ""
|
|
1699
|
+
let actions = actionNames(element)
|
|
1700
|
+
var valueSettable = DarwinBoolean(false)
|
|
1701
|
+
let valueStatus = AXUIElementIsAttributeSettable(element, kAXValueAttribute as CFString, &valueSettable)
|
|
1702
|
+
var focusedSettable = DarwinBoolean(false)
|
|
1703
|
+
let focusedStatus = AXUIElementIsAttributeSettable(element, kAXFocusedAttribute as CFString, &focusedSettable)
|
|
1704
|
+
let textRoles: Set<String> = ["AXTextField", "AXTextArea", "AXTextView", "AXSearchField", "AXComboBox", "AXEditableText", "AXSecureTextField"]
|
|
1705
|
+
let title = stringAttribute(element, attribute: kAXTitleAttribute as CFString) ?? ""
|
|
1706
|
+
let description = stringAttribute(element, attribute: kAXDescriptionAttribute as CFString) ?? ""
|
|
1707
|
+
let value = displayValue(element, role: role, subrole: subrole)
|
|
1708
|
+
let screenRect = frameForElement(element) ?? .zero
|
|
1709
|
+
let node = LookNode(
|
|
1710
|
+
element: element,
|
|
1711
|
+
ref: refStore.storeElement(element, snapshot: AXRefStore.Snapshot(
|
|
1712
|
+
role: role,
|
|
1713
|
+
identifier: stringAttribute(element, attribute: "AXIdentifier" as CFString) ?? "",
|
|
1714
|
+
label: normalizedLabel([title, description, value].joined(separator: " ")),
|
|
1715
|
+
rect: screenRect
|
|
1716
|
+
)),
|
|
1717
|
+
role: role,
|
|
1718
|
+
subrole: subrole,
|
|
1719
|
+
identifier: stringAttribute(element, attribute: "AXIdentifier" as CFString) ?? "",
|
|
1720
|
+
title: title,
|
|
1721
|
+
description: description,
|
|
1722
|
+
value: value,
|
|
1723
|
+
actions: actions,
|
|
1724
|
+
canPress: actions.contains(kAXPressAction as String),
|
|
1725
|
+
canFocus: focusedStatus == .success && focusedSettable.boolValue,
|
|
1726
|
+
canSetValue: valueStatus == .success && valueSettable.boolValue,
|
|
1727
|
+
canScroll: supportsAnyScrollAction(element),
|
|
1728
|
+
canIncrement: actions.contains(kAXIncrementAction as String),
|
|
1729
|
+
canDecrement: actions.contains(kAXDecrementAction as String),
|
|
1730
|
+
isTextInput: textRoles.contains(role),
|
|
1731
|
+
rect: transform(screenRect),
|
|
1732
|
+
focused: boolAttribute(element, attribute: kAXFocusedAttribute as CFString) == true,
|
|
1733
|
+
offscreen: offscreen
|
|
1734
|
+
)
|
|
1735
|
+
if node.canScroll {
|
|
1736
|
+
let rows = axElementArray(element, attribute: kAXRowsAttribute as CFString)
|
|
1737
|
+
let visibleRows = axElementArrayIfPresent(element, attribute: kAXVisibleRowsAttribute as CFString)
|
|
1738
|
+
if !rows.isEmpty, let visibleRows {
|
|
1739
|
+
node.scrollExtent = ["seen": visibleRows.count, "total": rows.count]
|
|
1740
|
+
}
|
|
1741
|
+
}
|
|
1742
|
+
return node
|
|
1743
|
+
}
|
|
1744
|
+
|
|
1745
|
+
private func visibleChildrenByKind(_ element: AXUIElement) -> [String: [AXUIElement]?] {
|
|
1746
|
+
[
|
|
1747
|
+
"AXRow": axElementArrayIfPresent(element, attribute: kAXVisibleRowsAttribute as CFString),
|
|
1748
|
+
"AXColumn": axElementArrayIfPresent(element, attribute: kAXVisibleColumnsAttribute as CFString),
|
|
1749
|
+
"AXCell": axElementArrayIfPresent(element, attribute: kAXVisibleCellsAttribute as CFString),
|
|
1750
|
+
"*": axElementArrayIfPresent(element, attribute: kAXVisibleChildrenAttribute as CFString),
|
|
1751
|
+
]
|
|
1752
|
+
}
|
|
1753
|
+
|
|
1754
|
+
private func childOffscreen(_ child: AXUIElement, role: String, visibleByKind: [String: [AXUIElement]?]) -> Bool {
|
|
1755
|
+
let key = role == "AXRow" || role == "AXColumn" || role == "AXCell" ? role : "*"
|
|
1756
|
+
guard let visible = visibleByKind[key] ?? nil else { return false }
|
|
1757
|
+
return !visible.contains { sameElement($0, child) }
|
|
1758
|
+
}
|
|
1759
|
+
|
|
1760
|
+
private func recognizeText(in image: CGImage, outputWidth: Int, outputHeight: Int) throws -> [OCRBox] {
|
|
1761
|
+
let semaphore = DispatchSemaphore(value: 0)
|
|
1762
|
+
let recognized = Box<[OCRBox]>([])
|
|
1763
|
+
let recognizedError = Box<Error?>(nil)
|
|
1764
|
+
let request = VNRecognizeTextRequest { request, error in
|
|
1765
|
+
defer { semaphore.signal() }
|
|
1766
|
+
if let error {
|
|
1767
|
+
recognizedError.value = error
|
|
1768
|
+
return
|
|
1769
|
+
}
|
|
1770
|
+
let observations = (request.results as? [VNRecognizedTextObservation]) ?? []
|
|
1771
|
+
recognized.value = observations.compactMap { observation in
|
|
1772
|
+
guard let candidate = observation.topCandidates(1).first else { return nil }
|
|
1773
|
+
let box = observation.boundingBox
|
|
1774
|
+
let x = box.origin.x * Double(outputWidth)
|
|
1775
|
+
let y = (1.0 - box.origin.y - box.height) * Double(outputHeight)
|
|
1776
|
+
let w = box.width * Double(outputWidth)
|
|
1777
|
+
let h = box.height * Double(outputHeight)
|
|
1778
|
+
return OCRBox(string: candidate.string, confidence: Double(candidate.confidence), rect: CGRect(x: x, y: y, width: w, height: h))
|
|
1779
|
+
}
|
|
1780
|
+
}
|
|
1781
|
+
request.recognitionLevel = .accurate
|
|
1782
|
+
request.usesLanguageCorrection = false
|
|
1783
|
+
try VNImageRequestHandler(cgImage: image, options: [:]).perform([request])
|
|
1784
|
+
if semaphore.wait(timeout: .now() + .seconds(8)) == .timedOut {
|
|
1785
|
+
throw BridgeFailure(message: "Text recognition timed out", code: "text_recognition_timeout")
|
|
1786
|
+
}
|
|
1787
|
+
if let error = recognizedError.value {
|
|
1788
|
+
throw BridgeFailure(message: "Text recognition failed: \(error.localizedDescription)", code: "text_recognition_failed")
|
|
1789
|
+
}
|
|
1790
|
+
return recognized.value
|
|
1791
|
+
}
|
|
1792
|
+
|
|
1793
|
+
private func attachOCR(_ boxes: [OCRBox], to root: LookNode) {
|
|
1794
|
+
var pictureOnlyIndex = 0
|
|
1795
|
+
for box in boxes {
|
|
1796
|
+
if ocrBoxDuplicatesAXLabel(box, in: root) { continue }
|
|
1797
|
+
let center = CGPoint(x: box.rect.midX, y: box.rect.midY)
|
|
1798
|
+
if let node = deepestNode(containing: center, in: root) {
|
|
1799
|
+
node.text.append(["string": box.string, "confidence": box.confidence, "rect": ["x": box.rect.origin.x, "y": box.rect.origin.y, "w": box.rect.width, "h": box.rect.height]])
|
|
1800
|
+
} else {
|
|
1801
|
+
pictureOnlyIndex += 1
|
|
1802
|
+
let parent = deepestContainer(containing: center, in: root) ?? root
|
|
1803
|
+
parent.children.append(LookNode(element: nil, ref: "pic_\(pictureOnlyIndex)", role: "AXImage", subrole: "", identifier: "", title: box.string, description: "", value: "", actions: [], canPress: false, canFocus: false, canSetValue: false, canScroll: false, canIncrement: false, canDecrement: false, isTextInput: false, rect: box.rect, pictureOnly: true))
|
|
1804
|
+
}
|
|
1805
|
+
}
|
|
1806
|
+
}
|
|
1807
|
+
|
|
1808
|
+
private func ocrBoxDuplicatesAXLabel(_ box: OCRBox, in root: LookNode) -> Bool {
|
|
1809
|
+
let boxLabel = normalizedLabel(box.string)
|
|
1810
|
+
if boxLabel.isEmpty { return true }
|
|
1811
|
+
var queue = [root]
|
|
1812
|
+
var index = 0
|
|
1813
|
+
while index < queue.count {
|
|
1814
|
+
let node = queue[index]
|
|
1815
|
+
index += 1
|
|
1816
|
+
if !node.pictureOnly, node.rect.intersects(box.rect) {
|
|
1817
|
+
let fields = [node.title, node.value, node.description]
|
|
1818
|
+
if fields.contains(where: { normalizedLabel($0) == boxLabel }) { return true }
|
|
1819
|
+
}
|
|
1820
|
+
queue.append(contentsOf: node.children)
|
|
1821
|
+
}
|
|
1822
|
+
return false
|
|
1823
|
+
}
|
|
1824
|
+
|
|
1825
|
+
private func deepestNode(containing point: CGPoint, in root: LookNode) -> LookNode? {
|
|
1826
|
+
guard root.rect.contains(point), !root.pictureOnly else { return nil }
|
|
1827
|
+
for child in root.children.reversed() {
|
|
1828
|
+
if let match = deepestNode(containing: point, in: child) { return match }
|
|
1829
|
+
}
|
|
1830
|
+
return root
|
|
1831
|
+
}
|
|
1832
|
+
|
|
1833
|
+
private func deepestContainer(containing point: CGPoint, in root: LookNode) -> LookNode? {
|
|
1834
|
+
guard root.rect.contains(point) else { return nil }
|
|
1835
|
+
for child in root.children.reversed() {
|
|
1836
|
+
if let match = deepestContainer(containing: point, in: child) { return match }
|
|
1837
|
+
}
|
|
1838
|
+
return root
|
|
1839
|
+
}
|
|
1840
|
+
|
|
1841
|
+
private func lookPoint(record: LookRecord, x: Double, y: Double) -> CGPoint {
|
|
1842
|
+
let relX = min(max(x / max(1.0, Double(record.imageWidth)), 0), 1)
|
|
1843
|
+
let relY = min(max(y / max(1.0, Double(record.imageHeight)), 0), 1)
|
|
1844
|
+
return CGPoint(x: record.windowFrame.origin.x + record.windowFrame.width * relX, y: record.windowFrame.origin.y + record.windowFrame.height * relY)
|
|
1845
|
+
}
|
|
1846
|
+
|
|
1847
|
+
private func payloadNode(element: AXUIElement) -> [String: Any] {
|
|
1848
|
+
let node = lookNode(element: element, transform: { $0 }, offscreen: false)
|
|
1849
|
+
var payload = node.payload()
|
|
1850
|
+
payload["children"] = []
|
|
1851
|
+
return payload
|
|
1852
|
+
}
|
|
1853
|
+
|
|
1854
|
+
private func normalizedLabel(_ value: String) -> String {
|
|
1855
|
+
value.lowercased().components(separatedBy: CharacterSet.whitespacesAndNewlines).filter { !$0.isEmpty }.joined(separator: " ")
|
|
1856
|
+
}
|
|
1857
|
+
|
|
1858
|
+
private func ensureRootObserver(pid: Int32) -> Bool {
|
|
1859
|
+
rootObserverLock.lock()
|
|
1860
|
+
if let existing = rootObservers[pid] {
|
|
1861
|
+
existing.lastUsed = Date().timeIntervalSince1970
|
|
1862
|
+
rootObserverLock.unlock()
|
|
1863
|
+
return true
|
|
1864
|
+
}
|
|
1865
|
+
rootObserverLock.unlock()
|
|
1866
|
+
|
|
1867
|
+
let appElement = AXUIElementCreateApplication(pid)
|
|
1868
|
+
AXUIElementSetMessagingTimeout(appElement, 0.25)
|
|
1869
|
+
var observer: AXObserver?
|
|
1870
|
+
let createStatus = AXObserverCreate(pid, { observer, element, notification, refcon in
|
|
1871
|
+
guard let refcon else { return }
|
|
1872
|
+
let bridge = Unmanaged<Bridge>.fromOpaque(refcon).takeUnretainedValue()
|
|
1873
|
+
bridge.recordRootAXEvent(observer: observer, notification: notification as String, element: element)
|
|
1874
|
+
}, &observer)
|
|
1875
|
+
guard createStatus == .success, let observer else { return false }
|
|
1876
|
+
|
|
1877
|
+
let state = RootAXObserverState(pid: pid, observer: observer)
|
|
1878
|
+
let context = UnsafeMutableRawPointer(Unmanaged.passUnretained(self).toOpaque())
|
|
1879
|
+
let notifications: [CFString] = [
|
|
1880
|
+
"AXWindowCreated" as CFString,
|
|
1881
|
+
"AXSheetCreated" as CFString,
|
|
1882
|
+
"AXMenuOpened" as CFString,
|
|
1883
|
+
"AXMenuClosed" as CFString,
|
|
1884
|
+
"AXUIElementDestroyed" as CFString,
|
|
1885
|
+
"AXFocusedWindowChanged" as CFString,
|
|
1886
|
+
kAXValueChangedNotification as CFString,
|
|
1887
|
+
kAXTitleChangedNotification as CFString,
|
|
1888
|
+
kAXSelectedChildrenChangedNotification as CFString,
|
|
1889
|
+
kAXLayoutChangedNotification as CFString,
|
|
1890
|
+
]
|
|
1891
|
+
var registered = false
|
|
1892
|
+
let observedElements = [appElement] + axElementArray(appElement, attribute: kAXWindowsAttribute as CFString)
|
|
1893
|
+
for observed in observedElements {
|
|
1894
|
+
for notification in notifications {
|
|
1895
|
+
let status = AXObserverAddNotification(observer, observed, notification, context)
|
|
1896
|
+
if status == .success || status == .notificationAlreadyRegistered { registered = true }
|
|
1897
|
+
}
|
|
1898
|
+
}
|
|
1899
|
+
guard registered else { return false }
|
|
1900
|
+
|
|
1901
|
+
rootObserverLock.lock()
|
|
1902
|
+
if rootObservers.count >= maxRootObservers,
|
|
1903
|
+
let evict = rootObservers.values.min(by: { $0.lastUsed < $1.lastUsed })?.pid
|
|
1904
|
+
{
|
|
1905
|
+
rootObservers.removeValue(forKey: evict)
|
|
1906
|
+
}
|
|
1907
|
+
rootObservers[pid] = state
|
|
1908
|
+
rootObserverLock.unlock()
|
|
1909
|
+
|
|
1910
|
+
let source = AXObserverGetRunLoopSource(observer)
|
|
1911
|
+
Thread.detachNewThread {
|
|
1912
|
+
// Keep each AXObserver on its own run loop so its callbacks never compete
|
|
1913
|
+
// with AppKit rendering on the helper's main thread.
|
|
1914
|
+
CFRunLoopAddSource(CFRunLoopGetCurrent(), source, .commonModes)
|
|
1915
|
+
CFRunLoopRun()
|
|
1916
|
+
}
|
|
1917
|
+
return true
|
|
1918
|
+
}
|
|
1919
|
+
|
|
1920
|
+
private func recordRootAXEvent(observer: AXObserver, notification: String, element: AXUIElement) {
|
|
1921
|
+
rootObserverLock.lock()
|
|
1922
|
+
let state = rootObservers.values.first { CFEqual($0.observer, observer) }
|
|
1923
|
+
if let state {
|
|
1924
|
+
state.events.append(RootAXEvent(sequence: state.nextSequence, timestamp: Date().timeIntervalSince1970, notification: notification, element: element))
|
|
1925
|
+
state.nextSequence += 1
|
|
1926
|
+
if state.events.count > 64 { state.events.removeFirst(state.events.count - 64) }
|
|
1927
|
+
}
|
|
1928
|
+
rootObserverLock.unlock()
|
|
1929
|
+
if let state {
|
|
1930
|
+
state.change.lock()
|
|
1931
|
+
state.changeGeneration += 1
|
|
1932
|
+
state.change.broadcast()
|
|
1933
|
+
state.change.unlock()
|
|
1934
|
+
}
|
|
1935
|
+
}
|
|
1936
|
+
|
|
1937
|
+
private func rootChangeGeneration(pid: Int32) -> UInt64 {
|
|
1938
|
+
rootObserverLock.lock()
|
|
1939
|
+
let state = rootObservers[pid]
|
|
1940
|
+
rootObserverLock.unlock()
|
|
1941
|
+
guard let state else { return 0 }
|
|
1942
|
+
state.change.lock()
|
|
1943
|
+
defer { state.change.unlock() }
|
|
1944
|
+
return state.changeGeneration
|
|
1945
|
+
}
|
|
1946
|
+
|
|
1947
|
+
private func waitForRootChange(pid: Int32, since generation: UInt64, until deadline: Date) {
|
|
1948
|
+
rootObserverLock.lock()
|
|
1949
|
+
let state = rootObservers[pid]
|
|
1950
|
+
rootObserverLock.unlock()
|
|
1951
|
+
guard let state else {
|
|
1952
|
+
Thread.sleep(forTimeInterval: min(0.2, max(0, deadline.timeIntervalSinceNow)))
|
|
1953
|
+
return
|
|
1954
|
+
}
|
|
1955
|
+
state.change.lock()
|
|
1956
|
+
if state.changeGeneration == generation {
|
|
1957
|
+
_ = state.change.wait(until: min(deadline, Date().addingTimeInterval(0.2)))
|
|
1958
|
+
}
|
|
1959
|
+
state.change.unlock()
|
|
1960
|
+
}
|
|
1961
|
+
|
|
1962
|
+
private func rootEventCursor(pid: Int32) -> UInt64 {
|
|
1963
|
+
rootObserverLock.lock()
|
|
1964
|
+
defer { rootObserverLock.unlock() }
|
|
1965
|
+
return rootObservers[pid]?.nextSequence ?? 1
|
|
1966
|
+
}
|
|
1967
|
+
|
|
1968
|
+
private func rootEvents(pid: Int32, since cursor: UInt64) -> [RootAXEvent] {
|
|
1969
|
+
rootObserverLock.lock()
|
|
1970
|
+
defer { rootObserverLock.unlock() }
|
|
1971
|
+
guard let events = rootObservers[pid]?.events else { return [] }
|
|
1972
|
+
return events.filter { $0.sequence >= cursor }
|
|
1973
|
+
}
|
|
1974
|
+
|
|
1975
|
+
// Onscreen CGWindowList id set for one pid: ~1-2ms per call, so it can be
|
|
1976
|
+
// polled tightly where a full AX enumeration cannot.
|
|
1977
|
+
private func cgRootSignature(pid: Int32) -> Set<UInt32> {
|
|
1978
|
+
guard let entries = CGWindowListCopyWindowInfo([.optionOnScreenOnly], kCGNullWindowID) as? [[String: Any]] else { return [] }
|
|
1979
|
+
var ids = Set<UInt32>()
|
|
1980
|
+
for entry in entries {
|
|
1981
|
+
guard let owner = (entry[kCGWindowOwnerPID as String] as? NSNumber)?.int32Value, owner == pid else { continue }
|
|
1982
|
+
guard let windowId = (entry[kCGWindowNumber as String] as? NSNumber)?.uint32Value else { continue }
|
|
1983
|
+
ids.insert(windowId)
|
|
1984
|
+
}
|
|
1985
|
+
return ids
|
|
1986
|
+
}
|
|
1987
|
+
|
|
1988
|
+
private func refindElement(ref: String, pid: Int32, windowId: UInt32) -> AXUIElement? {
|
|
1989
|
+
guard let snapshot = refStore.snapshot(for: ref),
|
|
1990
|
+
let window = windowElement(pid: pid, windowId: windowId)
|
|
1991
|
+
else { return nil }
|
|
1992
|
+
let targetCenter = CGPoint(x: snapshot.rect.midX, y: snapshot.rect.midY)
|
|
1993
|
+
let candidates = collectDescendants(startingAt: window, maxDepth: 8).filter { candidate in
|
|
1994
|
+
let role = stringAttribute(candidate, attribute: kAXRoleAttribute as CFString) ?? ""
|
|
1995
|
+
guard role == snapshot.role else { return false }
|
|
1996
|
+
let identifier = stringAttribute(candidate, attribute: "AXIdentifier" as CFString) ?? ""
|
|
1997
|
+
if !snapshot.identifier.isEmpty { return identifier == snapshot.identifier }
|
|
1998
|
+
let subrole = stringAttribute(candidate, attribute: kAXSubroleAttribute as CFString) ?? ""
|
|
1999
|
+
let title = stringAttribute(candidate, attribute: kAXTitleAttribute as CFString) ?? ""
|
|
2000
|
+
let description = stringAttribute(candidate, attribute: kAXDescriptionAttribute as CFString) ?? ""
|
|
2001
|
+
let value = displayValue(candidate, role: role, subrole: subrole)
|
|
2002
|
+
return normalizedLabel([title, description, value].joined(separator: " ")) == snapshot.label
|
|
2003
|
+
}
|
|
2004
|
+
return candidates.min { left, right in
|
|
2005
|
+
let leftFrame = frameForElement(left) ?? .zero
|
|
2006
|
+
let rightFrame = frameForElement(right) ?? .zero
|
|
2007
|
+
let leftDistance = hypot(leftFrame.midX - targetCenter.x, leftFrame.midY - targetCenter.y)
|
|
2008
|
+
let rightDistance = hypot(rightFrame.midX - targetCenter.x, rightFrame.midY - targetCenter.y)
|
|
2009
|
+
return leftDistance < rightDistance
|
|
2010
|
+
}
|
|
2011
|
+
}
|
|
2012
|
+
|
|
2013
|
+
private func hitTest(_ request: [String: Any]) throws -> [String: Any] {
|
|
2014
|
+
let lookId = try stringArg(request, "lookId")
|
|
2015
|
+
guard let record = lookRecord(for: lookId) else {
|
|
2016
|
+
throw BridgeFailure(message: "Look id '\(lookId)' is no longer available", code: "stale_look")
|
|
2017
|
+
}
|
|
2018
|
+
let point = lookPoint(record: record, x: try doubleArg(request, "x"), y: try doubleArg(request, "y"))
|
|
2019
|
+
guard let element = hitTestElement(at: point) else {
|
|
2020
|
+
throw BridgeFailure(message: "No element at point", code: "hit_test_failed")
|
|
2021
|
+
}
|
|
2022
|
+
return payloadNode(element: element)
|
|
2023
|
+
}
|
|
2024
|
+
|
|
2025
|
+
private func act(_ request: [String: Any]) throws -> [String: Any] {
|
|
2026
|
+
let lookId = try stringArg(request, "lookId")
|
|
2027
|
+
guard let record = lookRecord(for: lookId) else {
|
|
2028
|
+
throw BridgeFailure(message: "Look id '\(lookId)' is no longer available", code: "stale_look")
|
|
2029
|
+
}
|
|
2030
|
+
let pid = Int32(try intArg(request, "pid"))
|
|
2031
|
+
let action = try stringArg(request, "action")
|
|
2032
|
+
let target = request["target"] as? [String: Any] ?? [:]
|
|
2033
|
+
let params = request["params"] as? [String: Any] ?? [:]
|
|
2034
|
+
let policy = optionalStringArg(request, "policy") ?? "default"
|
|
2035
|
+
// Tangu:登记"正在操控这个窗口"(供 liveView),并按设置点亮边缘光效。放在最前面 —— 后面任何
|
|
2036
|
+
// 一步抛错都不该让用户失去"agent 正在动这个窗口"的可见信号。
|
|
2037
|
+
// 复用 cursorOverlay 这一个开关(vendor 的 act 请求里本就带着它),不为光效另开配置键:
|
|
2038
|
+
// 二者是同一件事——"把 agent 的动作画出来给人看"。要拆开时再加键不迟。
|
|
2039
|
+
noteControlledWindow(windowId: record.windowId, pid: pid, highlight: request["cursorOverlay"] as? Bool ?? true)
|
|
2040
|
+
let deferRootDelta = boolArg(request, "deferRootDelta") ?? false
|
|
2041
|
+
let delivery = policy == "background" ? "pid" : ((params["delivery"] as? String) == "pid" ? "pid" : "hid")
|
|
2042
|
+
var holdsPhysicalInput = false
|
|
2043
|
+
func acquirePhysicalInputIfNeeded() {
|
|
2044
|
+
if delivery == "hid" && !holdsPhysicalInput {
|
|
2045
|
+
physicalInputLock.lock(); ForegroundActivity.shared.enter()
|
|
2046
|
+
holdsPhysicalInput = true
|
|
2047
|
+
}
|
|
2048
|
+
}
|
|
2049
|
+
defer { if holdsPhysicalInput { ForegroundActivity.shared.leave(); physicalInputLock.unlock() } }
|
|
2050
|
+
var performed: [String: Any] = ["delivery": delivery]
|
|
2051
|
+
var element: AXUIElement?
|
|
2052
|
+
var rawPoint: CGPoint?
|
|
2053
|
+
var preflightCapsUnknown = false
|
|
2054
|
+
let eventsLive = !deferRootDelta && ensureRootObserver(pid: pid)
|
|
2055
|
+
let eventCursor = eventsLive ? rootEventCursor(pid: pid) : 0
|
|
2056
|
+
let beforeRootSnapshot = deferRootDelta ? [:] : rootMetadataSnapshot(pid: pid)
|
|
2057
|
+
let beforeCgSignature = deferRootDelta ? [] : cgRootSignature(pid: pid)
|
|
2058
|
+
let beforeFrontmostPid = deferRootDelta ? nil : NSWorkspace.shared.frontmostApplication?.processIdentifier
|
|
2059
|
+
let beforeSheetCount = windowElement(pid: pid, windowId: record.windowId).map { sheetElements(of: $0).count } ?? 0
|
|
2060
|
+
let beforeFocusedWindow = focusedWindowSummary(pid: pid)
|
|
2061
|
+
let beforeValue: String?
|
|
2062
|
+
let beforeSelected: String?
|
|
2063
|
+
func finish(_ response: [String: Any]) -> [String: Any] {
|
|
2064
|
+
if deferRootDelta { return response }
|
|
2065
|
+
return attachRootDelta(to: response, before: beforeRootSnapshot, beforeFrontmostPid: beforeFrontmostPid, pid: pid, eventsLive: eventsLive, eventCursor: eventCursor, beforeCgSignature: beforeCgSignature)
|
|
2066
|
+
}
|
|
2067
|
+
|
|
2068
|
+
if let ref = target["ref"] as? String {
|
|
2069
|
+
var refound = false
|
|
2070
|
+
let cached = refStore.element(for: ref)
|
|
2071
|
+
let cachedIsLive = cached.map {
|
|
2072
|
+
stringAttribute($0, attribute: kAXRoleAttribute as CFString) != nil && frameForElement($0) != nil
|
|
2073
|
+
} ?? false
|
|
2074
|
+
let resolved: AXUIElement?
|
|
2075
|
+
if cachedIsLive {
|
|
2076
|
+
resolved = cached
|
|
2077
|
+
} else {
|
|
2078
|
+
refound = true
|
|
2079
|
+
resolved = refindElement(ref: ref, pid: pid, windowId: record.windowId)
|
|
2080
|
+
}
|
|
2081
|
+
guard let stored = resolved else {
|
|
2082
|
+
throw BridgeFailure(message: "Element reference is stale", code: "stale_ref")
|
|
2083
|
+
}
|
|
2084
|
+
if refound { performed["refound"] = true }
|
|
2085
|
+
element = stored
|
|
2086
|
+
beforeValue = stringAttribute(stored, attribute: kAXValueAttribute as CFString)
|
|
2087
|
+
beforeSelected = stringAttribute(stored, attribute: kAXSelectedTextAttribute as CFString)
|
|
2088
|
+
} else if let xNumber = target["x"] as? NSNumber, let yNumber = target["y"] as? NSNumber {
|
|
2089
|
+
guard record.hasImage else {
|
|
2090
|
+
throw BridgeFailure(message: "Coordinate targeting is unavailable for this outline-only root", code: "coordinate_unavailable_for_root")
|
|
2091
|
+
}
|
|
2092
|
+
rawPoint = lookPoint(record: record, x: xNumber.doubleValue, y: yNumber.doubleValue)
|
|
2093
|
+
beforeValue = nil
|
|
2094
|
+
beforeSelected = nil
|
|
2095
|
+
} else {
|
|
2096
|
+
throw BridgeFailure(message: "act target must include ref or x/y", code: "invalid_args")
|
|
2097
|
+
}
|
|
2098
|
+
|
|
2099
|
+
func coordinatePoint() throws -> CGPoint {
|
|
2100
|
+
if let element, let frame = frameForElement(element) {
|
|
2101
|
+
return CGPoint(x: frame.midX, y: frame.midY)
|
|
2102
|
+
}
|
|
2103
|
+
if let rawPoint { return rawPoint }
|
|
2104
|
+
throw BridgeFailure(message: "No coordinate grounding is available", code: "coordinate_unavailable")
|
|
2105
|
+
}
|
|
2106
|
+
|
|
2107
|
+
/// `viaAx = true`:这一下是用 AX 直接按压完成的,压根没发物理事件 —— 无论 delivery 名义上
|
|
2108
|
+
/// 是什么,系统指针都没动过,所以虚拟光标该画。否则前台策略下用 AX 打成的那一下毫无反馈。
|
|
2109
|
+
func animateCursor(at point: CGPoint, viaAx: Bool = false) {
|
|
2110
|
+
guard supportsAgentCursor,
|
|
2111
|
+
(request["cursorOverlay"] as? Bool ?? true),
|
|
2112
|
+
viaAx || (delivery == "pid" && policy != "ax_only"),
|
|
2113
|
+
["press", "click", "moveMouse", "scroll", "drag"].contains(action)
|
|
2114
|
+
else { return }
|
|
2115
|
+
Task { @MainActor in AgentCursor.shared.animate(to: point, above: record.windowId) }
|
|
2116
|
+
}
|
|
2117
|
+
|
|
2118
|
+
/// Tangu 新增:**纯坐标点击先做一次 AX 命中测试**。命中可按压元素就直接 AXPress ——
|
|
2119
|
+
/// 不抢前台、不动真鼠标、画虚拟光标。命不中再照上游回落物理事件。
|
|
2120
|
+
///
|
|
2121
|
+
/// 为什么需要它:物理事件那条后台路(CGEventPostToPid)对非 key 窗口的 App 会被直接丢掉,
|
|
2122
|
+
/// 所以上游对坐标点击一律直接判 `needsForeground`(actions.ts)走 HID 前台。于是"看着截图
|
|
2123
|
+
/// 盲点"必然抢前台 —— 这是用户实报的症结。AX 按压是唯一能在后台真正落下去的点击。
|
|
2124
|
+
///
|
|
2125
|
+
/// ⚠️命中测试的根**必须是目标 App**,不能用 systemWide:目标窗口在后台时,那个屏幕坐标上压着的
|
|
2126
|
+
/// 是别人的窗口,systemWide 会把别人窗口里的控件返回给我们 —— 那一下就点到别的 App 上了。
|
|
2127
|
+
/// 再加一道窗口子树校验:同一个 App 开多个窗口时,同样会点错窗口。
|
|
2128
|
+
func backgroundPressAtPoint(_ point: CGPoint) -> Bool {
|
|
2129
|
+
guard action == "press" || action == "click" else { return false }
|
|
2130
|
+
// 右键/中键/双击的语义 AXPress 表达不了,老实走物理事件。
|
|
2131
|
+
guard (params["button"] as? String ?? "left") == "left",
|
|
2132
|
+
((params["clickCount"] as? NSNumber)?.intValue ?? 1) <= 1
|
|
2133
|
+
else { return false }
|
|
2134
|
+
var hitRef: AXUIElement?
|
|
2135
|
+
guard AXUIElementCopyElementAtPosition(AXUIElementCreateApplication(pid), Float(point.x), Float(point.y), &hitRef) == .success,
|
|
2136
|
+
let hit = hitRef
|
|
2137
|
+
else { return false }
|
|
2138
|
+
var hitPid: pid_t = 0
|
|
2139
|
+
guard AXUIElementGetPid(hit, &hitPid) == .success, hitPid == pid else { return false }
|
|
2140
|
+
guard let windowRoot = windowElement(pid: pid, windowId: record.windowId),
|
|
2141
|
+
sameElement(hit, windowRoot) || isElement(hit, descendantOf: windowRoot)
|
|
2142
|
+
else { return false }
|
|
2143
|
+
// 与 ref 路径同一条规则:网页内容与文本框要的是真实指针焦点,AXPress 打不动。
|
|
2144
|
+
let hitRole = stringAttribute(hit, attribute: kAXRoleAttribute as CFString) ?? ""
|
|
2145
|
+
let textRoles: Set<String> = ["AXTextField", "AXTextArea", "AXTextView", "AXSearchField", "AXComboBox", "AXEditableText", "AXSecureTextField"]
|
|
2146
|
+
guard !hasAncestorRole(hit, role: "AXWebArea"), !textRoles.contains(hitRole) else { return false }
|
|
2147
|
+
guard supportsAction(hit, action: kAXPressAction as CFString) else { return false }
|
|
2148
|
+
guard AXUIElementPerformAction(hit, kAXPressAction as CFString) == .success else { return false }
|
|
2149
|
+
performed["grounding"] = "hit_test"
|
|
2150
|
+
performed["delivery"] = "ax"
|
|
2151
|
+
performed["hitRole"] = hitRole
|
|
2152
|
+
// 坐标点击没有 before/after 值可比,和后台物理事件一样无从自证 —— 如实说要调用方自己查。
|
|
2153
|
+
performed["verification"] = "caller_required"
|
|
2154
|
+
animateCursor(at: point, viaAx: true)
|
|
2155
|
+
return true
|
|
2156
|
+
}
|
|
2157
|
+
|
|
2158
|
+
func focusTargetForPhysicalInput() {
|
|
2159
|
+
guard delivery == "hid" else { return }
|
|
2160
|
+
if let app = NSRunningApplication(processIdentifier: pid), !app.isActive {
|
|
2161
|
+
ForegroundActivity.shared.physicalInput()
|
|
2162
|
+
performed["activated"] = app.activate()
|
|
2163
|
+
}
|
|
2164
|
+
if let window = windowElement(pid: pid, windowId: record.windowId) {
|
|
2165
|
+
_ = AXUIElementSetAttributeValue(window, kAXMainAttribute as CFString, kCFBooleanTrue)
|
|
2166
|
+
_ = AXUIElementSetAttributeValue(window, kAXFocusedAttribute as CFString, kCFBooleanTrue)
|
|
2167
|
+
performed["raised"] = AXUIElementPerformAction(window, kAXRaiseAction as CFString) == .success
|
|
2168
|
+
}
|
|
2169
|
+
usleep(20_000)
|
|
2170
|
+
}
|
|
2171
|
+
|
|
2172
|
+
func preflight(_ point: CGPoint) throws {
|
|
2173
|
+
guard let element else { preflightCapsUnknown = true; return }
|
|
2174
|
+
for attempt in 0..<4 {
|
|
2175
|
+
guard let hit = hitTestElement(at: point) else { preflightCapsUnknown = true; return }
|
|
2176
|
+
if sameElement(hit, element) || isElement(hit, descendantOf: element) || isElement(element, descendantOf: hit) { return }
|
|
2177
|
+
let role = stringAttribute(hit, attribute: kAXRoleAttribute as CFString) ?? ""
|
|
2178
|
+
if role == "AXWindow" || role == "AXApplication" { preflightCapsUnknown = true; return }
|
|
2179
|
+
if delivery == "hid" && attempt < 3 {
|
|
2180
|
+
focusTargetForPhysicalInput()
|
|
2181
|
+
usleep(20_000)
|
|
2182
|
+
continue
|
|
2183
|
+
}
|
|
2184
|
+
throw BridgeFailure(message: "Target is occluded by \(payloadNode(element: hit))", code: "occluded_target")
|
|
2185
|
+
}
|
|
2186
|
+
}
|
|
2187
|
+
|
|
2188
|
+
func executeCoordinates(_ point: CGPoint) throws {
|
|
2189
|
+
guard element != nil || record.hasImage else {
|
|
2190
|
+
throw BridgeFailure(message: "Coordinate grounding is unavailable for this outline-only root", code: "coordinate_unavailable_for_root")
|
|
2191
|
+
}
|
|
2192
|
+
// 纯坐标点击先试后台 AX 按压。带 element 的路径不试:那边刚判过 AXPress 支不支持,
|
|
2193
|
+
// 走到这里就是已经失败过一次,再试一遍只是重复同一个失败。
|
|
2194
|
+
if element == nil, backgroundPressAtPoint(point) { return }
|
|
2195
|
+
performed["grounding"] = "coordinates"
|
|
2196
|
+
if delivery == "pid" { performed["verification"] = "caller_required" }
|
|
2197
|
+
acquirePhysicalInputIfNeeded()
|
|
2198
|
+
focusTargetForPhysicalInput()
|
|
2199
|
+
if delivery == "hid" { try preflight(point) }
|
|
2200
|
+
switch action {
|
|
2201
|
+
case "press", "click":
|
|
2202
|
+
animateCursor(at: point)
|
|
2203
|
+
try postMouseClick(at: point, pid: pid, button: mouseButton(params["button"] as? String ?? "left"), clickCount: max(1, min(3, (params["clickCount"] as? NSNumber)?.intValue ?? 1)), delivery: delivery)
|
|
2204
|
+
case "moveMouse":
|
|
2205
|
+
animateCursor(at: point)
|
|
2206
|
+
try postMouseMove(to: point, pid: pid, delivery: delivery)
|
|
2207
|
+
case "scroll":
|
|
2208
|
+
animateCursor(at: point)
|
|
2209
|
+
try postScrollWheel(at: point, deltaX: (params["scrollX"] as? NSNumber)?.intValue ?? 0, deltaY: (params["scrollY"] as? NSNumber)?.intValue ?? 0, pid: pid, delivery: delivery)
|
|
2210
|
+
case "drag":
|
|
2211
|
+
guard let rawPath = params["path"] as? [[String: Any]], rawPath.count >= 2 else {
|
|
2212
|
+
throw BridgeFailure(message: "drag requires path", code: "invalid_args")
|
|
2213
|
+
}
|
|
2214
|
+
let points = try rawPath.map { raw -> CGPoint in
|
|
2215
|
+
guard let x = (raw["x"] as? NSNumber)?.doubleValue, let y = (raw["y"] as? NSNumber)?.doubleValue else {
|
|
2216
|
+
throw BridgeFailure(message: "drag path entries require x and y", code: "invalid_args")
|
|
2217
|
+
}
|
|
2218
|
+
return lookPoint(record: record, x: x, y: y)
|
|
2219
|
+
}
|
|
2220
|
+
animateCursor(at: point)
|
|
2221
|
+
try postMouseDrag(points: points, pid: pid, delivery: delivery)
|
|
2222
|
+
default:
|
|
2223
|
+
throw BridgeFailure(message: "Action \(action) cannot use coordinate grounding", code: "invalid_args")
|
|
2224
|
+
}
|
|
2225
|
+
}
|
|
2226
|
+
|
|
2227
|
+
func refreshElement() -> AXUIElement? {
|
|
2228
|
+
guard let ref = target["ref"] as? String,
|
|
2229
|
+
let refreshed = refindElement(ref: ref, pid: pid, windowId: record.windowId)
|
|
2230
|
+
else { return nil }
|
|
2231
|
+
element = refreshed
|
|
2232
|
+
performed["refound"] = true
|
|
2233
|
+
return refreshed
|
|
2234
|
+
}
|
|
2235
|
+
|
|
2236
|
+
if let element, action == "press" || action == "click" {
|
|
2237
|
+
let elementRole = stringAttribute(element, attribute: kAXRoleAttribute as CFString) ?? ""
|
|
2238
|
+
let textRoles: Set<String> = ["AXTextField", "AXTextArea", "AXTextView", "AXSearchField", "AXComboBox", "AXEditableText", "AXSecureTextField"]
|
|
2239
|
+
let requiresPointerFocus = hasAncestorRole(element, role: "AXWebArea") || textRoles.contains(elementRole)
|
|
2240
|
+
if requiresPointerFocus && policy != "ax_only" {
|
|
2241
|
+
if policy == "foreground" {
|
|
2242
|
+
try executeCoordinates(coordinatePoint())
|
|
2243
|
+
} else {
|
|
2244
|
+
throw BridgeFailure(message: "Web content requires pointer input", code: "foreground_required")
|
|
2245
|
+
}
|
|
2246
|
+
} else if supportsAction(element, action: kAXPressAction as CFString) {
|
|
2247
|
+
let cursorPoint = try? coordinatePoint()
|
|
2248
|
+
var status = AXUIElementPerformAction(element, kAXPressAction as CFString)
|
|
2249
|
+
if status != .success, let refreshed = refreshElement(), supportsAction(refreshed, action: kAXPressAction as CFString) {
|
|
2250
|
+
status = AXUIElementPerformAction(refreshed, kAXPressAction as CFString)
|
|
2251
|
+
}
|
|
2252
|
+
if status == .success {
|
|
2253
|
+
performed["grounding"] = "description"
|
|
2254
|
+
performed["delivery"] = "ax"
|
|
2255
|
+
if let cursorPoint { animateCursor(at: cursorPoint, viaAx: true) }
|
|
2256
|
+
} else {
|
|
2257
|
+
try executeCoordinates(coordinatePoint())
|
|
2258
|
+
}
|
|
2259
|
+
} else {
|
|
2260
|
+
try executeCoordinates(coordinatePoint())
|
|
2261
|
+
}
|
|
2262
|
+
} else if let element, action == "setText" {
|
|
2263
|
+
let text = params["text"] as? String ?? ""
|
|
2264
|
+
if hasAncestorRole(element, role: "AXWebArea") {
|
|
2265
|
+
acquirePhysicalInputIfNeeded()
|
|
2266
|
+
focusTargetForPhysicalInput()
|
|
2267
|
+
_ = AXUIElementSetAttributeValue(element, kAXFocusedAttribute as CFString, kCFBooleanTrue)
|
|
2268
|
+
let currentValue = stringAttribute(element, attribute: kAXValueAttribute as CFString) ?? ""
|
|
2269
|
+
var range = CFRange(location: 0, length: (currentValue as NSString).length)
|
|
2270
|
+
let selected = AXValueCreate(.cfRange, &range).map {
|
|
2271
|
+
AXUIElementSetAttributeValue(element, kAXSelectedTextRangeAttribute as CFString, $0) == .success
|
|
2272
|
+
} ?? false
|
|
2273
|
+
if !selected {
|
|
2274
|
+
try postKeyPress(keys: ["cmd", "a"], pid: pid, delivery: delivery)
|
|
2275
|
+
usleep(20_000)
|
|
2276
|
+
}
|
|
2277
|
+
try postAtomicUnicodeText(text, pid: pid, delivery: delivery)
|
|
2278
|
+
usleep(40_000)
|
|
2279
|
+
let verificationElement = refreshElement() ?? element
|
|
2280
|
+
let value = stringAttribute(verificationElement, attribute: kAXValueAttribute as CFString) ?? ""
|
|
2281
|
+
performed["grounding"] = "keyboard-events"
|
|
2282
|
+
performed["delivery"] = delivery
|
|
2283
|
+
performed["selectionGrounding"] = selected ? "ax" : "keyboard"
|
|
2284
|
+
return finish(["outcome": value == text ? "worked" : "didnt", "performed": performed, "evidence": ["value": value]])
|
|
2285
|
+
}
|
|
2286
|
+
var targetElement = element
|
|
2287
|
+
var status = AXUIElementSetAttributeValue(targetElement, kAXValueAttribute as CFString, text as CFTypeRef)
|
|
2288
|
+
if status != .success, let refreshed = refreshElement() {
|
|
2289
|
+
targetElement = refreshed
|
|
2290
|
+
status = AXUIElementSetAttributeValue(targetElement, kAXValueAttribute as CFString, text as CFTypeRef)
|
|
2291
|
+
}
|
|
2292
|
+
if status == .success {
|
|
2293
|
+
performed["grounding"] = "description"
|
|
2294
|
+
performed["delivery"] = "ax"
|
|
2295
|
+
let value = stringAttribute(targetElement, attribute: kAXValueAttribute as CFString) ?? ""
|
|
2296
|
+
if value != text && policy != "foreground" {
|
|
2297
|
+
throw BridgeFailure(message: "The background accessibility value write was accepted but did not take effect", code: "foreground_required")
|
|
2298
|
+
}
|
|
2299
|
+
return finish(["outcome": value == text ? "worked" : "didnt", "performed": performed, "evidence": ["value": value]])
|
|
2300
|
+
}
|
|
2301
|
+
try executeCoordinates(coordinatePoint())
|
|
2302
|
+
} else if action == "typeText" {
|
|
2303
|
+
let preserveFocus = params["preserveFocus"] as? Bool ?? false
|
|
2304
|
+
if let element {
|
|
2305
|
+
let focused = AXUIElementSetAttributeValue(element, kAXFocusedAttribute as CFString, kCFBooleanTrue)
|
|
2306
|
+
if focused == .success { performed["focused"] = true }
|
|
2307
|
+
}
|
|
2308
|
+
acquirePhysicalInputIfNeeded()
|
|
2309
|
+
if delivery == "hid" && !preserveFocus { focusTargetForPhysicalInput() }
|
|
2310
|
+
let text = params["text"] as? String ?? ""
|
|
2311
|
+
try postUnicodeText(text, pid: pid, delivery: delivery)
|
|
2312
|
+
performed["grounding"] = "coordinates"
|
|
2313
|
+
if let element, !text.isEmpty {
|
|
2314
|
+
usleep(30_000)
|
|
2315
|
+
let afterValue = stringAttribute(element, attribute: kAXValueAttribute as CFString) ?? ""
|
|
2316
|
+
let changed = afterValue != (beforeValue ?? "")
|
|
2317
|
+
return finish(["outcome": changed ? "worked" : "didnt", "performed": performed, "evidence": ["value": afterValue, "valueChanged": changed]])
|
|
2318
|
+
}
|
|
2319
|
+
} else if action == "keypress" {
|
|
2320
|
+
let preserveFocus = params["preserveFocus"] as? Bool ?? false
|
|
2321
|
+
guard let keys = params["keys"] as? [String], !keys.isEmpty else {
|
|
2322
|
+
throw BridgeFailure(message: "keypress requires keys", code: "invalid_args")
|
|
2323
|
+
}
|
|
2324
|
+
if let element {
|
|
2325
|
+
let focused = AXUIElementSetAttributeValue(element, kAXFocusedAttribute as CFString, kCFBooleanTrue)
|
|
2326
|
+
if focused == .success { performed["focused"] = true }
|
|
2327
|
+
let normalizedKeys = keys.map { $0.trimmingCharacters(in: .whitespacesAndNewlines).lowercased() }
|
|
2328
|
+
if normalizedKeys.count == 2,
|
|
2329
|
+
normalizedKeys.last == "a",
|
|
2330
|
+
["cmd", "command", "meta"].contains(normalizedKeys.first ?? ""),
|
|
2331
|
+
let value = stringAttribute(element, attribute: kAXValueAttribute as CFString)
|
|
2332
|
+
{
|
|
2333
|
+
var range = CFRange(location: 0, length: (value as NSString).length)
|
|
2334
|
+
if let selection = AXValueCreate(.cfRange, &range),
|
|
2335
|
+
AXUIElementSetAttributeValue(element, kAXSelectedTextRangeAttribute as CFString, selection) == .success
|
|
2336
|
+
{
|
|
2337
|
+
performed["selectionGrounding"] = "ax"
|
|
2338
|
+
}
|
|
2339
|
+
}
|
|
2340
|
+
}
|
|
2341
|
+
acquirePhysicalInputIfNeeded()
|
|
2342
|
+
if delivery == "hid" && !preserveFocus { focusTargetForPhysicalInput() }
|
|
2343
|
+
try postKeyPress(keys: keys, pid: pid, delivery: delivery)
|
|
2344
|
+
performed["grounding"] = "coordinates"
|
|
2345
|
+
} else if let element, action == "scroll" {
|
|
2346
|
+
let cursorPoint = try? coordinatePoint()
|
|
2347
|
+
let before = scrollPositionSignature(element)
|
|
2348
|
+
let result = performScrollActionOrAncestor(startingAt: element, targetPid: pid, scrollX: (params["scrollX"] as? NSNumber)?.intValue ?? 0, scrollY: (params["scrollY"] as? NSNumber)?.intValue ?? 0, steps: 1)
|
|
2349
|
+
if (result["scrolled"] as? Bool) == true {
|
|
2350
|
+
performed["grounding"] = "description"
|
|
2351
|
+
performed["delivery"] = "ax"
|
|
2352
|
+
if let cursorPoint { animateCursor(at: cursorPoint) }
|
|
2353
|
+
let after = scrollPositionSignature(element)
|
|
2354
|
+
return finish(["outcome": before != after ? "worked" : "unknown", "performed": performed])
|
|
2355
|
+
}
|
|
2356
|
+
try executeCoordinates(coordinatePoint())
|
|
2357
|
+
} else {
|
|
2358
|
+
try executeCoordinates(coordinatePoint())
|
|
2359
|
+
}
|
|
2360
|
+
|
|
2361
|
+
let afterSheetCount = windowElement(pid: pid, windowId: record.windowId).map { sheetElements(of: $0).count } ?? beforeSheetCount
|
|
2362
|
+
let windowChanged = beforeFocusedWindow != focusedWindowSummary(pid: pid) || beforeSheetCount != afterSheetCount
|
|
2363
|
+
var outcome = preflightCapsUnknown ? "unknown" : "unknown"
|
|
2364
|
+
var evidence: [String: Any] = [:]
|
|
2365
|
+
if let element, action == "press" || action == "click" {
|
|
2366
|
+
let afterValue = stringAttribute(element, attribute: kAXValueAttribute as CFString)
|
|
2367
|
+
let afterSelected = stringAttribute(element, attribute: kAXSelectedTextAttribute as CFString)
|
|
2368
|
+
if beforeValue != afterValue || beforeSelected != afterSelected || windowChanged {
|
|
2369
|
+
outcome = "worked"
|
|
2370
|
+
}
|
|
2371
|
+
} else if windowChanged {
|
|
2372
|
+
outcome = "worked"
|
|
2373
|
+
}
|
|
2374
|
+
if windowChanged { evidence["windowChanged"] = true }
|
|
2375
|
+
var response: [String: Any] = ["outcome": outcome, "performed": performed]
|
|
2376
|
+
if !evidence.isEmpty { response["evidence"] = evidence }
|
|
2377
|
+
return finish(response)
|
|
2378
|
+
}
|
|
2379
|
+
|
|
2380
|
+
private func actBatch(_ request: [String: Any]) throws -> [String: Any] {
|
|
2381
|
+
guard let actions = request["actions"] as? [[String: Any]], !actions.isEmpty, actions.count <= 20 else {
|
|
2382
|
+
throw BridgeFailure(message: "actBatch requires 1...20 actions", code: "invalid_args")
|
|
2383
|
+
}
|
|
2384
|
+
let pid = Int32(try intArg(actions[0], "pid"))
|
|
2385
|
+
let lookId = try stringArg(actions[0], "lookId")
|
|
2386
|
+
guard actions.allSatisfy({ ($0["pid"] as? NSNumber)?.int32Value == pid }) else {
|
|
2387
|
+
throw BridgeFailure(message: "actBatch actions must target one pid", code: "invalid_args")
|
|
2388
|
+
}
|
|
2389
|
+
guard actions.allSatisfy({ ($0["lookId"] as? String) == lookId }) else {
|
|
2390
|
+
throw BridgeFailure(message: "actBatch actions must belong to one look", code: "invalid_args")
|
|
2391
|
+
}
|
|
2392
|
+
let eventsLive = ensureRootObserver(pid: pid)
|
|
2393
|
+
let eventCursor = eventsLive ? rootEventCursor(pid: pid) : 0
|
|
2394
|
+
let beforeRootSnapshot = rootMetadataSnapshot(pid: pid)
|
|
2395
|
+
let beforeCgSignature = cgRootSignature(pid: pid)
|
|
2396
|
+
let beforeFrontmostPid = NSWorkspace.shared.frontmostApplication?.processIdentifier
|
|
2397
|
+
let mayUsePhysicalInput = actions.contains { action in
|
|
2398
|
+
(optionalStringArg(action, "policy") ?? "default") != "ax_only"
|
|
2399
|
+
}
|
|
2400
|
+
if mayUsePhysicalInput { physicalInputLock.lock(); ForegroundActivity.shared.enter() }
|
|
2401
|
+
defer { if mayUsePhysicalInput { ForegroundActivity.shared.leave(); physicalInputLock.unlock() } }
|
|
2402
|
+
|
|
2403
|
+
var steps: [[String: Any]] = []
|
|
2404
|
+
var stoppedAt: Int?
|
|
2405
|
+
for (index, action) in actions.enumerated() {
|
|
2406
|
+
var deferred = action
|
|
2407
|
+
deferred["deferRootDelta"] = true
|
|
2408
|
+
do {
|
|
2409
|
+
let step = try act(deferred)
|
|
2410
|
+
steps.append(step)
|
|
2411
|
+
if (step["outcome"] as? String) == "didnt" { stoppedAt = index; break }
|
|
2412
|
+
} catch let failure as BridgeFailure {
|
|
2413
|
+
steps.append(["outcome": "didnt", "error": ["code": failure.code, "message": failure.message]])
|
|
2414
|
+
stoppedAt = index
|
|
2415
|
+
break
|
|
2416
|
+
}
|
|
2417
|
+
}
|
|
2418
|
+
let outcomes = steps.compactMap { $0["outcome"] as? String }
|
|
2419
|
+
let outcome = outcomes.contains("didnt") ? "didnt" : (outcomes.contains("unknown") ? "unknown" : "worked")
|
|
2420
|
+
var response: [String: Any] = ["outcome": outcome, "performed": ["transaction": true, "actionCount": steps.count], "steps": steps]
|
|
2421
|
+
if let stoppedAt { response["stoppedAt"] = stoppedAt }
|
|
2422
|
+
return attachRootDelta(to: response, before: beforeRootSnapshot, beforeFrontmostPid: beforeFrontmostPid, pid: pid, eventsLive: eventsLive, eventCursor: eventCursor, beforeCgSignature: beforeCgSignature)
|
|
2423
|
+
}
|
|
2424
|
+
|
|
2425
|
+
private func rootIdentity(_ root: [String: Any]) -> String {
|
|
2426
|
+
if let windowId = root["windowId"] as? Int, windowId > 0 { return "window:\(windowId)" }
|
|
2427
|
+
// AXUIElement CFEqual/CFHash are not stable after re-enumeration for all
|
|
2428
|
+
// transient roots; use the metadata tuple that comes from the cheap pass.
|
|
2429
|
+
let kind = root["kind"] as? String ?? "window"
|
|
2430
|
+
let title = root["title"] as? String ?? ""
|
|
2431
|
+
let role = root["role"] as? String ?? ""
|
|
2432
|
+
let frame = root["framePoints"] as? [String: Any] ?? [:]
|
|
2433
|
+
let x = Int((frame["x"] as? NSNumber)?.doubleValue ?? 0)
|
|
2434
|
+
let y = Int((frame["y"] as? NSNumber)?.doubleValue ?? 0)
|
|
2435
|
+
let w = Int((frame["w"] as? NSNumber)?.doubleValue ?? 0)
|
|
2436
|
+
let h = Int((frame["h"] as? NSNumber)?.doubleValue ?? 0)
|
|
2437
|
+
return "meta:\(kind):\(role):\(title):\(x),\(y),\(w),\(h)"
|
|
2438
|
+
}
|
|
2439
|
+
|
|
2440
|
+
private func rootMetadataSnapshot(pid: Int32) -> [String: [String: Any]] {
|
|
2441
|
+
let roots = ((try? listRoots(pid: pid)["roots"] as? [[String: Any]]) ?? [])
|
|
2442
|
+
return Dictionary(uniqueKeysWithValues: roots.map { (rootIdentity($0), $0) })
|
|
2443
|
+
}
|
|
2444
|
+
|
|
2445
|
+
private func rootDelta(before: [String: [String: Any]], beforeFrontmostPid: pid_t?, pid: Int32) -> [[String: Any]] {
|
|
2446
|
+
let after = rootMetadataSnapshot(pid: pid)
|
|
2447
|
+
var delta: [[String: Any]] = []
|
|
2448
|
+
for (key, root) in after where before[key] == nil {
|
|
2449
|
+
delta.append(rootDeltaItem(change: "appeared", root: root, pid: pid))
|
|
2450
|
+
}
|
|
2451
|
+
for (key, root) in before where after[key] == nil {
|
|
2452
|
+
delta.append(rootDeltaItem(change: "closed", root: root, pid: pid))
|
|
2453
|
+
}
|
|
2454
|
+
for (key, root) in after {
|
|
2455
|
+
if (root["isFocused"] as? Bool) == true && (before[key]?["isFocused"] as? Bool) != true {
|
|
2456
|
+
delta.append(rootDeltaItem(change: "focused", root: root, pid: pid))
|
|
2457
|
+
}
|
|
2458
|
+
}
|
|
2459
|
+
if let beforeFrontmostPid, beforeFrontmostPid != NSWorkspace.shared.frontmostApplication?.processIdentifier {
|
|
2460
|
+
if let frontmost = NSWorkspace.shared.frontmostApplication {
|
|
2461
|
+
delta.append(["change": "focused", "kind": "app", "title": frontmost.localizedName ?? processName(pid: frontmost.processIdentifier) ?? "Unknown App", "pid": Int(frontmost.processIdentifier)])
|
|
2462
|
+
}
|
|
2463
|
+
}
|
|
2464
|
+
return delta
|
|
2465
|
+
}
|
|
2466
|
+
|
|
2467
|
+
private func rootDeltaItem(change: String, root: [String: Any], pid: Int32) -> [String: Any] {
|
|
2468
|
+
var item: [String: Any] = ["change": change, "kind": root["kind"] as? String ?? "window", "title": root["title"] as? String ?? "", "pid": root["pid"] as? Int ?? Int(pid)]
|
|
2469
|
+
if let isModal = root["isModal"] as? Bool { item["isModal"] = isModal }
|
|
2470
|
+
if let metadata = root["metadata"] as? [String: Any] { item["metadata"] = metadata }
|
|
2471
|
+
if let ref = root["rootRef"] as? String ?? root["windowRef"] as? String { item["ref"] = ref }
|
|
2472
|
+
return item
|
|
2473
|
+
}
|
|
2474
|
+
|
|
2475
|
+
// The AX snapshot diff is authoritative: macOS emits no AXObserver
|
|
2476
|
+
// notification at all when a sheet appears (verified on macOS 26), so
|
|
2477
|
+
// events can only accelerate the decision, never make it. A cheap
|
|
2478
|
+
// CGWindowList id-set poll detects real-window appearance/closure early;
|
|
2479
|
+
// the AX diff runs once at the first signal or at timeout.
|
|
2480
|
+
private func attachRootDelta(to response: [String: Any], before: [String: [String: Any]], beforeFrontmostPid: pid_t?, pid: Int32, eventsLive: Bool, eventCursor: UInt64, beforeCgSignature: Set<UInt32>) -> [String: Any] {
|
|
2481
|
+
var output = response
|
|
2482
|
+
var performed = output["performed"] as? [String: Any] ?? [:]
|
|
2483
|
+
|
|
2484
|
+
// AXUIElementDestroyed is deliberately not a signal: it fires for every
|
|
2485
|
+
// rebuilt list row; a genuinely closed root also leaves the CG set.
|
|
2486
|
+
let signalNotifications: Set<String> = ["AXWindowCreated", "AXSheetCreated", "AXMenuOpened", "AXMenuClosed", "AXFocusedWindowChanged"]
|
|
2487
|
+
var source = "snapshot"
|
|
2488
|
+
let deadline = Date().addingTimeInterval(0.40)
|
|
2489
|
+
while Date() < deadline {
|
|
2490
|
+
if cgRootSignature(pid: pid) != beforeCgSignature { source = "cg-poll"; break }
|
|
2491
|
+
if let beforeFrontmostPid, NSWorkspace.shared.frontmostApplication?.processIdentifier != beforeFrontmostPid { source = "cg-poll"; break }
|
|
2492
|
+
if eventsLive && rootEvents(pid: pid, since: eventCursor).contains(where: { signalNotifications.contains($0.notification) }) { source = "events"; break }
|
|
2493
|
+
usleep(30_000)
|
|
2494
|
+
}
|
|
2495
|
+
|
|
2496
|
+
var delta = rootDelta(before: before, beforeFrontmostPid: beforeFrontmostPid, pid: pid)
|
|
2497
|
+
if delta.isEmpty && source != "snapshot" {
|
|
2498
|
+
// A signal fired but the AX tree can lag the CG window; give it a
|
|
2499
|
+
// bounded moment to catch up.
|
|
2500
|
+
for _ in 0..<3 where delta.isEmpty {
|
|
2501
|
+
usleep(80_000)
|
|
2502
|
+
delta = rootDelta(before: before, beforeFrontmostPid: beforeFrontmostPid, pid: pid)
|
|
2503
|
+
}
|
|
2504
|
+
}
|
|
2505
|
+
performed["deltaSource"] = source
|
|
2506
|
+
output["performed"] = performed
|
|
2507
|
+
if !delta.isEmpty { output["rootDelta"] = delta }
|
|
2508
|
+
return output
|
|
2509
|
+
}
|
|
2510
|
+
|
|
2511
|
+
private func focusedWindowSummary(pid: Int32) -> String {
|
|
2512
|
+
let app = AXUIElementCreateApplication(pid)
|
|
2513
|
+
guard let element = copyAttribute(app, attribute: kAXFocusedWindowAttribute as CFString).flatMap(asAXElement) else { return "none" }
|
|
2514
|
+
let role = stringAttribute(element, attribute: kAXRoleAttribute as CFString) ?? ""
|
|
2515
|
+
let title = stringAttribute(element, attribute: kAXTitleAttribute as CFString) ?? ""
|
|
2516
|
+
return "\(role):\(title)"
|
|
2517
|
+
}
|
|
2518
|
+
|
|
2519
|
+
private func scrollPositionSignature(_ element: AXUIElement) -> String {
|
|
2520
|
+
let names: [CFString] = ["AXVerticalScrollBar" as CFString, "AXHorizontalScrollBar" as CFString, kAXValueAttribute as CFString]
|
|
2521
|
+
return names.map { String(describing: copyAttribute(element, attribute: $0) ?? "" as CFTypeRef) }.joined(separator: "|")
|
|
2522
|
+
}
|
|
2523
|
+
|
|
2524
|
+
private func axWaitFor(_ request: [String: Any]) throws -> [String: Any] {
|
|
2525
|
+
let pid = Int32(try intArg(request, "pid"))
|
|
2526
|
+
ensureEnhancedAccessibility(pid: pid)
|
|
2527
|
+
let windowId = optionalIntArg(request, "windowId").map { UInt32($0) }
|
|
2528
|
+
let windowRef = optionalStringArg(request, "windowRef")
|
|
2529
|
+
let role = optionalStringArg(request, "role")?.trimmingCharacters(in: .whitespacesAndNewlines)
|
|
2530
|
+
let text = optionalStringArg(request, "text")?.trimmingCharacters(in: .whitespacesAndNewlines).lowercased()
|
|
2531
|
+
let expectedValue = optionalStringArg(request, "value")?.trimmingCharacters(in: .whitespacesAndNewlines).lowercased()
|
|
2532
|
+
let waitForGone = boolArg(request, "gone") ?? false
|
|
2533
|
+
let scopeExact = boolArg(request, "scopeExact") ?? false
|
|
2534
|
+
let timeoutMs = max(100, min(60_000, optionalIntArg(request, "timeoutMs") ?? 10_000))
|
|
2535
|
+
let deadline = Date().addingTimeInterval(Double(timeoutMs) / 1000.0)
|
|
2536
|
+
guard role?.isEmpty == false || text?.isEmpty == false || expectedValue?.isEmpty == false else {
|
|
2537
|
+
throw BridgeFailure(message: "axWaitFor requires role, text, or value", code: "invalid_args")
|
|
2538
|
+
}
|
|
2539
|
+
guard let window = windowElement(pid: pid, windowId: windowId, windowRef: windowRef) else {
|
|
2540
|
+
return ["found": false, "reason": "window_not_found"]
|
|
2541
|
+
}
|
|
2542
|
+
let rootElement: AXUIElement
|
|
2543
|
+
if let scopeRef = optionalStringArg(request, "scopeRef") {
|
|
2544
|
+
guard let scoped = refStore.element(for: scopeRef), isElement(scoped, descendantOf: window) else {
|
|
2545
|
+
throw BridgeFailure(message: "Condition scope ref is stale or outside the target root", code: "element_ref_invalid")
|
|
2546
|
+
}
|
|
2547
|
+
rootElement = scoped
|
|
2548
|
+
} else {
|
|
2549
|
+
rootElement = window
|
|
2550
|
+
}
|
|
2551
|
+
_ = ensureRootObserver(pid: pid)
|
|
2552
|
+
|
|
2553
|
+
func matches(_ element: AXUIElement) -> Bool {
|
|
2554
|
+
let candidateRole = self.stringAttribute(element, attribute: kAXRoleAttribute as CFString) ?? ""
|
|
2555
|
+
if let role, !role.isEmpty, candidateRole != role { return false }
|
|
2556
|
+
if let text, !text.isEmpty {
|
|
2557
|
+
let subrole = self.stringAttribute(element, attribute: kAXSubroleAttribute as CFString) ?? ""
|
|
2558
|
+
let haystack = [
|
|
2559
|
+
self.stringAttribute(element, attribute: kAXTitleAttribute as CFString) ?? "",
|
|
2560
|
+
self.stringAttribute(element, attribute: kAXDescriptionAttribute as CFString) ?? "",
|
|
2561
|
+
self.displayValue(element, role: candidateRole, subrole: subrole),
|
|
2562
|
+
].joined(separator: "\n").lowercased()
|
|
2563
|
+
if !haystack.contains(text) { return false }
|
|
2564
|
+
}
|
|
2565
|
+
if let expectedValue, !expectedValue.isEmpty {
|
|
2566
|
+
let subrole = self.stringAttribute(element, attribute: kAXSubroleAttribute as CFString) ?? ""
|
|
2567
|
+
if self.displayValue(element, role: candidateRole, subrole: subrole).trimmingCharacters(in: .whitespacesAndNewlines).lowercased() != expectedValue { return false }
|
|
2568
|
+
}
|
|
2569
|
+
return true
|
|
2570
|
+
}
|
|
2571
|
+
|
|
2572
|
+
var lastCount = 0
|
|
2573
|
+
repeat {
|
|
2574
|
+
let changeGeneration = rootChangeGeneration(pid: pid)
|
|
2575
|
+
let collected = collectDescendantsWithContext(startingAt: rootElement, maxDepth: 12, maxNodes: 2000)
|
|
2576
|
+
let descendants = scopeExact ? Array(collected.prefix(1)) : collected
|
|
2577
|
+
lastCount = descendants.count
|
|
2578
|
+
if let match = descendants.first(where: { matches($0.element) }) {
|
|
2579
|
+
if waitForGone {
|
|
2580
|
+
waitForRootChange(pid: pid, since: changeGeneration, until: deadline)
|
|
2581
|
+
continue
|
|
2582
|
+
}
|
|
2583
|
+
let candidateRole = self.stringAttribute(match.element, attribute: kAXRoleAttribute as CFString) ?? ""
|
|
2584
|
+
let isBrowser = isBrowser(pid: pid)
|
|
2585
|
+
let containsWebArea = descendants.contains { self.stringAttribute($0.element, attribute: kAXRoleAttribute as CFString) == "AXWebArea" }
|
|
2586
|
+
return [
|
|
2587
|
+
"found": true,
|
|
2588
|
+
"target": self.elementPayload(
|
|
2589
|
+
element: match.element,
|
|
2590
|
+
key: "target",
|
|
2591
|
+
source: self.axSource(role: candidateRole, insideWebArea: match.insideWebArea, isBrowser: isBrowser, containsWebArea: containsWebArea)
|
|
2592
|
+
),
|
|
2593
|
+
"nodeCount": lastCount,
|
|
2594
|
+
]
|
|
2595
|
+
}
|
|
2596
|
+
if waitForGone {
|
|
2597
|
+
return ["found": true, "gone": true, "nodeCount": lastCount]
|
|
2598
|
+
}
|
|
2599
|
+
waitForRootChange(pid: pid, since: changeGeneration, until: deadline)
|
|
2600
|
+
} while Date() < deadline
|
|
2601
|
+
|
|
2602
|
+
return ["found": false, "timedOut": true, "nodeCount": lastCount]
|
|
2603
|
+
}
|
|
2604
|
+
|
|
2605
|
+
private func hitTestElement(at point: CGPoint) -> AXUIElement? {
|
|
2606
|
+
let systemWide = AXUIElementCreateSystemWide()
|
|
2607
|
+
var hitElement: AXUIElement?
|
|
2608
|
+
let status = AXUIElementCopyElementAtPosition(systemWide, Float(point.x), Float(point.y), &hitElement)
|
|
2609
|
+
guard status == .success, let hitElement else { return nil }
|
|
2610
|
+
return hitElement
|
|
2611
|
+
}
|
|
2612
|
+
|
|
2613
|
+
private let axScrollDownAction = "AXScrollDown" as CFString
|
|
2614
|
+
private let axScrollUpAction = "AXScrollUp" as CFString
|
|
2615
|
+
private let axScrollLeftAction = "AXScrollLeft" as CFString
|
|
2616
|
+
private let axScrollRightAction = "AXScrollRight" as CFString
|
|
2617
|
+
|
|
2618
|
+
private func scrollActionNames(scrollX: Int, scrollY: Int) -> [CFString] {
|
|
2619
|
+
var actions: [CFString] = []
|
|
2620
|
+
if scrollY > 0 { actions.append(axScrollDownAction) }
|
|
2621
|
+
if scrollY < 0 { actions.append(axScrollUpAction) }
|
|
2622
|
+
if scrollX > 0 { actions.append(axScrollRightAction) }
|
|
2623
|
+
if scrollX < 0 { actions.append(axScrollLeftAction) }
|
|
2624
|
+
return actions
|
|
2625
|
+
}
|
|
2626
|
+
|
|
2627
|
+
private func supportsAnyScrollAction(_ element: AXUIElement) -> Bool {
|
|
2628
|
+
let actions = Set(actionNames(element))
|
|
2629
|
+
return actions.contains(axScrollDownAction as String) || actions.contains(axScrollUpAction as String) || actions.contains(axScrollLeftAction as String) || actions.contains(axScrollRightAction as String)
|
|
2630
|
+
}
|
|
2631
|
+
|
|
2632
|
+
private func performScrollActionOrAncestor(startingAt element: AXUIElement, targetPid: Int32, scrollX: Int, scrollY: Int, steps: Int) -> [String: Any] {
|
|
2633
|
+
let actions = scrollActionNames(scrollX: scrollX, scrollY: scrollY)
|
|
2634
|
+
guard !actions.isEmpty else { return ["scrolled": false, "reason": "zero_delta"] }
|
|
2635
|
+
var current: AXUIElement? = element
|
|
2636
|
+
var depth = 0
|
|
2637
|
+
|
|
2638
|
+
while let candidate = current, depth < 10 {
|
|
2639
|
+
if let pid = pidForElement(candidate), pid != targetPid {
|
|
2640
|
+
return ["scrolled": false, "reason": "pid_mismatch", "ownerPid": Int(pid)]
|
|
2641
|
+
}
|
|
2642
|
+
var didScroll = false
|
|
2643
|
+
for _ in 0..<steps {
|
|
2644
|
+
for action in actions where supportsAction(candidate, action: action) {
|
|
2645
|
+
let status = AXUIElementPerformAction(candidate, action)
|
|
2646
|
+
if status == .success { didScroll = true }
|
|
2647
|
+
}
|
|
2648
|
+
}
|
|
2649
|
+
if didScroll { return ["scrolled": true] }
|
|
2650
|
+
current = parentElement(candidate)
|
|
2651
|
+
depth += 1
|
|
2652
|
+
}
|
|
2653
|
+
|
|
2654
|
+
return ["scrolled": false, "reason": "no_scroll_action"]
|
|
2655
|
+
}
|
|
2656
|
+
|
|
2657
|
+
private func performActionOrAncestor(startingAt element: AXUIElement, action: CFString, targetPid: Int32) -> [String: Any] {
|
|
2658
|
+
var current: AXUIElement? = element
|
|
2659
|
+
var depth = 0
|
|
2660
|
+
|
|
2661
|
+
while let candidate = current, depth < 10 {
|
|
2662
|
+
if let pid = pidForElement(candidate), pid != targetPid {
|
|
2663
|
+
return ["performed": false, "reason": "pid_mismatch", "ownerPid": Int(pid)]
|
|
2664
|
+
}
|
|
2665
|
+
|
|
2666
|
+
if supportsAction(candidate, action: action) {
|
|
2667
|
+
let actionStatus = AXUIElementPerformAction(candidate, action)
|
|
2668
|
+
if actionStatus == .success {
|
|
2669
|
+
return ["performed": true]
|
|
2670
|
+
}
|
|
2671
|
+
}
|
|
2672
|
+
|
|
2673
|
+
current = parentElement(candidate)
|
|
2674
|
+
depth += 1
|
|
2675
|
+
}
|
|
2676
|
+
|
|
2677
|
+
return ["performed": false, "reason": "no_matching_action"]
|
|
2678
|
+
}
|
|
2679
|
+
|
|
2680
|
+
private func windowElement(pid: Int32, windowId: UInt32?, windowRef: String? = nil) -> AXUIElement? {
|
|
2681
|
+
if let windowRef, let stored = refStore.window(for: windowRef) {
|
|
2682
|
+
AXUIElementSetMessagingTimeout(stored, 1.0)
|
|
2683
|
+
var ownerPid: pid_t = 0
|
|
2684
|
+
if AXUIElementGetPid(stored, &ownerPid) == .success, ownerPid == pid {
|
|
2685
|
+
return stored
|
|
2686
|
+
}
|
|
2687
|
+
}
|
|
2688
|
+
|
|
2689
|
+
let appElement = AXUIElementCreateApplication(pid)
|
|
2690
|
+
AXUIElementSetMessagingTimeout(appElement, 1.0)
|
|
2691
|
+
let windows = Array(axElementArray(appElement, attribute: kAXWindowsAttribute as CFString).prefix(128))
|
|
2692
|
+
guard !windows.isEmpty else { return nil }
|
|
2693
|
+
guard let windowId else {
|
|
2694
|
+
return windows.first
|
|
2695
|
+
}
|
|
2696
|
+
let candidates = cgWindowCandidates(pid: pid)
|
|
2697
|
+
let pairings = windowPairings(windows: windows, candidates: candidates)
|
|
2698
|
+
for window in windows {
|
|
2699
|
+
if pairings[ObjectIdentifier(window)]?.candidate?.windowId == windowId {
|
|
2700
|
+
return window
|
|
2701
|
+
}
|
|
2702
|
+
for sheet in sheetElements(of: window) {
|
|
2703
|
+
if bestCandidate(for: sheet, candidates: candidates)?.windowId == windowId {
|
|
2704
|
+
return sheet
|
|
2705
|
+
}
|
|
2706
|
+
}
|
|
2707
|
+
}
|
|
2708
|
+
return nil
|
|
2709
|
+
}
|
|
2710
|
+
|
|
2711
|
+
private func findDescendant(startingAt root: AXUIElement, maxDepth: Int, predicate: (AXUIElement) -> Bool) -> AXUIElement? {
|
|
2712
|
+
collectDescendants(startingAt: root, maxDepth: maxDepth).first(where: predicate)
|
|
2713
|
+
}
|
|
2714
|
+
|
|
2715
|
+
private func ensureEnhancedAccessibility(pid: Int32) {
|
|
2716
|
+
enhancedAccessibilityLock.lock()
|
|
2717
|
+
let inserted = enhancedAccessibilityPids.insert(pid).inserted
|
|
2718
|
+
enhancedAccessibilityLock.unlock()
|
|
2719
|
+
if !inserted { return }
|
|
2720
|
+
let appElement = AXUIElementCreateApplication(pid)
|
|
2721
|
+
AXUIElementSetMessagingTimeout(appElement, 0.25)
|
|
2722
|
+
let enhancedStatus = AXUIElementSetAttributeValue(appElement, "AXEnhancedUserInterface" as CFString, kCFBooleanTrue)
|
|
2723
|
+
let manualStatus = AXUIElementSetAttributeValue(appElement, "AXManualAccessibility" as CFString, kCFBooleanTrue)
|
|
2724
|
+
// Chromium-family apps often materialize web-content AX asynchronously
|
|
2725
|
+
// after these toggles. Pay a small one-time settle cost per pid so the
|
|
2726
|
+
// first tree walk is less likely to see browser chrome only.
|
|
2727
|
+
if isBrowser(pid: pid) && (enhancedStatus == .success || manualStatus == .success) {
|
|
2728
|
+
Thread.sleep(forTimeInterval: 0.35)
|
|
2729
|
+
}
|
|
2730
|
+
}
|
|
2731
|
+
|
|
2732
|
+
private func isBrowser(pid: Int32) -> Bool {
|
|
2733
|
+
let app = NSRunningApplication(processIdentifier: pid)
|
|
2734
|
+
if browserBundleIds.contains(app?.bundleIdentifier ?? "") { return true }
|
|
2735
|
+
let name = (app?.localizedName ?? processName(pid: pid) ?? "").lowercased()
|
|
2736
|
+
return ["chrome", "chromium", "brave", "edge", "vivaldi", "opera", "firefox", "helium"].contains { name.contains($0) }
|
|
2737
|
+
}
|
|
2738
|
+
|
|
2739
|
+
private func collectDescendants(startingAt root: AXUIElement, maxDepth: Int, maxNodes: Int = 5000) -> [AXUIElement] {
|
|
2740
|
+
collectDescendantsWithContext(startingAt: root, maxDepth: maxDepth, maxNodes: maxNodes).map(\.element)
|
|
2741
|
+
}
|
|
2742
|
+
|
|
2743
|
+
private func collectDescendantsWithContext(startingAt root: AXUIElement, maxDepth: Int, maxNodes: Int = 5000) -> [AXDescendant] {
|
|
2744
|
+
let nodeLimit = max(1, maxNodes)
|
|
2745
|
+
var queue: [(AXUIElement, Int, Bool, Bool)] = [(root, 0, false, true)]
|
|
2746
|
+
var seen = Set<ObjectIdentifier>()
|
|
2747
|
+
var index = 0
|
|
2748
|
+
var output: [AXDescendant] = []
|
|
2749
|
+
while index < queue.count && output.count < nodeLimit {
|
|
2750
|
+
let (element, depth, parentInsideWebArea, inheritedVisible) = queue[index]
|
|
2751
|
+
index += 1
|
|
2752
|
+
let identity = ObjectIdentifier(element)
|
|
2753
|
+
if seen.contains(identity) { continue }
|
|
2754
|
+
seen.insert(identity)
|
|
2755
|
+
let role = stringAttribute(element, attribute: kAXRoleAttribute as CFString) ?? ""
|
|
2756
|
+
let insideWebArea = parentInsideWebArea || role == "AXWebArea"
|
|
2757
|
+
output.append(AXDescendant(element: element, depth: depth, insideWebArea: insideWebArea, axVisible: inheritedVisible))
|
|
2758
|
+
if depth >= maxDepth { continue }
|
|
2759
|
+
let children = axElementArray(element, attribute: kAXChildrenAttribute as CFString)
|
|
2760
|
+
let visibleChildren = visibleAXChildren(element)
|
|
2761
|
+
for child in children {
|
|
2762
|
+
if queue.count >= nodeLimit { break }
|
|
2763
|
+
let childVisible = inheritedVisible && (visibleChildren.map { set in set.contains { self.sameElement($0, child) } } ?? true)
|
|
2764
|
+
queue.append((child, depth + 1, insideWebArea, childVisible))
|
|
2765
|
+
}
|
|
2766
|
+
}
|
|
2767
|
+
return output
|
|
2768
|
+
}
|
|
2769
|
+
|
|
2770
|
+
private func visibleAXChildren(_ element: AXUIElement) -> [AXUIElement]? {
|
|
2771
|
+
let attributes: [CFString] = [
|
|
2772
|
+
kAXVisibleChildrenAttribute as CFString,
|
|
2773
|
+
kAXVisibleRowsAttribute as CFString,
|
|
2774
|
+
kAXVisibleColumnsAttribute as CFString,
|
|
2775
|
+
kAXVisibleCellsAttribute as CFString,
|
|
2776
|
+
]
|
|
2777
|
+
let visible = attributes.flatMap { axElementArray(element, attribute: $0) }
|
|
2778
|
+
return visible.isEmpty ? nil : visible
|
|
2779
|
+
}
|
|
2780
|
+
|
|
2781
|
+
private func insideWebAreaMap(_ descendants: [AXDescendant]) -> [ObjectIdentifier: Bool] {
|
|
2782
|
+
var output: [ObjectIdentifier: Bool] = [:]
|
|
2783
|
+
for descendant in descendants {
|
|
2784
|
+
let key = ObjectIdentifier(descendant.element)
|
|
2785
|
+
output[key] = (output[key] ?? false) || descendant.insideWebArea
|
|
2786
|
+
}
|
|
2787
|
+
return output
|
|
2788
|
+
}
|
|
2789
|
+
|
|
2790
|
+
private func axSource(role: String, insideWebArea: Bool, isBrowser: Bool, containsWebArea: Bool) -> String {
|
|
2791
|
+
if insideWebArea || role == "AXWebArea" { return "web_content_ax" }
|
|
2792
|
+
if isBrowser || containsWebArea { return "browser_chrome_ax" }
|
|
2793
|
+
return "desktop_ax"
|
|
2794
|
+
}
|
|
2795
|
+
|
|
2796
|
+
private func frameForElement(_ element: AXUIElement) -> CGRect? {
|
|
2797
|
+
let origin = pointAttribute(element, attribute: kAXPositionAttribute as CFString)
|
|
2798
|
+
let size = sizeAttribute(element, attribute: kAXSizeAttribute as CFString)
|
|
2799
|
+
guard let origin, let size, size.width > 0, size.height > 0 else { return nil }
|
|
2800
|
+
return CGRect(origin: origin, size: size)
|
|
2801
|
+
}
|
|
2802
|
+
|
|
2803
|
+
private func elementPayload(element: AXUIElement, key: String, score: Double? = nil, source: String? = nil, axVisible: Bool = true) -> [String: Any] {
|
|
2804
|
+
let role = stringAttribute(element, attribute: kAXRoleAttribute as CFString) ?? ""
|
|
2805
|
+
let subrole = stringAttribute(element, attribute: kAXSubroleAttribute as CFString) ?? ""
|
|
2806
|
+
let title = stringAttribute(element, attribute: kAXTitleAttribute as CFString) ?? ""
|
|
2807
|
+
let description = stringAttribute(element, attribute: kAXDescriptionAttribute as CFString) ?? ""
|
|
2808
|
+
let identifier = stringAttribute(element, attribute: "AXIdentifier" as CFString) ?? ""
|
|
2809
|
+
let value = displayValue(element, role: role, subrole: subrole)
|
|
2810
|
+
let frame = frameForElement(element)
|
|
2811
|
+
let parentFrame = copyAttribute(element, attribute: kAXParentAttribute as CFString).flatMap(asAXElement).flatMap(frameForElement)
|
|
2812
|
+
let centerX = frame.map { $0.midX } ?? 0
|
|
2813
|
+
let centerY = frame.map { $0.midY } ?? 0
|
|
2814
|
+
var valueSettable = DarwinBoolean(false)
|
|
2815
|
+
let valueStatus = AXUIElementIsAttributeSettable(element, kAXValueAttribute as CFString, &valueSettable)
|
|
2816
|
+
var focusedSettable = DarwinBoolean(false)
|
|
2817
|
+
let focusedStatus = AXUIElementIsAttributeSettable(element, kAXFocusedAttribute as CFString, &focusedSettable)
|
|
2818
|
+
let actions = actionNames(element)
|
|
2819
|
+
let canSetValue = valueStatus == .success && valueSettable.boolValue
|
|
2820
|
+
let textRoles: Set<String> = [
|
|
2821
|
+
"AXTextField", "AXTextArea", "AXTextView", "AXSearchField", "AXComboBox", "AXEditableText", "AXSecureTextField",
|
|
2822
|
+
]
|
|
2823
|
+
var payload: [String: Any] = [
|
|
2824
|
+
key: true,
|
|
2825
|
+
"elementRef": refStore.storeElement(element),
|
|
2826
|
+
"role": role,
|
|
2827
|
+
"subrole": subrole,
|
|
2828
|
+
"title": title,
|
|
2829
|
+
"description": description,
|
|
2830
|
+
"identifier": identifier,
|
|
2831
|
+
"value": value,
|
|
2832
|
+
"actions": actions,
|
|
2833
|
+
"isTextInput": textRoles.contains(role),
|
|
2834
|
+
"canSetValue": canSetValue,
|
|
2835
|
+
"canFocus": focusedStatus == .success && focusedSettable.boolValue,
|
|
2836
|
+
"canPress": actions.contains(kAXPressAction as String),
|
|
2837
|
+
"canScroll": supportsAnyScrollAction(element),
|
|
2838
|
+
"canIncrement": actions.contains(kAXIncrementAction as String),
|
|
2839
|
+
"canDecrement": actions.contains(kAXDecrementAction as String),
|
|
2840
|
+
"axVisible": axVisible,
|
|
2841
|
+
"x": centerX,
|
|
2842
|
+
"y": centerY,
|
|
2843
|
+
]
|
|
2844
|
+
if let frame {
|
|
2845
|
+
payload["frame"] = ["x": frame.origin.x, "y": frame.origin.y, "w": frame.width, "h": frame.height]
|
|
2846
|
+
}
|
|
2847
|
+
if let parentFrame {
|
|
2848
|
+
payload["parentFrame"] = ["x": parentFrame.origin.x, "y": parentFrame.origin.y, "w": parentFrame.width, "h": parentFrame.height]
|
|
2849
|
+
}
|
|
2850
|
+
if let score {
|
|
2851
|
+
payload["score"] = score
|
|
2852
|
+
}
|
|
2853
|
+
if let source {
|
|
2854
|
+
payload["source"] = source
|
|
2855
|
+
}
|
|
2856
|
+
return payload
|
|
2857
|
+
}
|
|
2858
|
+
|
|
2859
|
+
private func pidForElement(_ element: AXUIElement) -> Int32? {
|
|
2860
|
+
var pid: pid_t = 0
|
|
2861
|
+
let status = AXUIElementGetPid(element, &pid)
|
|
2862
|
+
guard status == .success else { return nil }
|
|
2863
|
+
return Int32(pid)
|
|
2864
|
+
}
|
|
2865
|
+
|
|
2866
|
+
private func parentElement(_ element: AXUIElement) -> AXUIElement? {
|
|
2867
|
+
guard let value = copyAttribute(element, attribute: kAXParentAttribute as CFString) else {
|
|
2868
|
+
return nil
|
|
2869
|
+
}
|
|
2870
|
+
return asAXElement(value)
|
|
2871
|
+
}
|
|
2872
|
+
|
|
2873
|
+
private func sameElement(_ lhs: AXUIElement, _ rhs: AXUIElement) -> Bool {
|
|
2874
|
+
CFEqual(lhs as CFTypeRef, rhs as CFTypeRef)
|
|
2875
|
+
}
|
|
2876
|
+
|
|
2877
|
+
private func isElement(_ element: AXUIElement, descendantOf ancestor: AXUIElement) -> Bool {
|
|
2878
|
+
var current: AXUIElement? = element
|
|
2879
|
+
var depth = 0
|
|
2880
|
+
while let candidate = current, depth < 20 {
|
|
2881
|
+
if sameElement(candidate, ancestor) {
|
|
2882
|
+
return true
|
|
2883
|
+
}
|
|
2884
|
+
current = parentElement(candidate)
|
|
2885
|
+
depth += 1
|
|
2886
|
+
}
|
|
2887
|
+
return false
|
|
2888
|
+
}
|
|
2889
|
+
|
|
2890
|
+
private func hasAncestorRole(_ element: AXUIElement, role: String) -> Bool {
|
|
2891
|
+
var current: AXUIElement? = element
|
|
2892
|
+
var depth = 0
|
|
2893
|
+
while let candidate = current, depth < 30 {
|
|
2894
|
+
if stringAttribute(candidate, attribute: kAXRoleAttribute as CFString) == role { return true }
|
|
2895
|
+
current = parentElement(candidate)
|
|
2896
|
+
depth += 1
|
|
2897
|
+
}
|
|
2898
|
+
return false
|
|
2899
|
+
}
|
|
2900
|
+
|
|
2901
|
+
private func actionNames(_ element: AXUIElement) -> [String] {
|
|
2902
|
+
var actionsValue: CFArray?
|
|
2903
|
+
let status = AXUIElementCopyActionNames(element, &actionsValue)
|
|
2904
|
+
guard status == .success else { return [] }
|
|
2905
|
+
guard let actionsArray = actionsValue as? [AnyObject] else { return [] }
|
|
2906
|
+
return actionsArray.compactMap { $0 as? String }
|
|
2907
|
+
}
|
|
2908
|
+
|
|
2909
|
+
private func supportsAction(_ element: AXUIElement, action: CFString) -> Bool {
|
|
2910
|
+
actionNames(element).contains(action as String)
|
|
2911
|
+
}
|
|
2912
|
+
|
|
2913
|
+
private func focusedElement(_ request: [String: Any]) throws -> [String: Any] {
|
|
2914
|
+
let pid = Int32(try intArg(request, "pid"))
|
|
2915
|
+
let windowId = optionalIntArg(request, "windowId").map { UInt32($0) }
|
|
2916
|
+
let windowRef = optionalStringArg(request, "windowRef")
|
|
2917
|
+
let app = AXUIElementCreateApplication(pid)
|
|
2918
|
+
guard let focusedValue = copyAttribute(app, attribute: kAXFocusedUIElementAttribute as CFString),
|
|
2919
|
+
let element = asAXElement(focusedValue)
|
|
2920
|
+
else {
|
|
2921
|
+
return ["exists": false]
|
|
2922
|
+
}
|
|
2923
|
+
if windowId != nil || windowRef != nil {
|
|
2924
|
+
guard let window = windowElement(pid: pid, windowId: windowId, windowRef: windowRef) else {
|
|
2925
|
+
return ["exists": false, "reason": "window_not_found"]
|
|
2926
|
+
}
|
|
2927
|
+
guard isElement(element, descendantOf: window) else {
|
|
2928
|
+
return ["exists": false, "reason": "focused_element_outside_window"]
|
|
2929
|
+
}
|
|
2930
|
+
}
|
|
2931
|
+
|
|
2932
|
+
let role = stringAttribute(element, attribute: kAXRoleAttribute as CFString) ?? ""
|
|
2933
|
+
let subrole = stringAttribute(element, attribute: kAXSubroleAttribute as CFString) ?? ""
|
|
2934
|
+
let secure = role == "AXSecureTextField" || subrole == "AXSecureTextField"
|
|
2935
|
+
|
|
2936
|
+
var settable = DarwinBoolean(false)
|
|
2937
|
+
let settableStatus = AXUIElementIsAttributeSettable(element, kAXValueAttribute as CFString, &settable)
|
|
2938
|
+
let canSetValue = settableStatus == .success && settable.boolValue
|
|
2939
|
+
|
|
2940
|
+
let textRoles: Set<String> = [
|
|
2941
|
+
"AXTextField",
|
|
2942
|
+
"AXTextArea",
|
|
2943
|
+
"AXTextView",
|
|
2944
|
+
"AXSearchField",
|
|
2945
|
+
"AXComboBox",
|
|
2946
|
+
"AXEditableText",
|
|
2947
|
+
"AXSecureTextField",
|
|
2948
|
+
]
|
|
2949
|
+
|
|
2950
|
+
let isTextInput = textRoles.contains(role) || canSetValue
|
|
2951
|
+
let elementRef = refStore.storeElement(element)
|
|
2952
|
+
|
|
2953
|
+
return [
|
|
2954
|
+
"exists": true,
|
|
2955
|
+
"elementRef": elementRef,
|
|
2956
|
+
"role": role,
|
|
2957
|
+
"subrole": subrole,
|
|
2958
|
+
"isTextInput": isTextInput,
|
|
2959
|
+
"isSecure": secure,
|
|
2960
|
+
"canSetValue": canSetValue,
|
|
2961
|
+
]
|
|
2962
|
+
}
|
|
2963
|
+
|
|
2964
|
+
private func axReadText(_ request: [String: Any]) throws -> [String: Any] {
|
|
2965
|
+
let elementRef = try stringArg(request, "elementRef")
|
|
2966
|
+
let offset = max(0, optionalIntArg(request, "offset") ?? 0)
|
|
2967
|
+
let limit = max(1, min(100_000, optionalIntArg(request, "limit") ?? 4_000))
|
|
2968
|
+
guard let element = refStore.element(for: elementRef) else {
|
|
2969
|
+
throw BridgeFailure(message: "Element reference is no longer valid", code: "element_ref_invalid")
|
|
2970
|
+
}
|
|
2971
|
+
let role = stringAttribute(element, attribute: kAXRoleAttribute as CFString) ?? ""
|
|
2972
|
+
let subrole = stringAttribute(element, attribute: kAXSubroleAttribute as CFString) ?? ""
|
|
2973
|
+
guard !isSecureTextElement(role: role, subrole: subrole) else {
|
|
2974
|
+
throw BridgeFailure(message: "Refers to a secure text field; refusing to read its value", code: "secure_text_unreadable")
|
|
2975
|
+
}
|
|
2976
|
+
guard let value = stringAttribute(element, attribute: kAXValueAttribute as CFString) else {
|
|
2977
|
+
throw BridgeFailure(message: "Element has no readable AXValue. Call snapshot/screenshot and choose a text-bearing ref.", code: "text_unavailable")
|
|
2978
|
+
}
|
|
2979
|
+
let characters = Array(value)
|
|
2980
|
+
if offset >= characters.count {
|
|
2981
|
+
return ["text": "", "offset": offset, "limit": limit, "totalChars": characters.count, "hasMore": false]
|
|
2982
|
+
}
|
|
2983
|
+
let end = min(characters.count, offset + limit)
|
|
2984
|
+
return [
|
|
2985
|
+
"text": String(characters[offset..<end]),
|
|
2986
|
+
"offset": offset,
|
|
2987
|
+
"limit": limit,
|
|
2988
|
+
"totalChars": characters.count,
|
|
2989
|
+
"hasMore": end < characters.count,
|
|
2990
|
+
]
|
|
2991
|
+
}
|
|
2992
|
+
|
|
2993
|
+
private func getMousePosition() -> [String: Any] {
|
|
2994
|
+
let position = NSEvent.mouseLocation
|
|
2995
|
+
return ["x": position.x, "y": position.y]
|
|
2996
|
+
}
|
|
2997
|
+
|
|
2998
|
+
private func copyAttribute(_ element: AXUIElement, attribute: CFString) -> AnyObject? {
|
|
2999
|
+
var value: AnyObject?
|
|
3000
|
+
let status = AXUIElementCopyAttributeValue(element, attribute, &value)
|
|
3001
|
+
guard status == .success else { return nil }
|
|
3002
|
+
return value
|
|
3003
|
+
}
|
|
3004
|
+
|
|
3005
|
+
private func boolAttribute(_ element: AXUIElement, attribute: CFString) -> Bool? {
|
|
3006
|
+
guard let value = copyAttribute(element, attribute: attribute) else { return nil }
|
|
3007
|
+
if let boolValue = value as? Bool {
|
|
3008
|
+
return boolValue
|
|
3009
|
+
}
|
|
3010
|
+
if let number = value as? NSNumber {
|
|
3011
|
+
return number.boolValue
|
|
3012
|
+
}
|
|
3013
|
+
return nil
|
|
3014
|
+
}
|
|
3015
|
+
|
|
3016
|
+
private func stringAttribute(_ element: AXUIElement, attribute: CFString) -> String? {
|
|
3017
|
+
copyAttribute(element, attribute: attribute) as? String
|
|
3018
|
+
}
|
|
3019
|
+
|
|
3020
|
+
// Secure fields can expose plaintext through AX value in non-native apps,
|
|
3021
|
+
// and serialized values flow into the model conversation. Never emit them.
|
|
3022
|
+
private func isSecureTextElement(role: String, subrole: String) -> Bool {
|
|
3023
|
+
role == "AXSecureTextField" || subrole == "AXSecureTextField"
|
|
3024
|
+
}
|
|
3025
|
+
|
|
3026
|
+
private func displayValue(_ element: AXUIElement, role: String, subrole: String) -> String {
|
|
3027
|
+
if isSecureTextElement(role: role, subrole: subrole) { return "" }
|
|
3028
|
+
return stringAttribute(element, attribute: kAXValueAttribute as CFString) ?? ""
|
|
3029
|
+
}
|
|
3030
|
+
|
|
3031
|
+
// kAXSheetsAttribute is unsupported (-25205) on recent macOS; sheets are
|
|
3032
|
+
// exposed only as AXSheet-role children. Merge both sources so sheet
|
|
3033
|
+
// discovery works across versions.
|
|
3034
|
+
private func sheetElements(of window: AXUIElement) -> [AXUIElement] {
|
|
3035
|
+
var sheets = axElementArray(window, attribute: "AXSheets" as CFString)
|
|
3036
|
+
for child in axElementArray(window, attribute: kAXChildrenAttribute as CFString) {
|
|
3037
|
+
guard (stringAttribute(child, attribute: kAXRoleAttribute as CFString) ?? "") == "AXSheet" else { continue }
|
|
3038
|
+
if !sheets.contains(where: { CFEqual($0, child) }) { sheets.append(child) }
|
|
3039
|
+
}
|
|
3040
|
+
return sheets
|
|
3041
|
+
}
|
|
3042
|
+
|
|
3043
|
+
private func axElementArray(_ element: AXUIElement, attribute: CFString) -> [AXUIElement] {
|
|
3044
|
+
guard let value = copyAttribute(element, attribute: attribute) else { return [] }
|
|
3045
|
+
if let array = value as? [AXUIElement] {
|
|
3046
|
+
return array
|
|
3047
|
+
}
|
|
3048
|
+
if let anyArray = value as? [AnyObject] {
|
|
3049
|
+
return anyArray.compactMap(asAXElement)
|
|
3050
|
+
}
|
|
3051
|
+
return []
|
|
3052
|
+
}
|
|
3053
|
+
|
|
3054
|
+
private func axElementArrayIfPresent(_ element: AXUIElement, attribute: CFString) -> [AXUIElement]? {
|
|
3055
|
+
var value: CFTypeRef?
|
|
3056
|
+
let status = AXUIElementCopyAttributeValue(element, attribute, &value)
|
|
3057
|
+
guard status == .success, let value else { return nil }
|
|
3058
|
+
if let array = value as? [AXUIElement] {
|
|
3059
|
+
return array
|
|
3060
|
+
}
|
|
3061
|
+
if let anyArray = value as? [AnyObject] {
|
|
3062
|
+
return anyArray.compactMap(asAXElement)
|
|
3063
|
+
}
|
|
3064
|
+
return []
|
|
3065
|
+
}
|
|
3066
|
+
|
|
3067
|
+
private func asAXElement(_ value: AnyObject) -> AXUIElement? {
|
|
3068
|
+
let cfValue = value as CFTypeRef
|
|
3069
|
+
guard CFGetTypeID(cfValue) == AXUIElementGetTypeID() else { return nil }
|
|
3070
|
+
return unsafeBitCast(cfValue, to: AXUIElement.self)
|
|
3071
|
+
}
|
|
3072
|
+
|
|
3073
|
+
private func pointAttribute(_ element: AXUIElement, attribute: CFString) -> CGPoint? {
|
|
3074
|
+
guard let value = copyAttribute(element, attribute: attribute) else { return nil }
|
|
3075
|
+
let cfValue = value as CFTypeRef
|
|
3076
|
+
guard CFGetTypeID(cfValue) == AXValueGetTypeID() else { return nil }
|
|
3077
|
+
let axValue = unsafeBitCast(cfValue, to: AXValue.self)
|
|
3078
|
+
guard AXValueGetType(axValue) == .cgPoint else { return nil }
|
|
3079
|
+
var point = CGPoint.zero
|
|
3080
|
+
guard AXValueGetValue(axValue, .cgPoint, &point) else { return nil }
|
|
3081
|
+
return point
|
|
3082
|
+
}
|
|
3083
|
+
|
|
3084
|
+
private func sizeAttribute(_ element: AXUIElement, attribute: CFString) -> CGSize? {
|
|
3085
|
+
guard let value = copyAttribute(element, attribute: attribute) else { return nil }
|
|
3086
|
+
let cfValue = value as CFTypeRef
|
|
3087
|
+
guard CFGetTypeID(cfValue) == AXValueGetTypeID() else { return nil }
|
|
3088
|
+
let axValue = unsafeBitCast(cfValue, to: AXValue.self)
|
|
3089
|
+
guard AXValueGetType(axValue) == .cgSize else { return nil }
|
|
3090
|
+
var size = CGSize.zero
|
|
3091
|
+
guard AXValueGetValue(axValue, .cgSize, &size) else { return nil }
|
|
3092
|
+
return size
|
|
3093
|
+
}
|
|
3094
|
+
|
|
3095
|
+
private func frameForWindow(_ window: AXUIElement) -> CGRect {
|
|
3096
|
+
let origin = pointAttribute(window, attribute: kAXPositionAttribute as CFString) ?? .zero
|
|
3097
|
+
let size = sizeAttribute(window, attribute: kAXSizeAttribute as CFString) ?? .zero
|
|
3098
|
+
return CGRect(origin: origin, size: size)
|
|
3099
|
+
}
|
|
3100
|
+
|
|
3101
|
+
private func allCGWindowEntries() -> [[String: Any]] {
|
|
3102
|
+
(CGWindowListCopyWindowInfo([.optionAll, .excludeDesktopElements], kCGNullWindowID) as? [[String: Any]]) ?? []
|
|
3103
|
+
}
|
|
3104
|
+
|
|
3105
|
+
private func cgWindowOwners(entries suppliedEntries: [[String: Any]]? = nil) -> [CGWindowOwnerSummary] {
|
|
3106
|
+
let entries = suppliedEntries ?? allCGWindowEntries()
|
|
3107
|
+
var seen = Set<Int32>()
|
|
3108
|
+
var owners: [CGWindowOwnerSummary] = []
|
|
3109
|
+
for entry in entries {
|
|
3110
|
+
guard let ownerPid = (entry[kCGWindowOwnerPID as String] as? NSNumber)?.int32Value else { continue }
|
|
3111
|
+
let layer = (entry[kCGWindowLayer as String] as? NSNumber)?.intValue ?? 0
|
|
3112
|
+
if layer != 0 || seen.contains(ownerPid) { continue }
|
|
3113
|
+
guard let boundsDict = entry[kCGWindowBounds as String] as? [String: Any],
|
|
3114
|
+
let bounds = CGRect(dictionaryRepresentation: boundsDict as CFDictionary),
|
|
3115
|
+
bounds.width >= 100,
|
|
3116
|
+
bounds.height >= 80
|
|
3117
|
+
else {
|
|
3118
|
+
continue
|
|
3119
|
+
}
|
|
3120
|
+
let ownerName = (entry[kCGWindowOwnerName as String] as? String) ?? processName(pid: ownerPid) ?? "Unknown App"
|
|
3121
|
+
seen.insert(ownerPid)
|
|
3122
|
+
owners.append(CGWindowOwnerSummary(pid: ownerPid, name: ownerName))
|
|
3123
|
+
}
|
|
3124
|
+
return owners
|
|
3125
|
+
}
|
|
3126
|
+
|
|
3127
|
+
private func pidForWindowId(_ windowId: UInt32) -> Int32? {
|
|
3128
|
+
windowInfo(windowId: windowId)?.pid
|
|
3129
|
+
}
|
|
3130
|
+
|
|
3131
|
+
private func cgWindowCandidates(pid: Int32, entries suppliedEntries: [[String: Any]]? = nil) -> [CGWindowCandidate] {
|
|
3132
|
+
let entries = suppliedEntries ?? allCGWindowEntries()
|
|
3133
|
+
var candidates: [CGWindowCandidate] = []
|
|
3134
|
+
for (zOrder, entry) in entries.enumerated() {
|
|
3135
|
+
guard let ownerPid = (entry[kCGWindowOwnerPID as String] as? NSNumber)?.int32Value,
|
|
3136
|
+
ownerPid == pid
|
|
3137
|
+
else {
|
|
3138
|
+
continue
|
|
3139
|
+
}
|
|
3140
|
+
let layer = (entry[kCGWindowLayer as String] as? NSNumber)?.intValue ?? 0
|
|
3141
|
+
if layer != 0 { continue }
|
|
3142
|
+
|
|
3143
|
+
guard let windowNumber = (entry[kCGWindowNumber as String] as? NSNumber)?.uint32Value else {
|
|
3144
|
+
continue
|
|
3145
|
+
}
|
|
3146
|
+
guard let boundsDict = entry[kCGWindowBounds as String] as? [String: Any],
|
|
3147
|
+
let bounds = CGRect(dictionaryRepresentation: boundsDict as CFDictionary)
|
|
3148
|
+
else {
|
|
3149
|
+
continue
|
|
3150
|
+
}
|
|
3151
|
+
|
|
3152
|
+
let title = (entry[kCGWindowName as String] as? String) ?? ""
|
|
3153
|
+
let isOnscreen = (entry[kCGWindowIsOnscreen as String] as? NSNumber)?.boolValue ?? true
|
|
3154
|
+
candidates.append(
|
|
3155
|
+
CGWindowCandidate(
|
|
3156
|
+
windowId: windowNumber,
|
|
3157
|
+
title: title,
|
|
3158
|
+
bounds: bounds,
|
|
3159
|
+
isOnscreen: isOnscreen,
|
|
3160
|
+
layer: layer,
|
|
3161
|
+
zOrder: zOrder
|
|
3162
|
+
)
|
|
3163
|
+
)
|
|
3164
|
+
if candidates.count == 128 { break }
|
|
3165
|
+
}
|
|
3166
|
+
return candidates
|
|
3167
|
+
}
|
|
3168
|
+
|
|
3169
|
+
private func cgBroadRootOwners(entries: [[String: Any]]) -> [CGWindowOwnerSummary] {
|
|
3170
|
+
let popupLevel = Int(CGWindowLevelForKey(.popUpMenuWindow))
|
|
3171
|
+
var seen = Set<Int32>()
|
|
3172
|
+
return entries.compactMap { entry -> CGWindowOwnerSummary? in
|
|
3173
|
+
let layer = (entry[kCGWindowLayer as String] as? NSNumber)?.intValue ?? 0
|
|
3174
|
+
guard layer == 0 || layer == popupLevel,
|
|
3175
|
+
let pid = (entry[kCGWindowOwnerPID as String] as? NSNumber)?.int32Value
|
|
3176
|
+
else { return nil }
|
|
3177
|
+
if layer == popupLevel {
|
|
3178
|
+
guard (entry[kCGWindowIsOnscreen as String] as? NSNumber)?.boolValue == true else { return nil }
|
|
3179
|
+
} else {
|
|
3180
|
+
guard let boundsDict = entry[kCGWindowBounds as String] as? [String: Any],
|
|
3181
|
+
let bounds = CGRect(dictionaryRepresentation: boundsDict as CFDictionary),
|
|
3182
|
+
bounds.width >= 100,
|
|
3183
|
+
bounds.height >= 80
|
|
3184
|
+
else { return nil }
|
|
3185
|
+
}
|
|
3186
|
+
guard seen.insert(pid).inserted else { return nil }
|
|
3187
|
+
let name = (entry[kCGWindowOwnerName as String] as? String) ?? processName(pid: pid) ?? "Unknown App"
|
|
3188
|
+
return CGWindowOwnerSummary(pid: pid, name: name)
|
|
3189
|
+
}
|
|
3190
|
+
}
|
|
3191
|
+
|
|
3192
|
+
private func cgPopupMenuCandidates(pid: Int32?, entries: [[String: Any]]) -> [CGWindowCandidate] {
|
|
3193
|
+
let popupLevel = Int(CGWindowLevelForKey(.popUpMenuWindow))
|
|
3194
|
+
var candidates: [CGWindowCandidate] = []
|
|
3195
|
+
for (zOrder, entry) in entries.enumerated() {
|
|
3196
|
+
guard let ownerPid = (entry[kCGWindowOwnerPID as String] as? NSNumber)?.int32Value else { continue }
|
|
3197
|
+
if let pid, ownerPid != pid { continue }
|
|
3198
|
+
let layer = (entry[kCGWindowLayer as String] as? NSNumber)?.intValue ?? 0
|
|
3199
|
+
if layer != popupLevel { continue }
|
|
3200
|
+
guard let windowNumber = (entry[kCGWindowNumber as String] as? NSNumber)?.uint32Value,
|
|
3201
|
+
let boundsDict = entry[kCGWindowBounds as String] as? [String: Any],
|
|
3202
|
+
let bounds = CGRect(dictionaryRepresentation: boundsDict as CFDictionary)
|
|
3203
|
+
else { continue }
|
|
3204
|
+
let title = (entry[kCGWindowName as String] as? String) ?? ""
|
|
3205
|
+
let isOnscreen = (entry[kCGWindowIsOnscreen as String] as? NSNumber)?.boolValue ?? false
|
|
3206
|
+
guard isOnscreen else { continue }
|
|
3207
|
+
candidates.append(CGWindowCandidate(windowId: windowNumber, title: title, bounds: bounds, isOnscreen: isOnscreen, layer: layer, zOrder: zOrder))
|
|
3208
|
+
if candidates.count == 128 { break }
|
|
3209
|
+
}
|
|
3210
|
+
return candidates
|
|
3211
|
+
}
|
|
3212
|
+
|
|
3213
|
+
private func openMenuElements(pid: Int32, messagingTimeout: Float = 1.0) -> [AXUIElement] {
|
|
3214
|
+
let app = AXUIElementCreateApplication(pid)
|
|
3215
|
+
AXUIElementSetMessagingTimeout(app, messagingTimeout)
|
|
3216
|
+
let descendants = collectDescendants(startingAt: app, maxDepth: 6)
|
|
3217
|
+
var menus = descendants.filter { (stringAttribute($0, attribute: kAXRoleAttribute as CFString) ?? "") == "AXMenu" }
|
|
3218
|
+
if menus.isEmpty,
|
|
3219
|
+
let focused = copyAttribute(app, attribute: kAXFocusedUIElementAttribute as CFString).flatMap(asAXElement)
|
|
3220
|
+
{
|
|
3221
|
+
var current: AXUIElement? = focused
|
|
3222
|
+
while let element = current {
|
|
3223
|
+
if (stringAttribute(element, attribute: kAXRoleAttribute as CFString) ?? "") == "AXMenu" {
|
|
3224
|
+
menus.append(element)
|
|
3225
|
+
break
|
|
3226
|
+
}
|
|
3227
|
+
current = copyAttribute(element, attribute: kAXParentAttribute as CFString).flatMap(asAXElement)
|
|
3228
|
+
}
|
|
3229
|
+
}
|
|
3230
|
+
return menus
|
|
3231
|
+
}
|
|
3232
|
+
|
|
3233
|
+
private func bestCandidate(for element: AXUIElement, candidates: [CGWindowCandidate]) -> CGWindowCandidate? {
|
|
3234
|
+
let title = stringAttribute(element, attribute: kAXTitleAttribute as CFString) ?? ""
|
|
3235
|
+
let frame = frameForWindow(element)
|
|
3236
|
+
return candidates.max { left, right in
|
|
3237
|
+
windowPairScore(frame: frame, title: title, candidate: left) < windowPairScore(frame: frame, title: title, candidate: right)
|
|
3238
|
+
}
|
|
3239
|
+
}
|
|
3240
|
+
|
|
3241
|
+
private func pairingForWindow(_ window: AXUIElement, pid: Int32) -> WindowPairing {
|
|
3242
|
+
windowPairings(windows: [window], candidates: cgWindowCandidates(pid: pid))[ObjectIdentifier(window)] ?? WindowPairing(candidate: nil, score: -Double.greatestFiniteMagnitude, confidence: "low")
|
|
3243
|
+
}
|
|
3244
|
+
|
|
3245
|
+
private func windowPairings(windows: [AXUIElement], candidates: [CGWindowCandidate]) -> [ObjectIdentifier: WindowPairing] {
|
|
3246
|
+
var pairs: [(window: AXUIElement, candidate: CGWindowCandidate, score: Double)] = []
|
|
3247
|
+
for window in windows {
|
|
3248
|
+
let title = stringAttribute(window, attribute: kAXTitleAttribute as CFString) ?? ""
|
|
3249
|
+
let frame = frameForWindow(window)
|
|
3250
|
+
for candidate in candidates {
|
|
3251
|
+
pairs.append((window, candidate, windowPairScore(frame: frame, title: title, candidate: candidate)))
|
|
3252
|
+
}
|
|
3253
|
+
}
|
|
3254
|
+
pairs.sort { $0.score > $1.score }
|
|
3255
|
+
var output: [ObjectIdentifier: WindowPairing] = [:]
|
|
3256
|
+
var usedWindows = Set<ObjectIdentifier>()
|
|
3257
|
+
var usedCandidateIds = Set<UInt32>()
|
|
3258
|
+
for pair in pairs {
|
|
3259
|
+
let key = ObjectIdentifier(pair.window)
|
|
3260
|
+
if usedWindows.contains(key) || usedCandidateIds.contains(pair.candidate.windowId) { continue }
|
|
3261
|
+
usedWindows.insert(key)
|
|
3262
|
+
usedCandidateIds.insert(pair.candidate.windowId)
|
|
3263
|
+
let frame = frameForWindow(pair.window)
|
|
3264
|
+
let title = stringAttribute(pair.window, attribute: kAXTitleAttribute as CFString) ?? ""
|
|
3265
|
+
output[key] = WindowPairing(candidate: pair.score >= 0 ? pair.candidate : nil, score: pair.score, confidence: pairingConfidence(frame: frame, title: title, candidate: pair.candidate, score: pair.score))
|
|
3266
|
+
}
|
|
3267
|
+
for window in windows {
|
|
3268
|
+
let key = ObjectIdentifier(window)
|
|
3269
|
+
if output[key] == nil {
|
|
3270
|
+
output[key] = WindowPairing(candidate: nil, score: -Double.greatestFiniteMagnitude, confidence: "low")
|
|
3271
|
+
}
|
|
3272
|
+
}
|
|
3273
|
+
return output
|
|
3274
|
+
}
|
|
3275
|
+
|
|
3276
|
+
private func windowPairScore(frame: CGRect, title: String, candidate: CGWindowCandidate) -> Double {
|
|
3277
|
+
var score = 0.0
|
|
3278
|
+
let normalizedTitle = title.trimmingCharacters(in: .whitespacesAndNewlines).lowercased()
|
|
3279
|
+
let candidateTitle = candidate.title.trimmingCharacters(in: .whitespacesAndNewlines).lowercased()
|
|
3280
|
+
if !normalizedTitle.isEmpty && !candidateTitle.isEmpty {
|
|
3281
|
+
if normalizedTitle == candidateTitle {
|
|
3282
|
+
score += 100
|
|
3283
|
+
} else if normalizedTitle.contains(candidateTitle) || candidateTitle.contains(normalizedTitle) {
|
|
3284
|
+
score += 50
|
|
3285
|
+
}
|
|
3286
|
+
}
|
|
3287
|
+
if frame.width > 1 && frame.height > 1 {
|
|
3288
|
+
let dx = abs(candidate.bounds.origin.x - frame.origin.x)
|
|
3289
|
+
let dy = abs(candidate.bounds.origin.y - frame.origin.y)
|
|
3290
|
+
let dw = abs(candidate.bounds.size.width - frame.size.width)
|
|
3291
|
+
let dh = abs(candidate.bounds.size.height - frame.size.height)
|
|
3292
|
+
score -= Double(dx + dy + dw + dh) / 20.0
|
|
3293
|
+
}
|
|
3294
|
+
if candidate.isOnscreen { score += 10 }
|
|
3295
|
+
return score
|
|
3296
|
+
}
|
|
3297
|
+
|
|
3298
|
+
private func pairingConfidence(frame: CGRect, title: String, candidate: CGWindowCandidate, score: Double) -> String {
|
|
3299
|
+
guard frame.width > 1 && frame.height > 1 else { return "low" }
|
|
3300
|
+
let normalizedTitle = title.trimmingCharacters(in: .whitespacesAndNewlines).lowercased()
|
|
3301
|
+
let candidateTitle = candidate.title.trimmingCharacters(in: .whitespacesAndNewlines).lowercased()
|
|
3302
|
+
let titleEqual = !normalizedTitle.isEmpty && !candidateTitle.isEmpty && normalizedTitle == candidateTitle
|
|
3303
|
+
let geometryExact = abs(frame.origin.x - candidate.bounds.origin.x) <= 2 && abs(frame.origin.y - candidate.bounds.origin.y) <= 2 && abs(frame.width - candidate.bounds.width) <= 2 && abs(frame.height - candidate.bounds.height) <= 2
|
|
3304
|
+
if titleEqual && geometryExact { return "exact" }
|
|
3305
|
+
if score >= 50 { return "high" }
|
|
3306
|
+
return "low"
|
|
3307
|
+
}
|
|
3308
|
+
|
|
3309
|
+
private func displayScaleFactor(for frame: CGRect) -> Double {
|
|
3310
|
+
var displayCount: UInt32 = 0
|
|
3311
|
+
guard CGGetOnlineDisplayList(0, nil, &displayCount) == .success, displayCount > 0 else {
|
|
3312
|
+
return Double(NSScreen.main?.backingScaleFactor ?? 1.0)
|
|
3313
|
+
}
|
|
3314
|
+
|
|
3315
|
+
var displays = Array(repeating: CGDirectDisplayID(), count: Int(displayCount))
|
|
3316
|
+
guard CGGetOnlineDisplayList(displayCount, &displays, &displayCount) == .success else {
|
|
3317
|
+
return Double(NSScreen.main?.backingScaleFactor ?? 1.0)
|
|
3318
|
+
}
|
|
3319
|
+
|
|
3320
|
+
var chosenDisplay: CGDirectDisplayID?
|
|
3321
|
+
var chosenArea: CGFloat = -1
|
|
3322
|
+
for display in displays {
|
|
3323
|
+
let bounds = CGDisplayBounds(display)
|
|
3324
|
+
let overlap = bounds.intersection(frame)
|
|
3325
|
+
let area = overlap.isNull ? 0 : overlap.width * overlap.height
|
|
3326
|
+
if area > chosenArea {
|
|
3327
|
+
chosenArea = area
|
|
3328
|
+
chosenDisplay = display
|
|
3329
|
+
}
|
|
3330
|
+
}
|
|
3331
|
+
|
|
3332
|
+
guard let display = chosenDisplay, let mode = CGDisplayCopyDisplayMode(display) else {
|
|
3333
|
+
return Double(NSScreen.main?.backingScaleFactor ?? 1.0)
|
|
3334
|
+
}
|
|
3335
|
+
|
|
3336
|
+
let width = Double(mode.width)
|
|
3337
|
+
guard width > 0 else { return 1.0 }
|
|
3338
|
+
let scale = Double(mode.pixelWidth) / width
|
|
3339
|
+
return scale > 0 ? scale : 1.0
|
|
3340
|
+
}
|
|
3341
|
+
|
|
3342
|
+
private func captureWindow(windowId: UInt32) throws -> CapturedWindowImage {
|
|
3343
|
+
if #available(macOS 14.0, *) {
|
|
3344
|
+
let semaphore = DispatchSemaphore(value: 0)
|
|
3345
|
+
let capturedImage = Box<CGImage?>(nil)
|
|
3346
|
+
let capturedError = Box<Error?>(nil)
|
|
3347
|
+
|
|
3348
|
+
let task = Task {
|
|
3349
|
+
defer { semaphore.signal() }
|
|
3350
|
+
do {
|
|
3351
|
+
if Task.isCancelled {
|
|
3352
|
+
return
|
|
3353
|
+
}
|
|
3354
|
+
let shareable = try await SCShareableContent.excludingDesktopWindows(false, onScreenWindowsOnly: false)
|
|
3355
|
+
guard let window = shareable.windows.first(where: { $0.windowID == windowId }) else {
|
|
3356
|
+
throw BridgeFailure(message: "Window \(windowId) is not available for capture", code: "window_not_found")
|
|
3357
|
+
}
|
|
3358
|
+
|
|
3359
|
+
let filter = SCContentFilter(desktopIndependentWindow: window)
|
|
3360
|
+
let config = SCStreamConfiguration()
|
|
3361
|
+
// Avoid ScreenCaptureKit's default 1920x1080 canvas for window captures.
|
|
3362
|
+
let scale = displayScaleFactor(for: window.frame)
|
|
3363
|
+
config.width = max(1, Int((window.frame.width * scale).rounded()))
|
|
3364
|
+
config.height = max(1, Int((window.frame.height * scale).rounded()))
|
|
3365
|
+
config.showsCursor = false
|
|
3366
|
+
config.ignoreShadowsSingleWindow = true
|
|
3367
|
+
|
|
3368
|
+
let image = try await SCScreenshotManager.captureImage(contentFilter: filter, configuration: config)
|
|
3369
|
+
capturedImage.value = image
|
|
3370
|
+
} catch {
|
|
3371
|
+
capturedError.value = error
|
|
3372
|
+
}
|
|
3373
|
+
}
|
|
3374
|
+
|
|
3375
|
+
if semaphore.wait(timeout: .now() + .seconds(8)) == .timedOut {
|
|
3376
|
+
task.cancel()
|
|
3377
|
+
if let payload = try cgWindowScreenshotFallback(windowId: windowId) {
|
|
3378
|
+
return payload
|
|
3379
|
+
}
|
|
3380
|
+
throw BridgeFailure(message: "Capture timed out while capturing window \(windowId)", code: "capture_timeout")
|
|
3381
|
+
}
|
|
3382
|
+
|
|
3383
|
+
if let error = capturedError.value {
|
|
3384
|
+
if let payload = try cgWindowScreenshotFallback(windowId: windowId) {
|
|
3385
|
+
return payload
|
|
3386
|
+
}
|
|
3387
|
+
if let failure = error as? BridgeFailure {
|
|
3388
|
+
throw failure
|
|
3389
|
+
}
|
|
3390
|
+
throw BridgeFailure(message: "Capture failed: \(error.localizedDescription)", code: "capture_failed")
|
|
3391
|
+
}
|
|
3392
|
+
|
|
3393
|
+
guard let image = capturedImage.value else {
|
|
3394
|
+
if let payload = try cgWindowScreenshotFallback(windowId: windowId) {
|
|
3395
|
+
return payload
|
|
3396
|
+
}
|
|
3397
|
+
throw BridgeFailure(message: "Capture failed", code: "capture_failed")
|
|
3398
|
+
}
|
|
3399
|
+
|
|
3400
|
+
return CapturedWindowImage(image: image, windowId: windowId, frame: currentWindowBounds(windowId: windowId) ?? CGRect(x: 0, y: 0, width: image.width, height: image.height))
|
|
3401
|
+
}
|
|
3402
|
+
if let payload = try cgWindowScreenshotFallback(windowId: windowId) {
|
|
3403
|
+
return payload
|
|
3404
|
+
}
|
|
3405
|
+
throw BridgeFailure(message: "Capture failed", code: "capture_failed")
|
|
3406
|
+
}
|
|
3407
|
+
|
|
3408
|
+
private func jpegData(image: CGImage, quality: Double) -> Data? {
|
|
3409
|
+
let data = NSMutableData()
|
|
3410
|
+
guard let destination = CGImageDestinationCreateWithData(data, "public.jpeg" as CFString, 1, nil) else { return nil }
|
|
3411
|
+
CGImageDestinationAddImage(destination, image, [kCGImageDestinationLossyCompressionQuality: quality] as CFDictionary)
|
|
3412
|
+
guard CGImageDestinationFinalize(destination) else { return nil }
|
|
3413
|
+
return data as Data
|
|
3414
|
+
}
|
|
3415
|
+
|
|
3416
|
+
private func downscaledImage(_ image: CGImage, maxDimension: Int?) -> CGImage? {
|
|
3417
|
+
guard let maxDimension, max(image.width, image.height) > maxDimension else { return nil }
|
|
3418
|
+
let scale = Double(maxDimension) / Double(max(image.width, image.height))
|
|
3419
|
+
let width = max(1, Int(Double(image.width) * scale))
|
|
3420
|
+
let height = max(1, Int(Double(image.height) * scale))
|
|
3421
|
+
guard let context = CGContext(
|
|
3422
|
+
data: nil,
|
|
3423
|
+
width: width,
|
|
3424
|
+
height: height,
|
|
3425
|
+
bitsPerComponent: image.bitsPerComponent,
|
|
3426
|
+
bytesPerRow: 0,
|
|
3427
|
+
space: image.colorSpace ?? CGColorSpaceCreateDeviceRGB(),
|
|
3428
|
+
bitmapInfo: image.bitmapInfo.rawValue
|
|
3429
|
+
) else { return nil }
|
|
3430
|
+
context.interpolationQuality = .medium
|
|
3431
|
+
context.draw(image, in: CGRect(x: 0, y: 0, width: width, height: height))
|
|
3432
|
+
return context.makeImage()
|
|
3433
|
+
}
|
|
3434
|
+
|
|
3435
|
+
private func cgWindowScreenshotFallback(windowId: UInt32) throws -> CapturedWindowImage? {
|
|
3436
|
+
if let payload = try systemScreenshotWindow(windowId: windowId) {
|
|
3437
|
+
return payload
|
|
3438
|
+
}
|
|
3439
|
+
return nil
|
|
3440
|
+
}
|
|
3441
|
+
|
|
3442
|
+
private func systemScreenshotWindow(windowId: UInt32) throws -> CapturedWindowImage? {
|
|
3443
|
+
let tempUrl = FileManager.default.temporaryDirectory.appendingPathComponent("pi-cu-\(UUID().uuidString).png")
|
|
3444
|
+
defer { try? FileManager.default.removeItem(at: tempUrl) }
|
|
3445
|
+
// Owner-only perms in case TMPDIR ever resolves to a shared directory.
|
|
3446
|
+
FileManager.default.createFile(atPath: tempUrl.path, contents: nil, attributes: [.posixPermissions: 0o600])
|
|
3447
|
+
|
|
3448
|
+
let process = Process()
|
|
3449
|
+
process.executableURL = URL(fileURLWithPath: "/usr/sbin/screencapture")
|
|
3450
|
+
process.arguments = ["-x", "-l", String(windowId), tempUrl.path]
|
|
3451
|
+
try process.run()
|
|
3452
|
+
let deadline = Date().addingTimeInterval(5)
|
|
3453
|
+
while process.isRunning && Date() < deadline {
|
|
3454
|
+
Thread.sleep(forTimeInterval: 0.05)
|
|
3455
|
+
}
|
|
3456
|
+
if process.isRunning {
|
|
3457
|
+
process.terminate()
|
|
3458
|
+
Thread.sleep(forTimeInterval: 0.1)
|
|
3459
|
+
if process.isRunning { process.interrupt() }
|
|
3460
|
+
return nil
|
|
3461
|
+
}
|
|
3462
|
+
guard process.terminationStatus == 0 else { return nil }
|
|
3463
|
+
guard let data = try? Data(contentsOf: tempUrl), !data.isEmpty else { return nil }
|
|
3464
|
+
guard let imageRep = NSBitmapImageRep(data: data), let cgImage = imageRep.cgImage else { return nil }
|
|
3465
|
+
return CapturedWindowImage(image: cgImage, windowId: windowId, frame: currentWindowBounds(windowId: windowId) ?? CGRect(x: 0, y: 0, width: cgImage.width, height: cgImage.height))
|
|
3466
|
+
}
|
|
3467
|
+
|
|
3468
|
+
private func currentWindowBounds(windowId: UInt32) -> CGRect? {
|
|
3469
|
+
if #available(macOS 14.0, *), let scBounds = currentWindowBoundsViaScreenCaptureKit(windowId: windowId) {
|
|
3470
|
+
return scBounds
|
|
3471
|
+
}
|
|
3472
|
+
return windowInfo(windowId: windowId)?.bounds
|
|
3473
|
+
}
|
|
3474
|
+
|
|
3475
|
+
private func windowInfo(windowId: UInt32) -> (pid: Int32, bounds: CGRect)? {
|
|
3476
|
+
func matchingEntry(_ entries: [[String: Any]]?) -> [String: Any]? {
|
|
3477
|
+
entries?.first {
|
|
3478
|
+
($0[kCGWindowNumber as String] as? NSNumber)?.uint32Value == windowId
|
|
3479
|
+
}
|
|
3480
|
+
}
|
|
3481
|
+
|
|
3482
|
+
let requestedIds = [NSNumber(value: windowId)] as CFArray
|
|
3483
|
+
let targetedEntries = CGWindowListCreateDescriptionFromArray(requestedIds) as? [[String: Any]]
|
|
3484
|
+
let entry = matchingEntry(targetedEntries) ?? matchingEntry(
|
|
3485
|
+
CGWindowListCopyWindowInfo([.optionAll], kCGNullWindowID) as? [[String: Any]]
|
|
3486
|
+
)
|
|
3487
|
+
guard let entry,
|
|
3488
|
+
let pid = (entry[kCGWindowOwnerPID as String] as? NSNumber)?.int32Value,
|
|
3489
|
+
let boundsDict = entry[kCGWindowBounds as String] as? [String: Any],
|
|
3490
|
+
let bounds = CGRect(dictionaryRepresentation: boundsDict as CFDictionary)
|
|
3491
|
+
else {
|
|
3492
|
+
return nil
|
|
3493
|
+
}
|
|
3494
|
+
return (pid, bounds)
|
|
3495
|
+
}
|
|
3496
|
+
|
|
3497
|
+
@available(macOS 14.0, *)
|
|
3498
|
+
private func currentWindowBoundsViaScreenCaptureKit(windowId: UInt32) -> CGRect? {
|
|
3499
|
+
let semaphore = DispatchSemaphore(value: 0)
|
|
3500
|
+
let output = Box<CGRect?>(nil)
|
|
3501
|
+
|
|
3502
|
+
let task = Task {
|
|
3503
|
+
defer { semaphore.signal() }
|
|
3504
|
+
do {
|
|
3505
|
+
if Task.isCancelled {
|
|
3506
|
+
return
|
|
3507
|
+
}
|
|
3508
|
+
let shareable = try await SCShareableContent.excludingDesktopWindows(false, onScreenWindowsOnly: false)
|
|
3509
|
+
if let window = shareable.windows.first(where: { $0.windowID == windowId }) {
|
|
3510
|
+
output.value = window.frame
|
|
3511
|
+
}
|
|
3512
|
+
} catch {
|
|
3513
|
+
output.value = nil
|
|
3514
|
+
}
|
|
3515
|
+
}
|
|
3516
|
+
|
|
3517
|
+
if semaphore.wait(timeout: .now() + .seconds(2)) == .timedOut {
|
|
3518
|
+
task.cancel()
|
|
3519
|
+
return nil
|
|
3520
|
+
}
|
|
3521
|
+
return output.value
|
|
3522
|
+
}
|
|
3523
|
+
|
|
3524
|
+
private func eventDelivery(_ request: [String: Any]) -> String {
|
|
3525
|
+
optionalStringArg(request, "delivery") == "pid" ? "pid" : "hid"
|
|
3526
|
+
}
|
|
3527
|
+
|
|
3528
|
+
private func postEvent(_ event: CGEvent, pid: Int32, delivery: String = "hid") {
|
|
3529
|
+
if delivery == "pid" {
|
|
3530
|
+
event.postToPid(pid)
|
|
3531
|
+
return
|
|
3532
|
+
}
|
|
3533
|
+
ForegroundActivity.shared.physicalInput()
|
|
3534
|
+
// Post as a real foreground HID event. AppKit views with mouseDown handlers
|
|
3535
|
+
// can ignore pid-targeted CGEvents even though postToPid reports success.
|
|
3536
|
+
// Keep the target app frontmost so the HID event is delivered to the intended
|
|
3537
|
+
// window, then post at the session event tap.
|
|
3538
|
+
if let app = NSRunningApplication(processIdentifier: pid), !app.isActive {
|
|
3539
|
+
if #available(macOS 14.0, *) {
|
|
3540
|
+
_ = app.activate()
|
|
3541
|
+
} else {
|
|
3542
|
+
_ = app.activate(options: [.activateIgnoringOtherApps])
|
|
3543
|
+
}
|
|
3544
|
+
usleep(20_000)
|
|
3545
|
+
}
|
|
3546
|
+
event.post(tap: .cghidEventTap)
|
|
3547
|
+
}
|
|
3548
|
+
|
|
3549
|
+
private func postMouseMove(to point: CGPoint, pid: Int32, delivery: String = "hid") throws {
|
|
3550
|
+
if delivery == "hid" { physicalInputLock.lock(); ForegroundActivity.shared.enter() }
|
|
3551
|
+
defer { if delivery == "hid" { ForegroundActivity.shared.leave(); physicalInputLock.unlock() } }
|
|
3552
|
+
guard let move = CGEvent(mouseEventSource: nil, mouseType: .mouseMoved, mouseCursorPosition: point, mouseButton: .left) else {
|
|
3553
|
+
throw BridgeFailure(message: "Failed to create mouse move event", code: "input_failed")
|
|
3554
|
+
}
|
|
3555
|
+
postEvent(move, pid: pid, delivery: delivery)
|
|
3556
|
+
}
|
|
3557
|
+
|
|
3558
|
+
private func mouseButton(_ name: String) -> CGMouseButton {
|
|
3559
|
+
switch name.trimmingCharacters(in: .whitespacesAndNewlines).lowercased() {
|
|
3560
|
+
case "right":
|
|
3561
|
+
return .right
|
|
3562
|
+
case "middle", "center":
|
|
3563
|
+
return .center
|
|
3564
|
+
default:
|
|
3565
|
+
return .left
|
|
3566
|
+
}
|
|
3567
|
+
}
|
|
3568
|
+
|
|
3569
|
+
private func mouseDownType(for button: CGMouseButton) -> CGEventType {
|
|
3570
|
+
switch button {
|
|
3571
|
+
case .right:
|
|
3572
|
+
return .rightMouseDown
|
|
3573
|
+
case .center:
|
|
3574
|
+
return .otherMouseDown
|
|
3575
|
+
default:
|
|
3576
|
+
return .leftMouseDown
|
|
3577
|
+
}
|
|
3578
|
+
}
|
|
3579
|
+
|
|
3580
|
+
private func mouseUpType(for button: CGMouseButton) -> CGEventType {
|
|
3581
|
+
switch button {
|
|
3582
|
+
case .right:
|
|
3583
|
+
return .rightMouseUp
|
|
3584
|
+
case .center:
|
|
3585
|
+
return .otherMouseUp
|
|
3586
|
+
default:
|
|
3587
|
+
return .leftMouseUp
|
|
3588
|
+
}
|
|
3589
|
+
}
|
|
3590
|
+
|
|
3591
|
+
private func mouseDraggedType(for button: CGMouseButton) -> CGEventType {
|
|
3592
|
+
switch button {
|
|
3593
|
+
case .right:
|
|
3594
|
+
return .rightMouseDragged
|
|
3595
|
+
case .center:
|
|
3596
|
+
return .otherMouseDragged
|
|
3597
|
+
default:
|
|
3598
|
+
return .leftMouseDragged
|
|
3599
|
+
}
|
|
3600
|
+
}
|
|
3601
|
+
|
|
3602
|
+
private func postMouseClick(at point: CGPoint, pid: Int32, button: CGMouseButton = .left, clickCount: Int = 1, delivery: String = "hid") throws {
|
|
3603
|
+
if delivery == "hid" { physicalInputLock.lock(); ForegroundActivity.shared.enter() }
|
|
3604
|
+
defer { if delivery == "hid" { ForegroundActivity.shared.leave(); physicalInputLock.unlock() } }
|
|
3605
|
+
// Tangu:HID 点击会把**用户的实物指针**挪到目标处并留在那里 —— 前台被抢已经够烦,
|
|
3606
|
+
// 鼠标还被丢在别处就更糟。点完把指针还回原位。
|
|
3607
|
+
// 后台(pid)路径不需要:那条本来就不动系统指针(只画 AgentCursor 虚拟光标)。
|
|
3608
|
+
// 守卫:只有指针仍停在我们投递的位置(= 这期间用户没自己动鼠标)才还原,绝不跟用户抢。
|
|
3609
|
+
let savedCursor: CGPoint? = delivery == "hid" ? CGEvent(source: nil)?.location : nil
|
|
3610
|
+
defer {
|
|
3611
|
+
// defer 里不能 return,只能靠嵌套 if
|
|
3612
|
+
if let savedCursor, let now = CGEvent(source: nil)?.location,
|
|
3613
|
+
abs(now.x - point.x) < 2, abs(now.y - point.y) < 2 {
|
|
3614
|
+
CGWarpMouseCursorPosition(savedCursor)
|
|
3615
|
+
// warp 之后必须重新关联,否则系统会短暂忽略真实鼠标移动(指针像卡住)。
|
|
3616
|
+
CGAssociateMouseAndMouseCursorPosition(1)
|
|
3617
|
+
}
|
|
3618
|
+
}
|
|
3619
|
+
try postMouseMove(to: point, pid: pid, delivery: delivery)
|
|
3620
|
+
for index in 1...max(1, clickCount) {
|
|
3621
|
+
guard let down = CGEvent(mouseEventSource: nil, mouseType: mouseDownType(for: button), mouseCursorPosition: point, mouseButton: button),
|
|
3622
|
+
let up = CGEvent(mouseEventSource: nil, mouseType: mouseUpType(for: button), mouseCursorPosition: point, mouseButton: button)
|
|
3623
|
+
else {
|
|
3624
|
+
throw BridgeFailure(message: "Failed to create mouse click event", code: "input_failed")
|
|
3625
|
+
}
|
|
3626
|
+
down.setIntegerValueField(.mouseEventClickState, value: Int64(index))
|
|
3627
|
+
up.setIntegerValueField(.mouseEventClickState, value: Int64(index))
|
|
3628
|
+
postEvent(down, pid: pid, delivery: delivery)
|
|
3629
|
+
usleep(12_000)
|
|
3630
|
+
postEvent(up, pid: pid, delivery: delivery)
|
|
3631
|
+
if index < clickCount {
|
|
3632
|
+
usleep(70_000)
|
|
3633
|
+
}
|
|
3634
|
+
}
|
|
3635
|
+
}
|
|
3636
|
+
|
|
3637
|
+
private func postMouseDrag(points: [CGPoint], pid: Int32, delivery: String = "hid") throws {
|
|
3638
|
+
if delivery == "hid" { physicalInputLock.lock(); ForegroundActivity.shared.enter() }
|
|
3639
|
+
defer { if delivery == "hid" { ForegroundActivity.shared.leave(); physicalInputLock.unlock() } }
|
|
3640
|
+
guard points.count >= 2, let first = points.first else {
|
|
3641
|
+
throw BridgeFailure(message: "Drag requires at least two points", code: "invalid_args")
|
|
3642
|
+
}
|
|
3643
|
+
try postMouseMove(to: first, pid: pid, delivery: delivery)
|
|
3644
|
+
guard let down = CGEvent(mouseEventSource: nil, mouseType: .leftMouseDown, mouseCursorPosition: first, mouseButton: .left) else {
|
|
3645
|
+
throw BridgeFailure(message: "Failed to create mouse down event", code: "input_failed")
|
|
3646
|
+
}
|
|
3647
|
+
postEvent(down, pid: pid, delivery: delivery)
|
|
3648
|
+
usleep(12_000)
|
|
3649
|
+
|
|
3650
|
+
for point in points.dropFirst() {
|
|
3651
|
+
guard let drag = CGEvent(mouseEventSource: nil, mouseType: mouseDraggedType(for: .left), mouseCursorPosition: point, mouseButton: .left) else {
|
|
3652
|
+
throw BridgeFailure(message: "Failed to create mouse drag event", code: "input_failed")
|
|
3653
|
+
}
|
|
3654
|
+
postEvent(drag, pid: pid, delivery: delivery)
|
|
3655
|
+
usleep(8_000)
|
|
3656
|
+
}
|
|
3657
|
+
|
|
3658
|
+
guard let last = points.last,
|
|
3659
|
+
let up = CGEvent(mouseEventSource: nil, mouseType: .leftMouseUp, mouseCursorPosition: last, mouseButton: .left)
|
|
3660
|
+
else {
|
|
3661
|
+
throw BridgeFailure(message: "Failed to create mouse up event", code: "input_failed")
|
|
3662
|
+
}
|
|
3663
|
+
postEvent(up, pid: pid, delivery: delivery)
|
|
3664
|
+
}
|
|
3665
|
+
|
|
3666
|
+
private func postScrollWheel(at point: CGPoint, deltaX: Int, deltaY: Int, pid: Int32, delivery: String = "hid") throws {
|
|
3667
|
+
if delivery == "hid" { physicalInputLock.lock(); ForegroundActivity.shared.enter() }
|
|
3668
|
+
defer { if delivery == "hid" { ForegroundActivity.shared.leave(); physicalInputLock.unlock() } }
|
|
3669
|
+
try postMouseMove(to: point, pid: pid, delivery: delivery)
|
|
3670
|
+
guard let event = CGEvent(
|
|
3671
|
+
scrollWheelEvent2Source: nil,
|
|
3672
|
+
units: .pixel,
|
|
3673
|
+
wheelCount: 2,
|
|
3674
|
+
wheel1: Int32(-deltaY),
|
|
3675
|
+
wheel2: Int32(deltaX),
|
|
3676
|
+
wheel3: 0
|
|
3677
|
+
) else {
|
|
3678
|
+
throw BridgeFailure(message: "Failed to create scroll event", code: "input_failed")
|
|
3679
|
+
}
|
|
3680
|
+
event.location = point
|
|
3681
|
+
postEvent(event, pid: pid, delivery: delivery)
|
|
3682
|
+
}
|
|
3683
|
+
|
|
3684
|
+
private func modifierFlag(_ key: String) -> CGEventFlags? {
|
|
3685
|
+
switch key.trimmingCharacters(in: .whitespacesAndNewlines).lowercased() {
|
|
3686
|
+
case "cmd", "command", "meta":
|
|
3687
|
+
return .maskCommand
|
|
3688
|
+
case "ctrl", "control":
|
|
3689
|
+
return .maskControl
|
|
3690
|
+
case "shift":
|
|
3691
|
+
return .maskShift
|
|
3692
|
+
case "option", "alt":
|
|
3693
|
+
return .maskAlternate
|
|
3694
|
+
default:
|
|
3695
|
+
return nil
|
|
3696
|
+
}
|
|
3697
|
+
}
|
|
3698
|
+
|
|
3699
|
+
private func keyCode(_ key: String) -> CGKeyCode? {
|
|
3700
|
+
let normalized = key.trimmingCharacters(in: .whitespacesAndNewlines).lowercased()
|
|
3701
|
+
let table: [String: CGKeyCode] = [
|
|
3702
|
+
"a": 0, "s": 1, "d": 2, "f": 3, "h": 4, "g": 5, "z": 6, "x": 7, "c": 8, "v": 9, "b": 11,
|
|
3703
|
+
"q": 12, "w": 13, "e": 14, "r": 15, "y": 16, "t": 17, "1": 18, "2": 19, "3": 20, "4": 21,
|
|
3704
|
+
"6": 22, "5": 23, "=": 24, "+": 24, "9": 25, "7": 26, "-": 27, "8": 28, "0": 29,
|
|
3705
|
+
"]": 30, "o": 31, "u": 32, "[": 33, "i": 34, "p": 35, "return": 36, "enter": 36,
|
|
3706
|
+
"l": 37, "j": 38, "'": 39, "\"": 39, "k": 40, ";": 41, "\\": 42, ",": 43, "/": 44,
|
|
3707
|
+
"n": 45, "m": 46, ".": 47, "tab": 48, "space": 49, " ": 49, "`": 50, "~": 50,
|
|
3708
|
+
"backspace": 51, "delete": 51, "del": 51, "esc": 53, "escape": 53,
|
|
3709
|
+
"f1": 122, "f2": 120, "f3": 99, "f4": 118, "f5": 96, "f6": 97, "f7": 98, "f8": 100,
|
|
3710
|
+
"f9": 101, "f10": 109, "f11": 103, "f12": 111,
|
|
3711
|
+
"home": 115, "pageup": 116, "page_up": 116, "page down": 121, "pagedown": 121, "page_down": 121,
|
|
3712
|
+
"forwarddelete": 117, "forward_delete": 117, "end": 119,
|
|
3713
|
+
"left": 123, "arrowleft": 123, "arrow_left": 123,
|
|
3714
|
+
"right": 124, "arrowright": 124, "arrow_right": 124,
|
|
3715
|
+
"down": 125, "arrowdown": 125, "arrow_down": 125,
|
|
3716
|
+
"up": 126, "arrowup": 126, "arrow_up": 126,
|
|
3717
|
+
]
|
|
3718
|
+
return table[normalized]
|
|
3719
|
+
}
|
|
3720
|
+
|
|
3721
|
+
private func keyChord(_ keys: [String]) -> (flags: CGEventFlags, key: String)? {
|
|
3722
|
+
guard keys.count >= 2 else { return nil }
|
|
3723
|
+
var flags = CGEventFlags()
|
|
3724
|
+
for key in keys.dropLast() {
|
|
3725
|
+
guard let flag = modifierFlag(key) else {
|
|
3726
|
+
return nil
|
|
3727
|
+
}
|
|
3728
|
+
flags.insert(flag)
|
|
3729
|
+
}
|
|
3730
|
+
return (flags, keys.last ?? "")
|
|
3731
|
+
}
|
|
3732
|
+
|
|
3733
|
+
private func postKeyPress(keys: [String], pid: Int32, delivery: String = "hid") throws {
|
|
3734
|
+
if delivery == "hid" { physicalInputLock.lock(); ForegroundActivity.shared.enter() }
|
|
3735
|
+
defer { if delivery == "hid" { ForegroundActivity.shared.leave(); physicalInputLock.unlock() } }
|
|
3736
|
+
if let chord = keyChord(keys) {
|
|
3737
|
+
try postKey(chord.key, flags: chord.flags, pid: pid, delivery: delivery)
|
|
3738
|
+
return
|
|
3739
|
+
}
|
|
3740
|
+
|
|
3741
|
+
for key in keys {
|
|
3742
|
+
let parts = key
|
|
3743
|
+
.split(separator: "+")
|
|
3744
|
+
.map { String($0).trimmingCharacters(in: .whitespacesAndNewlines) }
|
|
3745
|
+
.filter { !$0.isEmpty }
|
|
3746
|
+
if let chord = keyChord(parts) {
|
|
3747
|
+
try postKey(chord.key, flags: chord.flags, pid: pid, delivery: delivery)
|
|
3748
|
+
} else {
|
|
3749
|
+
try postKey(key, flags: [], pid: pid, delivery: delivery)
|
|
3750
|
+
}
|
|
3751
|
+
}
|
|
3752
|
+
}
|
|
3753
|
+
|
|
3754
|
+
private func postKey(_ key: String, flags: CGEventFlags, pid: Int32, delivery: String = "hid") throws {
|
|
3755
|
+
if delivery == "hid" { physicalInputLock.lock(); ForegroundActivity.shared.enter() }
|
|
3756
|
+
defer { if delivery == "hid" { ForegroundActivity.shared.leave(); physicalInputLock.unlock() } }
|
|
3757
|
+
guard let code = keyCode(key) else {
|
|
3758
|
+
if key.count == 1 {
|
|
3759
|
+
try postUnicodeText(key, pid: pid, delivery: delivery)
|
|
3760
|
+
return
|
|
3761
|
+
}
|
|
3762
|
+
throw BridgeFailure(message: "Unsupported key '\(key)'", code: "invalid_args")
|
|
3763
|
+
}
|
|
3764
|
+
guard let down = CGEvent(keyboardEventSource: nil, virtualKey: code, keyDown: true),
|
|
3765
|
+
let up = CGEvent(keyboardEventSource: nil, virtualKey: code, keyDown: false)
|
|
3766
|
+
else {
|
|
3767
|
+
throw BridgeFailure(message: "Failed to create key event", code: "input_failed")
|
|
3768
|
+
}
|
|
3769
|
+
down.flags = flags
|
|
3770
|
+
up.flags = flags
|
|
3771
|
+
postEvent(down, pid: pid, delivery: delivery)
|
|
3772
|
+
postEvent(up, pid: pid, delivery: delivery)
|
|
3773
|
+
usleep(8_000)
|
|
3774
|
+
}
|
|
3775
|
+
|
|
3776
|
+
private func postUnicodeText(_ text: String, pid: Int32, delivery: String = "hid") throws {
|
|
3777
|
+
if delivery == "hid" { physicalInputLock.lock(); ForegroundActivity.shared.enter() }
|
|
3778
|
+
defer { if delivery == "hid" { ForegroundActivity.shared.leave(); physicalInputLock.unlock() } }
|
|
3779
|
+
for scalar in text.unicodeScalars {
|
|
3780
|
+
let char = String(scalar)
|
|
3781
|
+
if let stroke = physicalKeyStroke(for: char) {
|
|
3782
|
+
try postKey(stroke.key, flags: stroke.flags, pid: pid, delivery: delivery)
|
|
3783
|
+
continue
|
|
3784
|
+
}
|
|
3785
|
+
guard let down = CGEvent(keyboardEventSource: nil, virtualKey: 0, keyDown: true),
|
|
3786
|
+
let up = CGEvent(keyboardEventSource: nil, virtualKey: 0, keyDown: false)
|
|
3787
|
+
else {
|
|
3788
|
+
throw BridgeFailure(message: "Failed to create unicode key event", code: "input_failed")
|
|
3789
|
+
}
|
|
3790
|
+
setUnicodeString(event: down, text: char)
|
|
3791
|
+
setUnicodeString(event: up, text: char)
|
|
3792
|
+
postEvent(down, pid: pid, delivery: delivery)
|
|
3793
|
+
postEvent(up, pid: pid, delivery: delivery)
|
|
3794
|
+
usleep(8_000)
|
|
3795
|
+
}
|
|
3796
|
+
}
|
|
3797
|
+
|
|
3798
|
+
private func postAtomicUnicodeText(_ text: String, pid: Int32, delivery: String = "hid") throws {
|
|
3799
|
+
if delivery == "hid" { physicalInputLock.lock(); ForegroundActivity.shared.enter() }
|
|
3800
|
+
defer { if delivery == "hid" { ForegroundActivity.shared.leave(); physicalInputLock.unlock() } }
|
|
3801
|
+
guard let down = CGEvent(keyboardEventSource: nil, virtualKey: 0, keyDown: true),
|
|
3802
|
+
let up = CGEvent(keyboardEventSource: nil, virtualKey: 0, keyDown: false)
|
|
3803
|
+
else { throw BridgeFailure(message: "Failed to create unicode text event", code: "input_failed") }
|
|
3804
|
+
setUnicodeString(event: down, text: text)
|
|
3805
|
+
setUnicodeString(event: up, text: text)
|
|
3806
|
+
postEvent(down, pid: pid, delivery: delivery)
|
|
3807
|
+
usleep(8_000)
|
|
3808
|
+
postEvent(up, pid: pid, delivery: delivery)
|
|
3809
|
+
}
|
|
3810
|
+
|
|
3811
|
+
/// Prefer physical key codes for characters represented by the US layout.
|
|
3812
|
+
/// AppKit field editors can observe synthetic Unicode events without applying
|
|
3813
|
+
/// them to the backing value, while normal key codes follow the same input
|
|
3814
|
+
/// path as a user keystroke. Unicode synthesis remains the fallback for text
|
|
3815
|
+
/// that has no direct key representation.
|
|
3816
|
+
private func physicalKeyStroke(for character: String) -> (key: String, flags: CGEventFlags)? {
|
|
3817
|
+
guard character.count == 1 else { return nil }
|
|
3818
|
+
if character >= "a" && character <= "z" { return (character, []) }
|
|
3819
|
+
if character >= "A" && character <= "Z" { return (character.lowercased(), [.maskShift]) }
|
|
3820
|
+
if character >= "0" && character <= "9" { return (character, []) }
|
|
3821
|
+
switch character {
|
|
3822
|
+
case " ": return ("space", [])
|
|
3823
|
+
case ".", ",", "/", "-", "=", ";", "'", "[", "]", "\\", "`": return (character, [])
|
|
3824
|
+
case "_": return ("-", [.maskShift])
|
|
3825
|
+
case "+": return ("=", [.maskShift])
|
|
3826
|
+
case ":": return (";", [.maskShift])
|
|
3827
|
+
case "\"": return ("'", [.maskShift])
|
|
3828
|
+
case "?": return ("/", [.maskShift])
|
|
3829
|
+
case "<": return (",", [.maskShift])
|
|
3830
|
+
case ">": return (".", [.maskShift])
|
|
3831
|
+
default: return nil
|
|
3832
|
+
}
|
|
3833
|
+
}
|
|
3834
|
+
|
|
3835
|
+
private func setUnicodeString(event: CGEvent, text: String) {
|
|
3836
|
+
var utf16 = Array(text.utf16)
|
|
3837
|
+
utf16.withUnsafeMutableBufferPointer { buffer in
|
|
3838
|
+
guard let base = buffer.baseAddress else { return }
|
|
3839
|
+
event.keyboardSetUnicodeString(stringLength: buffer.count, unicodeString: base)
|
|
3840
|
+
}
|
|
3841
|
+
}
|
|
3842
|
+
|
|
3843
|
+
}
|
|
3844
|
+
|
|
3845
|
+
@main
|
|
3846
|
+
struct PiComputerUseHelper {
|
|
3847
|
+
static func main() {
|
|
3848
|
+
_ = NSApplication.shared
|
|
3849
|
+
NSApp.setActivationPolicy(CommandLine.arguments.contains("serve") ? .accessory : .prohibited)
|
|
3850
|
+
Bridge().run()
|
|
3851
|
+
}
|
|
3852
|
+
}
|