@anionex/dsh-computer-use 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (120) hide show
  1. package/LICENSE +21 -0
  2. package/README.i18n.yaml +6 -0
  3. package/README.md +307 -0
  4. package/README.zh.md +307 -0
  5. package/assets/computer-use-fixture.png +0 -0
  6. package/assets/cursor.png +0 -0
  7. package/cordis.patch.yml +13 -0
  8. package/docs/interaction-policy.i18n.yaml +4 -0
  9. package/docs/interaction-policy.md +140 -0
  10. package/docs/interaction-policy.zh.md +140 -0
  11. package/lib/approval-policy.js +12 -0
  12. package/lib/approval-policy.js.map +1 -0
  13. package/lib/artifacts.js +79 -0
  14. package/lib/artifacts.js.map +1 -0
  15. package/lib/backend.js +3 -0
  16. package/lib/backend.js.map +1 -0
  17. package/lib/client.js +349 -0
  18. package/lib/client.js.map +1 -0
  19. package/lib/config.js +120 -0
  20. package/lib/config.js.map +1 -0
  21. package/lib/confirmations.js +99 -0
  22. package/lib/confirmations.js.map +1 -0
  23. package/lib/diff.js +58 -0
  24. package/lib/diff.js.map +1 -0
  25. package/lib/errors.js +22 -0
  26. package/lib/errors.js.map +1 -0
  27. package/lib/exposure.js +231 -0
  28. package/lib/exposure.js.map +1 -0
  29. package/lib/index.js +56 -0
  30. package/lib/index.js.map +1 -0
  31. package/lib/leases.js +259 -0
  32. package/lib/leases.js.map +1 -0
  33. package/lib/providers/macos.js +148 -0
  34. package/lib/providers/macos.js.map +1 -0
  35. package/lib/providers/native-helper.js +356 -0
  36. package/lib/providers/native-helper.js.map +1 -0
  37. package/lib/service.js +480 -0
  38. package/lib/service.js.map +1 -0
  39. package/lib/skill.js +119 -0
  40. package/lib/skill.js.map +1 -0
  41. package/lib/target-resolver.js +180 -0
  42. package/lib/target-resolver.js.map +1 -0
  43. package/lib/tools.js +493 -0
  44. package/lib/tools.js.map +1 -0
  45. package/lib/types/approval-policy.d.ts +13 -0
  46. package/lib/types/approval-policy.d.ts.map +1 -0
  47. package/lib/types/artifacts.d.ts +10 -0
  48. package/lib/types/artifacts.d.ts.map +1 -0
  49. package/lib/types/backend.d.ts +94 -0
  50. package/lib/types/backend.d.ts.map +1 -0
  51. package/lib/types/client/index.d.ts +78 -0
  52. package/lib/types/client/index.d.ts.map +1 -0
  53. package/lib/types/config.d.ts +75 -0
  54. package/lib/types/config.d.ts.map +1 -0
  55. package/lib/types/confirmations.d.ts +23 -0
  56. package/lib/types/confirmations.d.ts.map +1 -0
  57. package/lib/types/diff.d.ts +5 -0
  58. package/lib/types/diff.d.ts.map +1 -0
  59. package/lib/types/errors.d.ts +15 -0
  60. package/lib/types/errors.d.ts.map +1 -0
  61. package/lib/types/exposure.d.ts +30 -0
  62. package/lib/types/exposure.d.ts.map +1 -0
  63. package/lib/types/index.d.ts +21 -0
  64. package/lib/types/index.d.ts.map +1 -0
  65. package/lib/types/leases.d.ts +80 -0
  66. package/lib/types/leases.d.ts.map +1 -0
  67. package/lib/types/providers/macos.d.ts +15 -0
  68. package/lib/types/providers/macos.d.ts.map +1 -0
  69. package/lib/types/providers/native-helper.d.ts +36 -0
  70. package/lib/types/providers/native-helper.d.ts.map +1 -0
  71. package/lib/types/service.d.ts +52 -0
  72. package/lib/types/service.d.ts.map +1 -0
  73. package/lib/types/skill.d.ts +9 -0
  74. package/lib/types/skill.d.ts.map +1 -0
  75. package/lib/types/target-resolver.d.ts +38 -0
  76. package/lib/types/target-resolver.d.ts.map +1 -0
  77. package/lib/types/tools.d.ts +6 -0
  78. package/lib/types/tools.d.ts.map +1 -0
  79. package/lib/types/types.d.ts +246 -0
  80. package/lib/types/types.d.ts.map +1 -0
  81. package/lib/types/web.d.ts +32 -0
  82. package/lib/types/web.d.ts.map +1 -0
  83. package/lib/types.js +9 -0
  84. package/lib/types.js.map +1 -0
  85. package/lib/web.js +163 -0
  86. package/lib/web.js.map +1 -0
  87. package/native/macos/Sources/Fixture/main.swift +366 -0
  88. package/native/macos/Sources/Helper/CursorImage.swift +11 -0
  89. package/native/macos/Sources/Helper/CursorOverlay.swift +488 -0
  90. package/native/macos/Sources/Helper/TargetedPointer.swift +314 -0
  91. package/native/macos/Sources/Helper/main.swift +1140 -0
  92. package/native/macos/Sources/Monitor/main.swift +139 -0
  93. package/native/macos/bin/dsh-computer-use-helper +0 -0
  94. package/native/macos/manifest.json +14 -0
  95. package/package.json +128 -0
  96. package/scripts/build-client.mjs +24 -0
  97. package/scripts/build-native.mjs +158 -0
  98. package/scripts/check-native.mjs +92 -0
  99. package/scripts/model-e2e.mjs +291 -0
  100. package/scripts/session-transcript.mjs +117 -0
  101. package/scripts/validate.mjs +256 -0
  102. package/src/approval-policy.ts +16 -0
  103. package/src/artifacts.ts +93 -0
  104. package/src/backend.ts +103 -0
  105. package/src/client/index.tsx +539 -0
  106. package/src/config.ts +196 -0
  107. package/src/confirmations.ts +128 -0
  108. package/src/diff.ts +52 -0
  109. package/src/errors.ts +39 -0
  110. package/src/exposure.ts +236 -0
  111. package/src/index.ts +66 -0
  112. package/src/leases.ts +337 -0
  113. package/src/providers/macos.ts +185 -0
  114. package/src/providers/native-helper.ts +393 -0
  115. package/src/service.ts +583 -0
  116. package/src/skill.ts +123 -0
  117. package/src/target-resolver.ts +237 -0
  118. package/src/tools.ts +528 -0
  119. package/src/types.ts +299 -0
  120. package/src/web.ts +180 -0
@@ -0,0 +1,1140 @@
1
+ import AppKit
2
+ import ApplicationServices
3
+ import CoreGraphics
4
+ import CryptoKit
5
+ import Darwin
6
+ import Foundation
7
+ import ScreenCaptureKit
8
+
9
+ private let helperVersion = "0.1.0"
10
+
11
+ private struct HelperError: Error {
12
+ let code: String
13
+ let message: String
14
+ }
15
+
16
+ private struct ElementRecord {
17
+ let index: Int
18
+ let locator: [Int]
19
+ let element: AXUIElement
20
+ let json: [String: Any]
21
+ let hashLine: String
22
+ }
23
+
24
+ private struct ObservationSnapshot {
25
+ let app: NSRunningApplication
26
+ let appJSON: [String: Any]
27
+ let frontmost: Bool
28
+ let window: AXUIElement?
29
+ let windowJSON: [String: Any]?
30
+ let stateHash: String
31
+ let treeText: String
32
+ let truncated: Bool
33
+ let elements: [ElementRecord]
34
+ }
35
+
36
+ private func fail(_ code: String, _ message: String) -> HelperError {
37
+ HelperError(code: code, message: message)
38
+ }
39
+
40
+ private func string(_ object: Any?, _ field: String) throws -> String {
41
+ guard let value = object as? String, !value.isEmpty else {
42
+ throw fail("COMPUTER_PROVIDER_FAILURE", "missing or invalid \(field)")
43
+ }
44
+ return value
45
+ }
46
+
47
+ private func stringAllowingEmpty(_ object: Any?, _ field: String) throws -> String {
48
+ guard let value = object as? String else {
49
+ throw fail("COMPUTER_PROVIDER_FAILURE", "missing or invalid \(field)")
50
+ }
51
+ return value
52
+ }
53
+
54
+ private func int(_ object: Any?, _ field: String) throws -> Int {
55
+ guard let number = object as? NSNumber else {
56
+ throw fail("COMPUTER_PROVIDER_FAILURE", "missing or invalid \(field)")
57
+ }
58
+ return number.intValue
59
+ }
60
+
61
+ private func double(_ object: Any?, _ field: String) throws -> Double {
62
+ guard let number = object as? NSNumber else {
63
+ throw fail("COMPUTER_PROVIDER_FAILURE", "missing or invalid \(field)")
64
+ }
65
+ return number.doubleValue
66
+ }
67
+
68
+ private func bool(_ object: Any?, default fallback: Bool = false) -> Bool {
69
+ (object as? NSNumber)?.boolValue ?? fallback
70
+ }
71
+
72
+ private func dictionary(_ object: Any?, _ field: String) throws -> [String: Any] {
73
+ guard let value = object as? [String: Any] else {
74
+ throw fail("COMPUTER_PROVIDER_FAILURE", "missing or invalid \(field)")
75
+ }
76
+ return value
77
+ }
78
+
79
+ private func dictionaries(_ object: Any?) -> [[String: Any]] {
80
+ object as? [[String: Any]] ?? []
81
+ }
82
+
83
+ private enum TransportEndpoint: Hashable {
84
+ case pipe(handle: UInt64, peer: UInt64)
85
+ case unixSocket(handle: UInt64, peer: UInt64)
86
+ }
87
+
88
+ private func processDescriptors(_ pid: Int32) -> [proc_fdinfo] {
89
+ let capacity = Int(max(proc_pidinfo(pid, PROC_PIDLISTFDS, 0, nil, 0), 0))
90
+ guard capacity >= MemoryLayout<proc_fdinfo>.size else { return [] }
91
+ var descriptors = Array(repeating: proc_fdinfo(), count: capacity / MemoryLayout<proc_fdinfo>.size)
92
+ let bytes = descriptors.withUnsafeMutableBytes { buffer in
93
+ proc_pidinfo(pid, PROC_PIDLISTFDS, 0, buffer.baseAddress, Int32(buffer.count))
94
+ }
95
+ guard bytes > 0 else { return [] }
96
+ return Array(descriptors.prefix(Int(bytes) / MemoryLayout<proc_fdinfo>.size))
97
+ }
98
+
99
+ private func transportEndpoint(_ pid: Int32, _ fd: Int32) -> TransportEndpoint? {
100
+ var pipe = pipe_fdinfo()
101
+ let pipeSize = MemoryLayout<pipe_fdinfo>.size
102
+ if proc_pidfdinfo(pid, fd, PROC_PIDFDPIPEINFO, &pipe, Int32(pipeSize)) == pipeSize {
103
+ return .pipe(handle: pipe.pipeinfo.pipe_handle, peer: pipe.pipeinfo.pipe_peerhandle)
104
+ }
105
+
106
+ var socket = socket_fdinfo()
107
+ let socketSize = MemoryLayout<socket_fdinfo>.size
108
+ guard proc_pidfdinfo(pid, fd, PROC_PIDFDSOCKETINFO, &socket, Int32(socketSize)) == socketSize,
109
+ socket.psi.soi_family == AF_UNIX,
110
+ socket.psi.soi_type == SOCK_STREAM,
111
+ socket.psi.soi_kind == SOCKINFO_UN else { return nil }
112
+ return .unixSocket(
113
+ handle: socket.psi.soi_so,
114
+ peer: socket.psi.soi_proto.pri_un.unsi_conn_so
115
+ )
116
+ }
117
+
118
+ private func transportEndpointsMatch(_ child: TransportEndpoint, _ parent: TransportEndpoint) -> Bool {
119
+ switch (child, parent) {
120
+ case let (.pipe(childHandle, childPeer), .pipe(parentHandle, parentPeer)),
121
+ let (.unixSocket(childHandle, childPeer), .unixSocket(parentHandle, parentPeer)):
122
+ return childPeer != 0
123
+ && childHandle == parentPeer
124
+ && childPeer == parentHandle
125
+ default:
126
+ return false
127
+ }
128
+ }
129
+
130
+ private func disconnectedUnixSocket(_ endpoint: TransportEndpoint) -> Bool {
131
+ if case let .unixSocket(_, peer) = endpoint { return peer == 0 }
132
+ return false
133
+ }
134
+
135
+ private func parentOwnsStandardTransport() -> Bool {
136
+ let childEndpoints = [STDIN_FILENO, STDOUT_FILENO, STDERR_FILENO].compactMap { fd in
137
+ transportEndpoint(getpid(), fd).map { (fd, $0) }
138
+ }
139
+ guard childEndpoints.count == 3,
140
+ Set(childEndpoints.map(\.1)).count == 3 else { return false }
141
+
142
+ let parentEndpoints = processDescriptors(getppid()).compactMap { descriptor in
143
+ transportEndpoint(getppid(), descriptor.proc_fd)
144
+ }
145
+ return childEndpoints.allSatisfy { fd, child in
146
+ let matches = parentEndpoints.filter { transportEndpointsMatch(child, $0) }
147
+ if matches.count == 1 { return true }
148
+ // The batch stdin writer may close its parent endpoint before the helper
149
+ // reaches main. Node represents that standard stream as a disconnected
150
+ // Unix socket, while stdout and stderr remain provably parent-owned.
151
+ return fd == STDIN_FILENO && matches.isEmpty && disconnectedUnixSocket(child)
152
+ }
153
+ }
154
+
155
+ private func requireHostTransport() throws {
156
+ guard getpgrp() == getpid(),
157
+ parentOwnsStandardTransport() else {
158
+ throw fail("COMPUTER_ACTION_BLOCKED", "native helper requires managed parent transport; use the registered Computer Use Tools")
159
+ }
160
+ }
161
+
162
+ private func permissionAccessibility() -> String {
163
+ AXIsProcessTrusted() ? "granted" : "denied"
164
+ }
165
+
166
+ private func permissionScreenRecording() -> String {
167
+ CGPreflightScreenCaptureAccess() ? "granted" : "denied"
168
+ }
169
+
170
+ private func runningApps() -> [NSRunningApplication] {
171
+ NSWorkspace.shared.runningApplications
172
+ .filter { !$0.isTerminated && $0.activationPolicy == .regular && $0.bundleIdentifier != nil }
173
+ .sorted {
174
+ let lhs = $0.localizedName ?? $0.bundleIdentifier ?? ""
175
+ let rhs = $1.localizedName ?? $1.bundleIdentifier ?? ""
176
+ return lhs.localizedCaseInsensitiveCompare(rhs) == .orderedAscending
177
+ }
178
+ }
179
+
180
+ private func appJSON(_ app: NSRunningApplication) throws -> [String: Any] {
181
+ guard let bundleId = app.bundleIdentifier else {
182
+ throw fail("COMPUTER_APP_NOT_FOUND", "running application has no bundle identifier")
183
+ }
184
+ return [
185
+ "bundleId": bundleId,
186
+ "pid": Int(app.processIdentifier),
187
+ "name": app.localizedName ?? bundleId,
188
+ ]
189
+ }
190
+
191
+ private func resolveApp(_ selector: [String: Any]) throws -> NSRunningApplication {
192
+ let bundleId = selector["bundleId"] as? String
193
+ let pid = (selector["pid"] as? NSNumber)?.int32Value
194
+ let name = selector["name"] as? String
195
+ if bundleId == nil && pid == nil && name == nil {
196
+ throw fail("COMPUTER_APP_NOT_FOUND", "app selector needs bundleId, pid, or unique name")
197
+ }
198
+ let matches = runningApps().filter { app in
199
+ if let bundleId, app.bundleIdentifier != bundleId { return false }
200
+ if let pid, app.processIdentifier != pid { return false }
201
+ if let name {
202
+ let display = app.localizedName ?? ""
203
+ if display.compare(name, options: [.caseInsensitive, .diacriticInsensitive]) != .orderedSame { return false }
204
+ }
205
+ return true
206
+ }
207
+ guard matches.count == 1, let app = matches.first else {
208
+ if matches.isEmpty { throw fail("COMPUTER_APP_NOT_FOUND", "no running application uniquely matches the selector") }
209
+ if bundleId != nil {
210
+ throw fail("COMPUTER_APP_NOT_FOUND", "multiple running processes match the bundleId; add the exact pid from computer_list_apps")
211
+ }
212
+ throw fail("COMPUTER_APP_NOT_FOUND", "application name is ambiguous; use bundleId and, if necessary, pid")
213
+ }
214
+ return app
215
+ }
216
+
217
+ private func axCopy(_ element: AXUIElement, _ attribute: CFString) -> Any? {
218
+ var value: CFTypeRef?
219
+ let result = AXUIElementCopyAttributeValue(element, attribute, &value)
220
+ return result == .success ? value : nil
221
+ }
222
+
223
+ private func axString(_ element: AXUIElement, _ attribute: CFString) -> String? {
224
+ if let value = axCopy(element, attribute) as? String, !value.isEmpty { return value }
225
+ if let value = axCopy(element, attribute) as? NSAttributedString, !value.string.isEmpty { return value.string }
226
+ return nil
227
+ }
228
+
229
+ private func axBool(_ element: AXUIElement, _ attribute: CFString) -> Bool? {
230
+ (axCopy(element, attribute) as? NSNumber)?.boolValue
231
+ }
232
+
233
+ private func axPoint(_ element: AXUIElement, _ attribute: CFString) -> CGPoint? {
234
+ guard let value = axCopy(element, attribute), CFGetTypeID(value as CFTypeRef) == AXValueGetTypeID() else { return nil }
235
+ let axValue = value as! AXValue
236
+ guard AXValueGetType(axValue) == .cgPoint else { return nil }
237
+ var point = CGPoint.zero
238
+ return AXValueGetValue(axValue, .cgPoint, &point) ? point : nil
239
+ }
240
+
241
+ private func axSize(_ element: AXUIElement, _ attribute: CFString) -> CGSize? {
242
+ guard let value = axCopy(element, attribute), CFGetTypeID(value as CFTypeRef) == AXValueGetTypeID() else { return nil }
243
+ let axValue = value as! AXValue
244
+ guard AXValueGetType(axValue) == .cgSize else { return nil }
245
+ var size = CGSize.zero
246
+ return AXValueGetValue(axValue, .cgSize, &size) ? size : nil
247
+ }
248
+
249
+ private func axFrame(_ element: AXUIElement) -> CGRect? {
250
+ guard let position = axPoint(element, kAXPositionAttribute as CFString),
251
+ let size = axSize(element, kAXSizeAttribute as CFString),
252
+ size.width >= 0, size.height >= 0 else { return nil }
253
+ return CGRect(origin: position, size: size)
254
+ }
255
+
256
+ private func rectJSON(_ rect: CGRect) -> [String: Any] {
257
+ ["x": rect.origin.x, "y": rect.origin.y, "width": rect.size.width, "height": rect.size.height]
258
+ }
259
+
260
+ private func axActions(_ element: AXUIElement) -> [String] {
261
+ var names: CFArray?
262
+ guard AXUIElementCopyActionNames(element, &names) == .success, let names else { return [] }
263
+ return (names as NSArray).compactMap { $0 as? String }.sorted()
264
+ }
265
+
266
+ private func axChildren(_ element: AXUIElement) -> [AXUIElement] {
267
+ guard let values = axCopy(element, kAXChildrenAttribute as CFString) as? [AnyObject] else { return [] }
268
+ return values.compactMap { value in
269
+ guard CFGetTypeID(value) == AXUIElementGetTypeID() else { return nil }
270
+ return (value as! AXUIElement)
271
+ }
272
+ }
273
+
274
+ private func pressWithDescendantFallback(_ element: AXUIElement) -> Bool {
275
+ // Only descend when the selected element advertised AXPress; otherwise the
276
+ // element/coordinate fallback below owns the click decision.
277
+ guard axActions(element).contains(kAXPressAction as String) else { return false }
278
+ var visited = Set<CFHashCode>()
279
+ return pressRecursive(element, remainingDepth: 4, visited: &visited)
280
+ }
281
+
282
+ private func pressRecursive(_ element: AXUIElement, remainingDepth: Int, visited: inout Set<CFHashCode>) -> Bool {
283
+ guard remainingDepth >= 0 else { return false }
284
+ let identity = CFHash(element)
285
+ guard visited.insert(identity).inserted else { return false }
286
+ if axActions(element).contains(kAXPressAction as String),
287
+ AXUIElementPerformAction(element, kAXPressAction as CFString) == .success {
288
+ return true
289
+ }
290
+ for child in axChildren(element) {
291
+ if pressRecursive(child, remainingDepth: remainingDepth - 1, visited: &visited) { return true }
292
+ }
293
+ return false
294
+ }
295
+
296
+ private func sanitize(_ value: String, max: Int = 240) -> String {
297
+ let collapsed = value.replacingOccurrences(of: "\\s+", with: " ", options: .regularExpression).trimmingCharacters(in: .whitespacesAndNewlines)
298
+ guard collapsed.count > max else { return collapsed }
299
+ return String(collapsed.prefix(max)) + "..."
300
+ }
301
+
302
+ private func jsonString(_ value: String) -> String {
303
+ let data = try? JSONSerialization.data(withJSONObject: [value])
304
+ let encoded = data.flatMap { String(data: $0, encoding: .utf8) } ?? "[\"\"]"
305
+ return String(encoded.dropFirst().dropLast())
306
+ }
307
+
308
+ private func chosenWindow(_ appElement: AXUIElement) -> AXUIElement? {
309
+ if let focused = axCopy(appElement, kAXFocusedWindowAttribute as CFString) as AnyObject?,
310
+ CFGetTypeID(focused) == AXUIElementGetTypeID() {
311
+ return (focused as! AXUIElement)
312
+ }
313
+ if let main = axCopy(appElement, kAXMainWindowAttribute as CFString) as AnyObject?,
314
+ CFGetTypeID(main) == AXUIElementGetTypeID() {
315
+ return (main as! AXUIElement)
316
+ }
317
+ return axChildren(appElement).first
318
+ }
319
+
320
+ private func windowNumber(_ element: AXUIElement) -> Int? {
321
+ (axCopy(element, "AXWindowNumber" as CFString) as? NSNumber)?.intValue
322
+ }
323
+
324
+ private func windowNumber(
325
+ app: NSRunningApplication,
326
+ frame: CGRect?,
327
+ title: String?
328
+ ) -> Int? {
329
+ guard let windows = CGWindowListCopyWindowInfo(
330
+ [.optionOnScreenOnly, .excludeDesktopElements],
331
+ kCGNullWindowID
332
+ ) as? [[String: Any]] else { return nil }
333
+ let candidates = windows.filter { window in
334
+ guard (window[kCGWindowOwnerPID as String] as? NSNumber)?.int32Value == app.processIdentifier,
335
+ (window[kCGWindowLayer as String] as? NSNumber)?.intValue == 0 else { return false }
336
+ if let frame {
337
+ guard let bounds = window[kCGWindowBounds as String] as? [String: Any],
338
+ let candidate = CGRect(dictionaryRepresentation: bounds as CFDictionary) else { return false }
339
+ let tolerance: CGFloat = 2
340
+ guard abs(candidate.minX - frame.minX) <= tolerance,
341
+ abs(candidate.minY - frame.minY) <= tolerance,
342
+ abs(candidate.width - frame.width) <= tolerance,
343
+ abs(candidate.height - frame.height) <= tolerance else { return false }
344
+ }
345
+ if let title, !title.isEmpty {
346
+ guard let candidateTitle = window[kCGWindowName as String] as? String,
347
+ candidateTitle == title else { return false }
348
+ }
349
+ return true
350
+ }
351
+ guard candidates.count == 1 else { return nil }
352
+ return (candidates[0][kCGWindowNumber as String] as? NSNumber)?.intValue
353
+ }
354
+
355
+ private func sha256(_ value: String) -> String {
356
+ SHA256.hash(data: Data(value.utf8)).map { String(format: "%02x", $0) }.joined()
357
+ }
358
+
359
+ private func observeSnapshot(app: NSRunningApplication, limits: [String: Any]) throws -> ObservationSnapshot {
360
+ guard AXIsProcessTrusted() else {
361
+ throw fail("COMPUTER_PERMISSION_REQUIRED", "macOS Accessibility permission is required for the DSH helper")
362
+ }
363
+ let maxNodes = try int(limits["maxNodes"], "limits.maxNodes")
364
+ let maxDepth = try int(limits["maxDepth"], "limits.maxDepth")
365
+ let maxTextBytes = try int(limits["maxTextBytes"], "limits.maxTextBytes")
366
+ guard maxNodes > 0, maxDepth > 0, maxTextBytes > 0 else {
367
+ throw fail("COMPUTER_PROVIDER_FAILURE", "observation limits must be positive")
368
+ }
369
+ let appElement = AXUIElementCreateApplication(app.processIdentifier)
370
+ let rootWindow = chosenWindow(appElement)
371
+ let root = rootWindow ?? appElement
372
+ let frame = rootWindow.flatMap(axFrame)
373
+ let title = rootWindow.flatMap { axString($0, kAXTitleAttribute as CFString) }
374
+ let windowId = rootWindow.flatMap(windowNumber)
375
+ ?? windowNumber(app: app, frame: frame, title: title)
376
+ var windowJSON: [String: Any]?
377
+ if let frame {
378
+ var json: [String: Any] = ["frame": rectJSON(frame)]
379
+ if let title { json["title"] = title }
380
+ if let windowId { json["id"] = windowId }
381
+ windowJSON = json
382
+ }
383
+
384
+ var queue: [(AXUIElement, [Int], Int)] = [(root, [], 0)]
385
+ var cursor = 0
386
+ var seen = Set<CFHashCode>()
387
+ var records: [ElementRecord] = []
388
+ var lines: [String] = []
389
+ var textBytes = 0
390
+ var truncated = false
391
+
392
+ while cursor < queue.count {
393
+ if records.count >= maxNodes { truncated = true; break }
394
+ let (element, locator, depth) = queue[cursor]
395
+ cursor += 1
396
+ let identity = CFHash(element)
397
+ if seen.contains(identity) { continue }
398
+ seen.insert(identity)
399
+ let role = axString(element, kAXRoleAttribute as CFString) ?? "AXUnknown"
400
+ let subrole = axString(element, kAXSubroleAttribute as CFString)
401
+ let nativeIdentifier = axString(element, kAXIdentifierAttribute as CFString)
402
+ let titleValue = axString(element, kAXTitleAttribute as CFString)
403
+ let descriptionValue = axString(element, kAXDescriptionAttribute as CFString)
404
+ let labelValue = titleValue ?? descriptionValue
405
+ let secure = role == "AXSecureTextField" || subrole == "AXSecureTextField"
406
+ let valueValue: String? = secure ? "[secure]" : {
407
+ guard let raw = axCopy(element, kAXValueAttribute as CFString) else { return nil }
408
+ if let string = raw as? String { return sanitize(string) }
409
+ if let number = raw as? NSNumber { return number.stringValue }
410
+ return nil
411
+ }()
412
+ let enabled = axBool(element, kAXEnabledAttribute as CFString)
413
+ let focused = axBool(element, kAXFocusedAttribute as CFString)
414
+ let selected = axBool(element, kAXSelectedAttribute as CFString)
415
+ let elementFrame = axFrame(element)
416
+ let actions = axActions(element)
417
+ let index = records.count
418
+ var json: [String: Any] = [
419
+ "index": index,
420
+ "role": role,
421
+ "actions": actions,
422
+ "locator": locator,
423
+ ]
424
+ if let subrole { json["subrole"] = subrole }
425
+ if let nativeIdentifier { json["nativeIdentifier"] = sanitize(nativeIdentifier) }
426
+ if let titleValue { json["title"] = sanitize(titleValue) }
427
+ if let labelValue { json["label"] = sanitize(labelValue) }
428
+ if let valueValue { json["value"] = valueValue }
429
+ if let enabled { json["enabled"] = enabled }
430
+ if let focused { json["focused"] = focused }
431
+ if let selected { json["selected"] = selected }
432
+ if let elementFrame { json["frame"] = rectJSON(elementFrame) }
433
+ let frameKey = elementFrame.map { "\(Int($0.origin.x)),\(Int($0.origin.y)),\(Int($0.width)),\(Int($0.height))" } ?? ""
434
+ let hashLine = [locator.map(String.init).joined(separator: "."), nativeIdentifier ?? "", role, subrole ?? "", titleValue ?? "", labelValue ?? "", valueValue ?? "", String(enabled ?? true), String(focused ?? false), String(selected ?? false), frameKey, actions.joined(separator: ",")].joined(separator: "|")
435
+ var line = String(repeating: " ", count: min(depth, 20)) + "[\(index)] \(role)"
436
+ if let titleValue { line += " \(jsonString(sanitize(titleValue)))" }
437
+ else if let labelValue { line += " \(jsonString(sanitize(labelValue)))" }
438
+ if let valueValue { line += " value=\(jsonString(valueValue))" }
439
+ if enabled == false { line += " disabled" }
440
+ if focused == true { line += " focused" }
441
+ if selected == true { line += " selected" }
442
+ let lineBytes = line.lengthOfBytes(using: .utf8) + (lines.isEmpty ? 0 : 1)
443
+ if textBytes + lineBytes <= maxTextBytes {
444
+ lines.append(line)
445
+ textBytes += lineBytes
446
+ } else {
447
+ truncated = true
448
+ }
449
+ records.append(ElementRecord(index: index, locator: locator, element: element, json: json, hashLine: hashLine))
450
+ if depth >= maxDepth {
451
+ if !axChildren(element).isEmpty { truncated = true }
452
+ continue
453
+ }
454
+ for (childIndex, child) in axChildren(element).enumerated() {
455
+ queue.append((child, locator + [childIndex], depth + 1))
456
+ }
457
+ }
458
+ let appData = try appJSON(app)
459
+ let windowHash = [
460
+ title ?? "",
461
+ windowId.map(String.init) ?? "",
462
+ frame.map { "\(Int($0.origin.x)),\(Int($0.origin.y)),\(Int($0.width)),\(Int($0.height))" } ?? "",
463
+ ].joined(separator: "|")
464
+ let state = [appData["bundleId"] as? String ?? "", String(app.processIdentifier), windowHash, records.map(\.hashLine).joined(separator: "\n")].joined(separator: "\n")
465
+ return ObservationSnapshot(
466
+ app: app,
467
+ appJSON: appData,
468
+ frontmost: app.isActive,
469
+ window: rootWindow,
470
+ windowJSON: windowJSON,
471
+ stateHash: sha256(state),
472
+ treeText: lines.joined(separator: "\n"),
473
+ truncated: truncated,
474
+ elements: records
475
+ )
476
+ }
477
+
478
+ private func captureWindow(_ snapshot: ObservationSnapshot, path: String, required: Bool) async throws -> [String: Any]? {
479
+ guard CGPreflightScreenCaptureAccess() else {
480
+ if required { throw fail("COMPUTER_PERMISSION_REQUIRED", "macOS Screen Recording permission is required for screenshots") }
481
+ return nil
482
+ }
483
+ let content: SCShareableContent
484
+ do {
485
+ content = try await SCShareableContent.excludingDesktopWindows(false, onScreenWindowsOnly: true)
486
+ } catch {
487
+ if required { throw fail("COMPUTER_PROVIDER_FAILURE", "ScreenCaptureKit could not enumerate windows") }
488
+ return nil
489
+ }
490
+ let expectedId = (snapshot.windowJSON?["id"] as? NSNumber)?.uint32Value
491
+ let expectedTitle = snapshot.windowJSON?["title"] as? String
492
+ let windows = content.windows.filter { $0.owningApplication?.processID == snapshot.app.processIdentifier }
493
+ let selected = windows.first { window in
494
+ if let expectedId, window.windowID == expectedId { return true }
495
+ if let expectedTitle, !expectedTitle.isEmpty, window.title == expectedTitle { return true }
496
+ return false
497
+ } ?? windows.max { $0.frame.width * $0.frame.height < $1.frame.width * $1.frame.height }
498
+ guard let selected else {
499
+ if required { throw fail("COMPUTER_TARGET_UNAVAILABLE", "no capturable window belongs to the selected application") }
500
+ return nil
501
+ }
502
+ let configuration = SCStreamConfiguration()
503
+ configuration.width = max(1, Int(selected.frame.width.rounded()))
504
+ configuration.height = max(1, Int(selected.frame.height.rounded()))
505
+ configuration.showsCursor = false
506
+ let filter = SCContentFilter(desktopIndependentWindow: selected)
507
+ let image: CGImage
508
+ do {
509
+ image = try await SCScreenshotManager.captureImage(contentFilter: filter, configuration: configuration)
510
+ } catch {
511
+ if required { throw fail("COMPUTER_PROVIDER_FAILURE", "ScreenCaptureKit could not capture the selected window") }
512
+ return nil
513
+ }
514
+ let representation = NSBitmapImageRep(cgImage: image)
515
+ guard let data = representation.representation(using: .png, properties: [:]) else {
516
+ throw fail("COMPUTER_PROVIDER_FAILURE", "captured window could not be encoded as PNG")
517
+ }
518
+ do {
519
+ try data.write(to: URL(fileURLWithPath: path), options: .atomic)
520
+ } catch {
521
+ throw fail("COMPUTER_PROVIDER_FAILURE", "captured PNG could not be written")
522
+ }
523
+ return ["path": path, "width": image.width, "height": image.height]
524
+ }
525
+
526
+ private func observationJSON(_ snapshot: ObservationSnapshot, screenshot: [String: Any]?) -> [String: Any] {
527
+ var result: [String: Any] = [
528
+ "app": snapshot.appJSON,
529
+ "stateHash": snapshot.stateHash,
530
+ "frontmost": snapshot.frontmost,
531
+ "treeText": snapshot.treeText,
532
+ "truncated": snapshot.truncated,
533
+ "elements": snapshot.elements.map(\.json),
534
+ "permissions": [
535
+ "accessibility": permissionAccessibility(),
536
+ "screenRecording": permissionScreenRecording(),
537
+ ],
538
+ ]
539
+ if let window = snapshot.windowJSON { result["window"] = window }
540
+ if let screenshot { result["screenshot"] = screenshot }
541
+ return result
542
+ }
543
+
544
+ private func activate(_ app: NSRunningApplication, timeoutMs: Int) throws {
545
+ _ = app.activate(options: [.activateAllWindows])
546
+ let deadline = DispatchTime.now().uptimeNanoseconds + UInt64(timeoutMs) * 1_000_000
547
+ while !app.isActive {
548
+ if DispatchTime.now().uptimeNanoseconds >= deadline {
549
+ throw fail("COMPUTER_ACTION_BLOCKED", "the selected application did not become frontmost before the action timeout")
550
+ }
551
+ usleep(10_000)
552
+ }
553
+ }
554
+
555
+ private func targetedEventSource() throws -> CGEventSource {
556
+ guard let source = CGEventSource(stateID: .privateState) else {
557
+ throw fail("COMPUTER_ACTION_BLOCKED", "CoreGraphics input event source is unavailable")
558
+ }
559
+ return source
560
+ }
561
+
562
+ private func cgRect(_ frame: [String: Any]) throws -> CGRect {
563
+ CGRect(
564
+ x: try double(frame["x"], "window.x"),
565
+ y: try double(frame["y"], "window.y"),
566
+ width: try double(frame["width"], "window.width"),
567
+ height: try double(frame["height"], "window.height")
568
+ )
569
+ }
570
+
571
+ private func windowAtPoint(app: NSRunningApplication, point: CGPoint) throws -> (windowNumber: Int64, frame: CGRect) {
572
+ guard let windows = CGWindowListCopyWindowInfo(
573
+ [.optionOnScreenOnly, .excludeDesktopElements],
574
+ kCGNullWindowID
575
+ ) as? [[String: Any]] else {
576
+ throw fail("COMPUTER_TARGET_UNAVAILABLE", "CoreGraphics window list is unavailable")
577
+ }
578
+ // CGWindowListCopyWindowInfo orders on-screen windows front to back, so the
579
+ // first match is the topmost window of the selected app at the point.
580
+ guard let window = windows.first(where: { candidate in
581
+ guard (candidate[kCGWindowOwnerPID as String] as? NSNumber)?.int32Value == app.processIdentifier,
582
+ (candidate[kCGWindowLayer as String] as? NSNumber)?.intValue == 0,
583
+ let bounds = candidate[kCGWindowBounds as String] as? [String: Any],
584
+ let frame = CGRect(dictionaryRepresentation: bounds as CFDictionary),
585
+ frame.contains(point) else { return false }
586
+ return true
587
+ }),
588
+ let windowNumber = (window[kCGWindowNumber as String] as? NSNumber)?.int64Value,
589
+ let bounds = window[kCGWindowBounds as String] as? [String: Any],
590
+ let frame = CGRect(dictionaryRepresentation: bounds as CFDictionary) else {
591
+ throw fail("COMPUTER_TARGET_UNAVAILABLE", "no on-screen window of the selected app contains the requested coordinate")
592
+ }
593
+ return (windowNumber, frame)
594
+ }
595
+
596
+ private func pointerTarget(app: NSRunningApplication, window: [String: Any]?, at point: CGPoint) throws -> TargetedPointerTarget {
597
+ if let windowNumber = (window?["id"] as? NSNumber)?.int64Value,
598
+ let frame = window?["frame"] as? [String: Any],
599
+ let windowFrame = try? cgRect(frame),
600
+ windowFrame.contains(point) {
601
+ return TargetedPointerTarget(
602
+ pid: app.processIdentifier,
603
+ windowNumber: windowNumber,
604
+ windowFrame: windowFrame
605
+ )
606
+ }
607
+ let resolved = try windowAtPoint(app: app, point: point)
608
+ return TargetedPointerTarget(
609
+ pid: app.processIdentifier,
610
+ windowNumber: resolved.windowNumber,
611
+ windowFrame: resolved.frame
612
+ )
613
+ }
614
+
615
+ private func pointerAction(_ body: () throws -> Void) throws {
616
+ do {
617
+ try body()
618
+ } catch let error as TargetedPointerError {
619
+ throw fail("COMPUTER_ACTION_BLOCKED", error.message)
620
+ }
621
+ }
622
+
623
+ private func windowPoint(_ action: [String: Any], window: [String: Any]?, xKey: String, yKey: String, coordinateSpace: String = "window") throws -> CGPoint {
624
+ let x = try double(action[xKey], "action.\(xKey)")
625
+ let y = try double(action[yKey], "action.\(yKey)")
626
+ if coordinateSpace == "screen" {
627
+ return CGPoint(x: x, y: y)
628
+ }
629
+ guard coordinateSpace == "window" else {
630
+ throw fail("COMPUTER_PROVIDER_FAILURE", "action coordinateSpace must be one of window, screen")
631
+ }
632
+ guard let frame = window?["frame"] as? [String: Any] else {
633
+ throw fail("COMPUTER_TARGET_UNAVAILABLE", "the observation has no current window coordinate space")
634
+ }
635
+ let width = try double(frame["width"], "window.width")
636
+ let height = try double(frame["height"], "window.height")
637
+ guard x >= 0, y >= 0, x <= width, y <= height else {
638
+ throw fail("COMPUTER_TARGET_UNAVAILABLE", "coordinate is outside the observed window")
639
+ }
640
+ let origin = try cgRect(frame)
641
+ return CGPoint(x: origin.origin.x + x, y: origin.origin.y + y)
642
+ }
643
+
644
+ private func mouseButton(_ value: String?) throws -> CGMouseButton {
645
+ switch value ?? "left" {
646
+ case "left": return .left
647
+ case "right": return .right
648
+ case "middle": return .center
649
+ default: throw fail("COMPUTER_PROVIDER_FAILURE", "unsupported mouse button")
650
+ }
651
+ }
652
+
653
+ private let keyCodes: [String: CGKeyCode] = [
654
+ "a": 0, "s": 1, "d": 2, "f": 3, "h": 4, "g": 5, "z": 6, "x": 7, "c": 8, "v": 9,
655
+ "b": 11, "q": 12, "w": 13, "e": 14, "r": 15, "y": 16, "t": 17, "1": 18, "2": 19,
656
+ "3": 20, "4": 21, "6": 22, "5": 23, "=": 24, "9": 25, "7": 26, "-": 27, "8": 28,
657
+ "0": 29, "]": 30, "o": 31, "u": 32, "[": 33, "i": 34, "p": 35, "return": 36,
658
+ "l": 37, "j": 38, "'": 39, "k": 40, ";": 41, "\\": 42, ",": 43, "/": 44, "n": 45,
659
+ "m": 46, ".": 47, "tab": 48, "space": 49, "delete": 51, "escape": 53,
660
+ "home": 115, "pageup": 116, "forwarddelete": 117, "end": 119, "pagedown": 121,
661
+ "left": 123, "right": 124, "down": 125, "up": 126,
662
+ ]
663
+
664
+ private func flags(_ modifiers: [String]) throws -> CGEventFlags {
665
+ var result: CGEventFlags = []
666
+ for modifier in modifiers {
667
+ switch modifier {
668
+ case "command": result.insert(.maskCommand)
669
+ case "control": result.insert(.maskControl)
670
+ case "option": result.insert(.maskAlternate)
671
+ case "shift": result.insert(.maskShift)
672
+ default: throw fail("COMPUTER_PROVIDER_FAILURE", "unsupported key modifier")
673
+ }
674
+ }
675
+ return result
676
+ }
677
+
678
+ private func pressKey(_ key: String, modifiers: [String], app: NSRunningApplication) throws {
679
+ let normalized = key.lowercased()
680
+ guard let code = keyCodes[normalized] else {
681
+ throw fail("COMPUTER_PROVIDER_FAILURE", "unsupported key; use the documented key vocabulary")
682
+ }
683
+ let source = try targetedEventSource()
684
+ guard let down = CGEvent(keyboardEventSource: source, virtualKey: code, keyDown: true),
685
+ let up = CGEvent(keyboardEventSource: source, virtualKey: code, keyDown: false) else {
686
+ throw fail("COMPUTER_ACTION_BLOCKED", "CoreGraphics could not create keyboard events")
687
+ }
688
+ let eventFlags = try flags(modifiers)
689
+ down.flags = eventFlags
690
+ up.flags = eventFlags
691
+ down.postToPid(app.processIdentifier)
692
+ usleep(5_000)
693
+ up.postToPid(app.processIdentifier)
694
+ }
695
+
696
+ private func focusedElement(_ app: NSRunningApplication) -> AXUIElement? {
697
+ let appElement = AXUIElementCreateApplication(app.processIdentifier)
698
+ guard let focused = axCopy(appElement, kAXFocusedUIElementAttribute as CFString) as AnyObject?,
699
+ CFGetTypeID(focused) == AXUIElementGetTypeID() else { return nil }
700
+ return (focused as! AXUIElement)
701
+ }
702
+
703
+ private func setSelectedText(_ text: String, app: NSRunningApplication) -> Bool {
704
+ guard let focused = focusedElement(app) else { return false }
705
+ return AXUIElementSetAttributeValue(focused, kAXSelectedTextAttribute as CFString, text as CFTypeRef) == .success
706
+ }
707
+
708
+ private func typeTextWithKeyboard(_ text: String, app: NSRunningApplication) throws {
709
+ let source = try targetedEventSource()
710
+ guard let down = CGEvent(keyboardEventSource: source, virtualKey: 0, keyDown: true),
711
+ let up = CGEvent(keyboardEventSource: source, virtualKey: 0, keyDown: false) else {
712
+ throw fail("COMPUTER_ACTION_BLOCKED", "CoreGraphics could not create Unicode keyboard events")
713
+ }
714
+ let units = Array(text.utf16)
715
+ units.withUnsafeBufferPointer { buffer in
716
+ down.keyboardSetUnicodeString(stringLength: units.count, unicodeString: buffer.baseAddress!)
717
+ up.keyboardSetUnicodeString(stringLength: units.count, unicodeString: buffer.baseAddress!)
718
+ }
719
+ down.postToPid(app.processIdentifier)
720
+ usleep(5_000)
721
+ up.postToPid(app.processIdentifier)
722
+ }
723
+
724
+ private func targetRecord(_ request: [String: Any], snapshot: ObservationSnapshot) throws -> ElementRecord? {
725
+ guard let element = request["element"] as? [String: Any] else { return nil }
726
+ let locator = (element["locator"] as? [NSNumber])?.map(\.intValue) ?? []
727
+ guard let record = snapshot.elements.first(where: { $0.locator == locator }) else {
728
+ throw fail("COMPUTER_STALE_OBSERVATION", "the target element locator is no longer present")
729
+ }
730
+ return record
731
+ }
732
+
733
+ private func selectedIdentity(_ source: [String: Any], keys: [String]) -> [AnyHashable: Any] {
734
+ var identity: [AnyHashable: Any] = [:]
735
+ for key in keys {
736
+ if let value = source[key] { identity[key] = value }
737
+ }
738
+ return identity
739
+ }
740
+
741
+ private func targetMatches(_ expected: [String: Any], current: ElementRecord) -> Bool {
742
+ let keys = ["nativeIdentifier", "role", "subrole", "title", "label", "value", "enabled", "selected", "frame", "actions"]
743
+ return NSDictionary(dictionary: selectedIdentity(expected, keys: keys))
744
+ .isEqual(to: selectedIdentity(current.json, keys: keys))
745
+ }
746
+
747
+ private func windowMatches(_ expected: [String: Any]?, current: [String: Any]?) -> Bool {
748
+ guard let expected else { return current == nil }
749
+ guard let current else { return false }
750
+ let keys = ["id", "title", "frame"]
751
+ return NSDictionary(dictionary: selectedIdentity(expected, keys: keys))
752
+ .isEqual(to: selectedIdentity(current, keys: keys))
753
+ }
754
+
755
+ private func activationStateMatches(_ before: ObservationSnapshot, current: ObservationSnapshot) -> Bool {
756
+ guard windowMatches(before.windowJSON, current: current.windowJSON),
757
+ before.elements.count == current.elements.count else { return false }
758
+ return zip(before.elements, current.elements).allSatisfy { expected, actual in
759
+ expected.locator == actual.locator && targetMatches(expected.json, current: actual)
760
+ }
761
+ }
762
+
763
+ private func validateTarget(_ request: [String: Any], snapshot: ObservationSnapshot) throws -> ElementRecord? {
764
+ let record = try targetRecord(request, snapshot: snapshot)
765
+ if let record, let expectedElement = request["element"] as? [String: Any] {
766
+ guard windowMatches(request["window"] as? [String: Any], current: snapshot.windowJSON),
767
+ targetMatches(expectedElement, current: record) else {
768
+ throw fail("COMPUTER_STALE_OBSERVATION", "the target element or window changed after the referenced observation")
769
+ }
770
+ } else {
771
+ let expected = try string(request["expectedStateHash"], "request.expectedStateHash")
772
+ guard snapshot.stateHash == expected else {
773
+ throw fail("COMPUTER_STALE_OBSERVATION", "the application UI changed after the referenced observation")
774
+ }
775
+ }
776
+ return record
777
+ }
778
+
779
+ private func interactionValue(_ interaction: [String: Any], _ key: String, _ allowed: [String]) throws -> String {
780
+ let value = try string(interaction[key], "request.interaction.\(key)")
781
+ guard allowed.contains(value) else {
782
+ throw fail("COMPUTER_PROVIDER_FAILURE", "request.interaction.\(key) must be one of \(allowed.joined(separator: ", "))")
783
+ }
784
+ return value
785
+ }
786
+
787
+ private func requirePointerInput(_ policy: String) throws {
788
+ guard policy == "targeted" else {
789
+ throw fail("COMPUTER_ACTION_BLOCKED", "this action needs target-process pointer input, but interaction.pointerInputPolicy is deny")
790
+ }
791
+ }
792
+
793
+ private func requiresForegroundPermission(_ action: String) -> Bool {
794
+ action == (kAXRaiseAction as String)
795
+ }
796
+
797
+ private func inputContext(
798
+ app: NSRunningApplication,
799
+ request: [String: Any],
800
+ limits: [String: Any],
801
+ snapshot: ObservationSnapshot,
802
+ record: ElementRecord?,
803
+ focusPolicy: String,
804
+ keyboardPolicy: String,
805
+ actionKind: String,
806
+ timeoutMs: Int,
807
+ ) throws -> (snapshot: ObservationSnapshot, record: ElementRecord?, activation: String) {
808
+ let keyboardAction = actionKind == "type-text" || actionKind == "press-key"
809
+ let effectiveFocusPolicy = keyboardAction && keyboardPolicy == "activate" ? "activate" : focusPolicy
810
+ guard effectiveFocusPolicy == "activate" else {
811
+ return (snapshot, record, "not-requested")
812
+ }
813
+ if app.isActive {
814
+ return (snapshot, record, "already-frontmost")
815
+ }
816
+ try activate(app, timeoutMs: timeoutMs)
817
+ let refreshed = try observeSnapshot(app: app, limits: limits)
818
+ let refreshedRecord: ElementRecord?
819
+ if request["element"] != nil {
820
+ refreshedRecord = try validateTarget(request, snapshot: refreshed)
821
+ } else if keyboardAction {
822
+ // Activation may move focus to the app's default control; typing targets
823
+ // the refreshed focused element. The app identity is already revalidated
824
+ // by the fresh observation, so full pre-activation state equality would
825
+ // only make reliable keyboard input fail.
826
+ refreshedRecord = nil
827
+ } else {
828
+ guard activationStateMatches(snapshot, current: refreshed) else {
829
+ throw fail("COMPUTER_STALE_OBSERVATION", "the application UI changed while the target application was activated")
830
+ }
831
+ refreshedRecord = nil
832
+ }
833
+ return (refreshed, refreshedRecord, "activated")
834
+ }
835
+
836
+ private func actionResult(
837
+ channel: String,
838
+ activation: String,
839
+ pointerInput: Bool,
840
+ pointerRouting: String = "none"
841
+ ) -> [String: Any] {
842
+ [
843
+ "channel": channel,
844
+ "activation": activation,
845
+ "pointerInput": pointerInput,
846
+ "pointerRouting": pointerRouting,
847
+ ]
848
+ }
849
+
850
+ private func performAction(_ request: [String: Any]) throws -> [String: Any] {
851
+ let appIdentity = try dictionary(request["app"], "request.app")
852
+ let selector: [String: Any] = [
853
+ "bundleId": try string(appIdentity["bundleId"], "app.bundleId"),
854
+ "pid": try int(appIdentity["pid"], "app.pid"),
855
+ ]
856
+ let app = try resolveApp(selector)
857
+ let limits = try dictionary(request["limits"], "request.limits")
858
+ let snapshot = try observeSnapshot(app: app, limits: limits)
859
+ let action = try dictionary(request["action"], "request.action")
860
+ let kind = try string(action["kind"], "action.kind")
861
+ let actionTimeoutMs = try int(request["actionTimeoutMs"], "request.actionTimeoutMs")
862
+ guard actionTimeoutMs >= 1_000 && actionTimeoutMs <= 120_000 else {
863
+ throw fail("COMPUTER_PROVIDER_FAILURE", "request.actionTimeoutMs must be between 1000 and 120000")
864
+ }
865
+ let interaction = try dictionary(request["interaction"], "request.interaction")
866
+ let focusPolicy = try interactionValue(interaction, "focusPolicy", ["preserve", "activate"])
867
+ let keyboardPolicy = try interactionValue(interaction, "keyboardPolicy", ["preserve", "activate"])
868
+ let pointerInputPolicy = try interactionValue(interaction, "pointerInputPolicy", ["deny", "targeted"])
869
+ let record = try validateTarget(request, snapshot: snapshot)
870
+
871
+ switch kind {
872
+ case "click":
873
+ if let record, pressWithDescendantFallback(record.element) {
874
+ return actionResult(channel: "accessibility", activation: "not-requested", pointerInput: false)
875
+ }
876
+ let elementFallback = record != nil && bool(action["allowCoordinateFallback"])
877
+ let coordinateFallback = action["x"] != nil && action["y"] != nil
878
+ guard elementFallback || coordinateFallback else {
879
+ throw fail("COMPUTER_ELEMENT_UNAVAILABLE", "element does not support an actionable AXPress and coordinate fallback was not requested")
880
+ }
881
+ try requirePointerInput(pointerInputPolicy)
882
+ let current = try inputContext(
883
+ app: app,
884
+ request: request,
885
+ limits: limits,
886
+ snapshot: snapshot,
887
+ record: record,
888
+ focusPolicy: focusPolicy,
889
+ keyboardPolicy: keyboardPolicy,
890
+ actionKind: kind,
891
+ timeoutMs: actionTimeoutMs,
892
+ )
893
+ if let record = current.record, elementFallback, let frame = axFrame(record.element) {
894
+ let point = CGPoint(x: frame.midX, y: frame.midY)
895
+ let target = try pointerTarget(app: app, window: current.snapshot.windowJSON, at: point)
896
+ try pointerAction {
897
+ try targetedClick(
898
+ at: point,
899
+ button: try mouseButton(action["button"] as? String),
900
+ count: min(max((action["clickCount"] as? NSNumber)?.intValue ?? 1, 1), 3),
901
+ target: target
902
+ )
903
+ }
904
+ return actionResult(channel: "coordinates", activation: current.activation, pointerInput: true, pointerRouting: "target-process")
905
+ }
906
+ if action["x"] != nil, action["y"] != nil {
907
+ let coordinateSpace = action["coordinateSpace"] as? String ?? "window"
908
+ let point = try windowPoint(action, window: current.snapshot.windowJSON, xKey: "x", yKey: "y", coordinateSpace: coordinateSpace)
909
+ let target = try pointerTarget(app: app, window: current.snapshot.windowJSON, at: point)
910
+ try pointerAction {
911
+ try targetedClick(
912
+ at: point,
913
+ button: try mouseButton(action["button"] as? String),
914
+ count: min(max((action["clickCount"] as? NSNumber)?.intValue ?? 1, 1), 3),
915
+ target: target
916
+ )
917
+ }
918
+ return actionResult(channel: "coordinates", activation: current.activation, pointerInput: true, pointerRouting: "target-process")
919
+ }
920
+ throw fail("COMPUTER_ELEMENT_UNAVAILABLE", "coordinate fallback was requested but no current target coordinates were available")
921
+ case "set-value":
922
+ guard let record else { throw fail("COMPUTER_ELEMENT_UNAVAILABLE", "set-value requires an observed element") }
923
+ let value = try stringAllowingEmpty(action["value"], "action.value")
924
+ let result = AXUIElementSetAttributeValue(record.element, kAXValueAttribute as CFString, value as CFTypeRef)
925
+ guard result == .success else { throw fail("COMPUTER_ACTION_BLOCKED", "Accessibility value assignment was rejected") }
926
+ return actionResult(channel: "accessibility", activation: "not-requested", pointerInput: false)
927
+ case "type-text":
928
+ let text = try string(action["text"], "action.text")
929
+ if setSelectedText(text, app: app) {
930
+ return actionResult(channel: "accessibility", activation: "not-requested", pointerInput: false)
931
+ }
932
+ let current = try inputContext(
933
+ app: app,
934
+ request: request,
935
+ limits: limits,
936
+ snapshot: snapshot,
937
+ record: record,
938
+ focusPolicy: focusPolicy,
939
+ keyboardPolicy: keyboardPolicy,
940
+ actionKind: kind,
941
+ timeoutMs: actionTimeoutMs,
942
+ )
943
+ if current.activation == "activated", setSelectedText(text, app: app) {
944
+ return actionResult(channel: "accessibility", activation: current.activation, pointerInput: false)
945
+ }
946
+ try typeTextWithKeyboard(text, app: app)
947
+ return actionResult(channel: "keyboard", activation: current.activation, pointerInput: false)
948
+ case "press-key":
949
+ let current = try inputContext(
950
+ app: app,
951
+ request: request,
952
+ limits: limits,
953
+ snapshot: snapshot,
954
+ record: record,
955
+ focusPolicy: focusPolicy,
956
+ keyboardPolicy: keyboardPolicy,
957
+ actionKind: kind,
958
+ timeoutMs: actionTimeoutMs,
959
+ )
960
+ let modifiers = action["modifiers"] as? [String] ?? []
961
+ try pressKey(try string(action["key"], "action.key"), modifiers: modifiers, app: app)
962
+ return actionResult(channel: "keyboard", activation: current.activation, pointerInput: false)
963
+ case "scroll":
964
+ try requirePointerInput(pointerInputPolicy)
965
+ let current = try inputContext(
966
+ app: app,
967
+ request: request,
968
+ limits: limits,
969
+ snapshot: snapshot,
970
+ record: record,
971
+ focusPolicy: focusPolicy,
972
+ keyboardPolicy: keyboardPolicy,
973
+ actionKind: kind,
974
+ timeoutMs: actionTimeoutMs,
975
+ )
976
+ let point: CGPoint
977
+ if let record = current.record, let frame = axFrame(record.element) { point = CGPoint(x: frame.midX, y: frame.midY) }
978
+ else {
979
+ let coordinateSpace = action["coordinateSpace"] as? String ?? "window"
980
+ point = try windowPoint(action, window: current.snapshot.windowJSON, xKey: "x", yKey: "y", coordinateSpace: coordinateSpace)
981
+ }
982
+ let direction = try string(action["direction"], "action.direction")
983
+ let pages = min(max((action["pages"] as? NSNumber)?.intValue ?? 1, 1), 10)
984
+ let vertical = direction == "up" ? 10 * pages : direction == "down" ? -10 * pages : 0
985
+ let horizontal = direction == "left" ? 10 * pages : direction == "right" ? -10 * pages : 0
986
+ guard vertical != 0 || horizontal != 0 else { throw fail("COMPUTER_PROVIDER_FAILURE", "unsupported scroll direction") }
987
+ let target = try pointerTarget(app: app, window: current.snapshot.windowJSON, at: point)
988
+ try pointerAction {
989
+ try targetedScroll(
990
+ at: point,
991
+ vertical: Int32(vertical),
992
+ horizontal: Int32(horizontal),
993
+ target: target
994
+ )
995
+ }
996
+ return actionResult(channel: "coordinates", activation: current.activation, pointerInput: true, pointerRouting: "target-process")
997
+ case "drag":
998
+ try requirePointerInput(pointerInputPolicy)
999
+ let current = try inputContext(
1000
+ app: app,
1001
+ request: request,
1002
+ limits: limits,
1003
+ snapshot: snapshot,
1004
+ record: record,
1005
+ focusPolicy: focusPolicy,
1006
+ keyboardPolicy: keyboardPolicy,
1007
+ actionKind: kind,
1008
+ timeoutMs: actionTimeoutMs,
1009
+ )
1010
+ let coordinateSpace = action["coordinateSpace"] as? String ?? "window"
1011
+ let from = try windowPoint(action, window: current.snapshot.windowJSON, xKey: "fromX", yKey: "fromY", coordinateSpace: coordinateSpace)
1012
+ let to = try windowPoint(action, window: current.snapshot.windowJSON, xKey: "toX", yKey: "toY", coordinateSpace: coordinateSpace)
1013
+ let target = try pointerTarget(app: app, window: current.snapshot.windowJSON, at: from)
1014
+ try pointerAction {
1015
+ try targetedDrag(from: from, to: to, target: target)
1016
+ }
1017
+ return actionResult(channel: "coordinates", activation: current.activation, pointerInput: true, pointerRouting: "target-process")
1018
+ case "perform-action":
1019
+ guard let record else { throw fail("COMPUTER_ELEMENT_UNAVAILABLE", "perform-action requires an observed element") }
1020
+ let actionName = try string(action["action"], "action.action")
1021
+ let available = record.json["actions"] as? [String] ?? []
1022
+ guard available.contains(actionName) else { throw fail("COMPUTER_ELEMENT_UNAVAILABLE", "the element did not advertise the requested Accessibility action") }
1023
+ if requiresForegroundPermission(actionName), focusPolicy != "activate" {
1024
+ throw fail("COMPUTER_ACTION_BLOCKED", "the requested Accessibility action may raise the target window, but interaction.focusPolicy is preserve")
1025
+ }
1026
+ let current = requiresForegroundPermission(actionName)
1027
+ ? try inputContext(
1028
+ app: app,
1029
+ request: request,
1030
+ limits: limits,
1031
+ snapshot: snapshot,
1032
+ record: record,
1033
+ focusPolicy: focusPolicy,
1034
+ keyboardPolicy: keyboardPolicy,
1035
+ actionKind: kind,
1036
+ timeoutMs: actionTimeoutMs
1037
+ )
1038
+ : (snapshot, record, "not-requested")
1039
+ guard let currentRecord = current.record else {
1040
+ throw fail("COMPUTER_ELEMENT_UNAVAILABLE", "perform-action target is unavailable after foreground validation")
1041
+ }
1042
+ let result = AXUIElementPerformAction(currentRecord.element, actionName as CFString)
1043
+ guard result == .success else { throw fail("COMPUTER_ACTION_BLOCKED", "Accessibility action was rejected") }
1044
+ return actionResult(channel: "accessibility", activation: current.activation, pointerInput: false)
1045
+ default:
1046
+ throw fail("COMPUTER_PROVIDER_FAILURE", "unsupported action kind")
1047
+ }
1048
+ }
1049
+
1050
+ private func handle(_ request: [String: Any]) async throws -> Any {
1051
+ guard (request["protocolVersion"] as? NSNumber)?.intValue == 1 else {
1052
+ throw fail("COMPUTER_PROVIDER_FAILURE", "unsupported helper protocol version")
1053
+ }
1054
+ let command = try string(request["command"], "command")
1055
+ switch command {
1056
+ case "health":
1057
+ return [
1058
+ "helperVersion": helperVersion,
1059
+ "accessibility": permissionAccessibility(),
1060
+ "screenRecording": permissionScreenRecording(),
1061
+ ]
1062
+ case "list-apps":
1063
+ return try runningApps().map { app -> [String: Any] in
1064
+ var json = try appJSON(app)
1065
+ json["frontmost"] = app.isActive
1066
+ json["accessibility"] = permissionAccessibility()
1067
+ json["screenRecording"] = permissionScreenRecording()
1068
+ return json
1069
+ }
1070
+ case "resolve-app":
1071
+ return try appJSON(resolveApp(try dictionary(request["selector"], "selector")))
1072
+ case "open-settings":
1073
+ let kind = try string(request["kind"], "kind")
1074
+ let target: String
1075
+ switch kind {
1076
+ case "accessibility":
1077
+ target = "x-apple.systempreferences:com.apple.preference.security?Privacy_Accessibility"
1078
+ case "screen-recording":
1079
+ target = "x-apple.systempreferences:com.apple.preference.security?Privacy_ScreenCapture"
1080
+ default:
1081
+ throw fail("COMPUTER_PROVIDER_FAILURE", "unknown permission Settings pane")
1082
+ }
1083
+ guard let url = URL(string: target), NSWorkspace.shared.open(url) else {
1084
+ throw fail("COMPUTER_ACTION_BLOCKED", "macOS Settings could not be opened")
1085
+ }
1086
+ return NSNull()
1087
+ case "observe":
1088
+ let appIdentity = try dictionary(request["app"], "app")
1089
+ let app = try resolveApp([
1090
+ "bundleId": try string(appIdentity["bundleId"], "app.bundleId"),
1091
+ "pid": try int(appIdentity["pid"], "app.pid"),
1092
+ ])
1093
+ let options = try dictionary(request["options"], "options")
1094
+ let snapshot = try observeSnapshot(app: app, limits: options)
1095
+ let mode = options["screenshot"] as? String ?? "optional"
1096
+ var screenshot: [String: Any]?
1097
+ if mode != "none" {
1098
+ let path = try string(options["screenshotPath"], "options.screenshotPath")
1099
+ screenshot = try await captureWindow(snapshot, path: path, required: mode == "required")
1100
+ }
1101
+ return observationJSON(snapshot, screenshot: screenshot)
1102
+ case "act":
1103
+ return try performAction(try dictionary(request["request"], "request"))
1104
+ default:
1105
+ throw fail("COMPUTER_PROVIDER_FAILURE", "unknown helper command")
1106
+ }
1107
+ }
1108
+
1109
+ private func emit(_ payload: [String: Any], exitCode: Int32) -> Never {
1110
+ let data = (try? JSONSerialization.data(withJSONObject: payload, options: [.sortedKeys])) ?? Data("{\"ok\":false,\"error\":{\"code\":\"COMPUTER_PROVIDER_FAILURE\",\"message\":\"response serialization failed\"}}".utf8)
1111
+ FileHandle.standardOutput.write(data)
1112
+ FileHandle.standardOutput.write(Data("\n".utf8))
1113
+ Darwin.exit(exitCode)
1114
+ }
1115
+
1116
+ @main
1117
+ private struct HelperMain {
1118
+ @MainActor
1119
+ static func main() async {
1120
+ do {
1121
+ _ = NSApplication.shared
1122
+ NSApplication.shared.setActivationPolicy(.prohibited)
1123
+ try requireHostTransport()
1124
+ if ProcessInfo.processInfo.arguments.contains("--cursor-overlay") {
1125
+ CursorOverlayRuntime.run()
1126
+ return
1127
+ }
1128
+ guard let data = try FileHandle.standardInput.readToEnd(), !data.isEmpty,
1129
+ let request = try JSONSerialization.jsonObject(with: data) as? [String: Any] else {
1130
+ throw fail("COMPUTER_PROVIDER_FAILURE", "stdin must contain one JSON request")
1131
+ }
1132
+ let value = try await handle(request)
1133
+ emit(["ok": true, "value": value], exitCode: 0)
1134
+ } catch let error as HelperError {
1135
+ emit(["ok": false, "error": ["code": error.code, "message": error.message]], exitCode: 2)
1136
+ } catch {
1137
+ emit(["ok": false, "error": ["code": "COMPUTER_PROVIDER_FAILURE", "message": String(describing: error).prefix(1000).description]], exitCode: 2)
1138
+ }
1139
+ }
1140
+ }