@crosshands/platform-darwin 0.1.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/NOTICE +5 -0
- package/assets/.gitkeep +1 -0
- package/assets/CrossHands Computer Use.app/Contents/CodeResources +0 -0
- package/assets/CrossHands Computer Use.app/Contents/Info.plist +32 -0
- package/assets/CrossHands Computer Use.app/Contents/MacOS/crosshands-computer-use-macos +0 -0
- package/assets/CrossHands Computer Use.app/Contents/_CodeSignature/CodeResources +115 -0
- package/assets/payload.json +16 -0
- package/dist/index.d.ts +40 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +621 -0
- package/dist/index.js.map +1 -0
- package/package.json +46 -0
- package/src/index.ts +751 -0
package/src/index.ts
ADDED
|
@@ -0,0 +1,751 @@
|
|
|
1
|
+
import { execFile, spawn, type ChildProcess } from 'node:child_process'
|
|
2
|
+
import { createHash, randomBytes } from 'node:crypto'
|
|
3
|
+
import { chmod, mkdtemp, readFile, rm, writeFile } from 'node:fs/promises'
|
|
4
|
+
import { createConnection } from 'node:net'
|
|
5
|
+
import { tmpdir } from 'node:os'
|
|
6
|
+
import { dirname, isAbsolute, join } from 'node:path'
|
|
7
|
+
import { promisify } from 'node:util'
|
|
8
|
+
import { fileURLToPath } from 'node:url'
|
|
9
|
+
|
|
10
|
+
import {
|
|
11
|
+
COMPUTER_OPERATIONS,
|
|
12
|
+
CONTRACT_VERSIONS,
|
|
13
|
+
ProviderHandshakeSchema,
|
|
14
|
+
createComputerError,
|
|
15
|
+
type ComputerOperationName,
|
|
16
|
+
type ComputerProvider,
|
|
17
|
+
type ProviderHandshake,
|
|
18
|
+
type ProviderRequest,
|
|
19
|
+
type ProviderResponse,
|
|
20
|
+
type ReferenceBindings,
|
|
21
|
+
type TargetReference
|
|
22
|
+
} from '@crosshands/contract'
|
|
23
|
+
|
|
24
|
+
type JsonObject = Record<string, unknown>
|
|
25
|
+
|
|
26
|
+
export const packageVersion = CONTRACT_VERSIONS.product
|
|
27
|
+
const MAX_NATIVE_RESPONSE_BYTES = 4 * 1024 * 1024
|
|
28
|
+
|
|
29
|
+
type NativeClient = {
|
|
30
|
+
request(method: string, params?: JsonObject): Promise<unknown>
|
|
31
|
+
close(): Promise<void>
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
type NativeClientFactory = (graphicalSessionId: string) => Promise<NativeClient>
|
|
35
|
+
|
|
36
|
+
type NativeApp = {
|
|
37
|
+
name: string
|
|
38
|
+
bundleId: string | null
|
|
39
|
+
pid: number
|
|
40
|
+
processStartedAt: string
|
|
41
|
+
executableId: string
|
|
42
|
+
isRunning: boolean
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
type NativeWindow = {
|
|
46
|
+
id: number
|
|
47
|
+
index: number
|
|
48
|
+
title: string
|
|
49
|
+
x: number
|
|
50
|
+
y: number
|
|
51
|
+
width: number
|
|
52
|
+
height: number
|
|
53
|
+
isMinimized: boolean
|
|
54
|
+
app?: { bundleId?: string | null; pid?: number }
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
class NativeProviderError extends Error {
|
|
58
|
+
constructor(
|
|
59
|
+
readonly code: string,
|
|
60
|
+
message: string
|
|
61
|
+
) {
|
|
62
|
+
super(message)
|
|
63
|
+
}
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
const HELPER_PATH = fileURLToPath(
|
|
67
|
+
new URL(
|
|
68
|
+
'../assets/CrossHands%20Computer%20Use.app/Contents/MacOS/crosshands-computer-use-macos',
|
|
69
|
+
import.meta.url
|
|
70
|
+
)
|
|
71
|
+
)
|
|
72
|
+
const MANIFEST_PATH = fileURLToPath(new URL('../assets/payload.json', import.meta.url))
|
|
73
|
+
const execFileAsync = promisify(execFile)
|
|
74
|
+
|
|
75
|
+
export function resolveHelperPath(): string {
|
|
76
|
+
if (!isAbsolute(HELPER_PATH)) throw new Error('CrossHands helper path must be absolute')
|
|
77
|
+
return HELPER_PATH
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
export async function verifyDarwinPayload(
|
|
81
|
+
helperPath = HELPER_PATH,
|
|
82
|
+
manifestPath = MANIFEST_PATH,
|
|
83
|
+
verifySignature = process.platform === 'darwin'
|
|
84
|
+
): Promise<void> {
|
|
85
|
+
if (!isAbsolute(helperPath) || !isAbsolute(manifestPath)) {
|
|
86
|
+
throw createComputerError('provider_unavailable', 'macOS payload paths must be absolute')
|
|
87
|
+
}
|
|
88
|
+
const manifest = JSON.parse(await readFile(manifestPath, 'utf8')) as {
|
|
89
|
+
productVersion?: unknown
|
|
90
|
+
bundleIdentifier?: unknown
|
|
91
|
+
files?: Record<string, unknown>
|
|
92
|
+
signing?: {
|
|
93
|
+
required?: unknown
|
|
94
|
+
authority?: unknown
|
|
95
|
+
teamIdentifier?: unknown
|
|
96
|
+
notarized?: unknown
|
|
97
|
+
}
|
|
98
|
+
}
|
|
99
|
+
if (manifest.productVersion !== packageVersion) {
|
|
100
|
+
throw createComputerError('version_incompatible', 'macOS payload product version mismatch')
|
|
101
|
+
}
|
|
102
|
+
if (manifest.bundleIdentifier !== 'ai.crosshands.ComputerUse') {
|
|
103
|
+
throw createComputerError('provider_unavailable', 'macOS helper bundle identity mismatch')
|
|
104
|
+
}
|
|
105
|
+
const expected = manifest.files?.['crosshands-computer-use-macos']
|
|
106
|
+
if (typeof expected !== 'string' || !/^[a-f0-9]{64}$/.test(expected)) {
|
|
107
|
+
throw createComputerError('provider_unavailable', 'macOS payload manifest is malformed')
|
|
108
|
+
}
|
|
109
|
+
const actual = createHash('sha256')
|
|
110
|
+
.update(await readFile(helperPath))
|
|
111
|
+
.digest('hex')
|
|
112
|
+
if (actual !== expected) {
|
|
113
|
+
throw createComputerError('provider_unavailable', 'macOS provider payload hash mismatch')
|
|
114
|
+
}
|
|
115
|
+
if (!verifySignature) return
|
|
116
|
+
const signing = manifest.signing
|
|
117
|
+
if (
|
|
118
|
+
signing?.required !== true ||
|
|
119
|
+
typeof signing.authority !== 'string' ||
|
|
120
|
+
signing.authority.length === 0 ||
|
|
121
|
+
typeof signing.teamIdentifier !== 'string' ||
|
|
122
|
+
!/^[A-Z0-9]{10}$/.test(signing.teamIdentifier) ||
|
|
123
|
+
signing.notarized !== true
|
|
124
|
+
) {
|
|
125
|
+
throw createComputerError(
|
|
126
|
+
'provider_unavailable',
|
|
127
|
+
'macOS release payload has no valid signing and notarization policy'
|
|
128
|
+
)
|
|
129
|
+
}
|
|
130
|
+
const appPath = dirname(dirname(dirname(helperPath)))
|
|
131
|
+
try {
|
|
132
|
+
await execFileAsync('/usr/bin/codesign', ['--verify', '--strict', appPath])
|
|
133
|
+
const requirement = await execFileAsync('/usr/bin/codesign', ['-d', '-r-', appPath])
|
|
134
|
+
if (!requirement.stderr.includes('identifier "ai.crosshands.ComputerUse"')) {
|
|
135
|
+
throw new Error('designated requirement does not bind the stable bundle identifier')
|
|
136
|
+
}
|
|
137
|
+
const details = await execFileAsync('/usr/bin/codesign', ['-d', '--verbose=4', appPath])
|
|
138
|
+
if (
|
|
139
|
+
!details.stderr.includes(`Authority=${signing.authority}`) ||
|
|
140
|
+
!details.stderr.includes(`TeamIdentifier=${signing.teamIdentifier}`)
|
|
141
|
+
) {
|
|
142
|
+
throw new Error('code-signing authority or team identifier does not match the manifest')
|
|
143
|
+
}
|
|
144
|
+
await execFileAsync('/usr/sbin/spctl', ['--assess', '--type', 'execute', appPath])
|
|
145
|
+
} catch (cause) {
|
|
146
|
+
throw createComputerError(
|
|
147
|
+
'provider_unavailable',
|
|
148
|
+
'macOS provider signature verification failed',
|
|
149
|
+
{
|
|
150
|
+
cause: cause instanceof Error ? cause.message : 'unknown'
|
|
151
|
+
}
|
|
152
|
+
)
|
|
153
|
+
}
|
|
154
|
+
}
|
|
155
|
+
|
|
156
|
+
function currentGraphicalSessionId(): string {
|
|
157
|
+
const uid = process.getuid?.()
|
|
158
|
+
const osIdentity = uid === undefined ? `user:${process.env.USER ?? 'unknown'}` : `uid:${uid}`
|
|
159
|
+
return (
|
|
160
|
+
process.env.CROSSHANDS_GRAPHICAL_SESSION_ID ??
|
|
161
|
+
process.env.SECURITYSESSIONID ??
|
|
162
|
+
`interactive:${osIdentity}`
|
|
163
|
+
)
|
|
164
|
+
}
|
|
165
|
+
|
|
166
|
+
class SocketNativeClient implements NativeClient {
|
|
167
|
+
#sequence = 0
|
|
168
|
+
|
|
169
|
+
private constructor(
|
|
170
|
+
private readonly child: ChildProcess,
|
|
171
|
+
private readonly directory: string,
|
|
172
|
+
private readonly socketPath: string,
|
|
173
|
+
private readonly token: string
|
|
174
|
+
) {}
|
|
175
|
+
|
|
176
|
+
static async start(graphicalSessionId: string): Promise<SocketNativeClient> {
|
|
177
|
+
const directory = await mkdtemp(join(tmpdir(), 'crosshands-darwin-'))
|
|
178
|
+
await chmod(directory, 0o700)
|
|
179
|
+
const socketPath = join(directory, 'provider.sock')
|
|
180
|
+
const tokenPath = join(directory, 'provider.token')
|
|
181
|
+
const token = randomBytes(32).toString('base64url')
|
|
182
|
+
await writeFile(tokenPath, `${token}\n`, { mode: 0o600, flag: 'wx' })
|
|
183
|
+
const child = spawn(resolveHelperPath(), ['--agent', socketPath, '--token-file', tokenPath], {
|
|
184
|
+
env: { ...process.env, CROSSHANDS_GRAPHICAL_SESSION_ID: graphicalSessionId },
|
|
185
|
+
stdio: 'ignore'
|
|
186
|
+
})
|
|
187
|
+
const client = new SocketNativeClient(child, directory, socketPath, token)
|
|
188
|
+
const deadline = Date.now() + 3_000
|
|
189
|
+
let lastError: unknown
|
|
190
|
+
while (Date.now() < deadline) {
|
|
191
|
+
if (child.exitCode !== null) break
|
|
192
|
+
try {
|
|
193
|
+
// oxlint-disable-next-line no-await-in-loop -- readiness requires ordered retries.
|
|
194
|
+
await client.request('handshake')
|
|
195
|
+
return client
|
|
196
|
+
} catch (cause) {
|
|
197
|
+
lastError = cause
|
|
198
|
+
// oxlint-disable-next-line no-await-in-loop -- readiness requires ordered retries.
|
|
199
|
+
await new Promise<void>((resolve) => setTimeout(resolve, 40))
|
|
200
|
+
}
|
|
201
|
+
}
|
|
202
|
+
await client.close()
|
|
203
|
+
throw createComputerError('provider_unavailable', 'CrossHands macOS helper did not start', {
|
|
204
|
+
cause: lastError instanceof Error ? lastError.message : `exit ${String(child.exitCode)}`
|
|
205
|
+
})
|
|
206
|
+
}
|
|
207
|
+
|
|
208
|
+
request(method: string, params: JsonObject = {}): Promise<unknown> {
|
|
209
|
+
const id = ++this.#sequence
|
|
210
|
+
const payload = JSON.stringify({ id, method, params, token: this.token }) + '\n'
|
|
211
|
+
return new Promise((resolve, reject) => {
|
|
212
|
+
const socket = createConnection(this.socketPath)
|
|
213
|
+
let buffer = ''
|
|
214
|
+
let receivedBytes = 0
|
|
215
|
+
let settled = false
|
|
216
|
+
const timer = setTimeout(() => {
|
|
217
|
+
settled = true
|
|
218
|
+
socket.destroy()
|
|
219
|
+
reject(new NativeProviderError('timeout', `macOS helper timed out during ${method}`))
|
|
220
|
+
}, 30_000)
|
|
221
|
+
timer.unref()
|
|
222
|
+
const finish = (): boolean => {
|
|
223
|
+
if (settled) return false
|
|
224
|
+
settled = true
|
|
225
|
+
clearTimeout(timer)
|
|
226
|
+
return true
|
|
227
|
+
}
|
|
228
|
+
socket.setEncoding('utf8')
|
|
229
|
+
socket.once('connect', () => socket.end(payload))
|
|
230
|
+
socket.on('data', (chunk: string) => {
|
|
231
|
+
receivedBytes += Buffer.byteLength(chunk)
|
|
232
|
+
if (receivedBytes > MAX_NATIVE_RESPONSE_BYTES) {
|
|
233
|
+
if (finish()) {
|
|
234
|
+
socket.destroy()
|
|
235
|
+
reject(
|
|
236
|
+
new NativeProviderError('payload_too_large', 'macOS helper response is too large')
|
|
237
|
+
)
|
|
238
|
+
}
|
|
239
|
+
return
|
|
240
|
+
}
|
|
241
|
+
buffer += chunk
|
|
242
|
+
const newline = buffer.indexOf('\n')
|
|
243
|
+
if (newline < 0) return
|
|
244
|
+
if (!finish()) return
|
|
245
|
+
socket.destroy()
|
|
246
|
+
try {
|
|
247
|
+
const response = JSON.parse(buffer.slice(0, newline)) as JsonObject
|
|
248
|
+
if (response.id !== id) throw new Error('macOS helper response id mismatch')
|
|
249
|
+
if (response.ok === true) resolve(response.result)
|
|
250
|
+
else {
|
|
251
|
+
const error = asObject(response.error)
|
|
252
|
+
reject(
|
|
253
|
+
new NativeProviderError(
|
|
254
|
+
typeof error.code === 'string' ? error.code : 'accessibility_error',
|
|
255
|
+
typeof error.message === 'string' ? error.message : 'macOS helper failed'
|
|
256
|
+
)
|
|
257
|
+
)
|
|
258
|
+
}
|
|
259
|
+
} catch (cause) {
|
|
260
|
+
reject(cause)
|
|
261
|
+
}
|
|
262
|
+
})
|
|
263
|
+
socket.once('error', (cause) => {
|
|
264
|
+
if (finish()) reject(cause)
|
|
265
|
+
})
|
|
266
|
+
})
|
|
267
|
+
}
|
|
268
|
+
|
|
269
|
+
async close(): Promise<void> {
|
|
270
|
+
if (this.child.exitCode === null) {
|
|
271
|
+
await this.request('terminate').catch(() => undefined)
|
|
272
|
+
this.child.kill('SIGTERM')
|
|
273
|
+
}
|
|
274
|
+
await rm(this.directory, { recursive: true, force: true })
|
|
275
|
+
}
|
|
276
|
+
}
|
|
277
|
+
|
|
278
|
+
function asObject(value: unknown): JsonObject {
|
|
279
|
+
if (value === null || typeof value !== 'object' || Array.isArray(value)) {
|
|
280
|
+
throw new TypeError('Expected an object from the macOS helper')
|
|
281
|
+
}
|
|
282
|
+
return value as JsonObject
|
|
283
|
+
}
|
|
284
|
+
|
|
285
|
+
function asString(value: unknown, field: string): string {
|
|
286
|
+
if (typeof value !== 'string' || value.length === 0) throw new TypeError(`Missing ${field}`)
|
|
287
|
+
return value
|
|
288
|
+
}
|
|
289
|
+
|
|
290
|
+
function asNumber(value: unknown, field: string): number {
|
|
291
|
+
if (typeof value !== 'number' || !Number.isFinite(value)) throw new TypeError(`Missing ${field}`)
|
|
292
|
+
return value
|
|
293
|
+
}
|
|
294
|
+
|
|
295
|
+
function nativeApp(value: unknown): NativeApp {
|
|
296
|
+
const app = asObject(value)
|
|
297
|
+
return {
|
|
298
|
+
name: asString(app.name, 'app.name'),
|
|
299
|
+
bundleId: typeof app.bundleId === 'string' ? app.bundleId : null,
|
|
300
|
+
pid: asNumber(app.pid, 'app.pid'),
|
|
301
|
+
processStartedAt: asString(app.processStartedAt, 'app.processStartedAt'),
|
|
302
|
+
executableId: asString(app.executableId, 'app.executableId'),
|
|
303
|
+
isRunning: app.isRunning !== false
|
|
304
|
+
}
|
|
305
|
+
}
|
|
306
|
+
|
|
307
|
+
function appId(app: NativeApp): string {
|
|
308
|
+
return app.bundleId ?? app.executableId
|
|
309
|
+
}
|
|
310
|
+
|
|
311
|
+
function normalizeApp(app: NativeApp): JsonObject {
|
|
312
|
+
return {
|
|
313
|
+
id: appId(app),
|
|
314
|
+
name: app.name,
|
|
315
|
+
bundleId: app.bundleId,
|
|
316
|
+
pid: app.pid,
|
|
317
|
+
isRunning: app.isRunning
|
|
318
|
+
}
|
|
319
|
+
}
|
|
320
|
+
|
|
321
|
+
function nativeWindow(value: unknown): NativeWindow {
|
|
322
|
+
const window = asObject(value)
|
|
323
|
+
const normalized: NativeWindow = {
|
|
324
|
+
id: asNumber(window.id, 'window.id'),
|
|
325
|
+
index: asNumber(window.index, 'window.index'),
|
|
326
|
+
title: typeof window.title === 'string' ? window.title : '',
|
|
327
|
+
x: asNumber(window.x, 'window.x'),
|
|
328
|
+
y: asNumber(window.y, 'window.y'),
|
|
329
|
+
width: asNumber(window.width, 'window.width'),
|
|
330
|
+
height: asNumber(window.height, 'window.height'),
|
|
331
|
+
isMinimized: window.isMinimized === true
|
|
332
|
+
}
|
|
333
|
+
if (window.app !== undefined) {
|
|
334
|
+
normalized.app = asObject(window.app) as NonNullable<NativeWindow['app']>
|
|
335
|
+
}
|
|
336
|
+
return normalized
|
|
337
|
+
}
|
|
338
|
+
|
|
339
|
+
function normalizeWindow(window: NativeWindow, owner: NativeApp): JsonObject {
|
|
340
|
+
return {
|
|
341
|
+
id: String(window.id),
|
|
342
|
+
appId: appId(owner),
|
|
343
|
+
title: window.title,
|
|
344
|
+
index: window.index,
|
|
345
|
+
bounds: { x: window.x, y: window.y, width: window.width, height: window.height },
|
|
346
|
+
minimized: window.isMinimized
|
|
347
|
+
}
|
|
348
|
+
}
|
|
349
|
+
|
|
350
|
+
function bindings(
|
|
351
|
+
app: NativeApp,
|
|
352
|
+
window: NativeWindow,
|
|
353
|
+
snapshotId: string,
|
|
354
|
+
generation: string,
|
|
355
|
+
graphicalSessionId: string
|
|
356
|
+
): ReferenceBindings {
|
|
357
|
+
return {
|
|
358
|
+
brokerGeneration: 'pending-broker',
|
|
359
|
+
providerGeneration: generation,
|
|
360
|
+
graphicalSessionId,
|
|
361
|
+
process: { pid: app.pid, startedAt: app.processStartedAt, executableId: app.executableId },
|
|
362
|
+
appId: appId(app),
|
|
363
|
+
window: { id: String(window.id), ownerPid: app.pid },
|
|
364
|
+
snapshotId,
|
|
365
|
+
desktopEpoch: 0
|
|
366
|
+
}
|
|
367
|
+
}
|
|
368
|
+
|
|
369
|
+
function normalizeSnapshot(
|
|
370
|
+
value: unknown,
|
|
371
|
+
generation: string,
|
|
372
|
+
graphicalSessionId: string
|
|
373
|
+
): JsonObject {
|
|
374
|
+
const raw = asObject(value)
|
|
375
|
+
const snapshot = asObject(raw.snapshot)
|
|
376
|
+
const app = nativeApp(snapshot.app)
|
|
377
|
+
const window = nativeWindow(snapshot.window)
|
|
378
|
+
const id = asString(snapshot.id, 'snapshot.id')
|
|
379
|
+
return {
|
|
380
|
+
bindings: bindings(app, window, id, generation, graphicalSessionId),
|
|
381
|
+
snapshot: {
|
|
382
|
+
id,
|
|
383
|
+
app: normalizeApp(app),
|
|
384
|
+
window: normalizeWindow(window, app),
|
|
385
|
+
treeText: typeof snapshot.treeText === 'string' ? snapshot.treeText : '',
|
|
386
|
+
elementCount: asNumber(snapshot.elementCount, 'snapshot.elementCount'),
|
|
387
|
+
focusedElementRef:
|
|
388
|
+
typeof snapshot.focusedElementId === 'number'
|
|
389
|
+
? `element:${snapshot.focusedElementId}`
|
|
390
|
+
: null,
|
|
391
|
+
desktopEpoch: 0
|
|
392
|
+
},
|
|
393
|
+
screenshot:
|
|
394
|
+
raw.screenshot === null || raw.screenshot === undefined
|
|
395
|
+
? null
|
|
396
|
+
: normalizeScreenshot(raw.screenshot),
|
|
397
|
+
issues: normalizeScreenshotIssues(raw.screenshotStatus)
|
|
398
|
+
}
|
|
399
|
+
}
|
|
400
|
+
|
|
401
|
+
export function normalizeScreenshotIssues(value: unknown): JsonObject[] {
|
|
402
|
+
if (value === null || value === undefined) return []
|
|
403
|
+
const status = asObject(value)
|
|
404
|
+
if (status.state !== 'failed') return []
|
|
405
|
+
const message = asString(status.message, 'screenshotStatus.message')
|
|
406
|
+
const code =
|
|
407
|
+
status.code === 'permission_denied' || status.code === 'screenshot_failed'
|
|
408
|
+
? status.code
|
|
409
|
+
: 'screenshot_failed'
|
|
410
|
+
return [
|
|
411
|
+
createComputerError(code, message, {
|
|
412
|
+
component: 'screenshots',
|
|
413
|
+
...(status.metadata === undefined ? {} : { native: status.metadata })
|
|
414
|
+
}).toJSON()
|
|
415
|
+
]
|
|
416
|
+
}
|
|
417
|
+
|
|
418
|
+
function normalizeScreenshot(value: unknown): JsonObject {
|
|
419
|
+
const screenshot = asObject(value)
|
|
420
|
+
return {
|
|
421
|
+
format: 'png',
|
|
422
|
+
width: asNumber(screenshot.width, 'screenshot.width'),
|
|
423
|
+
height: asNumber(screenshot.height, 'screenshot.height'),
|
|
424
|
+
scale: asNumber(screenshot.scale, 'screenshot.scale'),
|
|
425
|
+
data: asString(screenshot.data, 'screenshot.data')
|
|
426
|
+
}
|
|
427
|
+
}
|
|
428
|
+
|
|
429
|
+
function targetReference(input: JsonObject): TargetReference | undefined {
|
|
430
|
+
for (const key of ['target', 'from', 'to']) {
|
|
431
|
+
const target = input[key]
|
|
432
|
+
if (target === null || typeof target !== 'object') continue
|
|
433
|
+
const record = target as JsonObject
|
|
434
|
+
const candidate =
|
|
435
|
+
'contextToken' in record
|
|
436
|
+
? target
|
|
437
|
+
: record.kind === 'element'
|
|
438
|
+
? record.ref
|
|
439
|
+
: (record.window ?? target)
|
|
440
|
+
if (candidate !== null && typeof candidate === 'object' && 'snapshotId' in candidate) {
|
|
441
|
+
return candidate as TargetReference
|
|
442
|
+
}
|
|
443
|
+
}
|
|
444
|
+
return undefined
|
|
445
|
+
}
|
|
446
|
+
|
|
447
|
+
function nativeTarget(target: unknown, prefix = ''): JsonObject {
|
|
448
|
+
const value = asObject(target)
|
|
449
|
+
const elementKey = prefix.length === 0 ? 'elementIndex' : `${prefix}ElementIndex`
|
|
450
|
+
const xKey = prefix.length === 0 ? 'x' : `${prefix}X`
|
|
451
|
+
const yKey = prefix.length === 0 ? 'y' : `${prefix}Y`
|
|
452
|
+
if (value.kind === 'element') {
|
|
453
|
+
const ref = asObject(value.ref)
|
|
454
|
+
const index = Number(asString(ref.ref, 'element ref').replace(/^element:/, ''))
|
|
455
|
+
if (!Number.isInteger(index) || index < 0) throw new TypeError('Invalid element reference')
|
|
456
|
+
return { [elementKey]: index, snapshotId: ref.snapshotId }
|
|
457
|
+
}
|
|
458
|
+
if (value.kind === 'coordinate') {
|
|
459
|
+
return {
|
|
460
|
+
[xKey]: asNumber(value.x, `${prefix}x`),
|
|
461
|
+
[yKey]: asNumber(value.y, `${prefix}y`),
|
|
462
|
+
...(value.window === undefined ? {} : { snapshotId: asObject(value.window).snapshotId })
|
|
463
|
+
}
|
|
464
|
+
}
|
|
465
|
+
if ('snapshotId' in value) return { snapshotId: value.snapshotId }
|
|
466
|
+
throw new TypeError('Unsupported macOS action target')
|
|
467
|
+
}
|
|
468
|
+
|
|
469
|
+
function nativeInput(operation: ComputerOperationName, input: unknown): JsonObject {
|
|
470
|
+
const value = asObject(input)
|
|
471
|
+
const common = {
|
|
472
|
+
...(typeof value.app === 'string' ? { app: value.app } : {}),
|
|
473
|
+
...(value.restoreWindow === true ? { restoreWindow: true } : {}),
|
|
474
|
+
...(value.captureScreenshot === false ? { noScreenshot: true } : {})
|
|
475
|
+
}
|
|
476
|
+
if (operation === 'getAppState') {
|
|
477
|
+
const window = value.window === undefined ? undefined : asObject(value.window)
|
|
478
|
+
return {
|
|
479
|
+
...common,
|
|
480
|
+
app: value.app,
|
|
481
|
+
...(window?.id === undefined ? {} : { windowId: Number(window.id) }),
|
|
482
|
+
...(window?.index === undefined ? {} : { windowIndex: window.index })
|
|
483
|
+
}
|
|
484
|
+
}
|
|
485
|
+
if (operation === 'listWindows') return { app: value.app }
|
|
486
|
+
if (!COMPUTER_OPERATIONS[operation].mutation) return value
|
|
487
|
+
const reference = targetReference(value)
|
|
488
|
+
const app =
|
|
489
|
+
typeof value.app === 'string'
|
|
490
|
+
? value.app
|
|
491
|
+
: reference
|
|
492
|
+
? `pid:${reference.process.pid}`
|
|
493
|
+
: undefined
|
|
494
|
+
const actionCommon = {
|
|
495
|
+
...common,
|
|
496
|
+
...(app === undefined ? {} : { app }),
|
|
497
|
+
...(reference === undefined
|
|
498
|
+
? {}
|
|
499
|
+
: {
|
|
500
|
+
expectedProcessStartedAt: reference.process.startedAt,
|
|
501
|
+
expectedExecutableId: reference.process.executableId
|
|
502
|
+
})
|
|
503
|
+
}
|
|
504
|
+
if (operation === 'drag') {
|
|
505
|
+
return {
|
|
506
|
+
...actionCommon,
|
|
507
|
+
...nativeTarget(value.from, 'from'),
|
|
508
|
+
...nativeTarget(value.to, 'to'),
|
|
509
|
+
...(typeof value.durationMs === 'number' ? { durationMs: value.durationMs } : {})
|
|
510
|
+
}
|
|
511
|
+
}
|
|
512
|
+
const target = nativeTarget(value.target)
|
|
513
|
+
switch (operation) {
|
|
514
|
+
case 'click':
|
|
515
|
+
return {
|
|
516
|
+
...actionCommon,
|
|
517
|
+
...target,
|
|
518
|
+
mouseButton: value.button,
|
|
519
|
+
clickCount: value.clickCount,
|
|
520
|
+
modifiers: value.modifiers
|
|
521
|
+
}
|
|
522
|
+
case 'performSecondaryAction':
|
|
523
|
+
return { ...actionCommon, ...target, action: value.action }
|
|
524
|
+
case 'scroll':
|
|
525
|
+
return { ...actionCommon, ...target, direction: value.direction, pages: value.pages }
|
|
526
|
+
case 'hotkey':
|
|
527
|
+
return { ...actionCommon, ...target, key: (value.keys as string[]).join('+') }
|
|
528
|
+
case 'typeText':
|
|
529
|
+
case 'pasteText':
|
|
530
|
+
return { ...actionCommon, ...target, text: value.text }
|
|
531
|
+
case 'pressKey':
|
|
532
|
+
return { ...actionCommon, ...target, key: value.key }
|
|
533
|
+
case 'setValue':
|
|
534
|
+
return { ...actionCommon, ...target, value: value.value }
|
|
535
|
+
default:
|
|
536
|
+
return { ...actionCommon, ...target }
|
|
537
|
+
}
|
|
538
|
+
}
|
|
539
|
+
|
|
540
|
+
function errorCode(code: string): Parameters<typeof createComputerError>[0] {
|
|
541
|
+
const aliases: Record<string, Parameters<typeof createComputerError>[0]> = {
|
|
542
|
+
window_stale: 'stale_target',
|
|
543
|
+
action_timeout: 'timeout'
|
|
544
|
+
}
|
|
545
|
+
if (code in aliases) return aliases[code]!
|
|
546
|
+
const known = [
|
|
547
|
+
'app_not_found',
|
|
548
|
+
'app_blocked',
|
|
549
|
+
'window_not_found',
|
|
550
|
+
'window_not_focused',
|
|
551
|
+
'permission_denied',
|
|
552
|
+
'element_not_found',
|
|
553
|
+
'element_not_clickable',
|
|
554
|
+
'action_not_supported',
|
|
555
|
+
'value_not_settable',
|
|
556
|
+
'invalid_argument',
|
|
557
|
+
'timeout',
|
|
558
|
+
'screenshot_failed',
|
|
559
|
+
'accessibility_error'
|
|
560
|
+
]
|
|
561
|
+
return known.includes(code)
|
|
562
|
+
? (code as Parameters<typeof createComputerError>[0])
|
|
563
|
+
: 'accessibility_error'
|
|
564
|
+
}
|
|
565
|
+
|
|
566
|
+
function mutationResult(
|
|
567
|
+
value: unknown,
|
|
568
|
+
generation: string,
|
|
569
|
+
graphicalSessionId: string
|
|
570
|
+
): JsonObject {
|
|
571
|
+
const raw = asObject(value)
|
|
572
|
+
const action = asObject(raw.action)
|
|
573
|
+
const verification = action.verification === undefined ? undefined : asObject(action.verification)
|
|
574
|
+
return {
|
|
575
|
+
outcome:
|
|
576
|
+
verification?.state === 'verified'
|
|
577
|
+
? { state: 'verified', evidence: action }
|
|
578
|
+
: {
|
|
579
|
+
state: 'indeterminate',
|
|
580
|
+
reason: String(verification?.reason ?? 'verification unavailable')
|
|
581
|
+
},
|
|
582
|
+
freshState: normalizeSnapshot(raw, generation, graphicalSessionId)
|
|
583
|
+
}
|
|
584
|
+
}
|
|
585
|
+
|
|
586
|
+
export class DarwinComputerProvider implements ComputerProvider {
|
|
587
|
+
#generation = 'darwin-starting'
|
|
588
|
+
#client: NativeClient | undefined
|
|
589
|
+
#handshake: ProviderHandshake | undefined
|
|
590
|
+
|
|
591
|
+
constructor(
|
|
592
|
+
private readonly graphicalSessionId = currentGraphicalSessionId(),
|
|
593
|
+
private readonly clientFactory: NativeClientFactory = SocketNativeClient.start,
|
|
594
|
+
private readonly verifyPayload = clientFactory === SocketNativeClient.start
|
|
595
|
+
) {}
|
|
596
|
+
|
|
597
|
+
get generation(): string {
|
|
598
|
+
return this.#generation
|
|
599
|
+
}
|
|
600
|
+
|
|
601
|
+
async start(): Promise<ProviderHandshake> {
|
|
602
|
+
if (this.#handshake !== undefined) return this.#handshake
|
|
603
|
+
if (this.verifyPayload) await verifyDarwinPayload()
|
|
604
|
+
this.#client = await this.clientFactory(this.graphicalSessionId)
|
|
605
|
+
const handshake = ProviderHandshakeSchema.parse(await this.#client.request('handshake'))
|
|
606
|
+
this.#generation = handshake.generation
|
|
607
|
+
this.#handshake = handshake
|
|
608
|
+
return handshake
|
|
609
|
+
}
|
|
610
|
+
|
|
611
|
+
async dispatch(request: ProviderRequest): Promise<ProviderResponse> {
|
|
612
|
+
const client = this.#client ?? (await this.start(), this.#client)
|
|
613
|
+
if (client === undefined) throw new Error('macOS provider failed to start')
|
|
614
|
+
try {
|
|
615
|
+
const raw = await client.request(
|
|
616
|
+
request.operation,
|
|
617
|
+
nativeInput(request.operation, request.input)
|
|
618
|
+
)
|
|
619
|
+
let result: unknown = raw
|
|
620
|
+
if (request.operation === 'listApps') {
|
|
621
|
+
result = {
|
|
622
|
+
apps: (asObject(raw).apps as unknown[]).map((app) => normalizeApp(nativeApp(app)))
|
|
623
|
+
}
|
|
624
|
+
} else if (request.operation === 'listWindows') {
|
|
625
|
+
const response = asObject(raw)
|
|
626
|
+
const owner = nativeApp(response.app)
|
|
627
|
+
result = {
|
|
628
|
+
windows: (response.windows as unknown[]).map((window) =>
|
|
629
|
+
normalizeWindow(nativeWindow(window), owner)
|
|
630
|
+
)
|
|
631
|
+
}
|
|
632
|
+
} else if (request.operation === 'getAppState') {
|
|
633
|
+
result = normalizeSnapshot(raw, this.generation, this.graphicalSessionId)
|
|
634
|
+
} else if (COMPUTER_OPERATIONS[request.operation].mutation) {
|
|
635
|
+
result = mutationResult(raw, this.generation, this.graphicalSessionId)
|
|
636
|
+
}
|
|
637
|
+
return {
|
|
638
|
+
requestId: request.requestId,
|
|
639
|
+
dispatched: COMPUTER_OPERATIONS[request.operation].mutation,
|
|
640
|
+
result
|
|
641
|
+
}
|
|
642
|
+
} catch (cause) {
|
|
643
|
+
const native =
|
|
644
|
+
cause instanceof NativeProviderError
|
|
645
|
+
? cause
|
|
646
|
+
: new NativeProviderError(
|
|
647
|
+
'accessibility_error',
|
|
648
|
+
cause instanceof Error ? cause.message : 'macOS provider failed'
|
|
649
|
+
)
|
|
650
|
+
const error = createComputerError(errorCode(native.code), native.message)
|
|
651
|
+
const notDispatched = new Set([
|
|
652
|
+
'permission_denied',
|
|
653
|
+
'invalid_argument',
|
|
654
|
+
'app_not_found',
|
|
655
|
+
'app_blocked',
|
|
656
|
+
'window_not_found',
|
|
657
|
+
'window_stale',
|
|
658
|
+
'element_not_found',
|
|
659
|
+
'action_not_supported',
|
|
660
|
+
'value_not_settable'
|
|
661
|
+
])
|
|
662
|
+
return {
|
|
663
|
+
requestId: request.requestId,
|
|
664
|
+
dispatched: !notDispatched.has(native.code),
|
|
665
|
+
error: error.toJSON()
|
|
666
|
+
}
|
|
667
|
+
}
|
|
668
|
+
}
|
|
669
|
+
|
|
670
|
+
async cancel(_requestId: string): Promise<void> {
|
|
671
|
+
await this.close()
|
|
672
|
+
}
|
|
673
|
+
|
|
674
|
+
async close(): Promise<void> {
|
|
675
|
+
const client = this.#client
|
|
676
|
+
this.#client = undefined
|
|
677
|
+
this.#handshake = undefined
|
|
678
|
+
await client?.close()
|
|
679
|
+
}
|
|
680
|
+
|
|
681
|
+
async inspect(
|
|
682
|
+
operation: ComputerOperationName,
|
|
683
|
+
input: unknown
|
|
684
|
+
): Promise<{
|
|
685
|
+
bindings: ReferenceBindings
|
|
686
|
+
appIdentity: { appId: string; executableId: string }
|
|
687
|
+
} | null> {
|
|
688
|
+
if (operation === 'capabilities' || operation === 'permissions' || operation === 'listApps')
|
|
689
|
+
return null
|
|
690
|
+
const client = this.#client ?? (await this.start(), this.#client)
|
|
691
|
+
if (client === undefined) return null
|
|
692
|
+
const value = asObject(input)
|
|
693
|
+
const reference = targetReference(value)
|
|
694
|
+
const query =
|
|
695
|
+
typeof value.app === 'string'
|
|
696
|
+
? value.app
|
|
697
|
+
: reference
|
|
698
|
+
? `pid:${reference.process.pid}`
|
|
699
|
+
: undefined
|
|
700
|
+
if (query === undefined) return null
|
|
701
|
+
const apps = (asObject(await client.request('listApps')).apps as unknown[]).map(nativeApp)
|
|
702
|
+
const app = apps.find(
|
|
703
|
+
(candidate) =>
|
|
704
|
+
candidate.name.toLowerCase() === query.toLowerCase() ||
|
|
705
|
+
candidate.bundleId?.toLowerCase() === query.toLowerCase() ||
|
|
706
|
+
`pid:${candidate.pid}` === query
|
|
707
|
+
)
|
|
708
|
+
if (app === undefined) return null
|
|
709
|
+
const windows = asObject(await client.request('listWindows', { app: `pid:${app.pid}` }))
|
|
710
|
+
.windows as unknown[]
|
|
711
|
+
const requestedWindow =
|
|
712
|
+
reference?.window.id ??
|
|
713
|
+
(value.window === undefined ? undefined : String(asObject(value.window).id ?? ''))
|
|
714
|
+
const window = windows
|
|
715
|
+
.map(nativeWindow)
|
|
716
|
+
.find(
|
|
717
|
+
(candidate) => requestedWindow === undefined || String(candidate.id) === requestedWindow
|
|
718
|
+
)
|
|
719
|
+
if (window === undefined) return null
|
|
720
|
+
const current = bindings(
|
|
721
|
+
app,
|
|
722
|
+
window,
|
|
723
|
+
reference?.snapshotId ?? 'inspection',
|
|
724
|
+
this.generation,
|
|
725
|
+
this.graphicalSessionId
|
|
726
|
+
)
|
|
727
|
+
return {
|
|
728
|
+
bindings:
|
|
729
|
+
reference === undefined
|
|
730
|
+
? current
|
|
731
|
+
: {
|
|
732
|
+
...current,
|
|
733
|
+
brokerGeneration: reference.brokerGeneration,
|
|
734
|
+
snapshotId: reference.snapshotId,
|
|
735
|
+
desktopEpoch: reference.desktopEpoch
|
|
736
|
+
},
|
|
737
|
+
appIdentity: { appId: appId(app), executableId: app.executableId }
|
|
738
|
+
}
|
|
739
|
+
}
|
|
740
|
+
}
|
|
741
|
+
|
|
742
|
+
let activeProvider: DarwinComputerProvider | undefined
|
|
743
|
+
|
|
744
|
+
export function createProvider(): ComputerProvider {
|
|
745
|
+
activeProvider = new DarwinComputerProvider()
|
|
746
|
+
return activeProvider
|
|
747
|
+
}
|
|
748
|
+
|
|
749
|
+
export async function inspectTarget(operation: ComputerOperationName, input: unknown) {
|
|
750
|
+
return activeProvider?.inspect(operation, input) ?? null
|
|
751
|
+
}
|