@crosshands/platform-linux 0.1.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +8 -0
- package/assets/payload.json +9 -0
- package/assets/runtime.py +1490 -0
- package/dist/index.d.ts +42 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +632 -0
- package/dist/index.js.map +1 -0
- package/package.json +45 -0
- package/src/index.ts +777 -0
package/src/index.ts
ADDED
|
@@ -0,0 +1,777 @@
|
|
|
1
|
+
import { spawn, type ChildProcessWithoutNullStreams } from 'node:child_process'
|
|
2
|
+
import { createHash, randomUUID } from 'node:crypto'
|
|
3
|
+
import { access, readFile, readlink, stat } from 'node:fs/promises'
|
|
4
|
+
import { isAbsolute } from 'node:path'
|
|
5
|
+
import { createInterface, type Interface } from 'node:readline'
|
|
6
|
+
import { fileURLToPath } from 'node:url'
|
|
7
|
+
|
|
8
|
+
import {
|
|
9
|
+
COMPUTER_OPERATIONS,
|
|
10
|
+
CONTRACT_VERSIONS,
|
|
11
|
+
createComputerError,
|
|
12
|
+
type ComputerError,
|
|
13
|
+
type ComputerOperationName,
|
|
14
|
+
type ComputerProvider,
|
|
15
|
+
type ProviderHandshake,
|
|
16
|
+
type ProviderRequest,
|
|
17
|
+
type ProviderResponse,
|
|
18
|
+
type ReferenceBindings,
|
|
19
|
+
type TargetReference
|
|
20
|
+
} from '@crosshands/contract'
|
|
21
|
+
|
|
22
|
+
const FIXED_SYSTEM_PATH = '/usr/local/bin:/usr/bin:/bin'
|
|
23
|
+
export const packageVersion = CONTRACT_VERSIONS.product
|
|
24
|
+
const MAX_NATIVE_FRAME_BYTES = 1_048_576
|
|
25
|
+
const PACKAGED_RUNTIME = fileURLToPath(new URL('../assets/runtime.py', import.meta.url))
|
|
26
|
+
const PACKAGED_MANIFEST = fileURLToPath(new URL('../assets/payload.json', import.meta.url))
|
|
27
|
+
|
|
28
|
+
type NativeReadiness = {
|
|
29
|
+
available: boolean
|
|
30
|
+
sessionType: string
|
|
31
|
+
graphicalSessionId: string
|
|
32
|
+
issues: Array<{ code: string; component: string; message: string }>
|
|
33
|
+
capabilities: Record<string, boolean>
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
type NativeHandshake = {
|
|
37
|
+
type: 'handshake'
|
|
38
|
+
provider: string
|
|
39
|
+
providerVersion: string
|
|
40
|
+
providerProtocol: number
|
|
41
|
+
publicContract: string
|
|
42
|
+
generation: string
|
|
43
|
+
graphicalSessionId: string
|
|
44
|
+
capabilities: {
|
|
45
|
+
readiness: NativeReadiness
|
|
46
|
+
supports: { actions: Record<string, boolean>; observation: Record<string, boolean> }
|
|
47
|
+
}
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
type NativeResponse =
|
|
51
|
+
| { type: 'response'; requestId: string; ok: true; result: Record<string, unknown> }
|
|
52
|
+
| { type: 'response'; requestId: string; ok: false; error: string; dispatched?: boolean }
|
|
53
|
+
type NativeFrame = NativeHandshake | NativeResponse | { type: 'fatal'; error: string }
|
|
54
|
+
type Pending = { resolve: (frame: NativeResponse) => void; reject: (cause: unknown) => void }
|
|
55
|
+
|
|
56
|
+
type Environment = Readonly<Record<string, string | undefined>>
|
|
57
|
+
|
|
58
|
+
export function linuxProviderEnvironment(source: Environment = process.env): NodeJS.ProcessEnv {
|
|
59
|
+
const result: NodeJS.ProcessEnv = {
|
|
60
|
+
PATH: FIXED_SYSTEM_PATH,
|
|
61
|
+
LANG: source.LANG ?? 'C.UTF-8',
|
|
62
|
+
LC_ALL: source.LC_ALL ?? 'C.UTF-8',
|
|
63
|
+
PYTHONNOUSERSITE: '1'
|
|
64
|
+
}
|
|
65
|
+
for (const name of [
|
|
66
|
+
'DISPLAY',
|
|
67
|
+
'WAYLAND_DISPLAY',
|
|
68
|
+
'XDG_SESSION_TYPE',
|
|
69
|
+
'XDG_SESSION_ID',
|
|
70
|
+
'XDG_RUNTIME_DIR',
|
|
71
|
+
'DBUS_SESSION_BUS_ADDRESS'
|
|
72
|
+
] as const) {
|
|
73
|
+
const value = source[name]
|
|
74
|
+
if (value !== undefined && value.length > 0) result[name] = value
|
|
75
|
+
}
|
|
76
|
+
return result
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
function record(value: unknown): Record<string, unknown> {
|
|
80
|
+
return value !== null && typeof value === 'object' && !Array.isArray(value)
|
|
81
|
+
? (value as Record<string, unknown>)
|
|
82
|
+
: {}
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
function captureOptions(input: Record<string, unknown>): Record<string, unknown> {
|
|
86
|
+
return {
|
|
87
|
+
...(input.captureScreenshot === false ? { noScreenshot: true } : {}),
|
|
88
|
+
...(input.restoreWindow === true ? { restoreWindow: true } : {})
|
|
89
|
+
}
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
function targetFields(targetValue: unknown, prefix = ''): Record<string, unknown> {
|
|
93
|
+
const target = record(targetValue)
|
|
94
|
+
if (target.kind === 'coordinate') {
|
|
95
|
+
return { [`${prefix}x`]: target.x, [`${prefix}y`]: target.y }
|
|
96
|
+
}
|
|
97
|
+
const reference = record(
|
|
98
|
+
target.ref !== null && typeof target.ref === 'object' ? target.ref : target
|
|
99
|
+
)
|
|
100
|
+
if (reference.kind === 'element') {
|
|
101
|
+
const match = /^element:(\d+)$/.exec(String(reference.ref ?? ''))
|
|
102
|
+
return match === null ? {} : { [`${prefix}elementIndex`]: Number(match[1]) }
|
|
103
|
+
}
|
|
104
|
+
if (target.kind === 'element' && typeof target.elementIndex === 'number') {
|
|
105
|
+
return { [`${prefix}elementIndex`]: target.elementIndex }
|
|
106
|
+
}
|
|
107
|
+
return {}
|
|
108
|
+
}
|
|
109
|
+
|
|
110
|
+
function targetReferenceValue(inputValue: unknown): TargetReference | undefined {
|
|
111
|
+
const source = record(inputValue)
|
|
112
|
+
for (const candidate of [source.target, source.from, source.to]) {
|
|
113
|
+
const target = record(candidate)
|
|
114
|
+
const nested = target.ref !== null && typeof target.ref === 'object' ? target.ref : undefined
|
|
115
|
+
const coordinateWindow =
|
|
116
|
+
target.kind === 'coordinate' && target.window !== null && typeof target.window === 'object'
|
|
117
|
+
? target.window
|
|
118
|
+
: undefined
|
|
119
|
+
const reference = record(nested ?? coordinateWindow ?? target)
|
|
120
|
+
if (typeof reference.contextToken === 'string' && typeof reference.ref === 'string')
|
|
121
|
+
return reference as TargetReference
|
|
122
|
+
}
|
|
123
|
+
return undefined
|
|
124
|
+
}
|
|
125
|
+
|
|
126
|
+
export function mapNativeOperation(
|
|
127
|
+
operation: ComputerOperationName,
|
|
128
|
+
inputValue: unknown
|
|
129
|
+
): Record<string, unknown> {
|
|
130
|
+
const input = record(inputValue)
|
|
131
|
+
const reference = targetReferenceValue(input)
|
|
132
|
+
const base = reference
|
|
133
|
+
? { app: `pid:${reference.process.pid}`, expectedIdentity: reference.process }
|
|
134
|
+
: typeof input.app === 'string'
|
|
135
|
+
? { app: input.app }
|
|
136
|
+
: {}
|
|
137
|
+
switch (operation) {
|
|
138
|
+
case 'capabilities':
|
|
139
|
+
return { tool: 'handshake' }
|
|
140
|
+
case 'permissions':
|
|
141
|
+
return { tool: 'handshake' }
|
|
142
|
+
case 'listApps':
|
|
143
|
+
return { tool: 'list_apps' }
|
|
144
|
+
case 'listWindows':
|
|
145
|
+
return { tool: 'list_windows', app: input.app }
|
|
146
|
+
case 'getAppState': {
|
|
147
|
+
const window = record(input.window)
|
|
148
|
+
return {
|
|
149
|
+
tool: 'get_app_state',
|
|
150
|
+
app: input.app,
|
|
151
|
+
...(typeof window.id === 'string' ? { windowId: window.id } : {}),
|
|
152
|
+
...(typeof window.index === 'number' ? { windowIndex: window.index } : {}),
|
|
153
|
+
...captureOptions(input)
|
|
154
|
+
}
|
|
155
|
+
}
|
|
156
|
+
case 'click':
|
|
157
|
+
return {
|
|
158
|
+
tool: 'click',
|
|
159
|
+
...base,
|
|
160
|
+
...targetFields(input.target),
|
|
161
|
+
...(typeof input.clickCount === 'number' ? { click_count: input.clickCount } : {}),
|
|
162
|
+
...(typeof input.button === 'string' ? { mouse_button: input.button } : {}),
|
|
163
|
+
...(Array.isArray(input.modifiers) ? { modifiers: input.modifiers } : {}),
|
|
164
|
+
...captureOptions(input)
|
|
165
|
+
}
|
|
166
|
+
case 'performSecondaryAction':
|
|
167
|
+
return {
|
|
168
|
+
tool: 'perform_secondary_action',
|
|
169
|
+
...base,
|
|
170
|
+
...targetFields(input.target),
|
|
171
|
+
action: input.action,
|
|
172
|
+
...captureOptions(input)
|
|
173
|
+
}
|
|
174
|
+
case 'scroll':
|
|
175
|
+
return {
|
|
176
|
+
tool: 'scroll',
|
|
177
|
+
...base,
|
|
178
|
+
...targetFields(input.target),
|
|
179
|
+
direction: input.direction,
|
|
180
|
+
...(typeof input.pages === 'number' ? { pages: input.pages } : {}),
|
|
181
|
+
...captureOptions(input)
|
|
182
|
+
}
|
|
183
|
+
case 'drag':
|
|
184
|
+
return {
|
|
185
|
+
tool: 'drag',
|
|
186
|
+
...base,
|
|
187
|
+
...targetFields(input.from, 'from_'),
|
|
188
|
+
...targetFields(input.to, 'to_'),
|
|
189
|
+
...(typeof input.durationMs === 'number' ? { duration_ms: input.durationMs } : {}),
|
|
190
|
+
...captureOptions(input)
|
|
191
|
+
}
|
|
192
|
+
case 'typeText':
|
|
193
|
+
return { tool: 'type_text', ...base, text: input.text, ...captureOptions(input) }
|
|
194
|
+
case 'pressKey':
|
|
195
|
+
return { tool: 'press_key', ...base, key: input.key, ...captureOptions(input) }
|
|
196
|
+
case 'hotkey':
|
|
197
|
+
return {
|
|
198
|
+
tool: 'hotkey',
|
|
199
|
+
...base,
|
|
200
|
+
key: Array.isArray(input.keys) ? input.keys.join('+') : input.keys,
|
|
201
|
+
...captureOptions(input)
|
|
202
|
+
}
|
|
203
|
+
case 'pasteText':
|
|
204
|
+
return { tool: 'paste_text', ...base, text: input.text, ...captureOptions(input) }
|
|
205
|
+
case 'setValue':
|
|
206
|
+
return {
|
|
207
|
+
tool: 'set_value',
|
|
208
|
+
...base,
|
|
209
|
+
...targetFields(input.target),
|
|
210
|
+
value: input.value,
|
|
211
|
+
...captureOptions(input)
|
|
212
|
+
}
|
|
213
|
+
default:
|
|
214
|
+
throw createComputerError('invalid_argument', `Unknown Linux operation: ${operation}`)
|
|
215
|
+
}
|
|
216
|
+
}
|
|
217
|
+
|
|
218
|
+
export function normalizeNativeError(message: string): ComputerError {
|
|
219
|
+
const normalized = message.toLowerCase()
|
|
220
|
+
if (normalized.includes('appblocked'))
|
|
221
|
+
return createComputerError('app_blocked', 'Sensitive applications are blocked by default')
|
|
222
|
+
if (normalized.includes('appnotfound')) return createComputerError('app_not_found', message)
|
|
223
|
+
if (normalized.includes('windownotfound') || normalized.includes('no top-level'))
|
|
224
|
+
return createComputerError('window_not_found', message)
|
|
225
|
+
if (normalized.includes('window_not_focused'))
|
|
226
|
+
return createComputerError('window_not_focused', message)
|
|
227
|
+
if (normalized.includes('stale')) return createComputerError('stale_target', message)
|
|
228
|
+
if (normalized.includes('unsupported_capability'))
|
|
229
|
+
return createComputerError('unsupported_capability', message)
|
|
230
|
+
if (normalized.includes('provider_unavailable'))
|
|
231
|
+
return createComputerError('provider_unavailable', message)
|
|
232
|
+
if (normalized.includes('not settable')) return createComputerError('value_not_settable', message)
|
|
233
|
+
if (normalized.includes('not a valid secondary action'))
|
|
234
|
+
return createComputerError('action_not_supported', message)
|
|
235
|
+
return createComputerError('accessibility_error', message)
|
|
236
|
+
}
|
|
237
|
+
|
|
238
|
+
export async function verifyLinuxPayload(
|
|
239
|
+
runtimePath = PACKAGED_RUNTIME,
|
|
240
|
+
manifestPath = PACKAGED_MANIFEST
|
|
241
|
+
): Promise<void> {
|
|
242
|
+
if (!isAbsolute(runtimePath) || !isAbsolute(manifestPath))
|
|
243
|
+
throw createComputerError('provider_unavailable', 'Linux payload paths must be absolute')
|
|
244
|
+
const manifest = JSON.parse(await readFile(manifestPath, 'utf8')) as {
|
|
245
|
+
productVersion?: unknown
|
|
246
|
+
files?: Record<string, unknown>
|
|
247
|
+
}
|
|
248
|
+
if (manifest.productVersion !== packageVersion)
|
|
249
|
+
throw createComputerError('version_incompatible', 'Linux payload product version mismatch')
|
|
250
|
+
const expected = manifest.files?.['runtime.py']
|
|
251
|
+
if (typeof expected !== 'string' || !/^[a-f0-9]{64}$/.test(expected))
|
|
252
|
+
throw createComputerError('provider_unavailable', 'Linux payload manifest is malformed')
|
|
253
|
+
const actual = createHash('sha256')
|
|
254
|
+
.update(await readFile(runtimePath))
|
|
255
|
+
.digest('hex')
|
|
256
|
+
if (actual !== expected)
|
|
257
|
+
throw createComputerError('provider_unavailable', 'Linux provider payload hash mismatch')
|
|
258
|
+
}
|
|
259
|
+
|
|
260
|
+
async function selectPython(explicit?: string): Promise<string> {
|
|
261
|
+
const candidates =
|
|
262
|
+
explicit === undefined ? ['/usr/bin/python3', '/usr/local/bin/python3'] : [explicit]
|
|
263
|
+
for (const candidate of candidates) {
|
|
264
|
+
if (!isAbsolute(candidate)) continue
|
|
265
|
+
try {
|
|
266
|
+
// oxlint-disable-next-line no-await-in-loop -- fixed candidates are checked in order.
|
|
267
|
+
await access(candidate)
|
|
268
|
+
return candidate
|
|
269
|
+
} catch {
|
|
270
|
+
// Continue to the next fixed absolute candidate.
|
|
271
|
+
}
|
|
272
|
+
}
|
|
273
|
+
throw createComputerError(
|
|
274
|
+
'provider_unavailable',
|
|
275
|
+
'Python 3 was not found at a supported absolute path; install python3'
|
|
276
|
+
)
|
|
277
|
+
}
|
|
278
|
+
|
|
279
|
+
function operations(readiness: NativeReadiness): Record<string, boolean> {
|
|
280
|
+
const capability = readiness.capabilities
|
|
281
|
+
const observation = readiness.available && capability.accessibility === true
|
|
282
|
+
return {
|
|
283
|
+
capabilities: true,
|
|
284
|
+
permissions: true,
|
|
285
|
+
listApps: observation,
|
|
286
|
+
listWindows: observation,
|
|
287
|
+
getAppState: observation,
|
|
288
|
+
click:
|
|
289
|
+
observation && (capability.semanticActions === true || capability.syntheticPointer === true),
|
|
290
|
+
performSecondaryAction: observation && capability.semanticActions === true,
|
|
291
|
+
scroll: observation && capability.syntheticPointer === true,
|
|
292
|
+
drag: observation && capability.syntheticPointer === true,
|
|
293
|
+
typeText: observation && capability.syntheticKeyboard === true,
|
|
294
|
+
pressKey: observation && capability.syntheticKeyboard === true,
|
|
295
|
+
hotkey: observation && capability.hotkey === true,
|
|
296
|
+
pasteText:
|
|
297
|
+
observation && capability.clipboard === true && capability.syntheticKeyboard === true,
|
|
298
|
+
setValue: observation && capability.semanticActions === true
|
|
299
|
+
}
|
|
300
|
+
}
|
|
301
|
+
|
|
302
|
+
function targetReference(input: unknown): TargetReference | undefined {
|
|
303
|
+
return targetReferenceValue(input)
|
|
304
|
+
}
|
|
305
|
+
|
|
306
|
+
async function processIdentity(pid: number): Promise<ReferenceBindings['process']> {
|
|
307
|
+
const [executable, executableInfo, processInfo, processStat, bootId] = await Promise.all([
|
|
308
|
+
readlink(`/proc/${pid}/exe`),
|
|
309
|
+
stat(`/proc/${pid}/exe`),
|
|
310
|
+
stat(`/proc/${pid}`),
|
|
311
|
+
readFile(`/proc/${pid}/stat`, 'utf8'),
|
|
312
|
+
readFile('/proc/sys/kernel/random/boot_id', 'utf8')
|
|
313
|
+
])
|
|
314
|
+
const afterCommand = processStat
|
|
315
|
+
.slice(processStat.lastIndexOf(')') + 2)
|
|
316
|
+
.trim()
|
|
317
|
+
.split(/\s+/)
|
|
318
|
+
// /proc/<pid>/stat field 22 is starttime; the sliced sequence starts at field 3.
|
|
319
|
+
const startTicks = afterCommand[19]
|
|
320
|
+
if (startTicks === undefined || !/^\d+$/.test(startTicks))
|
|
321
|
+
throw createComputerError('provider_unavailable', 'Linux process start identity is unavailable')
|
|
322
|
+
return {
|
|
323
|
+
pid,
|
|
324
|
+
startedAt: processInfo.ctime.toISOString(),
|
|
325
|
+
executableId: `${executable}:${executableInfo.dev}:${executableInfo.ino}:${bootId.trim()}:${startTicks}`
|
|
326
|
+
}
|
|
327
|
+
}
|
|
328
|
+
|
|
329
|
+
function appInfo(rawValue: unknown): {
|
|
330
|
+
id: string
|
|
331
|
+
name: string
|
|
332
|
+
bundleId: string
|
|
333
|
+
pid: number
|
|
334
|
+
isRunning: true
|
|
335
|
+
} {
|
|
336
|
+
const raw = record(rawValue)
|
|
337
|
+
const pid = Number(raw.pid)
|
|
338
|
+
const name = String(raw.name ?? 'Unknown')
|
|
339
|
+
return { id: `linux:${pid}:${name}`, name, bundleId: name, pid, isRunning: true }
|
|
340
|
+
}
|
|
341
|
+
|
|
342
|
+
function nativeWindowId(pid: number, index: number): string {
|
|
343
|
+
return `linux:${pid}:window:${index}`
|
|
344
|
+
}
|
|
345
|
+
|
|
346
|
+
export type LinuxComputerProviderOptions = {
|
|
347
|
+
runtimePath?: string
|
|
348
|
+
manifestPath?: string
|
|
349
|
+
pythonPath?: string
|
|
350
|
+
environment?: Environment
|
|
351
|
+
skipPayloadVerification?: boolean
|
|
352
|
+
}
|
|
353
|
+
|
|
354
|
+
export class LinuxComputerProvider implements ComputerProvider {
|
|
355
|
+
readonly runtimePath: string
|
|
356
|
+
readonly #manifestPath: string
|
|
357
|
+
readonly #pythonPath: string | undefined
|
|
358
|
+
readonly #environment: Environment
|
|
359
|
+
readonly #skipPayloadVerification: boolean
|
|
360
|
+
readonly #pending = new Map<string, Pending>()
|
|
361
|
+
readonly #elements = new Map<string, unknown[]>()
|
|
362
|
+
#child: ChildProcessWithoutNullStreams | undefined
|
|
363
|
+
#lines: Interface | undefined
|
|
364
|
+
#handshake: NativeHandshake | undefined
|
|
365
|
+
#handshakeWait: Promise<NativeHandshake> | undefined
|
|
366
|
+
#resolveHandshake: ((value: NativeHandshake) => void) | undefined
|
|
367
|
+
#rejectHandshake: ((cause: unknown) => void) | undefined
|
|
368
|
+
#generation = `linux-unstarted-${randomUUID()}`
|
|
369
|
+
#stderr = ''
|
|
370
|
+
|
|
371
|
+
constructor(options: LinuxComputerProviderOptions = {}) {
|
|
372
|
+
this.runtimePath = options.runtimePath ?? PACKAGED_RUNTIME
|
|
373
|
+
this.#manifestPath = options.manifestPath ?? PACKAGED_MANIFEST
|
|
374
|
+
this.#pythonPath = options.pythonPath
|
|
375
|
+
this.#environment = options.environment ?? process.env
|
|
376
|
+
this.#skipPayloadVerification = options.skipPayloadVerification ?? false
|
|
377
|
+
}
|
|
378
|
+
|
|
379
|
+
get generation(): string {
|
|
380
|
+
return this.#generation
|
|
381
|
+
}
|
|
382
|
+
|
|
383
|
+
async start(): Promise<ProviderHandshake> {
|
|
384
|
+
if (this.#handshake !== undefined) return this.#contractHandshake(this.#handshake)
|
|
385
|
+
if (!this.#skipPayloadVerification)
|
|
386
|
+
await verifyLinuxPayload(this.runtimePath, this.#manifestPath)
|
|
387
|
+
const python = await selectPython(this.#pythonPath)
|
|
388
|
+
this.#handshakeWait = new Promise<NativeHandshake>((resolve, reject) => {
|
|
389
|
+
this.#resolveHandshake = resolve
|
|
390
|
+
this.#rejectHandshake = reject
|
|
391
|
+
})
|
|
392
|
+
const child = spawn(python, ['-I', '-u', this.runtimePath], {
|
|
393
|
+
cwd: '/',
|
|
394
|
+
env: linuxProviderEnvironment(this.#environment),
|
|
395
|
+
stdio: ['pipe', 'pipe', 'pipe'],
|
|
396
|
+
windowsHide: true
|
|
397
|
+
})
|
|
398
|
+
this.#child = child
|
|
399
|
+
child.stderr.setEncoding('utf8')
|
|
400
|
+
child.stderr.on('data', (chunk: string) => {
|
|
401
|
+
this.#stderr = (this.#stderr + chunk).slice(-4096)
|
|
402
|
+
})
|
|
403
|
+
child.once('error', (cause) => this.#failAll(cause))
|
|
404
|
+
child.once('exit', (code, signal) =>
|
|
405
|
+
this.#failAll(
|
|
406
|
+
createComputerError('provider_crashed', 'Linux provider process exited', { code, signal })
|
|
407
|
+
)
|
|
408
|
+
)
|
|
409
|
+
this.#lines = createInterface({ input: child.stdout, crlfDelay: Infinity })
|
|
410
|
+
this.#lines.on('line', (line) => this.#handleLine(line))
|
|
411
|
+
const handshake = await this.#handshakeWait
|
|
412
|
+
this.#handshake = handshake
|
|
413
|
+
this.#generation = handshake.generation
|
|
414
|
+
return this.#contractHandshake(handshake)
|
|
415
|
+
}
|
|
416
|
+
|
|
417
|
+
#contractHandshake(handshake: NativeHandshake): ProviderHandshake {
|
|
418
|
+
const readiness = handshake.capabilities.readiness
|
|
419
|
+
return {
|
|
420
|
+
provider: handshake.provider,
|
|
421
|
+
generation: handshake.generation,
|
|
422
|
+
graphicalSessionId: handshake.graphicalSessionId,
|
|
423
|
+
providerProtocol: handshake.providerProtocol,
|
|
424
|
+
publicContract: handshake.publicContract,
|
|
425
|
+
capabilities: {
|
|
426
|
+
platform: 'linux',
|
|
427
|
+
provider: handshake.provider,
|
|
428
|
+
providerVersion: handshake.providerVersion,
|
|
429
|
+
operations: operations(readiness),
|
|
430
|
+
permissions: {
|
|
431
|
+
accessibility: readiness.capabilities.accessibility === true ? 'granted' : 'denied',
|
|
432
|
+
screenshots:
|
|
433
|
+
readiness.sessionType === 'wayland'
|
|
434
|
+
? 'not_required'
|
|
435
|
+
: readiness.capabilities.screenshots === true
|
|
436
|
+
? 'granted'
|
|
437
|
+
: 'denied'
|
|
438
|
+
}
|
|
439
|
+
}
|
|
440
|
+
}
|
|
441
|
+
}
|
|
442
|
+
|
|
443
|
+
#handleLine(line: string): void {
|
|
444
|
+
if (Buffer.byteLength(line, 'utf8') > MAX_NATIVE_FRAME_BYTES) {
|
|
445
|
+
this.#failAll(
|
|
446
|
+
createComputerError('provider_crashed', 'Linux provider emitted an oversized frame')
|
|
447
|
+
)
|
|
448
|
+
void this.close()
|
|
449
|
+
return
|
|
450
|
+
}
|
|
451
|
+
let frame: NativeFrame
|
|
452
|
+
try {
|
|
453
|
+
frame = JSON.parse(line) as NativeFrame
|
|
454
|
+
} catch {
|
|
455
|
+
this.#failAll(
|
|
456
|
+
createComputerError('provider_crashed', 'Linux provider emitted malformed JSON')
|
|
457
|
+
)
|
|
458
|
+
void this.close()
|
|
459
|
+
return
|
|
460
|
+
}
|
|
461
|
+
if (frame.type === 'handshake') {
|
|
462
|
+
this.#resolveHandshake?.(frame)
|
|
463
|
+
return
|
|
464
|
+
}
|
|
465
|
+
if (frame.type === 'fatal') {
|
|
466
|
+
this.#failAll(createComputerError('provider_crashed', frame.error))
|
|
467
|
+
return
|
|
468
|
+
}
|
|
469
|
+
const pending = this.#pending.get(frame.requestId)
|
|
470
|
+
if (pending === undefined) return
|
|
471
|
+
this.#pending.delete(frame.requestId)
|
|
472
|
+
pending.resolve(frame)
|
|
473
|
+
}
|
|
474
|
+
|
|
475
|
+
#failAll(cause: unknown): void {
|
|
476
|
+
this.#rejectHandshake?.(cause)
|
|
477
|
+
this.#rejectHandshake = undefined
|
|
478
|
+
this.#resolveHandshake = undefined
|
|
479
|
+
for (const pending of this.#pending.values()) pending.reject(cause)
|
|
480
|
+
this.#pending.clear()
|
|
481
|
+
}
|
|
482
|
+
|
|
483
|
+
async dispatch(request: ProviderRequest): Promise<ProviderResponse> {
|
|
484
|
+
const handshake = this.#handshake ?? (await this.start(), this.#handshake)
|
|
485
|
+
const child = this.#child
|
|
486
|
+
if (handshake === undefined || child === undefined)
|
|
487
|
+
throw createComputerError('provider_unavailable', 'Linux provider failed to start')
|
|
488
|
+
if (request.operation !== 'capabilities' && request.operation !== 'permissions') {
|
|
489
|
+
const supported = operations(handshake.capabilities.readiness)[request.operation]
|
|
490
|
+
if (supported !== true) {
|
|
491
|
+
return {
|
|
492
|
+
requestId: request.requestId,
|
|
493
|
+
dispatched: false,
|
|
494
|
+
error: createComputerError(
|
|
495
|
+
handshake.capabilities.readiness.available
|
|
496
|
+
? 'unsupported_capability'
|
|
497
|
+
: 'provider_unavailable',
|
|
498
|
+
handshake.capabilities.readiness.issues.map((issue) => issue.message).join('; ') ||
|
|
499
|
+
`${request.operation} is unavailable in this session`
|
|
500
|
+
).toJSON()
|
|
501
|
+
}
|
|
502
|
+
}
|
|
503
|
+
const input = record(request.input)
|
|
504
|
+
const requestsScreenshot =
|
|
505
|
+
(request.operation === 'getAppState' || COMPUTER_OPERATIONS[request.operation].mutation) &&
|
|
506
|
+
input.captureScreenshot !== false
|
|
507
|
+
if (
|
|
508
|
+
requestsScreenshot &&
|
|
509
|
+
handshake.capabilities.readiness.capabilities.screenshots !== true
|
|
510
|
+
) {
|
|
511
|
+
return {
|
|
512
|
+
requestId: request.requestId,
|
|
513
|
+
dispatched: false,
|
|
514
|
+
error: createComputerError(
|
|
515
|
+
'unsupported_capability',
|
|
516
|
+
`Window screenshots are unavailable in ${handshake.capabilities.readiness.sessionType} sessions; retry with captureScreenshot=false`
|
|
517
|
+
).toJSON()
|
|
518
|
+
}
|
|
519
|
+
}
|
|
520
|
+
if (
|
|
521
|
+
request.operation === 'click' &&
|
|
522
|
+
record(input.target).kind === 'coordinate' &&
|
|
523
|
+
handshake.capabilities.readiness.capabilities.syntheticPointer !== true
|
|
524
|
+
) {
|
|
525
|
+
return {
|
|
526
|
+
requestId: request.requestId,
|
|
527
|
+
dispatched: false,
|
|
528
|
+
error: createComputerError(
|
|
529
|
+
'unsupported_capability',
|
|
530
|
+
`Coordinate clicking is unavailable in ${handshake.capabilities.readiness.sessionType} sessions`
|
|
531
|
+
).toJSON()
|
|
532
|
+
}
|
|
533
|
+
}
|
|
534
|
+
}
|
|
535
|
+
|
|
536
|
+
const nativeOperation = this.#attachElementRecords(
|
|
537
|
+
mapNativeOperation(request.operation, request.input),
|
|
538
|
+
request.input
|
|
539
|
+
)
|
|
540
|
+
const frame = await new Promise<NativeResponse>((resolve, reject) => {
|
|
541
|
+
this.#pending.set(request.requestId, { resolve, reject })
|
|
542
|
+
child.stdin.write(
|
|
543
|
+
`${JSON.stringify({ type: 'request', requestId: request.requestId, operation: nativeOperation })}\n`,
|
|
544
|
+
(cause) => {
|
|
545
|
+
if (cause === null || cause === undefined) return
|
|
546
|
+
this.#pending.delete(request.requestId)
|
|
547
|
+
reject(cause)
|
|
548
|
+
}
|
|
549
|
+
)
|
|
550
|
+
})
|
|
551
|
+
if (!frame.ok) {
|
|
552
|
+
return {
|
|
553
|
+
requestId: request.requestId,
|
|
554
|
+
dispatched: frame.dispatched === true,
|
|
555
|
+
error: normalizeNativeError(frame.error).toJSON()
|
|
556
|
+
}
|
|
557
|
+
}
|
|
558
|
+
return {
|
|
559
|
+
requestId: request.requestId,
|
|
560
|
+
dispatched: COMPUTER_OPERATIONS[request.operation].mutation,
|
|
561
|
+
result: await this.#normalizeResult(request.operation, frame.result)
|
|
562
|
+
}
|
|
563
|
+
}
|
|
564
|
+
|
|
565
|
+
#attachElementRecords(
|
|
566
|
+
operation: Record<string, unknown>,
|
|
567
|
+
input: unknown
|
|
568
|
+
): Record<string, unknown> {
|
|
569
|
+
const result = { ...operation }
|
|
570
|
+
const source = record(input)
|
|
571
|
+
for (const [targetName, nativeName] of [
|
|
572
|
+
['target', 'element'],
|
|
573
|
+
['from', 'fromElement'],
|
|
574
|
+
['to', 'toElement']
|
|
575
|
+
] as const) {
|
|
576
|
+
const target = record(source[targetName])
|
|
577
|
+
const reference = record(target.ref ?? target)
|
|
578
|
+
const match = /^element:(\d+)$/.exec(String(reference.ref ?? ''))
|
|
579
|
+
const elements = this.#elements.get(String(reference.snapshotId ?? ''))
|
|
580
|
+
const index = match === null ? undefined : Number(match[1])
|
|
581
|
+
if (elements !== undefined && index !== undefined && elements[index] !== undefined)
|
|
582
|
+
result[nativeName] = elements[index]
|
|
583
|
+
}
|
|
584
|
+
return result
|
|
585
|
+
}
|
|
586
|
+
|
|
587
|
+
async #normalizeResult(
|
|
588
|
+
operation: ComputerOperationName,
|
|
589
|
+
native: Record<string, unknown>
|
|
590
|
+
): Promise<unknown> {
|
|
591
|
+
const handshake = this.#handshake!
|
|
592
|
+
if (operation === 'capabilities') return this.#contractHandshake(handshake).capabilities
|
|
593
|
+
if (operation === 'permissions')
|
|
594
|
+
return { permissions: this.#contractHandshake(handshake).capabilities.permissions }
|
|
595
|
+
if (operation === 'listApps') {
|
|
596
|
+
const apps = Array.isArray(native.apps) ? native.apps.map(appInfo) : []
|
|
597
|
+
return { apps }
|
|
598
|
+
}
|
|
599
|
+
if (operation === 'listWindows') {
|
|
600
|
+
const windows = Array.isArray(native.windows) ? native.windows : []
|
|
601
|
+
return {
|
|
602
|
+
windows: windows.map((value) => {
|
|
603
|
+
const window = record(value)
|
|
604
|
+
const app = appInfo(window.app)
|
|
605
|
+
return {
|
|
606
|
+
id: nativeWindowId(app.pid, Number(window.index)),
|
|
607
|
+
appId: app.id,
|
|
608
|
+
title: String(window.title ?? ''),
|
|
609
|
+
index: Number(window.index),
|
|
610
|
+
bounds: {
|
|
611
|
+
x: Number(window.x),
|
|
612
|
+
y: Number(window.y),
|
|
613
|
+
width: Number(window.width),
|
|
614
|
+
height: Number(window.height)
|
|
615
|
+
},
|
|
616
|
+
minimized: window.isMinimized === true
|
|
617
|
+
}
|
|
618
|
+
})
|
|
619
|
+
}
|
|
620
|
+
}
|
|
621
|
+
|
|
622
|
+
const rawSnapshot = record(native.snapshot)
|
|
623
|
+
const normalizedSnapshot = await this.#normalizeSnapshot(rawSnapshot)
|
|
624
|
+
if (operation === 'getAppState') return normalizedSnapshot
|
|
625
|
+
const action = record(native.action)
|
|
626
|
+
const verification = record(action.verification)
|
|
627
|
+
const verified = action.path === 'accessibility' && verification.state === 'verified'
|
|
628
|
+
return {
|
|
629
|
+
outcome: verified
|
|
630
|
+
? {
|
|
631
|
+
state: 'verified',
|
|
632
|
+
evidence: { action: action.actionName, snapshotId: normalizedSnapshot.snapshot.id }
|
|
633
|
+
}
|
|
634
|
+
: {
|
|
635
|
+
state: 'indeterminate',
|
|
636
|
+
reason:
|
|
637
|
+
typeof verification.reason === 'string'
|
|
638
|
+
? verification.reason
|
|
639
|
+
: 'fresh state returned but the provider could not prove the requested effect'
|
|
640
|
+
},
|
|
641
|
+
freshState: normalizedSnapshot
|
|
642
|
+
}
|
|
643
|
+
}
|
|
644
|
+
|
|
645
|
+
async #normalizeSnapshot(raw: Record<string, unknown>): Promise<{
|
|
646
|
+
bindings: ReferenceBindings
|
|
647
|
+
snapshot: Record<string, unknown>
|
|
648
|
+
screenshot: Record<string, unknown> | null
|
|
649
|
+
issues: Record<string, unknown>[]
|
|
650
|
+
}> {
|
|
651
|
+
const app = appInfo(raw.app)
|
|
652
|
+
const snapshotId = String(raw.snapshotId)
|
|
653
|
+
const index = Number(raw.windowIndex)
|
|
654
|
+
const bounds = record(raw.windowBounds)
|
|
655
|
+
const process = await processIdentity(app.pid)
|
|
656
|
+
const windowId = nativeWindowId(app.pid, index)
|
|
657
|
+
const elements = Array.isArray(raw.elements) ? raw.elements : []
|
|
658
|
+
this.#elements.set(snapshotId, elements)
|
|
659
|
+
if (this.#elements.size > 32) this.#elements.delete(this.#elements.keys().next().value!)
|
|
660
|
+
const bindings: ReferenceBindings = {
|
|
661
|
+
brokerGeneration: 'broker-pending',
|
|
662
|
+
providerGeneration: this.generation,
|
|
663
|
+
graphicalSessionId: this.#handshake!.graphicalSessionId,
|
|
664
|
+
process,
|
|
665
|
+
appId: app.id,
|
|
666
|
+
window: { id: windowId, ownerPid: app.pid },
|
|
667
|
+
snapshotId,
|
|
668
|
+
desktopEpoch: 0
|
|
669
|
+
}
|
|
670
|
+
const width = Number(raw.screenshotWidth)
|
|
671
|
+
const height = Number(raw.screenshotHeight)
|
|
672
|
+
const screenshotData = raw.screenshotPngBase64
|
|
673
|
+
return {
|
|
674
|
+
bindings,
|
|
675
|
+
snapshot: {
|
|
676
|
+
id: snapshotId,
|
|
677
|
+
app,
|
|
678
|
+
window: {
|
|
679
|
+
id: windowId,
|
|
680
|
+
appId: app.id,
|
|
681
|
+
title: String(raw.windowTitle ?? ''),
|
|
682
|
+
index,
|
|
683
|
+
bounds: {
|
|
684
|
+
x: Number(bounds.x),
|
|
685
|
+
y: Number(bounds.y),
|
|
686
|
+
width: Number(bounds.width),
|
|
687
|
+
height: Number(bounds.height)
|
|
688
|
+
},
|
|
689
|
+
minimized: false
|
|
690
|
+
},
|
|
691
|
+
treeText: Array.isArray(raw.treeLines) ? raw.treeLines.join('\n') : '',
|
|
692
|
+
elementCount: elements.length,
|
|
693
|
+
focusedElementRef: null,
|
|
694
|
+
desktopEpoch: 0
|
|
695
|
+
},
|
|
696
|
+
screenshot:
|
|
697
|
+
typeof screenshotData === 'string' && screenshotData.length > 0 && width > 0 && height > 0
|
|
698
|
+
? {
|
|
699
|
+
format: 'png',
|
|
700
|
+
width,
|
|
701
|
+
height,
|
|
702
|
+
scale: Number(raw.screenshotScale),
|
|
703
|
+
data: screenshotData
|
|
704
|
+
}
|
|
705
|
+
: null,
|
|
706
|
+
issues: normalizeScreenshotIssues(raw.screenshotError)
|
|
707
|
+
}
|
|
708
|
+
}
|
|
709
|
+
|
|
710
|
+
async inspectTarget(
|
|
711
|
+
_operation: ComputerOperationName,
|
|
712
|
+
input: unknown
|
|
713
|
+
): Promise<{
|
|
714
|
+
bindings: ReferenceBindings
|
|
715
|
+
appIdentity: { appId: string; executableId: string }
|
|
716
|
+
} | null> {
|
|
717
|
+
const reference = targetReference(input)
|
|
718
|
+
if (reference === undefined) return null
|
|
719
|
+
let current
|
|
720
|
+
try {
|
|
721
|
+
current = await processIdentity(reference.process.pid)
|
|
722
|
+
} catch {
|
|
723
|
+
return null
|
|
724
|
+
}
|
|
725
|
+
if (
|
|
726
|
+
current.startedAt !== reference.process.startedAt ||
|
|
727
|
+
current.executableId !== reference.process.executableId
|
|
728
|
+
)
|
|
729
|
+
return null
|
|
730
|
+
return {
|
|
731
|
+
bindings: { ...reference, process: current },
|
|
732
|
+
appIdentity: { appId: reference.appId, executableId: current.executableId }
|
|
733
|
+
}
|
|
734
|
+
}
|
|
735
|
+
|
|
736
|
+
async cancel(requestId: string): Promise<void> {
|
|
737
|
+
if (!this.#pending.has(requestId)) return
|
|
738
|
+
this.#child?.stdin.write(`${JSON.stringify({ type: 'cancel', requestId })}\n`)
|
|
739
|
+
}
|
|
740
|
+
|
|
741
|
+
async close(): Promise<void> {
|
|
742
|
+
const child = this.#child
|
|
743
|
+
this.#child = undefined
|
|
744
|
+
this.#handshake = undefined
|
|
745
|
+
this.#lines?.close()
|
|
746
|
+
this.#lines = undefined
|
|
747
|
+
this.#failAll(createComputerError('provider_crashed', 'Linux provider closed'))
|
|
748
|
+
if (child === undefined) return
|
|
749
|
+
child.stdin.end()
|
|
750
|
+
if (child.exitCode === null) child.kill('SIGTERM')
|
|
751
|
+
}
|
|
752
|
+
}
|
|
753
|
+
|
|
754
|
+
export function normalizeScreenshotIssues(value: unknown): Record<string, unknown>[] {
|
|
755
|
+
const error = record(value)
|
|
756
|
+
if (typeof error.message !== 'string' || error.message.length === 0) return []
|
|
757
|
+
return [
|
|
758
|
+
createComputerError('screenshot_failed', error.message, { component: 'screenshots' }).toJSON()
|
|
759
|
+
]
|
|
760
|
+
}
|
|
761
|
+
|
|
762
|
+
let activeProvider: LinuxComputerProvider | undefined
|
|
763
|
+
|
|
764
|
+
export function createProvider(): LinuxComputerProvider {
|
|
765
|
+
activeProvider = new LinuxComputerProvider()
|
|
766
|
+
return activeProvider
|
|
767
|
+
}
|
|
768
|
+
|
|
769
|
+
export async function inspectTarget(
|
|
770
|
+
operation: ComputerOperationName,
|
|
771
|
+
input: unknown
|
|
772
|
+
): Promise<{
|
|
773
|
+
bindings: ReferenceBindings
|
|
774
|
+
appIdentity: { appId: string; executableId: string }
|
|
775
|
+
} | null> {
|
|
776
|
+
return activeProvider?.inspectTarget(operation, input) ?? null
|
|
777
|
+
}
|