@anionex/dsh-computer-use 0.2.1 → 0.3.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +13 -7
- package/README.zh.md +13 -7
- package/cordis.patch.yml +3 -0
- package/docs/interaction-policy.md +6 -4
- package/docs/interaction-policy.zh.md +7 -5
- package/lib/client.js +31 -11
- package/lib/client.js.map +1 -1
- package/lib/config.js +13 -3
- package/lib/config.js.map +1 -1
- package/lib/providers/macos.js +79 -14
- package/lib/providers/macos.js.map +1 -1
- package/lib/providers/native-helper.js +240 -17
- package/lib/providers/native-helper.js.map +1 -1
- package/lib/providers/unsupported.js +6 -0
- package/lib/providers/unsupported.js.map +1 -1
- package/lib/service.js +74 -32
- package/lib/service.js.map +1 -1
- package/lib/types/backend.d.ts +18 -3
- package/lib/types/backend.d.ts.map +1 -1
- package/lib/types/client/index.d.ts +7 -2
- package/lib/types/client/index.d.ts.map +1 -1
- package/lib/types/config.d.ts +7 -1
- package/lib/types/config.d.ts.map +1 -1
- package/lib/types/providers/macos.d.ts +8 -2
- package/lib/types/providers/macos.d.ts.map +1 -1
- package/lib/types/providers/native-helper.d.ts +9 -1
- package/lib/types/providers/native-helper.d.ts.map +1 -1
- package/lib/types/providers/unsupported.d.ts +6 -2
- package/lib/types/providers/unsupported.d.ts.map +1 -1
- package/lib/types/service.d.ts.map +1 -1
- package/native/macos/Sources/Fixture/main.swift +46 -0
- package/native/macos/Sources/Helper/CursorMotion.swift +67 -0
- package/native/macos/Sources/Helper/CursorOverlay.swift +334 -60
- package/native/macos/Sources/Helper/TargetedPointer.swift +23 -10
- package/native/macos/Sources/Helper/main.swift +80 -5
- package/native/macos/Sources/Monitor/main.swift +15 -1
- package/native/macos/bin/dsh-computer-use-helper +0 -0
- package/native/macos/manifest.json +3 -3
- package/package.json +16 -18
- package/scripts/build-client.mjs +0 -0
- package/scripts/build-native.mjs +0 -0
- package/scripts/model-e2e.mjs +0 -0
- package/scripts/validate.mjs +0 -0
- package/src/backend.ts +18 -3
- package/src/client/index.tsx +50 -13
- package/src/config.ts +20 -4
- package/src/providers/macos.ts +102 -14
- package/src/providers/native-helper.ts +234 -17
- package/src/providers/unsupported.ts +11 -1
- package/src/service.ts +100 -37
package/src/providers/macos.ts
CHANGED
|
@@ -9,6 +9,8 @@ import type {} from '@deepseek-ai/dsh-user-approval'
|
|
|
9
9
|
import type {
|
|
10
10
|
BackendActionRequest,
|
|
11
11
|
BackendActionResult,
|
|
12
|
+
BackendCursorActivation,
|
|
13
|
+
BackendTrackedDragResult,
|
|
12
14
|
BackendCursorAction,
|
|
13
15
|
BackendHealth,
|
|
14
16
|
BackendObservation,
|
|
@@ -36,6 +38,11 @@ interface NativeHealth {
|
|
|
36
38
|
|
|
37
39
|
interface NativeObservation extends BackendObservation {}
|
|
38
40
|
|
|
41
|
+
interface NativeCursorActivation {
|
|
42
|
+
observation: NativeObservation
|
|
43
|
+
activation: BackendCursorActivation['activation']
|
|
44
|
+
}
|
|
45
|
+
|
|
39
46
|
function createBackend(ctx: Context, config: ResolvedComputerUseConfig): ComputerUseBackend {
|
|
40
47
|
return process.platform === 'darwin'
|
|
41
48
|
? new MacOSBackend(ctx, config)
|
|
@@ -71,7 +78,27 @@ export class MacOSBackend implements ComputerUseBackend {
|
|
|
71
78
|
}, signal)
|
|
72
79
|
}
|
|
73
80
|
|
|
81
|
+
async activateForCursor(
|
|
82
|
+
app: ComputerAppIdentity,
|
|
83
|
+
expectedStateHash: string,
|
|
84
|
+
options: BackendObserveOptions,
|
|
85
|
+
signal: AbortSignal,
|
|
86
|
+
): Promise<BackendCursorActivation> {
|
|
87
|
+
return await this.client.invoke<NativeCursorActivation>({
|
|
88
|
+
command: 'activate-for-cursor',
|
|
89
|
+
app,
|
|
90
|
+
expectedStateHash,
|
|
91
|
+
options,
|
|
92
|
+
actionTimeoutMs: this.config.actionTimeoutMs,
|
|
93
|
+
}, signal)
|
|
94
|
+
}
|
|
95
|
+
|
|
74
96
|
async act(request: BackendActionRequest, signal: AbortSignal): Promise<BackendActionResult> {
|
|
97
|
+
if (request.action.kind === 'drag') {
|
|
98
|
+
const execution = await this.prepareNativeDrag(request, signal)
|
|
99
|
+
await execution.start()
|
|
100
|
+
return await execution.result
|
|
101
|
+
}
|
|
75
102
|
return await this.client.invoke<BackendActionResult>({
|
|
76
103
|
command: 'act',
|
|
77
104
|
request: {
|
|
@@ -86,7 +113,58 @@ export class MacOSBackend implements ComputerUseBackend {
|
|
|
86
113
|
}, signal)
|
|
87
114
|
}
|
|
88
115
|
|
|
89
|
-
async
|
|
116
|
+
async actDragWithCursor(
|
|
117
|
+
request: BackendActionRequest,
|
|
118
|
+
cursor: BackendCursorAction & { kind: 'drag' },
|
|
119
|
+
signal: AbortSignal,
|
|
120
|
+
): Promise<BackendTrackedDragResult> {
|
|
121
|
+
const execution = await this.prepareNativeDrag(request, signal)
|
|
122
|
+
let started = false
|
|
123
|
+
let cursorResult: CursorVisibility
|
|
124
|
+
try {
|
|
125
|
+
cursorResult = await this.visualizeCursorPhase(cursor, 'during', signal, async () => {
|
|
126
|
+
await execution.start()
|
|
127
|
+
started = true
|
|
128
|
+
})
|
|
129
|
+
} catch (error) {
|
|
130
|
+
if (!started) {
|
|
131
|
+
execution.cancel()
|
|
132
|
+
await execution.result.catch(() => undefined)
|
|
133
|
+
throw error
|
|
134
|
+
}
|
|
135
|
+
cursorResult = {
|
|
136
|
+
visible: false,
|
|
137
|
+
reason: `the agent cursor could not track the drag action: ${error instanceof Error ? error.message : String(error)}`,
|
|
138
|
+
}
|
|
139
|
+
}
|
|
140
|
+
return { action: await execution.result, cursor: cursorResult }
|
|
141
|
+
}
|
|
142
|
+
|
|
143
|
+
private async prepareNativeDrag(request: BackendActionRequest, signal: AbortSignal) {
|
|
144
|
+
return await this.client.prepareDrag<BackendActionResult>({
|
|
145
|
+
command: 'act',
|
|
146
|
+
request: {
|
|
147
|
+
...request,
|
|
148
|
+
actionTimeoutMs: this.config.actionTimeoutMs,
|
|
149
|
+
limits: {
|
|
150
|
+
maxNodes: this.config.maxNodes,
|
|
151
|
+
maxDepth: this.config.maxDepth,
|
|
152
|
+
maxTextBytes: this.config.maxTextBytes,
|
|
153
|
+
},
|
|
154
|
+
},
|
|
155
|
+
}, signal, this.config.actionTimeoutMs + 2_000)
|
|
156
|
+
}
|
|
157
|
+
|
|
158
|
+
async visualizeCursor(action: BackendCursorAction, phase: 'before' | 'during' | 'after', signal: AbortSignal): Promise<CursorVisibility> {
|
|
159
|
+
return await this.visualizeCursorPhase(action, phase, signal)
|
|
160
|
+
}
|
|
161
|
+
|
|
162
|
+
private async visualizeCursorPhase(
|
|
163
|
+
action: BackendCursorAction,
|
|
164
|
+
phase: 'before' | 'during' | 'after',
|
|
165
|
+
signal: AbortSignal,
|
|
166
|
+
onMoveWritten?: () => void | Promise<void>,
|
|
167
|
+
): Promise<CursorVisibility> {
|
|
90
168
|
if (this.config.interaction.cursorVisualization !== 'visible') return { visible: false, reason: 'the agent cursor is disabled by configuration' }
|
|
91
169
|
// The overlay answers per command; the least visible outcome wins, because
|
|
92
170
|
// a cursor that vanished partway through is a cursor the user cannot follow.
|
|
@@ -95,17 +173,23 @@ export class MacOSBackend implements ComputerUseBackend {
|
|
|
95
173
|
if (!response.visible && outcome.visible) outcome = response
|
|
96
174
|
}
|
|
97
175
|
const autoHideMs = this.config.interaction.cursorAutoHideMs
|
|
98
|
-
const move = async (point: { x: number; y: number },
|
|
99
|
-
|
|
176
|
+
const move = async (point: { x: number; y: number }, onWritten?: () => void | Promise<void>): Promise<void> => {
|
|
177
|
+
const command = {
|
|
100
178
|
op: 'move',
|
|
101
179
|
x: point.x,
|
|
102
180
|
y: point.y,
|
|
103
|
-
|
|
104
|
-
|
|
181
|
+
speedPxPerSecond: this.config.interaction.cursorSpeedPxPerSecond,
|
|
182
|
+
accelerationPxPerSecondSquared: this.config.interaction.cursorAccelerationPxPerSecondSquared,
|
|
183
|
+
// The after phase arms the configured timeout once native input has
|
|
184
|
+
// completed. Hiding during travel or dwell would invalidate press.
|
|
185
|
+
autoHideMs: 0,
|
|
105
186
|
targetPid: action.targetPid,
|
|
106
187
|
targetWindowNumber: action.targetWindowNumber,
|
|
107
188
|
targetWindowFrame: action.targetWindowFrame,
|
|
108
|
-
}
|
|
189
|
+
}
|
|
190
|
+
record(await (onWritten === undefined
|
|
191
|
+
? this.client.cursorCommand(command, signal)
|
|
192
|
+
: this.client.cursorCommand(command, signal, onWritten)))
|
|
109
193
|
}
|
|
110
194
|
if (phase === 'after') {
|
|
111
195
|
// Every action validates the bound target after native input. Only drag
|
|
@@ -119,24 +203,28 @@ export class MacOSBackend implements ComputerUseBackend {
|
|
|
119
203
|
}, signal))
|
|
120
204
|
return outcome
|
|
121
205
|
}
|
|
206
|
+
if (phase === 'during') {
|
|
207
|
+
if (action.kind !== 'drag') return { visible: false, reason: 'only drag has a during-action cursor phase' }
|
|
208
|
+
await move(action.to, onMoveWritten)
|
|
209
|
+
return outcome
|
|
210
|
+
}
|
|
122
211
|
const start = action.kind === 'drag' ? action.from : action.to
|
|
123
212
|
if (start === undefined) return { visible: false, reason: 'this action has no cursor position to show' }
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
213
|
+
// A move response means the native overlay reached its destination. Keep
|
|
214
|
+
// the configurable dwell after arrival and before the visual/native press.
|
|
215
|
+
await move(start)
|
|
216
|
+
if (!outcome.visible || action.kind === 'scroll') return outcome
|
|
217
|
+
if (this.config.interaction.cursorClickDelayMs > 0) {
|
|
218
|
+
await delay(this.config.interaction.cursorClickDelayMs, undefined, { signal })
|
|
127
219
|
}
|
|
128
|
-
if (action.kind === 'scroll') return outcome
|
|
129
220
|
record(await this.client.cursorCommand({
|
|
130
221
|
op: 'press',
|
|
131
|
-
autoHideMs,
|
|
222
|
+
autoHideMs: 0,
|
|
132
223
|
targetPid: action.targetPid,
|
|
133
224
|
targetWindowNumber: action.targetWindowNumber,
|
|
134
225
|
targetWindowFrame: action.targetWindowFrame,
|
|
135
226
|
sustainedPress: action.kind === 'drag',
|
|
136
227
|
}, signal))
|
|
137
|
-
if (action.kind === 'drag') {
|
|
138
|
-
await move(action.to, Math.max(this.config.interaction.cursorMotionMs, 240))
|
|
139
|
-
}
|
|
140
228
|
return outcome
|
|
141
229
|
}
|
|
142
230
|
|
|
@@ -5,6 +5,7 @@ import { access, chmod, lstat, readFile, realpath, stat } from 'node:fs/promises
|
|
|
5
5
|
import { constants, type Stats } from 'node:fs'
|
|
6
6
|
import type { Writable } from 'node:stream'
|
|
7
7
|
import { dirname, resolve } from 'node:path'
|
|
8
|
+
import { createInterface } from 'node:readline'
|
|
8
9
|
import { fileURLToPath } from 'node:url'
|
|
9
10
|
import type { Context } from '@deepseek-ai/cordis'
|
|
10
11
|
import type { SubprocessHandle, SubprocessOutcome, SubprocessOutputReader } from '@deepseek-ai/dsh-subprocess'
|
|
@@ -39,6 +40,12 @@ interface HelperSuccess<T> {
|
|
|
39
40
|
|
|
40
41
|
type HelperEnvelope<T> = HelperFailure | HelperSuccess<T>
|
|
41
42
|
|
|
43
|
+
export interface PreparedNativeDrag<T> {
|
|
44
|
+
readonly result: Promise<T>
|
|
45
|
+
start(): Promise<void>
|
|
46
|
+
cancel(): void
|
|
47
|
+
}
|
|
48
|
+
|
|
42
49
|
/**
|
|
43
50
|
* The response includes synchronous WindowServer validation, which can exceed
|
|
44
51
|
* a scheduler tick under load even though the protocol itself is local.
|
|
@@ -46,6 +53,7 @@ type HelperEnvelope<T> = HelperFailure | HelperSuccess<T>
|
|
|
46
53
|
// Keep the bound finite. A timed-out generation is discarded before the next
|
|
47
54
|
// serialized command, so a late frame cannot satisfy a later command.
|
|
48
55
|
const CURSOR_RESPONSE_TIMEOUT_MS = 1_000
|
|
56
|
+
const CURSOR_MOTION_RESPONSE_TIMEOUT_MS = 4_000
|
|
49
57
|
|
|
50
58
|
const CURSOR_READY_TIMEOUT_MS = 2_000
|
|
51
59
|
const CURSOR_PROTOCOL_MAX_BYTES = 64 * 1024
|
|
@@ -60,7 +68,16 @@ interface CursorProcess {
|
|
|
60
68
|
terminate: () => void
|
|
61
69
|
waitForExit: SubprocessHandle['waitForExit']
|
|
62
70
|
/** Next response line from this process generation, in command order. */
|
|
63
|
-
nextResponse: () => Promise<CursorProtocolFrame>
|
|
71
|
+
nextResponse: (timeoutMs: number, signal: AbortSignal) => Promise<CursorProtocolFrame>
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
function cursorResponseTimeout(command: Record<string, unknown>): number {
|
|
75
|
+
if (command.op !== 'move') return CURSOR_RESPONSE_TIMEOUT_MS
|
|
76
|
+
if (typeof command.speedPxPerSecond === 'number') return CURSOR_MOTION_RESPONSE_TIMEOUT_MS
|
|
77
|
+
if (typeof command.durationMs === 'number' && Number.isFinite(command.durationMs)) {
|
|
78
|
+
return Math.max(CURSOR_RESPONSE_TIMEOUT_MS, Math.trunc(command.durationMs) + 2 * CURSOR_RESPONSE_TIMEOUT_MS)
|
|
79
|
+
}
|
|
80
|
+
return CURSOR_RESPONSE_TIMEOUT_MS
|
|
64
81
|
}
|
|
65
82
|
|
|
66
83
|
function collected(reader: SubprocessOutputReader | undefined): string {
|
|
@@ -115,9 +132,13 @@ function normalizeCursorResponse(
|
|
|
115
132
|
}
|
|
116
133
|
if (response.visible === true) return { result: { visible: true }, discardGeneration: false }
|
|
117
134
|
if (response.visible === false) {
|
|
135
|
+
const reasonCode = response.reasonCode === 'target-not-frontmost' || response.reasonCode === 'target-invalid'
|
|
136
|
+
? response.reasonCode
|
|
137
|
+
: undefined
|
|
118
138
|
return {
|
|
119
139
|
result: {
|
|
120
140
|
visible: false,
|
|
141
|
+
...(reasonCode === undefined ? {} : { reasonCode }),
|
|
121
142
|
...(typeof response.reason === 'string' && response.reason.length > 0
|
|
122
143
|
? { reason: response.reason.slice(0, 1000) }
|
|
123
144
|
: { reason: 'the native cursor overlay reported that the cursor is not visible' }),
|
|
@@ -144,6 +165,7 @@ export class NativeHelperClient {
|
|
|
144
165
|
private cursor: CursorProcess | undefined
|
|
145
166
|
private cursorStart: { promise: Promise<CursorProcess> } | undefined
|
|
146
167
|
private cursorCommandTail: Promise<void> = Promise.resolve()
|
|
168
|
+
private disposed = false
|
|
147
169
|
|
|
148
170
|
constructor(
|
|
149
171
|
private readonly ctx: Context,
|
|
@@ -255,17 +277,179 @@ export class NativeHelperClient {
|
|
|
255
277
|
return envelope.value
|
|
256
278
|
}
|
|
257
279
|
|
|
280
|
+
/** Prepare and validate a drag, then wait for an explicit pre-mouse-down start barrier. */
|
|
281
|
+
async prepareDrag<T>(
|
|
282
|
+
request: Record<string, unknown>,
|
|
283
|
+
signal: AbortSignal,
|
|
284
|
+
readinessTimeoutMs: number,
|
|
285
|
+
): Promise<PreparedNativeDrag<T>> {
|
|
286
|
+
const prepared = this.prepared ?? await this.prepare(signal)
|
|
287
|
+
if (signal.aborted) throw new ComputerUseError('COMPUTER_CANCELLED', 'native drag preparation was cancelled')
|
|
288
|
+
const processAbort = new AbortController()
|
|
289
|
+
const handle = this.ctx.subprocess.spawn({
|
|
290
|
+
argv: [prepared.path, '--drag-action'],
|
|
291
|
+
cwd: dirname(prepared.path),
|
|
292
|
+
stdio: {
|
|
293
|
+
stdin: 'pipe',
|
|
294
|
+
stdout: 'pipe',
|
|
295
|
+
stderr: { maxBytes: 64 * 1024 },
|
|
296
|
+
},
|
|
297
|
+
graceMs: 1000,
|
|
298
|
+
signal: processAbort.signal,
|
|
299
|
+
env: {
|
|
300
|
+
LANG: process.env.LANG ?? 'en_US.UTF-8',
|
|
301
|
+
LC_ALL: process.env.LC_ALL ?? 'en_US.UTF-8',
|
|
302
|
+
},
|
|
303
|
+
})
|
|
304
|
+
if (handle.stdin === undefined || handle.stdout === undefined) {
|
|
305
|
+
handle.terminate()
|
|
306
|
+
await handle.waitForExit()
|
|
307
|
+
throw new ComputerUseError('COMPUTER_PROVIDER_FAILURE', 'native drag barrier protocol pipes are unavailable')
|
|
308
|
+
}
|
|
309
|
+
const stdin = handle.stdin
|
|
310
|
+
stdin.on('error', () => {
|
|
311
|
+
// Write callbacks and process completion own protocol error reporting.
|
|
312
|
+
})
|
|
313
|
+
const lines = createInterface({ input: handle.stdout, crlfDelay: Infinity })
|
|
314
|
+
const iterator = lines[Symbol.asyncIterator]()
|
|
315
|
+
const readLine = async (readSignal: AbortSignal): Promise<string> => await new Promise((resolveLine, rejectLine) => {
|
|
316
|
+
const onAbort = (): void => { rejectLine(readSignal.reason) }
|
|
317
|
+
if (readSignal.aborted) {
|
|
318
|
+
onAbort()
|
|
319
|
+
return
|
|
320
|
+
}
|
|
321
|
+
readSignal.addEventListener('abort', onAbort, { once: true })
|
|
322
|
+
void iterator.next().then(next => {
|
|
323
|
+
readSignal.removeEventListener('abort', onAbort)
|
|
324
|
+
if (next.done === true) rejectLine(new Error('native drag helper closed before replying'))
|
|
325
|
+
else if (Buffer.byteLength(next.value) > 4 * 1024 * 1024) rejectLine(new Error('native drag helper response exceeded its protocol limit'))
|
|
326
|
+
else resolveLine(next.value)
|
|
327
|
+
}, error => {
|
|
328
|
+
readSignal.removeEventListener('abort', onAbort)
|
|
329
|
+
rejectLine(error)
|
|
330
|
+
})
|
|
331
|
+
})
|
|
332
|
+
const writeLine = async (line: string): Promise<void> => await new Promise((resolveWrite, rejectWrite) => {
|
|
333
|
+
stdin.write(`${line}\n`, error => {
|
|
334
|
+
if (error === undefined || error === null) resolveWrite()
|
|
335
|
+
else rejectWrite(error)
|
|
336
|
+
})
|
|
337
|
+
})
|
|
338
|
+
let started = false
|
|
339
|
+
const onCallerAbort = (): void => {
|
|
340
|
+
if (!started) processAbort.abort()
|
|
341
|
+
}
|
|
342
|
+
signal.addEventListener('abort', onCallerAbort)
|
|
343
|
+
const readinessTimeout = AbortSignal.timeout(readinessTimeoutMs)
|
|
344
|
+
try {
|
|
345
|
+
await writeLine(JSON.stringify({ protocolVersion: 1, ...request }))
|
|
346
|
+
const readySignal = AbortSignal.any([signal, readinessTimeout, processAbort.signal])
|
|
347
|
+
const ready = JSON.parse(await readLine(readySignal)) as Record<string, unknown>
|
|
348
|
+
if (ready.ok !== true || ready.event !== 'drag-ready') {
|
|
349
|
+
throw new Error('native drag helper returned an invalid ready frame')
|
|
350
|
+
}
|
|
351
|
+
} catch (error) {
|
|
352
|
+
processAbort.abort()
|
|
353
|
+
await handle.done.catch(() => undefined)
|
|
354
|
+
signal.removeEventListener('abort', onCallerAbort)
|
|
355
|
+
lines.close()
|
|
356
|
+
if (signal.aborted) throw new ComputerUseError('COMPUTER_CANCELLED', 'native drag preparation was cancelled', { cause: error })
|
|
357
|
+
if (readinessTimeout.aborted) {
|
|
358
|
+
throw new ComputerUseError('COMPUTER_TIMEOUT', `native drag preparation exceeded ${readinessTimeoutMs} milliseconds`, { cause: error })
|
|
359
|
+
}
|
|
360
|
+
throw computerUseError(error, 'native drag preparation failed')
|
|
361
|
+
}
|
|
362
|
+
|
|
363
|
+
const result = (async (): Promise<T> => {
|
|
364
|
+
try {
|
|
365
|
+
const raw = await readLine(processAbort.signal)
|
|
366
|
+
const outcome = await handle.done
|
|
367
|
+
let envelope: HelperEnvelope<T>
|
|
368
|
+
try {
|
|
369
|
+
envelope = JSON.parse(raw) as HelperEnvelope<T>
|
|
370
|
+
} catch (error) {
|
|
371
|
+
throw new ComputerUseError('COMPUTER_PROVIDER_FAILURE', 'native drag helper returned invalid JSON', { cause: error })
|
|
372
|
+
}
|
|
373
|
+
if (envelope.ok !== true) throw new ComputerUseError(envelope.error.code, envelope.error.message.slice(0, 1000))
|
|
374
|
+
if (outcome.exitCode !== 0) {
|
|
375
|
+
throw new ComputerUseError('COMPUTER_PROVIDER_FAILURE', `native drag helper exited ${String(outcome.exitCode)}`)
|
|
376
|
+
}
|
|
377
|
+
if (signal.aborted) throw new ComputerUseError('COMPUTER_CANCELLED', 'native drag action was cancelled after safe mouse release')
|
|
378
|
+
return envelope.value
|
|
379
|
+
} catch (error) {
|
|
380
|
+
if (error instanceof ComputerUseError && error.code === 'COMPUTER_CANCELLED') throw error
|
|
381
|
+
if (signal.aborted) {
|
|
382
|
+
throw new ComputerUseError('COMPUTER_CANCELLED', 'native drag action was cancelled', { cause: error })
|
|
383
|
+
}
|
|
384
|
+
throw computerUseError(error, 'native drag action failed')
|
|
385
|
+
} finally {
|
|
386
|
+
signal.removeEventListener('abort', onCallerAbort)
|
|
387
|
+
lines.close()
|
|
388
|
+
}
|
|
389
|
+
})()
|
|
390
|
+
void result.catch(() => undefined)
|
|
391
|
+
return {
|
|
392
|
+
result,
|
|
393
|
+
start: async () => {
|
|
394
|
+
if (started) return
|
|
395
|
+
if (signal.aborted) {
|
|
396
|
+
processAbort.abort()
|
|
397
|
+
throw new ComputerUseError('COMPUTER_CANCELLED', 'native drag action was cancelled before mouse-down')
|
|
398
|
+
}
|
|
399
|
+
started = true
|
|
400
|
+
signal.removeEventListener('abort', onCallerAbort)
|
|
401
|
+
// Once native mouse-down is possible, caller cancellation no longer
|
|
402
|
+
// owns this process. The bounded native motion must reach mouse-up.
|
|
403
|
+
try {
|
|
404
|
+
await writeLine('START')
|
|
405
|
+
} catch (error) {
|
|
406
|
+
processAbort.abort()
|
|
407
|
+
throw computerUseError(error, 'native drag start barrier failed')
|
|
408
|
+
}
|
|
409
|
+
},
|
|
410
|
+
cancel: () => {
|
|
411
|
+
if (!started) processAbort.abort()
|
|
412
|
+
},
|
|
413
|
+
}
|
|
414
|
+
}
|
|
415
|
+
|
|
258
416
|
/** Send one serialized command to the persistent, click-through Agent cursor overlay. */
|
|
259
|
-
cursorCommand(
|
|
260
|
-
|
|
417
|
+
cursorCommand(
|
|
418
|
+
command: Record<string, unknown>,
|
|
419
|
+
signal: AbortSignal,
|
|
420
|
+
onWritten?: () => void | Promise<void>,
|
|
421
|
+
): Promise<CursorVisibility> {
|
|
422
|
+
const run = this.cursorCommandTail.then(async () => {
|
|
423
|
+
if (signal.aborted) throw new ComputerUseError('COMPUTER_CANCELLED', 'native cursor overlay command was cancelled')
|
|
424
|
+
if (this.disposed) throw new ComputerUseError('COMPUTER_PROVIDER_FAILURE', 'native cursor overlay client is disposed')
|
|
425
|
+
return await this.executeCursorCommand(command, signal, onWritten)
|
|
426
|
+
})
|
|
261
427
|
this.cursorCommandTail = run.then(() => undefined, () => undefined)
|
|
262
|
-
return
|
|
428
|
+
return new Promise<CursorVisibility>((resolveCommand, rejectCommand) => {
|
|
429
|
+
const onAbort = (): void => {
|
|
430
|
+
rejectCommand(new ComputerUseError('COMPUTER_CANCELLED', 'native cursor overlay command was cancelled'))
|
|
431
|
+
}
|
|
432
|
+
if (signal.aborted) onAbort()
|
|
433
|
+
else signal.addEventListener('abort', onAbort, { once: true })
|
|
434
|
+
void run.then(resolveCommand, rejectCommand).finally(() => {
|
|
435
|
+
signal.removeEventListener('abort', onAbort)
|
|
436
|
+
})
|
|
437
|
+
})
|
|
263
438
|
}
|
|
264
439
|
|
|
265
|
-
private async executeCursorCommand(
|
|
440
|
+
private async executeCursorCommand(
|
|
441
|
+
command: Record<string, unknown>,
|
|
442
|
+
signal: AbortSignal,
|
|
443
|
+
onWritten?: () => void | Promise<void>,
|
|
444
|
+
): Promise<CursorVisibility> {
|
|
266
445
|
const prepared = this.prepared ?? await this.prepare(signal)
|
|
446
|
+
if (this.disposed) throw new ComputerUseError('COMPUTER_PROVIDER_FAILURE', 'native cursor overlay client is disposed')
|
|
267
447
|
const cursor = await this.getCursor(prepared, signal)
|
|
268
448
|
signal.throwIfAborted()
|
|
449
|
+
if (this.disposed) {
|
|
450
|
+
this.discardCursor(cursor)
|
|
451
|
+
throw new ComputerUseError('COMPUTER_PROVIDER_FAILURE', 'native cursor overlay client is disposed')
|
|
452
|
+
}
|
|
269
453
|
try {
|
|
270
454
|
await new Promise<void>((resolveWrite, rejectWrite) => {
|
|
271
455
|
cursor.stdin.write(`${JSON.stringify(command)}\n`, error => {
|
|
@@ -273,7 +457,8 @@ export class NativeHelperClient {
|
|
|
273
457
|
else rejectWrite(error)
|
|
274
458
|
})
|
|
275
459
|
})
|
|
276
|
-
|
|
460
|
+
await onWritten?.()
|
|
461
|
+
const frame = await cursor.nextResponse(cursorResponseTimeout(command), signal)
|
|
277
462
|
if (frame.kind === 'failure') {
|
|
278
463
|
this.discardCursor(cursor)
|
|
279
464
|
return { visible: false, reason: frame.reason }
|
|
@@ -283,6 +468,9 @@ export class NativeHelperClient {
|
|
|
283
468
|
return normalized.result
|
|
284
469
|
} catch (error) {
|
|
285
470
|
this.discardCursor(cursor)
|
|
471
|
+
if (signal.aborted) {
|
|
472
|
+
throw new ComputerUseError('COMPUTER_CANCELLED', 'native cursor overlay command was cancelled', { cause: error })
|
|
473
|
+
}
|
|
286
474
|
throw computerUseError(error, 'native cursor overlay command failed')
|
|
287
475
|
}
|
|
288
476
|
}
|
|
@@ -294,18 +482,24 @@ export class NativeHelperClient {
|
|
|
294
482
|
|
|
295
483
|
/** Stop the cursor process before a provider generation is replaced or disposed. */
|
|
296
484
|
async dispose(): Promise<void> {
|
|
485
|
+
this.disposed = true
|
|
297
486
|
const cursor = this.cursor ?? await this.cursorStart?.promise.catch(() => undefined)
|
|
298
487
|
this.cursorStart = undefined
|
|
299
488
|
this.cursor = undefined
|
|
300
|
-
if (cursor === undefined)
|
|
489
|
+
if (cursor === undefined) {
|
|
490
|
+
await this.cursorCommandTail.catch(() => undefined)
|
|
491
|
+
return
|
|
492
|
+
}
|
|
301
493
|
try {
|
|
302
494
|
cursor.stdin.end(`${JSON.stringify({ op: 'stop' })}\n`)
|
|
303
495
|
} catch {
|
|
304
496
|
// A child that already closed its pipe is handled by the tree-exit wait below.
|
|
305
497
|
}
|
|
306
|
-
if (await cursor.waitForExit(AbortSignal.timeout(1_000)))
|
|
307
|
-
|
|
308
|
-
|
|
498
|
+
if (!await cursor.waitForExit(AbortSignal.timeout(1_000))) {
|
|
499
|
+
cursor.terminate()
|
|
500
|
+
await cursor.waitForExit()
|
|
501
|
+
}
|
|
502
|
+
await this.cursorCommandTail.catch(() => undefined)
|
|
309
503
|
}
|
|
310
504
|
|
|
311
505
|
/** Prepared integrity facts used by provider health. */
|
|
@@ -339,8 +533,10 @@ export class NativeHelperClient {
|
|
|
339
533
|
|
|
340
534
|
private async getCursor(prepared: PreparedNativeHelper, signal: AbortSignal): Promise<CursorProcess> {
|
|
341
535
|
signal.throwIfAborted()
|
|
536
|
+
if (this.disposed) throw new ComputerUseError('COMPUTER_PROVIDER_FAILURE', 'native cursor overlay client is disposed')
|
|
342
537
|
if (this.cursor !== undefined) return this.cursor
|
|
343
538
|
if (this.cursorStart === undefined) {
|
|
539
|
+
if (this.disposed) throw new ComputerUseError('COMPUTER_PROVIDER_FAILURE', 'native cursor overlay client is disposed')
|
|
344
540
|
const start: { promise: Promise<CursorProcess> } = { promise: Promise.resolve(undefined as never) }
|
|
345
541
|
start.promise = this.spawnCursor(prepared).then(cursor => {
|
|
346
542
|
this.cursor = cursor
|
|
@@ -355,6 +551,10 @@ export class NativeHelperClient {
|
|
|
355
551
|
}
|
|
356
552
|
const cursor = await this.cursorStart.promise
|
|
357
553
|
signal.throwIfAborted()
|
|
554
|
+
if (this.disposed) {
|
|
555
|
+
this.discardCursor(cursor)
|
|
556
|
+
throw new ComputerUseError('COMPUTER_PROVIDER_FAILURE', 'native cursor overlay client is disposed')
|
|
557
|
+
}
|
|
358
558
|
return cursor
|
|
359
559
|
}
|
|
360
560
|
|
|
@@ -446,20 +646,37 @@ export class NativeHelperClient {
|
|
|
446
646
|
return {
|
|
447
647
|
stdin: handle.stdin,
|
|
448
648
|
done: handle.done,
|
|
449
|
-
nextResponse: async () => {
|
|
649
|
+
nextResponse: async (timeoutMs, signal) => {
|
|
450
650
|
const ready = buffered.shift()
|
|
451
|
-
if (ready !== undefined)
|
|
452
|
-
|
|
651
|
+
if (ready !== undefined) {
|
|
652
|
+
signal.throwIfAborted()
|
|
653
|
+
return ready
|
|
654
|
+
}
|
|
655
|
+
return await new Promise<CursorProtocolFrame>((resolveResponse, rejectResponse) => {
|
|
656
|
+
let timer: ReturnType<typeof setTimeout> | undefined
|
|
453
657
|
const settle = (frame: CursorProtocolFrame): void => {
|
|
454
658
|
const index = pending.indexOf(settle)
|
|
455
659
|
if (index >= 0) pending.splice(index, 1)
|
|
456
|
-
clearTimeout(timer)
|
|
660
|
+
if (timer !== undefined) clearTimeout(timer)
|
|
661
|
+
signal.removeEventListener('abort', onAbort)
|
|
457
662
|
resolveResponse(frame)
|
|
458
663
|
}
|
|
459
|
-
const
|
|
664
|
+
const onAbort = (): void => {
|
|
665
|
+
const index = pending.indexOf(settle)
|
|
666
|
+
if (index >= 0) pending.splice(index, 1)
|
|
667
|
+
if (timer !== undefined) clearTimeout(timer)
|
|
668
|
+
signal.removeEventListener('abort', onAbort)
|
|
669
|
+
rejectResponse(signal.reason)
|
|
670
|
+
}
|
|
671
|
+
timer = setTimeout(() => settle({
|
|
460
672
|
kind: 'failure',
|
|
461
|
-
reason: `the native cursor overlay did not respond within ${
|
|
462
|
-
}),
|
|
673
|
+
reason: `the native cursor overlay did not respond within ${timeoutMs} milliseconds`,
|
|
674
|
+
}), timeoutMs)
|
|
675
|
+
if (signal.aborted) {
|
|
676
|
+
onAbort()
|
|
677
|
+
return
|
|
678
|
+
}
|
|
679
|
+
signal.addEventListener('abort', onAbort, { once: true })
|
|
463
680
|
pending.push(settle)
|
|
464
681
|
})
|
|
465
682
|
},
|
|
@@ -3,6 +3,8 @@
|
|
|
3
3
|
import type {
|
|
4
4
|
BackendActionRequest,
|
|
5
5
|
BackendActionResult,
|
|
6
|
+
BackendCursorActivation,
|
|
7
|
+
BackendTrackedDragResult,
|
|
6
8
|
BackendCursorAction,
|
|
7
9
|
BackendHealth,
|
|
8
10
|
BackendObservation,
|
|
@@ -51,11 +53,19 @@ export class UnsupportedPlatformBackend implements ComputerUseBackend {
|
|
|
51
53
|
return Promise.reject(unsupported(this.platform))
|
|
52
54
|
}
|
|
53
55
|
|
|
56
|
+
activateForCursor(_app: ComputerAppIdentity, _expectedStateHash: string, _options: BackendObserveOptions): Promise<BackendCursorActivation> {
|
|
57
|
+
return Promise.reject(unsupported(this.platform))
|
|
58
|
+
}
|
|
59
|
+
|
|
54
60
|
act(_request: BackendActionRequest): Promise<BackendActionResult> {
|
|
55
61
|
return Promise.reject(unsupported(this.platform))
|
|
56
62
|
}
|
|
57
63
|
|
|
58
|
-
|
|
64
|
+
actDragWithCursor(_request: BackendActionRequest, _cursor: BackendCursorAction & { kind: 'drag' }): Promise<BackendTrackedDragResult> {
|
|
65
|
+
return Promise.reject(unsupported(this.platform))
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
visualizeCursor(_action: BackendCursorAction, _phase: 'before' | 'during' | 'after'): Promise<CursorVisibility> {
|
|
59
69
|
return Promise.reject(unsupported(this.platform))
|
|
60
70
|
}
|
|
61
71
|
|