@anionex/dsh-computer-use 0.2.1 → 0.3.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (50) hide show
  1. package/README.md +13 -7
  2. package/README.zh.md +13 -7
  3. package/cordis.patch.yml +3 -0
  4. package/docs/interaction-policy.md +6 -4
  5. package/docs/interaction-policy.zh.md +7 -5
  6. package/lib/client.js +31 -11
  7. package/lib/client.js.map +1 -1
  8. package/lib/config.js +13 -3
  9. package/lib/config.js.map +1 -1
  10. package/lib/providers/macos.js +79 -14
  11. package/lib/providers/macos.js.map +1 -1
  12. package/lib/providers/native-helper.js +240 -17
  13. package/lib/providers/native-helper.js.map +1 -1
  14. package/lib/providers/unsupported.js +6 -0
  15. package/lib/providers/unsupported.js.map +1 -1
  16. package/lib/service.js +74 -32
  17. package/lib/service.js.map +1 -1
  18. package/lib/types/backend.d.ts +18 -3
  19. package/lib/types/backend.d.ts.map +1 -1
  20. package/lib/types/client/index.d.ts +7 -2
  21. package/lib/types/client/index.d.ts.map +1 -1
  22. package/lib/types/config.d.ts +7 -1
  23. package/lib/types/config.d.ts.map +1 -1
  24. package/lib/types/providers/macos.d.ts +8 -2
  25. package/lib/types/providers/macos.d.ts.map +1 -1
  26. package/lib/types/providers/native-helper.d.ts +9 -1
  27. package/lib/types/providers/native-helper.d.ts.map +1 -1
  28. package/lib/types/providers/unsupported.d.ts +6 -2
  29. package/lib/types/providers/unsupported.d.ts.map +1 -1
  30. package/lib/types/service.d.ts.map +1 -1
  31. package/native/macos/Sources/Fixture/main.swift +46 -0
  32. package/native/macos/Sources/Helper/CursorMotion.swift +67 -0
  33. package/native/macos/Sources/Helper/CursorOverlay.swift +334 -60
  34. package/native/macos/Sources/Helper/TargetedPointer.swift +23 -10
  35. package/native/macos/Sources/Helper/main.swift +80 -5
  36. package/native/macos/Sources/Monitor/main.swift +15 -1
  37. package/native/macos/bin/dsh-computer-use-helper +0 -0
  38. package/native/macos/manifest.json +3 -3
  39. package/package.json +16 -18
  40. package/scripts/build-client.mjs +0 -0
  41. package/scripts/build-native.mjs +0 -0
  42. package/scripts/model-e2e.mjs +0 -0
  43. package/scripts/validate.mjs +0 -0
  44. package/src/backend.ts +18 -3
  45. package/src/client/index.tsx +50 -13
  46. package/src/config.ts +20 -4
  47. package/src/providers/macos.ts +102 -14
  48. package/src/providers/native-helper.ts +234 -17
  49. package/src/providers/unsupported.ts +11 -1
  50. package/src/service.ts +100 -37
@@ -9,6 +9,8 @@ import type {} from '@deepseek-ai/dsh-user-approval'
9
9
  import type {
10
10
  BackendActionRequest,
11
11
  BackendActionResult,
12
+ BackendCursorActivation,
13
+ BackendTrackedDragResult,
12
14
  BackendCursorAction,
13
15
  BackendHealth,
14
16
  BackendObservation,
@@ -36,6 +38,11 @@ interface NativeHealth {
36
38
 
37
39
  interface NativeObservation extends BackendObservation {}
38
40
 
41
+ interface NativeCursorActivation {
42
+ observation: NativeObservation
43
+ activation: BackendCursorActivation['activation']
44
+ }
45
+
39
46
  function createBackend(ctx: Context, config: ResolvedComputerUseConfig): ComputerUseBackend {
40
47
  return process.platform === 'darwin'
41
48
  ? new MacOSBackend(ctx, config)
@@ -71,7 +78,27 @@ export class MacOSBackend implements ComputerUseBackend {
71
78
  }, signal)
72
79
  }
73
80
 
81
+ async activateForCursor(
82
+ app: ComputerAppIdentity,
83
+ expectedStateHash: string,
84
+ options: BackendObserveOptions,
85
+ signal: AbortSignal,
86
+ ): Promise<BackendCursorActivation> {
87
+ return await this.client.invoke<NativeCursorActivation>({
88
+ command: 'activate-for-cursor',
89
+ app,
90
+ expectedStateHash,
91
+ options,
92
+ actionTimeoutMs: this.config.actionTimeoutMs,
93
+ }, signal)
94
+ }
95
+
74
96
  async act(request: BackendActionRequest, signal: AbortSignal): Promise<BackendActionResult> {
97
+ if (request.action.kind === 'drag') {
98
+ const execution = await this.prepareNativeDrag(request, signal)
99
+ await execution.start()
100
+ return await execution.result
101
+ }
75
102
  return await this.client.invoke<BackendActionResult>({
76
103
  command: 'act',
77
104
  request: {
@@ -86,7 +113,58 @@ export class MacOSBackend implements ComputerUseBackend {
86
113
  }, signal)
87
114
  }
88
115
 
89
- async visualizeCursor(action: BackendCursorAction, phase: 'before' | 'after', signal: AbortSignal): Promise<CursorVisibility> {
116
+ async actDragWithCursor(
117
+ request: BackendActionRequest,
118
+ cursor: BackendCursorAction & { kind: 'drag' },
119
+ signal: AbortSignal,
120
+ ): Promise<BackendTrackedDragResult> {
121
+ const execution = await this.prepareNativeDrag(request, signal)
122
+ let started = false
123
+ let cursorResult: CursorVisibility
124
+ try {
125
+ cursorResult = await this.visualizeCursorPhase(cursor, 'during', signal, async () => {
126
+ await execution.start()
127
+ started = true
128
+ })
129
+ } catch (error) {
130
+ if (!started) {
131
+ execution.cancel()
132
+ await execution.result.catch(() => undefined)
133
+ throw error
134
+ }
135
+ cursorResult = {
136
+ visible: false,
137
+ reason: `the agent cursor could not track the drag action: ${error instanceof Error ? error.message : String(error)}`,
138
+ }
139
+ }
140
+ return { action: await execution.result, cursor: cursorResult }
141
+ }
142
+
143
+ private async prepareNativeDrag(request: BackendActionRequest, signal: AbortSignal) {
144
+ return await this.client.prepareDrag<BackendActionResult>({
145
+ command: 'act',
146
+ request: {
147
+ ...request,
148
+ actionTimeoutMs: this.config.actionTimeoutMs,
149
+ limits: {
150
+ maxNodes: this.config.maxNodes,
151
+ maxDepth: this.config.maxDepth,
152
+ maxTextBytes: this.config.maxTextBytes,
153
+ },
154
+ },
155
+ }, signal, this.config.actionTimeoutMs + 2_000)
156
+ }
157
+
158
+ async visualizeCursor(action: BackendCursorAction, phase: 'before' | 'during' | 'after', signal: AbortSignal): Promise<CursorVisibility> {
159
+ return await this.visualizeCursorPhase(action, phase, signal)
160
+ }
161
+
162
+ private async visualizeCursorPhase(
163
+ action: BackendCursorAction,
164
+ phase: 'before' | 'during' | 'after',
165
+ signal: AbortSignal,
166
+ onMoveWritten?: () => void | Promise<void>,
167
+ ): Promise<CursorVisibility> {
90
168
  if (this.config.interaction.cursorVisualization !== 'visible') return { visible: false, reason: 'the agent cursor is disabled by configuration' }
91
169
  // The overlay answers per command; the least visible outcome wins, because
92
170
  // a cursor that vanished partway through is a cursor the user cannot follow.
@@ -95,17 +173,23 @@ export class MacOSBackend implements ComputerUseBackend {
95
173
  if (!response.visible && outcome.visible) outcome = response
96
174
  }
97
175
  const autoHideMs = this.config.interaction.cursorAutoHideMs
98
- const move = async (point: { x: number; y: number }, durationMs: number): Promise<void> => {
99
- record(await this.client.cursorCommand({
176
+ const move = async (point: { x: number; y: number }, onWritten?: () => void | Promise<void>): Promise<void> => {
177
+ const command = {
100
178
  op: 'move',
101
179
  x: point.x,
102
180
  y: point.y,
103
- durationMs,
104
- autoHideMs,
181
+ speedPxPerSecond: this.config.interaction.cursorSpeedPxPerSecond,
182
+ accelerationPxPerSecondSquared: this.config.interaction.cursorAccelerationPxPerSecondSquared,
183
+ // The after phase arms the configured timeout once native input has
184
+ // completed. Hiding during travel or dwell would invalidate press.
185
+ autoHideMs: 0,
105
186
  targetPid: action.targetPid,
106
187
  targetWindowNumber: action.targetWindowNumber,
107
188
  targetWindowFrame: action.targetWindowFrame,
108
- }, signal))
189
+ }
190
+ record(await (onWritten === undefined
191
+ ? this.client.cursorCommand(command, signal)
192
+ : this.client.cursorCommand(command, signal, onWritten)))
109
193
  }
110
194
  if (phase === 'after') {
111
195
  // Every action validates the bound target after native input. Only drag
@@ -119,24 +203,28 @@ export class MacOSBackend implements ComputerUseBackend {
119
203
  }, signal))
120
204
  return outcome
121
205
  }
206
+ if (phase === 'during') {
207
+ if (action.kind !== 'drag') return { visible: false, reason: 'only drag has a during-action cursor phase' }
208
+ await move(action.to, onMoveWritten)
209
+ return outcome
210
+ }
122
211
  const start = action.kind === 'drag' ? action.from : action.to
123
212
  if (start === undefined) return { visible: false, reason: 'this action has no cursor position to show' }
124
- await move(start, this.config.interaction.cursorMotionMs)
125
- if (this.config.interaction.cursorMotionMs > 0) {
126
- await delay(this.config.interaction.cursorMotionMs, undefined, { signal })
213
+ // A move response means the native overlay reached its destination. Keep
214
+ // the configurable dwell after arrival and before the visual/native press.
215
+ await move(start)
216
+ if (!outcome.visible || action.kind === 'scroll') return outcome
217
+ if (this.config.interaction.cursorClickDelayMs > 0) {
218
+ await delay(this.config.interaction.cursorClickDelayMs, undefined, { signal })
127
219
  }
128
- if (action.kind === 'scroll') return outcome
129
220
  record(await this.client.cursorCommand({
130
221
  op: 'press',
131
- autoHideMs,
222
+ autoHideMs: 0,
132
223
  targetPid: action.targetPid,
133
224
  targetWindowNumber: action.targetWindowNumber,
134
225
  targetWindowFrame: action.targetWindowFrame,
135
226
  sustainedPress: action.kind === 'drag',
136
227
  }, signal))
137
- if (action.kind === 'drag') {
138
- await move(action.to, Math.max(this.config.interaction.cursorMotionMs, 240))
139
- }
140
228
  return outcome
141
229
  }
142
230
 
@@ -5,6 +5,7 @@ import { access, chmod, lstat, readFile, realpath, stat } from 'node:fs/promises
5
5
  import { constants, type Stats } from 'node:fs'
6
6
  import type { Writable } from 'node:stream'
7
7
  import { dirname, resolve } from 'node:path'
8
+ import { createInterface } from 'node:readline'
8
9
  import { fileURLToPath } from 'node:url'
9
10
  import type { Context } from '@deepseek-ai/cordis'
10
11
  import type { SubprocessHandle, SubprocessOutcome, SubprocessOutputReader } from '@deepseek-ai/dsh-subprocess'
@@ -39,6 +40,12 @@ interface HelperSuccess<T> {
39
40
 
40
41
  type HelperEnvelope<T> = HelperFailure | HelperSuccess<T>
41
42
 
43
+ export interface PreparedNativeDrag<T> {
44
+ readonly result: Promise<T>
45
+ start(): Promise<void>
46
+ cancel(): void
47
+ }
48
+
42
49
  /**
43
50
  * The response includes synchronous WindowServer validation, which can exceed
44
51
  * a scheduler tick under load even though the protocol itself is local.
@@ -46,6 +53,7 @@ type HelperEnvelope<T> = HelperFailure | HelperSuccess<T>
46
53
  // Keep the bound finite. A timed-out generation is discarded before the next
47
54
  // serialized command, so a late frame cannot satisfy a later command.
48
55
  const CURSOR_RESPONSE_TIMEOUT_MS = 1_000
56
+ const CURSOR_MOTION_RESPONSE_TIMEOUT_MS = 4_000
49
57
 
50
58
  const CURSOR_READY_TIMEOUT_MS = 2_000
51
59
  const CURSOR_PROTOCOL_MAX_BYTES = 64 * 1024
@@ -60,7 +68,16 @@ interface CursorProcess {
60
68
  terminate: () => void
61
69
  waitForExit: SubprocessHandle['waitForExit']
62
70
  /** Next response line from this process generation, in command order. */
63
- nextResponse: () => Promise<CursorProtocolFrame>
71
+ nextResponse: (timeoutMs: number, signal: AbortSignal) => Promise<CursorProtocolFrame>
72
+ }
73
+
74
+ function cursorResponseTimeout(command: Record<string, unknown>): number {
75
+ if (command.op !== 'move') return CURSOR_RESPONSE_TIMEOUT_MS
76
+ if (typeof command.speedPxPerSecond === 'number') return CURSOR_MOTION_RESPONSE_TIMEOUT_MS
77
+ if (typeof command.durationMs === 'number' && Number.isFinite(command.durationMs)) {
78
+ return Math.max(CURSOR_RESPONSE_TIMEOUT_MS, Math.trunc(command.durationMs) + 2 * CURSOR_RESPONSE_TIMEOUT_MS)
79
+ }
80
+ return CURSOR_RESPONSE_TIMEOUT_MS
64
81
  }
65
82
 
66
83
  function collected(reader: SubprocessOutputReader | undefined): string {
@@ -115,9 +132,13 @@ function normalizeCursorResponse(
115
132
  }
116
133
  if (response.visible === true) return { result: { visible: true }, discardGeneration: false }
117
134
  if (response.visible === false) {
135
+ const reasonCode = response.reasonCode === 'target-not-frontmost' || response.reasonCode === 'target-invalid'
136
+ ? response.reasonCode
137
+ : undefined
118
138
  return {
119
139
  result: {
120
140
  visible: false,
141
+ ...(reasonCode === undefined ? {} : { reasonCode }),
121
142
  ...(typeof response.reason === 'string' && response.reason.length > 0
122
143
  ? { reason: response.reason.slice(0, 1000) }
123
144
  : { reason: 'the native cursor overlay reported that the cursor is not visible' }),
@@ -144,6 +165,7 @@ export class NativeHelperClient {
144
165
  private cursor: CursorProcess | undefined
145
166
  private cursorStart: { promise: Promise<CursorProcess> } | undefined
146
167
  private cursorCommandTail: Promise<void> = Promise.resolve()
168
+ private disposed = false
147
169
 
148
170
  constructor(
149
171
  private readonly ctx: Context,
@@ -255,17 +277,179 @@ export class NativeHelperClient {
255
277
  return envelope.value
256
278
  }
257
279
 
280
+ /** Prepare and validate a drag, then wait for an explicit pre-mouse-down start barrier. */
281
+ async prepareDrag<T>(
282
+ request: Record<string, unknown>,
283
+ signal: AbortSignal,
284
+ readinessTimeoutMs: number,
285
+ ): Promise<PreparedNativeDrag<T>> {
286
+ const prepared = this.prepared ?? await this.prepare(signal)
287
+ if (signal.aborted) throw new ComputerUseError('COMPUTER_CANCELLED', 'native drag preparation was cancelled')
288
+ const processAbort = new AbortController()
289
+ const handle = this.ctx.subprocess.spawn({
290
+ argv: [prepared.path, '--drag-action'],
291
+ cwd: dirname(prepared.path),
292
+ stdio: {
293
+ stdin: 'pipe',
294
+ stdout: 'pipe',
295
+ stderr: { maxBytes: 64 * 1024 },
296
+ },
297
+ graceMs: 1000,
298
+ signal: processAbort.signal,
299
+ env: {
300
+ LANG: process.env.LANG ?? 'en_US.UTF-8',
301
+ LC_ALL: process.env.LC_ALL ?? 'en_US.UTF-8',
302
+ },
303
+ })
304
+ if (handle.stdin === undefined || handle.stdout === undefined) {
305
+ handle.terminate()
306
+ await handle.waitForExit()
307
+ throw new ComputerUseError('COMPUTER_PROVIDER_FAILURE', 'native drag barrier protocol pipes are unavailable')
308
+ }
309
+ const stdin = handle.stdin
310
+ stdin.on('error', () => {
311
+ // Write callbacks and process completion own protocol error reporting.
312
+ })
313
+ const lines = createInterface({ input: handle.stdout, crlfDelay: Infinity })
314
+ const iterator = lines[Symbol.asyncIterator]()
315
+ const readLine = async (readSignal: AbortSignal): Promise<string> => await new Promise((resolveLine, rejectLine) => {
316
+ const onAbort = (): void => { rejectLine(readSignal.reason) }
317
+ if (readSignal.aborted) {
318
+ onAbort()
319
+ return
320
+ }
321
+ readSignal.addEventListener('abort', onAbort, { once: true })
322
+ void iterator.next().then(next => {
323
+ readSignal.removeEventListener('abort', onAbort)
324
+ if (next.done === true) rejectLine(new Error('native drag helper closed before replying'))
325
+ else if (Buffer.byteLength(next.value) > 4 * 1024 * 1024) rejectLine(new Error('native drag helper response exceeded its protocol limit'))
326
+ else resolveLine(next.value)
327
+ }, error => {
328
+ readSignal.removeEventListener('abort', onAbort)
329
+ rejectLine(error)
330
+ })
331
+ })
332
+ const writeLine = async (line: string): Promise<void> => await new Promise((resolveWrite, rejectWrite) => {
333
+ stdin.write(`${line}\n`, error => {
334
+ if (error === undefined || error === null) resolveWrite()
335
+ else rejectWrite(error)
336
+ })
337
+ })
338
+ let started = false
339
+ const onCallerAbort = (): void => {
340
+ if (!started) processAbort.abort()
341
+ }
342
+ signal.addEventListener('abort', onCallerAbort)
343
+ const readinessTimeout = AbortSignal.timeout(readinessTimeoutMs)
344
+ try {
345
+ await writeLine(JSON.stringify({ protocolVersion: 1, ...request }))
346
+ const readySignal = AbortSignal.any([signal, readinessTimeout, processAbort.signal])
347
+ const ready = JSON.parse(await readLine(readySignal)) as Record<string, unknown>
348
+ if (ready.ok !== true || ready.event !== 'drag-ready') {
349
+ throw new Error('native drag helper returned an invalid ready frame')
350
+ }
351
+ } catch (error) {
352
+ processAbort.abort()
353
+ await handle.done.catch(() => undefined)
354
+ signal.removeEventListener('abort', onCallerAbort)
355
+ lines.close()
356
+ if (signal.aborted) throw new ComputerUseError('COMPUTER_CANCELLED', 'native drag preparation was cancelled', { cause: error })
357
+ if (readinessTimeout.aborted) {
358
+ throw new ComputerUseError('COMPUTER_TIMEOUT', `native drag preparation exceeded ${readinessTimeoutMs} milliseconds`, { cause: error })
359
+ }
360
+ throw computerUseError(error, 'native drag preparation failed')
361
+ }
362
+
363
+ const result = (async (): Promise<T> => {
364
+ try {
365
+ const raw = await readLine(processAbort.signal)
366
+ const outcome = await handle.done
367
+ let envelope: HelperEnvelope<T>
368
+ try {
369
+ envelope = JSON.parse(raw) as HelperEnvelope<T>
370
+ } catch (error) {
371
+ throw new ComputerUseError('COMPUTER_PROVIDER_FAILURE', 'native drag helper returned invalid JSON', { cause: error })
372
+ }
373
+ if (envelope.ok !== true) throw new ComputerUseError(envelope.error.code, envelope.error.message.slice(0, 1000))
374
+ if (outcome.exitCode !== 0) {
375
+ throw new ComputerUseError('COMPUTER_PROVIDER_FAILURE', `native drag helper exited ${String(outcome.exitCode)}`)
376
+ }
377
+ if (signal.aborted) throw new ComputerUseError('COMPUTER_CANCELLED', 'native drag action was cancelled after safe mouse release')
378
+ return envelope.value
379
+ } catch (error) {
380
+ if (error instanceof ComputerUseError && error.code === 'COMPUTER_CANCELLED') throw error
381
+ if (signal.aborted) {
382
+ throw new ComputerUseError('COMPUTER_CANCELLED', 'native drag action was cancelled', { cause: error })
383
+ }
384
+ throw computerUseError(error, 'native drag action failed')
385
+ } finally {
386
+ signal.removeEventListener('abort', onCallerAbort)
387
+ lines.close()
388
+ }
389
+ })()
390
+ void result.catch(() => undefined)
391
+ return {
392
+ result,
393
+ start: async () => {
394
+ if (started) return
395
+ if (signal.aborted) {
396
+ processAbort.abort()
397
+ throw new ComputerUseError('COMPUTER_CANCELLED', 'native drag action was cancelled before mouse-down')
398
+ }
399
+ started = true
400
+ signal.removeEventListener('abort', onCallerAbort)
401
+ // Once native mouse-down is possible, caller cancellation no longer
402
+ // owns this process. The bounded native motion must reach mouse-up.
403
+ try {
404
+ await writeLine('START')
405
+ } catch (error) {
406
+ processAbort.abort()
407
+ throw computerUseError(error, 'native drag start barrier failed')
408
+ }
409
+ },
410
+ cancel: () => {
411
+ if (!started) processAbort.abort()
412
+ },
413
+ }
414
+ }
415
+
258
416
  /** Send one serialized command to the persistent, click-through Agent cursor overlay. */
259
- cursorCommand(command: Record<string, unknown>, signal: AbortSignal): Promise<CursorVisibility> {
260
- const run = this.cursorCommandTail.then(async () => await this.executeCursorCommand(command, signal))
417
+ cursorCommand(
418
+ command: Record<string, unknown>,
419
+ signal: AbortSignal,
420
+ onWritten?: () => void | Promise<void>,
421
+ ): Promise<CursorVisibility> {
422
+ const run = this.cursorCommandTail.then(async () => {
423
+ if (signal.aborted) throw new ComputerUseError('COMPUTER_CANCELLED', 'native cursor overlay command was cancelled')
424
+ if (this.disposed) throw new ComputerUseError('COMPUTER_PROVIDER_FAILURE', 'native cursor overlay client is disposed')
425
+ return await this.executeCursorCommand(command, signal, onWritten)
426
+ })
261
427
  this.cursorCommandTail = run.then(() => undefined, () => undefined)
262
- return run
428
+ return new Promise<CursorVisibility>((resolveCommand, rejectCommand) => {
429
+ const onAbort = (): void => {
430
+ rejectCommand(new ComputerUseError('COMPUTER_CANCELLED', 'native cursor overlay command was cancelled'))
431
+ }
432
+ if (signal.aborted) onAbort()
433
+ else signal.addEventListener('abort', onAbort, { once: true })
434
+ void run.then(resolveCommand, rejectCommand).finally(() => {
435
+ signal.removeEventListener('abort', onAbort)
436
+ })
437
+ })
263
438
  }
264
439
 
265
- private async executeCursorCommand(command: Record<string, unknown>, signal: AbortSignal): Promise<CursorVisibility> {
440
+ private async executeCursorCommand(
441
+ command: Record<string, unknown>,
442
+ signal: AbortSignal,
443
+ onWritten?: () => void | Promise<void>,
444
+ ): Promise<CursorVisibility> {
266
445
  const prepared = this.prepared ?? await this.prepare(signal)
446
+ if (this.disposed) throw new ComputerUseError('COMPUTER_PROVIDER_FAILURE', 'native cursor overlay client is disposed')
267
447
  const cursor = await this.getCursor(prepared, signal)
268
448
  signal.throwIfAborted()
449
+ if (this.disposed) {
450
+ this.discardCursor(cursor)
451
+ throw new ComputerUseError('COMPUTER_PROVIDER_FAILURE', 'native cursor overlay client is disposed')
452
+ }
269
453
  try {
270
454
  await new Promise<void>((resolveWrite, rejectWrite) => {
271
455
  cursor.stdin.write(`${JSON.stringify(command)}\n`, error => {
@@ -273,7 +457,8 @@ export class NativeHelperClient {
273
457
  else rejectWrite(error)
274
458
  })
275
459
  })
276
- const frame = await cursor.nextResponse()
460
+ await onWritten?.()
461
+ const frame = await cursor.nextResponse(cursorResponseTimeout(command), signal)
277
462
  if (frame.kind === 'failure') {
278
463
  this.discardCursor(cursor)
279
464
  return { visible: false, reason: frame.reason }
@@ -283,6 +468,9 @@ export class NativeHelperClient {
283
468
  return normalized.result
284
469
  } catch (error) {
285
470
  this.discardCursor(cursor)
471
+ if (signal.aborted) {
472
+ throw new ComputerUseError('COMPUTER_CANCELLED', 'native cursor overlay command was cancelled', { cause: error })
473
+ }
286
474
  throw computerUseError(error, 'native cursor overlay command failed')
287
475
  }
288
476
  }
@@ -294,18 +482,24 @@ export class NativeHelperClient {
294
482
 
295
483
  /** Stop the cursor process before a provider generation is replaced or disposed. */
296
484
  async dispose(): Promise<void> {
485
+ this.disposed = true
297
486
  const cursor = this.cursor ?? await this.cursorStart?.promise.catch(() => undefined)
298
487
  this.cursorStart = undefined
299
488
  this.cursor = undefined
300
- if (cursor === undefined) return
489
+ if (cursor === undefined) {
490
+ await this.cursorCommandTail.catch(() => undefined)
491
+ return
492
+ }
301
493
  try {
302
494
  cursor.stdin.end(`${JSON.stringify({ op: 'stop' })}\n`)
303
495
  } catch {
304
496
  // A child that already closed its pipe is handled by the tree-exit wait below.
305
497
  }
306
- if (await cursor.waitForExit(AbortSignal.timeout(1_000))) return
307
- cursor.terminate()
308
- await cursor.waitForExit()
498
+ if (!await cursor.waitForExit(AbortSignal.timeout(1_000))) {
499
+ cursor.terminate()
500
+ await cursor.waitForExit()
501
+ }
502
+ await this.cursorCommandTail.catch(() => undefined)
309
503
  }
310
504
 
311
505
  /** Prepared integrity facts used by provider health. */
@@ -339,8 +533,10 @@ export class NativeHelperClient {
339
533
 
340
534
  private async getCursor(prepared: PreparedNativeHelper, signal: AbortSignal): Promise<CursorProcess> {
341
535
  signal.throwIfAborted()
536
+ if (this.disposed) throw new ComputerUseError('COMPUTER_PROVIDER_FAILURE', 'native cursor overlay client is disposed')
342
537
  if (this.cursor !== undefined) return this.cursor
343
538
  if (this.cursorStart === undefined) {
539
+ if (this.disposed) throw new ComputerUseError('COMPUTER_PROVIDER_FAILURE', 'native cursor overlay client is disposed')
344
540
  const start: { promise: Promise<CursorProcess> } = { promise: Promise.resolve(undefined as never) }
345
541
  start.promise = this.spawnCursor(prepared).then(cursor => {
346
542
  this.cursor = cursor
@@ -355,6 +551,10 @@ export class NativeHelperClient {
355
551
  }
356
552
  const cursor = await this.cursorStart.promise
357
553
  signal.throwIfAborted()
554
+ if (this.disposed) {
555
+ this.discardCursor(cursor)
556
+ throw new ComputerUseError('COMPUTER_PROVIDER_FAILURE', 'native cursor overlay client is disposed')
557
+ }
358
558
  return cursor
359
559
  }
360
560
 
@@ -446,20 +646,37 @@ export class NativeHelperClient {
446
646
  return {
447
647
  stdin: handle.stdin,
448
648
  done: handle.done,
449
- nextResponse: async () => {
649
+ nextResponse: async (timeoutMs, signal) => {
450
650
  const ready = buffered.shift()
451
- if (ready !== undefined) return ready
452
- return await new Promise<CursorProtocolFrame>((resolveResponse) => {
651
+ if (ready !== undefined) {
652
+ signal.throwIfAborted()
653
+ return ready
654
+ }
655
+ return await new Promise<CursorProtocolFrame>((resolveResponse, rejectResponse) => {
656
+ let timer: ReturnType<typeof setTimeout> | undefined
453
657
  const settle = (frame: CursorProtocolFrame): void => {
454
658
  const index = pending.indexOf(settle)
455
659
  if (index >= 0) pending.splice(index, 1)
456
- clearTimeout(timer)
660
+ if (timer !== undefined) clearTimeout(timer)
661
+ signal.removeEventListener('abort', onAbort)
457
662
  resolveResponse(frame)
458
663
  }
459
- const timer = setTimeout(() => settle({
664
+ const onAbort = (): void => {
665
+ const index = pending.indexOf(settle)
666
+ if (index >= 0) pending.splice(index, 1)
667
+ if (timer !== undefined) clearTimeout(timer)
668
+ signal.removeEventListener('abort', onAbort)
669
+ rejectResponse(signal.reason)
670
+ }
671
+ timer = setTimeout(() => settle({
460
672
  kind: 'failure',
461
- reason: `the native cursor overlay did not respond within ${CURSOR_RESPONSE_TIMEOUT_MS} milliseconds`,
462
- }), CURSOR_RESPONSE_TIMEOUT_MS)
673
+ reason: `the native cursor overlay did not respond within ${timeoutMs} milliseconds`,
674
+ }), timeoutMs)
675
+ if (signal.aborted) {
676
+ onAbort()
677
+ return
678
+ }
679
+ signal.addEventListener('abort', onAbort, { once: true })
463
680
  pending.push(settle)
464
681
  })
465
682
  },
@@ -3,6 +3,8 @@
3
3
  import type {
4
4
  BackendActionRequest,
5
5
  BackendActionResult,
6
+ BackendCursorActivation,
7
+ BackendTrackedDragResult,
6
8
  BackendCursorAction,
7
9
  BackendHealth,
8
10
  BackendObservation,
@@ -51,11 +53,19 @@ export class UnsupportedPlatformBackend implements ComputerUseBackend {
51
53
  return Promise.reject(unsupported(this.platform))
52
54
  }
53
55
 
56
+ activateForCursor(_app: ComputerAppIdentity, _expectedStateHash: string, _options: BackendObserveOptions): Promise<BackendCursorActivation> {
57
+ return Promise.reject(unsupported(this.platform))
58
+ }
59
+
54
60
  act(_request: BackendActionRequest): Promise<BackendActionResult> {
55
61
  return Promise.reject(unsupported(this.platform))
56
62
  }
57
63
 
58
- visualizeCursor(_action: BackendCursorAction, _phase: 'before' | 'after'): Promise<CursorVisibility> {
64
+ actDragWithCursor(_request: BackendActionRequest, _cursor: BackendCursorAction & { kind: 'drag' }): Promise<BackendTrackedDragResult> {
65
+ return Promise.reject(unsupported(this.platform))
66
+ }
67
+
68
+ visualizeCursor(_action: BackendCursorAction, _phase: 'before' | 'during' | 'after'): Promise<CursorVisibility> {
59
69
  return Promise.reject(unsupported(this.platform))
60
70
  }
61
71