dsh-codex-style-computer-use 1.3.0 → 2.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (41) hide show
  1. package/README.md +53 -22
  2. package/README.zh.md +46 -21
  3. package/icon.svg +29 -0
  4. package/lib/audit.js +7 -0
  5. package/lib/batch.js +14 -19
  6. package/lib/bin/attention-worker.exe +0 -0
  7. package/lib/bin/attention-worker.exe.json +1 -1
  8. package/lib/compact-feedback.js +175 -0
  9. package/lib/continuation.js +1 -1
  10. package/lib/core/bin/dsh-computer-use-worker.exe +0 -0
  11. package/lib/core/bin/dsh-computer-use-worker.exe.json +2 -2
  12. package/lib/core/compile-worker.ps1 +5 -1
  13. package/lib/core/index.js +452 -129
  14. package/lib/core/native-runtime.js +23 -1
  15. package/lib/core/winmd/Windows.winmd +0 -0
  16. package/lib/core/worker.cs +2415 -356
  17. package/lib/feedback-routes.json +11 -0
  18. package/lib/feedback.js +49 -16
  19. package/lib/image-output.js +59 -11
  20. package/lib/index.js +42 -7
  21. package/lib/key-command.js +37 -0
  22. package/lib/media-contract.js +43 -0
  23. package/lib/observations.js +267 -13
  24. package/lib/policy.js +30 -10
  25. package/lib/tool-contract.js +56 -0
  26. package/lib/tools.js +174 -51
  27. package/lib/verification-region.js +17 -0
  28. package/mcp/package.json +2 -1
  29. package/mcp/src/execution.js +8 -5
  30. package/mcp/src/index.js +16 -267
  31. package/mcp/src/tools.js +271 -0
  32. package/package.json +5 -2
  33. package/scripts/check-behavior.mjs +83 -0
  34. package/scripts/measure-tool-tokens.mjs +21 -0
  35. package/scripts/render-promo.py +81 -0
  36. package/skills/computer-use/SKILL.md +35 -8
  37. package/skills/computer-use/references/lifecycle.md +11 -7
  38. package/skills/computer-use/references/maintenance.md +9 -5
  39. package/skills/computer-use/references/precision.md +39 -14
  40. package/skills/computer-use/references/progressive-control.md +10 -4
  41. package/skills/computer-use/references/zero-uia-apps.md +54 -0
package/lib/core/index.js CHANGED
@@ -16,12 +16,19 @@ import { AsyncLocalStorage } from 'node:async_hooks'
16
16
  import { existsSync, readFileSync, statSync } from 'node:fs'
17
17
  import { join } from 'node:path'
18
18
  import { fileURLToPath } from 'node:url'
19
- import { prepareNativeBinary, workerReferences } from './native-runtime.js'
19
+ import { prepareNativeBinary, workerReferences, nativeRuntimeIdentity } from './native-runtime.js'
20
+ import { parseKeyCommand } from '../key-command.js'
21
+ import { verificationRegion } from '../verification-region.js'
20
22
 
21
23
  const __dirname = fileURLToPath(new URL('.', import.meta.url))
22
24
 
23
25
  const DEFAULT_TIMEOUT_MS = 15_000
24
26
  const CAPTURE_TIMEOUT_MS = 30_000
27
+ const INPUT_CLEANUP_GRACE_MS = 1500
28
+ // A UIA tree walk can block the resident's synchronous COM thread. Bound the
29
+ // caller's wait separately from recovery of a worker that never returns.
30
+ const UIA_READS = new Set(['uia', 'uiaFromPoint', 'uiaFocused'])
31
+ const URGENT_CONTROL = new Set(['cancelInput', 'abort', 'panic', 'exit', 'endAsk'])
25
32
  const INPUT_OPS = new Set(['click', 'shiftClick', 'move', 'drag', 'scroll', 'selectRange',
26
33
  'type', 'key', 'uiaAct', 'windowOp', 'activate', 'clipWrite', 'calibrate', 'selftest'])
27
34
  const isInputOp = (op, args = {}) => INPUT_OPS.has(op) || (op === 'focus' && args.refocus === true)
@@ -46,7 +53,7 @@ const ASK_DEADLINE_MS = 15_000
46
53
  const END_ASK_DEADLINE_MS = 5_000
47
54
  export function workerCallError (message, code, op, requestId, outcome = 'unknown') {
48
55
  return Object.assign(new Error(`${message} [${code}; outcome=${outcome}]. ${outcome === 'unknown' ? 'Inspect current state before retrying a write.' : 'No input was dispatched by this operation.'}`),
49
- { code, op, requestId, outcome })
56
+ { code, op, requestId, outcome, detail: message })
50
57
  }
51
58
 
52
59
  /** A handoff interrupts an input operation, never the authorized task. */
@@ -71,6 +78,19 @@ function humanReobserveError (error = {}) {
71
78
  { transient: true, taskStatus: 'continuing', continuation })
72
79
  }
73
80
 
81
+ function workerReobserveError (workerGeneration, interrupted = {}) {
82
+ const outcome = interrupted.outcome === 'unknown' ? 'unknown' : 'not-dispatched'
83
+ const continuation = {
84
+ required: true, taskStatus: 'continuing', reason: 'worker-restarted',
85
+ nextAction: 'observe-current-target-then-continue-original-task',
86
+ replayInterruptedInput: false, outcome, workerGeneration,
87
+ interruptedOperation: interrupted.op || null, requestId: interrupted.requestId ?? null,
88
+ instruction: 'The native worker restarted and previous desktop evidence is invalid. Obtain fresh target evidence, inspect any interrupted operation, and continue the original authorized task. This is worker recovery, not a human handback or permission to replay a write.',
89
+ }
90
+ return Object.assign(workerCallError(continuation.instruction, 'WORKER_REOBSERVE', continuation.interruptedOperation, continuation.requestId, outcome),
91
+ { transient: true, taskStatus: 'continuing', continuation })
92
+ }
93
+
74
94
  /** Interactive roles that receive Set-of-Marks labels. */
75
95
  const SOM_ROLES = new Set([
76
96
  'Button', 'MenuItem', 'ListItem', 'TreeItem', 'DataItem', 'TabItem',
@@ -106,6 +126,13 @@ export class ComputerUse extends EventEmitter {
106
126
  this.stopped = false // true while the human's brake is engaged (see 'panic' event)
107
127
  this.exited = false // true once the human ended the session (see 'exit' event)
108
128
  this.cooperation = { phase: 'idle', active: false, revision: 0 }
129
+ this.workerGeneration = null
130
+ this.nativeStateSeq = -1
131
+ this.workerStateInvalid = false
132
+ this.runtime = null
133
+ this.inputCleanupGraceMs = INPUT_CLEANUP_GRACE_MS
134
+ this.shutdownGraceMs = INPUT_CLEANUP_GRACE_MS
135
+ this.activeActions = new Map()
109
136
  this.inputContext = new AsyncLocalStorage()
110
137
  // Monotonic counter of UNSOLICITED state transitions (panic/ask/resume/exit). Anything that
111
138
  // learns the machine's state asynchronously — above all the `ask` op, which can take seconds and
@@ -141,13 +168,17 @@ export class ComputerUse extends EventEmitter {
141
168
  // app (main vs renderer/GPU), so the worker is also given the executable path, which
142
169
  // recognises every process of the app at once. The worker refuses to drive any window of
143
170
  // either; one-shot CLI runs (no env) keep the guard disabled.
144
- const env = { ...process.env, DSH_CU_HOST_PID: String(process.pid), DSH_CU_HOST_EXE: process.execPath || '' }
171
+ this.spawnRuntime = nativeRuntimeIdentity(this.workerExe)
172
+ const env = { ...process.env, DSH_CU_HOST_PID: String(process.pid), DSH_CU_HOST_EXE: process.execPath || '',
173
+ DSH_CU_PACKAGE_VERSION: this.spawnRuntime.packageVersion, DSH_CU_BUILD_ID: this.spawnRuntime.buildId,
174
+ DSH_CU_SOURCE_HASH: this.spawnRuntime.sourceHash || '', DSH_CU_BINARY_HASH: this.spawnRuntime.binaryHash }
145
175
  this.log(`[core] host guard armed for pid ${process.pid} (${process.execPath})`)
146
176
  this.proc = spawn(this.workerExe, [], { env, windowsHide: true, stdio: ['pipe', 'pipe', 'pipe'] })
147
177
  // A CHILD THAT FAILS TO START MUST NAME ITSELF (2026-09-13). Every computer_* tool died with a
148
178
  // bare `Error: spawn UNKNOWN` — no path, no syscall, no Windows code — because NOTHING listened
149
179
  // for this child's 'error' event: an unhandled 'error' on an EventEmitter is thrown, so the rich
150
180
  // error object (which names the file Windows refused) was discarded on the way out.
181
+ const child = this.proc
151
182
  this.proc.on('error', (e) => {
152
183
  let file = '?'
153
184
  try { file = `${existsSync(this.workerExe)} exists, ${statSync(this.workerExe).size} bytes` } catch { file = 'stat failed' }
@@ -165,27 +196,40 @@ export class ComputerUse extends EventEmitter {
165
196
  const line = buf.slice(0, nl).trim()
166
197
  buf = buf.slice(nl + 1)
167
198
  if (!line) continue
168
- this.onLine(line)
199
+ if (this.proc === child) this.onLine(line)
169
200
  }
170
201
  })
171
202
  this.proc.stderr.on('data', (d) => this.log(`[worker:stderr] ${d}`))
172
- this.proc.on('exit', this.onWorkerExit)
203
+ this.proc.on('exit', (code) => { if (this.proc === child) this.onWorkerExit(code) })
173
204
  return new Promise((resolve, reject) => {
174
- const t = setTimeout(() => reject(new Error('worker ready timeout')), 10_000)
175
- // surface the real reason instead of letting the raw emitter error escape
176
- this.proc.once('error', (e) => {
205
+ const finish = (error) => {
177
206
  clearTimeout(t)
178
- reject(new Error(`worker could not be started: code=${e.code} syscall=${e.syscall} path=${e.path || this.workerExe}`))
179
- })
180
- this.once('ready', () => { clearTimeout(t); resolve() })
181
- this.once('dead', (code) => { clearTimeout(t); reject(new Error(`worker died at startup (code ${code})`)) })
207
+ this.off('ready', ready); this.off('dead', dead); this.off('worker-incompatible', incompatible)
208
+ child.off('error', failed)
209
+ if (error) reject(error); else resolve()
210
+ }
211
+ const ready = () => finish()
212
+ const dead = code => finish(new Error(`worker died at startup (code ${code})`))
213
+ const incompatible = error => finish(error)
214
+ const failed = e => finish(new Error(`worker could not be started: code=${e.code} syscall=${e.syscall} path=${e.path || this.workerExe}`))
215
+ const t = setTimeout(() => {
216
+ finish(new Error('worker ready timeout'))
217
+ try { if (this.proc === child) child.kill() } catch { /* already gone */ }
218
+ }, 10_000)
219
+ // surface the real reason instead of letting the raw emitter error escape
220
+ child.once('error', failed)
221
+ this.once('ready', ready); this.once('dead', dead); this.once('worker-incompatible', incompatible)
182
222
  })
183
223
  }
184
224
 
185
225
  onWorkerExit (code) {
186
226
  this.ready = false
187
227
  this.proc = null
188
- for (const [id, p] of this.pending) { clearTimeout(p.timer); p.cleanup?.(); p.reject(workerCallError('worker exited', 'worker-exited', p.op, id)) }
228
+ this.workerStateInvalid = true
229
+ this.cooperation = { phase: 'unknown', active: false, revision: 0, valid: false, workerGeneration: this.workerGeneration }
230
+ this.maps.clear(); this.lastMarks = []; this.lastMarkFrame = null
231
+ this.finishUiaRead(false)
232
+ for (const [id, p] of this.pending) { clearTimeout(p.timer); p.cleanup?.(); p.reject(Object.assign(workerCallError('worker exited', 'worker-exited', p.op, id), { workerGeneration: p.workerGeneration, cleanup: { cleanupConfirmed: false, cleanupState: 'worker-exited' } })) }
189
233
  this.pending.clear()
190
234
  this.emit('dead', code)
191
235
  }
@@ -194,6 +238,17 @@ export class ComputerUse extends EventEmitter {
194
238
  let msg
195
239
  try { msg = JSON.parse(line) } catch { this.log(`[worker:badline] ${line.slice(0, 200)}`); return }
196
240
  if (msg.event === 'ready') {
241
+ // Production spawns must speak the ordered protocol. An old custom binary
242
+ // cannot safely participate in cancellation or generation-aware recovery.
243
+ if (this.spawnRuntime && (msg.protocolVersion !== 2 || typeof msg.workerGeneration !== 'string' ||
244
+ !msg.workerGeneration || !Number.isSafeInteger(msg.cooperation?.stateSeq) ||
245
+ msg.cooperation.workerGeneration !== msg.workerGeneration)) {
246
+ this.ready = false; this.workerStateInvalid = true
247
+ this.emit('worker-incompatible', workerCallError('native worker protocol is incompatible; reinstall or rebuild the current helper',
248
+ 'WORKER_PROTOCOL_INCOMPATIBLE', null, null, 'not-dispatched'))
249
+ try { this.proc?.kill() } catch { /* already gone */ }
250
+ return
251
+ }
197
252
  this.ready = true
198
253
  // COLD-START STATE SYNC (cold-start bug, 2026-09-13): the worker has already adopted any STOP
199
254
  // or EXITED record that predates this process, and it reports that state with `ready`. Without
@@ -203,14 +258,32 @@ export class ComputerUse extends EventEmitter {
203
258
  // released — the mirror simply starts out agreeing with the machine.
204
259
  if (typeof msg.stopped === 'boolean') this.stopped = msg.stopped
205
260
  if (typeof msg.exited === 'boolean') this.exited = msg.exited
206
- this.cooperation = msg.cooperation || { phase: 'idle', active: false, revision: 0 }
261
+ this.workerGeneration = msg.workerGeneration || msg.runtime?.workerGeneration || msg.cooperation?.workerGeneration || null
262
+ this.nativeStateSeq = -1
263
+ this.runtime = this.spawnRuntime ? { ...this.spawnRuntime, ...msg.runtime, workerGeneration: this.workerGeneration } : msg.runtime || null
264
+ this.cooperation = { phase: 'idle', active: false, revision: 0 }
265
+ this.workerStateInvalid = false
266
+ if (msg.cooperation) this.syncCooperation(msg.cooperation, false)
267
+ this.maps.clear(); this.lastMarks = []; this.lastMarkFrame = null
207
268
  this.emit('ready', msg)
208
269
  return
209
270
  }
271
+ // stdout from a replaced child is already filtered; generation also guards
272
+ // protocol-level delayed events and records restored by an observer.
273
+ const generation = msg.workerGeneration || msg.cooperation?.workerGeneration || msg.data?.cooperation?.workerGeneration
274
+ if (generation && this.workerGeneration && generation !== this.workerGeneration) return
210
275
  if (msg.event === 'handoff') {
211
276
  this.syncCooperation(msg.cooperation)
212
277
  return
213
278
  }
279
+ // Secure-desktop transitions (UAC prompt / lock screen, human request 2026-11-04): the
280
+ // worker refuses to dispatch while it stands and clears it the moment the normal desktop
281
+ // returns. Mirror the state so callers can show it, and let the plugin notify the human.
282
+ if (msg.event === 'secure') {
283
+ this.secureDesktop = msg.why === 'active'
284
+ this.emit('secure', msg)
285
+ return
286
+ }
214
287
  // Durable brake events are separate from automatic physical-input handoff.
215
288
  // panic — ESC: the human ABORTED. The machine is stopped; wrap up briefly, then wait.
216
289
  // resume — Ctrl+Alt+R: work may continue (a PAUSE ends; it never ends the cycle).
@@ -236,6 +309,7 @@ export class ComputerUse extends EventEmitter {
236
309
  this.emit(msg.event, msg)
237
310
  return
238
311
  }
312
+ if (this.uiaRead?.id === msg.id) this.finishUiaRead(true)
239
313
  const p = this.pending.get(msg.id)
240
314
  if (!p) return
241
315
  this.pending.delete(msg.id)
@@ -246,6 +320,8 @@ export class ComputerUse extends EventEmitter {
246
320
  else {
247
321
  const error = workerCallError(msg.error || 'worker error', msg.code || 'worker-refusal-or-error', p.op, msg.id,
248
322
  msg.outcome === 'not-dispatched' ? 'not-dispatched' : 'unknown')
323
+ if (msg.glowHealth) error.glowHealth = msg.glowHealth
324
+ for (const key of ['dragStarted', 'expectedCursor', 'actualCursor', 'cursorCheck']) if (msg[key] !== undefined) error[key] = msg[key]
249
325
  p.reject(error.code === 'HUMAN_REOBSERVE' ? humanReobserveError(error) : error)
250
326
  }
251
327
  }
@@ -257,96 +333,260 @@ export class ComputerUse extends EventEmitter {
257
333
  * @param {boolean} [killOnTimeout] kill a wedged worker when that cap expires (default true)
258
334
  * @returns {Promise<any>} worker `data` payload
259
335
  */
336
+ // Cleanup runs under its own explicit authority. A cancelled user context must
337
+ // not block key release or the closing of a question; ordinary tools never use it.
338
+ callInternal (op, args = {}, timeoutMs = DEFAULT_TIMEOUT_MS, killOnTimeout = true, beforeDispatch) {
339
+ if (!['cancelInput', 'calm', 'endAsk', 'abort'].includes(op)) throw new Error('unsupported internal cleanup operation')
340
+ return this.inputContext.run({ internalCleanup: true }, () => this.call(op, args, timeoutMs, killOnTimeout, beforeDispatch))
341
+ }
342
+
343
+ async cancelRequest (requestId, child, reason, killOnFailure = true) {
344
+ const cleanup = { requestId, workerGeneration: this.workerGeneration, requested: true,
345
+ cleanupConfirmed: false, cleanupState: 'unconfirmed', workerKilled: false }
346
+ const sameWorker = () => {
347
+ if (this.proc !== child) throw Object.assign(new Error('original worker exited before cleanup'), { code: 'worker-replaced' })
348
+ }
349
+ try {
350
+ sameWorker()
351
+ const ack = await this.callInternal('cancelInput', { requestId }, this.inputCleanupGraceMs, false, sameWorker)
352
+ if (ack?.requestId !== undefined && ack.requestId !== requestId) throw new Error('cancellation acknowledgement named another request')
353
+ if (ack?.workerGeneration && ack.workerGeneration !== cleanup.workerGeneration) throw new Error('cancellation acknowledgement named another worker')
354
+ cleanup.cancelled = ack?.cancelled === true
355
+ cleanup.cleanupConfirmed = ack?.cleanupConfirmed === true && ack?.ownedInputCount === 0
356
+ cleanup.cleanupState = ack?.cleanupState || (cleanup.cleanupConfirmed ? 'released' : 'unconfirmed')
357
+ if (Number.isInteger(ack?.ownedInputCount)) cleanup.ownedInputCount = ack.ownedInputCount
358
+ } catch (error) {
359
+ cleanup.cleanupState = this.proc !== child ? 'worker-exited' : 'ack-failed'
360
+ cleanup.errorCode = error.code || 'cleanup-error'
361
+ }
362
+ if (!cleanup.cleanupConfirmed && killOnFailure && this.proc === child) {
363
+ cleanup.killReason = reason
364
+ try { cleanup.workerKilled = child.kill() !== false } catch { cleanup.workerKilled = false }
365
+ }
366
+ return cleanup
367
+ }
368
+
260
369
  async call (op, args = {}, timeoutMs = DEFAULT_TIMEOUT_MS, killOnTimeout = true, beforeDispatch) {
261
370
  const input = isInputOp(op, args)
262
- const context = input ? this.inputContext.getStore() : undefined
371
+ const priorGeneration = this.workerGeneration
372
+ const invalidAtStart = this.workerStateInvalid
373
+ const context = this.inputContext.getStore()
263
374
  const signal = context?.signal
264
- if (signal?.aborted) throw workerCallError('owning tool cancelled', 'INPUT_CANCELLED', op, null, 'not-dispatched')
265
- beforeDispatch?.()
266
- context?.assertOwnership?.()
375
+ const check = () => {
376
+ if (signal?.aborted) throw workerCallError('owning tool cancelled', 'INPUT_CANCELLED', op, null, 'not-dispatched')
377
+ beforeDispatch?.()
378
+ context?.assertOwnership?.()
379
+ }
380
+ check()
267
381
  try { await this.ensureWorker() }
268
382
  catch (error) { throw workerCallError(error.message || String(error), 'worker-unavailable', op, null, 'not-dispatched') }
269
- if (signal?.aborted) throw workerCallError('owning tool cancelled', 'INPUT_CANCELLED', op, null, 'not-dispatched')
270
- beforeDispatch?.()
271
- context?.assertOwnership?.()
383
+ const child = this.proc
384
+ if (!child) throw workerCallError('worker exited before dispatch', 'worker-exited', op, null, 'not-dispatched')
385
+ const queueDeadline = Date.now() + timeoutMs
386
+ while (this.uiaRead && !URGENT_CONTROL.has(op)) {
387
+ await this.waitForUiaRead(op, input, signal, queueDeadline)
388
+ if (this.proc !== child) throw workerCallError('worker exited before dispatch', 'worker-exited', op, null, 'not-dispatched')
389
+ if (Date.now() >= queueDeadline) throw workerCallError('deadline expired waiting for the UIA read', 'worker-uia-busy', op, null, 'not-dispatched')
390
+ }
391
+ check()
392
+ if (input && (invalidAtStart || (priorGeneration && priorGeneration !== this.workerGeneration))) {
393
+ if (this.stopped || this.exited) throw workerCallError('manual pause/exit remains authoritative', 'ABORTED', op, null, 'not-dispatched')
394
+ throw workerReobserveError(this.workerGeneration, { op })
395
+ }
272
396
  const id = ++this.seq
273
- // `__cycle` is this layer's SIGNATURE on a call made on the agent's behalf, and the worker
274
- // opens (or keeps lit) a computer-use CYCLE only for signed calls. A one-shot
275
- // `worker.exe --op click` from a shell script, or a deploy-time health probe, carries no
276
- // signature — and so can never resurrect a session the human ended with Ctrl+Alt+Q. Stamped
277
- // LAST so an op's own args can never forge it.
278
- const signed = { ...args, __cycle: true }
279
- signed.__kickoff = this.kickoffRequired
280
- // One warning per external call/batch, with a fresh identity for each context.
281
- // Never let a caller-supplied argument reuse another tool's announcement.
282
- if (input) signed.__inputSeries = this.inputContext.getStore()?.inputSeries || randomUUID()
397
+ const generation = this.workerGeneration
398
+ if (input && context?.actionRecord) Object.assign(context.actionRecord, {
399
+ requestId: id, workerGeneration: generation, runtime: this.runtime,
400
+ })
401
+ // __cycle is signed last. A caller cannot forge another input series or turn.
402
+ const signed = { ...args, __cycle: context?.internalCleanup !== true, __kickoff: this.kickoffRequired }
403
+ if (input) signed.__inputSeries = context?.inputSeries || randomUUID()
283
404
  return new Promise((resolve, reject) => {
284
405
  let written = false
406
+ const read = UIA_READS.has(op) && killOnTimeout ? (this.uiaRead = { id, child, timedOut: false }) : null
285
407
  const cleanup = () => signal?.removeEventListener('abort', abort)
286
- const abort = () => {
408
+ const deferReadRecovery = () => {
409
+ if (!read || this.uiaRead !== read) return
410
+ read.timedOut = true
411
+ this.emit('uia-read-settled', false)
412
+ const grace = Math.min(timeoutMs, DEFAULT_TIMEOUT_MS)
413
+ read.watchdog = setTimeout(() => {
414
+ if (this.proc !== child || this.uiaRead !== read) return
415
+ this.log(`[core] UIA read ${id} (${op}) still blocked after ${grace}ms recovery grace; stopping worker pid=${child.pid}`)
416
+ try { child.kill() } catch { /* already gone */ }
417
+ }, grace)
418
+ }
419
+ const fail = async (code, message, timeout) => {
287
420
  if (!this.pending.delete(id)) return
288
421
  clearTimeout(timer); cleanup()
289
- if (written) void this.call('cancelInput', { requestId: id }, 2000, false).catch(() => {})
290
- reject(workerCallError('owning tool cancelled; accepted input must be inspected before any retry',
291
- 'INPUT_CANCELLED', op, id, written ? 'unknown' : 'not-dispatched'))
422
+ const error = workerCallError(message, code, op, id, input && written ? 'unknown' : 'not-dispatched')
423
+ error.operationDispatched = written
424
+ error.workerGeneration = generation
425
+ if (input && written) {
426
+ // Stop the original request out of band, then allow a bounded cleanup
427
+ // acknowledgement before killing a worker that may still own a key down.
428
+ error.cleanup = await this.cancelRequest(id, child, timeout ? 'input-timeout' : 'caller-cancelled', killOnTimeout)
429
+ } else if (read) {
430
+ deferReadRecovery()
431
+ } else if (timeout && killOnTimeout) {
432
+ try { if (this.proc === child) child.kill() } catch { /* already gone */ }
433
+ }
434
+ reject(error)
435
+ }
436
+ const abort = () => {
437
+ // beginAsk is a receipt-producing transaction: after dispatch we need
438
+ // its askId to release the exact brake. ask() checks cancellation before
439
+ // showing the card and always closes that receipt in its finally.
440
+ if (op === 'beginAsk' && written) return
441
+ void fail('INPUT_CANCELLED', 'owning tool cancelled; inspect any accepted input before retrying', false)
292
442
  }
293
443
  const timer = setTimeout(() => {
294
- this.pending.delete(id)
295
- cleanup()
296
- // A CALL THAT RELEASES THE BRAKE MUST NOT BE KILLED FOR BEING SLOW. `proc.kill()` deletes no
297
- // STOP file (there is no ProcessExit handler in worker.cs; `abort` is Environment.Exit(0) at
298
- // worker.cs:3045), so killing here would end a question with the brake still up — the exact
299
- // defect this design removes. Without the kill the worker is still alive, so the caller's
300
- // retry, or the next op's SyncFromFile, can still finish the release.
301
- //
302
- // Everything else keeps today's behaviour: a timed-out op means the worker is wedged (a hung
303
- // UIA/COM provider call blocks its single-threaded loop forever), so kill it and let the NEXT
304
- // call respawn fresh instead of queueing behind a corpse; pending calls are rejected via
305
- // onWorkerExit.
306
- if (killOnTimeout) { try { this.proc && this.proc.kill() } catch { /* already gone */ } }
307
- reject(workerCallError(`worker timeout: ${op} (${timeoutMs}ms)` +
308
- (killOnTimeout
309
- ? ' — worker killed, next call respawns'
310
- : ' — worker left alive so the brake can still be released'),
311
- 'worker-timeout', op, id))
312
- }, timeoutMs)
313
- this.pending.set(id, { resolve, reject, timer, op, input, cleanup, abort })
444
+ const message = read
445
+ ? `UIA observation timed out: ${op} (${timeoutMs}ms); worker still finishing this read. Narrow the tree or use a point query once it returns.`
446
+ : `worker timeout: ${op} (${timeoutMs}ms)` + (input ? ' — cancellation and owned-input cleanup requested before worker recovery' : killOnTimeout ? ' — worker recovery requested' : ' — worker left alive so cleanup can finish')
447
+ void fail('worker-timeout', message, true)
448
+ }, Math.max(0, queueDeadline - Date.now()))
449
+ this.pending.set(id, { resolve, reject, timer, op, input, cleanup, abort, child, workerGeneration: generation })
314
450
  signal?.addEventListener('abort', abort, { once: true })
315
451
  if (signal?.aborted) { abort(); return }
316
- written = true
317
- this.proc.stdin.write(JSON.stringify({ id, op, args: signed }) + '\r\n')
452
+ try {
453
+ written = true // a synchronous pipe failure still leaves dispatch uncertain
454
+ child.stdin.write(JSON.stringify({ id, op, args: signed }) + '\r\n')
455
+ } catch (error) {
456
+ void fail('worker-write-failed', 'worker input pipe rejected the request: ' + error.message, false)
457
+ }
458
+ })
459
+ }
460
+
461
+ finishUiaRead (completed) {
462
+ const read = this.uiaRead
463
+ if (!read) return
464
+ this.uiaRead = null
465
+ clearTimeout(read.watchdog)
466
+ this.emit('uia-read-settled', completed && !read.timedOut)
467
+ }
468
+
469
+ // Only the stdin reader's cancellation command is out of band. Preserve
470
+ // emergency/release requests, but never queue ordinary work behind a slow
471
+ // observation after its caller has given up. Do not replay any request.
472
+ waitForUiaRead (op, input, signal, deadline) {
473
+ const busy = () => workerCallError('native UIA observation still busy; this request was not sent',
474
+ 'worker-uia-busy', op, null, 'not-dispatched')
475
+ if (this.uiaRead.timedOut || Date.now() >= deadline) return Promise.reject(busy())
476
+ return new Promise((resolve, reject) => {
477
+ const finish = (error) => {
478
+ clearTimeout(timer)
479
+ this.off('uia-read-settled', settled)
480
+ this.off('input-cancel', cancel)
481
+ signal?.removeEventListener('abort', cancel)
482
+ if (error) reject(error); else resolve()
483
+ }
484
+ const settled = (completed) => finish(completed ? null : busy())
485
+ const cancel = () => finish(workerCallError('cancelled before dispatch', 'INPUT_CANCELLED', op, null, 'not-dispatched'))
486
+ const timer = setTimeout(() => finish(busy()), Math.max(0, deadline - Date.now()))
487
+ this.once('uia-read-settled', settled)
488
+ if (input) this.once('input-cancel', cancel)
489
+ signal?.addEventListener('abort', cancel, { once: true })
490
+ if (signal?.aborted) cancel()
318
491
  })
319
492
  }
320
493
 
321
494
  async kill () {
495
+ const ownAttempt = this.inputContext.getStore()?.actionRecord?.attemptId
496
+ const actions = [...this.activeActions.entries()].filter(([id]) => id !== ownAttempt).map(([, settled]) => settled)
322
497
  this.emit('input-cancel')
323
- if (!this.proc) return
324
- try { await this.call('abort', {}, 1000) } catch { /* best effort */ }
325
- try { this.proc.kill() } catch { /* already gone */ }
498
+ const child = this.proc
499
+ const bounded = async promise => {
500
+ let timer
501
+ try { return await Promise.race([promise.then(() => true), new Promise(resolve => { timer = setTimeout(() => resolve(false), this.shutdownGraceMs) })]) }
502
+ finally { clearTimeout(timer) }
503
+ }
504
+ if (child) {
505
+ let exited, detach
506
+ const death = new Promise(resolve => {
507
+ exited = resolve
508
+ if (typeof child.once === 'function') {
509
+ child.once('exit', exited); detach = () => child.off('exit', exited)
510
+ } else {
511
+ this.once('dead', exited); detach = () => this.off('dead', exited)
512
+ }
513
+ })
514
+ try {
515
+ const sameWorker = () => { if (this.proc !== child) throw new Error('original worker already exited') }
516
+ try { await this.callInternal('abort', {}, 1000, false, sameWorker) } catch { /* best effort */ }
517
+ if (this.proc === child) {
518
+ try { child.kill() } catch { /* failure is reported as unknown below */ }
519
+ if (!await bounded(death)) {
520
+ // Termination itself is uncertain. Settle local callers honestly so
521
+ // disposal cannot silently discard their final audit records.
522
+ for (const [id, pending] of this.pending) if (pending.child === child) {
523
+ this.pending.delete(id); clearTimeout(pending.timer); pending.cleanup?.()
524
+ pending.reject(Object.assign(workerCallError('worker shutdown did not confirm process exit', 'worker-shutdown-unconfirmed', pending.op, id),
525
+ { cleanup: { cleanupConfirmed: false, cleanupState: 'shutdown-unconfirmed' } }))
526
+ }
527
+ }
528
+ }
529
+ } finally { detach() }
530
+ }
531
+ // In-flight actuate() finally blocks emit the persistent settlement. Waiting
532
+ // for process exit alone is too early; never wait for the action invoking us.
533
+ await bounded(Promise.all(actions))
326
534
  }
327
535
 
328
536
  // ---------------- screen observation ----------------
329
- syncCooperation (state) {
330
- if (!state || Number(state.revision) < Number(this.cooperation.revision)) return
537
+ syncCooperation (state, notify = true) {
538
+ if (!state || this.workerStateInvalid) return false
539
+ if (state.workerGeneration && this.workerGeneration && state.workerGeneration !== this.workerGeneration) return false
540
+ if (Number.isSafeInteger(state.stateSeq)) {
541
+ if (state.stateSeq <= this.nativeStateSeq) return false
542
+ } else if (this.nativeStateSeq >= 0) return false // never downgrade an ordered worker
543
+ if (Number(state.revision) < Number(this.cooperation.revision)) return false
331
544
  if (Number(state.revision) === Number(this.cooperation.revision) && Number.isFinite(state.inputEpoch) &&
332
- state.inputEpoch < (this.cooperation.inputEpoch || 0)) return
545
+ state.inputEpoch < (this.cooperation.inputEpoch || 0)) return false
546
+ if (Number.isSafeInteger(state.stateSeq)) this.nativeStateSeq = state.stateSeq
333
547
  const inputChanged = Number.isFinite(state.inputEpoch) && state.inputEpoch !== (this.cooperation.inputEpoch || 0)
334
548
  this.cooperation = state
335
549
  if (state.active || inputChanged) { this.maps.clear(); this.lastMarks = []; this.lastMarkFrame = null }
336
- this.emit('handoff', state)
550
+ if (notify) this.emit('handoff', state)
551
+ return true
337
552
  }
338
553
 
339
554
  withInputContext (context, fn) {
340
- return this.inputContext.run({ signal: context?.signal, assertOwnership: context?.assertOwnership, inputSeries: randomUUID() }, fn)
555
+ return this.inputContext.run({ signal: context?.signal, assertOwnership: context?.assertOwnership,
556
+ cycleId: context?.cycleId ?? null, sessionKey: context?.sessionKey ?? null, agentKey: context?.agentKey ?? null,
557
+ inputSeries: randomUUID() }, fn)
341
558
  }
342
559
 
343
560
  waitForHuman () {
344
561
  const signal = this.inputContext.getStore()?.signal
562
+ const started = performance.now()
563
+ const state = () => Object.fromEntries(['phase', 'active', 'revision', 'heldCount', 'lastKind', 'quietRemainingMs', 'inputEpoch']
564
+ .filter(key => this.cooperation[key] !== undefined).map(key => [key, this.cooperation[key]]))
565
+ const report = (phase, extra = {}) => {
566
+ const record = { ts: Date.now(), op: 'handoff-wait', phase, waitedMs: Math.round(performance.now() - started), cooperation: state(), ...extra }
567
+ try { this.emit('handoff-wait', record) } catch { /* diagnostics never own the wait */ }
568
+ return record
569
+ }
345
570
  return new Promise((resolve, reject) => {
571
+ let finished = false
346
572
  const events = ['handoff', 'panic', 'ask', 'exit', 'dead', 'input-cancel']
347
573
  const cleanup = () => { for (const event of events) this.off(event, listeners[event]); signal?.removeEventListener('abort', cancelled) }
348
- const finish = (error) => { cleanup(); if (error) reject(error); else resolve() }
349
- const cancelled = () => finish(workerCallError('waiting for the human was cancelled', 'INPUT_CANCELLED', null, null, 'not-dispatched'))
574
+ const finish = (error) => {
575
+ if (finished) return
576
+ finished = true
577
+ cleanup()
578
+ const record = report(error ? 'ended-with-error' : 'handback', error ? { code: error.code, cancelSource: error.cancelSource || null } : {})
579
+ if (error) { error.handoffWait = record; reject(error) } else resolve()
580
+ }
581
+ const cancelled = () => {
582
+ const source = signal?.aborted ? 'host-signal' : 'core-input-cancel'
583
+ const code = signal?.reason?.code || signal?.reason?.name
584
+ const reasonCode = typeof code === 'string' && /^[A-Za-z0-9_-]{1,64}$/.test(code) ? code : 'unspecified'
585
+ const snapshot = state()
586
+ const error = workerCallError(`waiting for the human was cancelled; source=${source}; reasonCode=${reasonCode}; waitedMs=${Math.round(performance.now() - started)}; phase=${snapshot.phase}; heldCount=${snapshot.heldCount ?? 'unknown'}. Cancellation ended the local wait; it does not explain why handback was delayed`, 'INPUT_CANCELLED', null, null, 'not-dispatched')
587
+ error.cancelSource = source
588
+ finish(error)
589
+ }
350
590
  const check = () => {
351
591
  if (signal?.aborted) return cancelled()
352
592
  if (this.stopped || this.exited) return finish(workerCallError('manual pause/exit remains authoritative', 'ABORTED', null, null, 'not-dispatched'))
@@ -357,11 +597,23 @@ export class ComputerUse extends EventEmitter {
357
597
  event === 'input-cancel' ? cancelled : check]))
358
598
  for (const event of events) this.on(event, listeners[event])
359
599
  signal?.addEventListener('abort', cancelled, { once: true })
600
+ report('started')
360
601
  check()
361
602
  })
362
603
  }
363
604
 
364
605
  async waitInputReady () {
606
+ if (this.workerStateInvalid) {
607
+ const context = this.inputContext.getStore()
608
+ if (context?.signal?.aborted) throw workerCallError('owning tool cancelled', 'INPUT_CANCELLED', null, null, 'not-dispatched')
609
+ context?.assertOwnership?.()
610
+ await this.ensureWorker()
611
+ if (context?.signal?.aborted) throw workerCallError('owning tool cancelled', 'INPUT_CANCELLED', null, null, 'not-dispatched')
612
+ context?.assertOwnership?.()
613
+ if (this.workerStateInvalid) throw workerCallError('native worker has not supplied a fresh ready snapshot', 'WORKER_STATE_UNAVAILABLE', null, null, 'not-dispatched')
614
+ if (this.stopped || this.exited) throw workerCallError('manual pause/exit remains authoritative', 'ABORTED', null, null, 'not-dispatched')
615
+ throw workerReobserveError(this.workerGeneration)
616
+ }
365
617
  if (!this.cooperation.active) return
366
618
  await this.waitForHuman()
367
619
  this.inputContext.getStore()?.assertOwnership?.()
@@ -409,8 +661,10 @@ export class ComputerUse extends EventEmitter {
409
661
  const active = (windows.windows || []).find(w => w.active) || null
410
662
  return {
411
663
  active, windows: windows.windows, cursor,
664
+ cooperation: windows.cooperation, indicator: windows.indicator, virtualScreen: windows.virtualScreen,
665
+ runtime: this.runtime, workerGeneration: this.workerGeneration,
412
666
  uiaRoot: uia && uia.root, uiaFlat: (uia && uia.flat) || [],
413
- uiaCount: uia && uia.count, uiaTruncated: !uia || uia.truncated !== false, ts: Date.now(),
667
+ uiaCount: uia && uia.count, uiaTruncated: !uia || uia.truncated !== false, uiaBudget: uia?.budget, ts: Date.now(),
414
668
  }
415
669
  }
416
670
 
@@ -420,7 +674,8 @@ export class ComputerUse extends EventEmitter {
420
674
  * mark ids refer to the same observed map as the text table.
421
675
  */
422
676
  async marks (opts = {}) {
423
- const map = opts.landmarkMap || await this.landmarks({ hwnd: opts.hwnd, depth: opts.uiaDepth ?? 14, maxNodes: opts.uiaMaxNodes ?? 1500 })
677
+ const map = opts.landmarkMap || await this.landmarks({ hwnd: opts.hwnd, depth: opts.uiaDepth ?? 6,
678
+ maxNodes: opts.uiaMaxNodes ?? 300, maxMarks: opts.maxMarks })
424
679
  await this.validateMarkFrame()
425
680
  const state = { active: { hwnd: map.hwnd }, cursor: await this.call('cursor', {}) }
426
681
  // capture FIRST so the visible region drives the ghost-mark filter:
@@ -446,17 +701,24 @@ export class ComputerUse extends EventEmitter {
446
701
  }, CAPTURE_TIMEOUT_MS)
447
702
  annotated = Buffer.from(res.image, 'base64')
448
703
  }
449
- return { marks: markList, annotated, raw: opts.raw ? shot.image : undefined, state: { active: state.active, cursor: state.cursor }, ts: Date.now() }
704
+ return { marks: markList, annotated, raw: opts.raw || !annotated ? shot.image : undefined,
705
+ mime: annotated ? 'image/jpeg' : shot.mime, rawMime: shot.mime,
706
+ region: shot.region, imageSize: shot.imageSize, capturedAt: shot.capturedAt,
707
+ cursorInImage: shot.cursorInImage, scaledTo: shot.scaledTo,
708
+ settled: shot.settled, settleMs: shot.settleMs,
709
+ state: { active: state.active, cursor: state.cursor }, ts: shot.ts ?? Date.now() }
450
710
  }
451
711
 
452
712
  // ---------------- actions (all audited) ----------------
453
713
  async move (x, y) { return this.actuate('move', { x, y }) }
454
714
 
455
715
  async click (target, opts = {}) {
716
+ const witness = verificationRegion(opts.verificationRegion)
456
717
  const pt = await this.resolveTarget(target) // {x,y} | {mark:'M3'} | [x,y]
457
718
  return this.actuate('click', {
458
719
  x: pt.x, y: pt.y, button: opts.button || 'left', clicks: opts.clicks || 1,
459
720
  settleMs: opts.settleMs ?? 40,
721
+ ...(witness ? { verificationRegion: witness } : {}),
460
722
  ...((pt.expectHwnd || opts.expectHwnd) ? { expectHwnd: pt.expectHwnd || opts.expectHwnd } : {}),
461
723
  })
462
724
  }
@@ -464,7 +726,8 @@ export class ComputerUse extends EventEmitter {
464
726
  async drag (from, to, opts = {}) {
465
727
  const a = await this.resolveTarget(from)
466
728
  const b = await this.resolveTarget(to)
467
- return this.actuate('drag', { fromX: a.x, fromY: a.y, toX: b.x, toY: b.y, ...opts })
729
+ return this.actuate('drag', { fromX: a.x, fromY: a.y, toX: b.x, toY: b.y, ...opts,
730
+ ...((a.expectHwnd || opts.expectHwnd) ? { expectHwnd: a.expectHwnd || opts.expectHwnd } : {}) })
468
731
  }
469
732
 
470
733
  async scroll (direction, lines, opts = {}) {
@@ -481,7 +744,9 @@ export class ComputerUse extends EventEmitter {
481
744
  // Anchor-based selection: click the START, scroll freely, then shift+click the END.
482
745
  // Replaces drag-select, whose end point depends on drag duration and auto-scroll speed.
483
746
  async shiftClick (x, y, opts = {}) {
747
+ const witness = verificationRegion(opts.verificationRegion)
484
748
  return this.actuate('shiftClick', { x, y, clicks: opts.clicks || 1, settleMs: opts.settleMs ?? 90,
749
+ ...(witness ? { verificationRegion: witness } : {}),
485
750
  ...(opts.expectHwnd ? { expectHwnd: opts.expectHwnd } : {}) })
486
751
  }
487
752
 
@@ -503,7 +768,7 @@ export class ComputerUse extends EventEmitter {
503
768
  * emergency brake. Called when the agent's turn ends, so a lingering blue border can never
504
769
  * claim the machine is still under control after control actually stopped.
505
770
  */
506
- async calm ({ beforeDispatch } = {}) { return this.call('calm', {}, DEFAULT_TIMEOUT_MS, true, beforeDispatch) }
771
+ async calm ({ beforeDispatch } = {}) { return this.callInternal('calm', {}, DEFAULT_TIMEOUT_MS, true, beforeDispatch) }
507
772
 
508
773
  /// **Ask the human one question, machine paused — the ONE question path.**
509
774
  ///
@@ -548,10 +813,17 @@ export class ComputerUse extends EventEmitter {
548
813
  this._askOpen = null
549
814
  }
550
815
  const askId = b && b.askId
816
+ const askWorker = this.proc
817
+ const sameAskWorker = () => {
818
+ if (this.proc !== askWorker) throw Object.assign(new Error('question worker exited; its askId cannot release a replacement worker'), { code: 'worker-replaced' })
819
+ }
551
820
  let answer = null
552
821
  let rel = null
553
822
  let via = 'ok'
554
823
  try {
824
+ const context = this.inputContext.getStore()
825
+ if (context?.signal?.aborted) throw workerCallError('owning tool cancelled after the ask brake was engaged', 'INPUT_CANCELLED', 'beginAsk', null, 'unknown')
826
+ context?.assertOwnership?.()
555
827
  answer = await opts.onAsk({ askId, hostRaised: !!(b && b.hostRaised), caretPlaced: !!(b && b.caretPlaced) })
556
828
  // `via` is an AUDIT LABEL ONLY — it decides nothing. The pause decision is `keepPause`, and it
557
829
  // comes from the human: ①让你停 keeps the brake, and every other ending (②放你走, a typed custom
@@ -576,14 +848,14 @@ export class ComputerUse extends EventEmitter {
576
848
  // reply describes a machine that no longer exists and must not be applied on top of it.
577
849
  const beforeRelease = this.stateSeq
578
850
  try {
579
- rel = await this.call('endAsk', { askId, via, keepPause }, END_ASK_DEADLINE_MS, /* killOnTimeout */ false)
851
+ rel = await this.callInternal('endAsk', { askId, via, keepPause }, END_ASK_DEADLINE_MS, /* killOnTimeout */ false, sameAskWorker)
580
852
  } catch (e) {
581
853
  // ONE retry: the first attempt may have timed out against a briefly busy worker. A second
582
854
  // failure is not swallowed — the audit pair (ENGAGE-ASK with no ASK-RELEASE) is the signal,
583
855
  // and the orphan sweep is the backstop. NEVER a kill here (killOnTimeout:false above): the
584
856
  // dead worker's STOP file would outlive every waiter, which is the defect being removed.
585
857
  try {
586
- rel = await this.call('endAsk', { askId, via: via + '+retry', keepPause }, END_ASK_DEADLINE_MS, false)
858
+ rel = await this.callInternal('endAsk', { askId, via: via + '+retry', keepPause }, END_ASK_DEADLINE_MS, false, sameAskWorker)
587
859
  } catch (e2) {
588
860
  this.log(`[ask] endAsk failed twice: ${e2.message} — the brake may need the orphan sweep (the next worker call sweeps it, or the human's Ctrl+Alt+R)`)
589
861
  }
@@ -643,25 +915,32 @@ export class ComputerUse extends EventEmitter {
643
915
  async landmarks (opts = {}) {
644
916
  const step = opts.step ?? 16
645
917
  const insetPct = opts.insetPct ?? 10
918
+ // UIA-ONLY MODE (test report 2026-11-04): an animated neighbour (DSH Pet) keeps flipping the
919
+ // sampled pixel signature, so every refresh came back STALE_MARK. uiaOnly pins identity by
920
+ // window handle / process / geometry alone — the sampled signature is read (for the report)
921
+ // but never compared; dispatch-time occlusion checking still guards the actual click.
922
+ const uiaOnly = opts.uiaOnly === true
646
923
  const fs = await this.frameSig({ hwnd: opts.hwnd, step, insetPct })
924
+ if (uiaOnly) fs.sig = null
647
925
  const hwnd = fs.hwnd
648
- if (!hwnd || !fs.sig) throw staleMark('window snapshot unavailable')
926
+ if (!hwnd || (!fs.sig && !uiaOnly)) throw staleMark('window snapshot unavailable')
649
927
  const cached = this.maps.get(String(hwnd))
650
- const key = JSON.stringify([opts.depth ?? 14, opts.maxNodes ?? 1500, opts.maxMarks ?? 80, opts.minSize ?? 8, opts.role, opts.nameContains])
928
+ const key = JSON.stringify([opts.depth ?? 6, opts.maxNodes ?? 300, opts.maxMarks ?? 80, opts.minSize ?? 8, opts.role, opts.nameContains, uiaOnly])
651
929
  if (cached && sameMarkFrame(cached.frame, fs) && cached.key === key && !opts.refresh) {
652
930
  this.lastMarks = cached.marks
653
931
  this.lastMarkFrame = { ...fs, step, insetPct }
654
- return { hwnd, marks: cached.marks, count: cached.marks.length, reused: true, ageMs: Date.now() - cached.ts, sig: fs.sig }
932
+ return { hwnd, marks: cached.marks, count: cached.marks.length, reused: true, ageMs: Date.now() - cached.ts, sig: fs.sig, uiaOnly }
655
933
  }
656
934
  const uia = await this.call('uia', {
657
935
  hwnd: hwnd || undefined,
658
- depth: opts.depth ?? 14,
659
- maxNodes: opts.maxNodes ?? 1500,
936
+ depth: opts.depth ?? 6,
937
+ maxNodes: opts.maxNodes ?? 300,
660
938
  role: opts.role,
661
939
  nameContains: opts.nameContains,
662
940
  })
663
941
  const marks = buildMarks(uia.flat || [], opts.maxMarks ?? 80, opts.minSize ?? 8)
664
942
  const after = await this.frameSig({ hwnd, step, insetPct })
943
+ if (uiaOnly) after.sig = null
665
944
  if (!sameMarkFrame(fs, after)) {
666
945
  const changed = ['hwnd', 'pid'].filter(k => fs[k] !== after[k])
667
946
  for (const k of ['x', 'y', 'width', 'height']) if (fs.region?.[k] !== after.region?.[k]) changed.push(`region.${k}`)
@@ -674,7 +953,7 @@ export class ComputerUse extends EventEmitter {
674
953
  this.lastMarkFrame = { ...after, step, insetPct }
675
954
  this.maps.set(String(hwnd), { marks, frame: after, sig: after.sig, key, ts: Date.now() })
676
955
  if (this.maps.size > 16) this.maps.delete(this.maps.keys().next().value)
677
- return { hwnd, marks, count: marks.length, reused: false, sig: fs ? fs.sig : null, elements: (uia.flat || []).length }
956
+ return { hwnd, marks, count: marks.length, reused: false, sig: fs ? fs.sig : null, uiaOnly, elements: (uia.flat || []).length }
678
957
  }
679
958
 
680
959
  // The human's emergency brake, reachable from the model side too (ESC / Ctrl+Alt+Q are
@@ -716,7 +995,7 @@ export class ComputerUse extends EventEmitter {
716
995
  async calibrate (opts = {}) { return this.actuate('calibrate', opts) }
717
996
 
718
997
  async key (combo, opts = {}) {
719
- return this.actuate('key', { combo, holdMs: opts.holdMs ?? 60, ...(opts.expectHwnd ? { expectHwnd: opts.expectHwnd } : {}) })
998
+ return this.actuate('key', { combo: parseKeyCommand(combo).combo, holdMs: opts.holdMs ?? 60, ...(opts.expectHwnd ? { expectHwnd: opts.expectHwnd } : {}) })
720
999
  }
721
1000
 
722
1001
  async type (text, opts = {}) {
@@ -748,6 +1027,8 @@ export class ComputerUse extends EventEmitter {
748
1027
  }
749
1028
 
750
1029
  async clipRead () { return this.call('clipRead', {}) }
1030
+ /** Worker-side OCR: text + per-word boxes in ABSOLUTE screen pixels (2026-11-04). */
1031
+ async ocr (region) { return this.call('ocr', region ? { ...region } : {}) }
751
1032
  async clipWrite (text) { return this.actuate('clipWrite', { text }) }
752
1033
 
753
1034
  async activateWindow (sel) { return this.actuate('activate', sel) }
@@ -783,52 +1064,73 @@ export class ComputerUse extends EventEmitter {
783
1064
 
784
1065
  /** Every state-changing op funnels through here for the audit trail. */
785
1066
  async actuate (op, args) {
786
- await this.waitInputReady()
787
- if (this.dryRun) {
788
- const rec = { op, args: redact(args), ts: Date.now(), dryRun: true }
789
- this.emit('action', rec)
790
- return { dryRun: true, op }
791
- }
792
- const now = Date.now()
793
- this.rateWindow = this.rateWindow.filter(t => now - t < this.rateLimit.windowMs)
794
- if (this.rateWindow.length >= this.rateLimit.max) {
795
- throw new Error(`rate limit: ${this.rateLimit.max} actuations/${this.rateLimit.windowMs}ms exceeded`)
796
- }
797
- this.rateWindow.push(now)
798
- const rec = { op, args: redact(args), ts: Date.now() }
799
- this.emit('action', rec)
800
- try {
801
- const data = await this.call(op, args)
802
- rec.result = data
803
- return data
804
- } catch (err) {
805
- if (err.code === 'HUMAN_YIELD') {
806
- // The hook's event can arrive just after its operation refusal. One cheap local
807
- // status read closes that race; waiting spends no model requests and never replays input.
808
- try {
809
- await this.call('ping', {})
810
- await this.waitForHuman()
811
- this.inputContext.getStore()?.assertOwnership?.()
812
- err = humanReobserveError({ ...err, op })
813
- } catch (waitError) {
814
- // Cancelling the local wait cannot erase input already accepted before handoff.
815
- err = workerCallError(waitError.message || String(waitError), waitError.code || 'handoff-wait-failed',
816
- op, err.requestId ?? null, err.outcome === 'not-dispatched' ? 'not-dispatched' : 'unknown')
1067
+ const parent = this.inputContext.getStore() || {}
1068
+ const rec = { attemptId: randomUUID(), requestId: null, op, args: redact(args),
1069
+ phase: 'started', ts: Date.now(), cycleId: parent.cycleId ?? null,
1070
+ sessionKey: parent.sessionKey ?? null, agentKey: parent.agentKey ?? null,
1071
+ workerGeneration: this.workerGeneration, runtime: this.runtime }
1072
+ this.emit('action', { ...rec })
1073
+ let settleAction
1074
+ this.activeActions.set(rec.attemptId, new Promise(resolve => { settleAction = resolve }))
1075
+ return this.inputContext.run({ ...parent, inputSeries: parent.inputSeries || randomUUID(), actionRecord: rec }, async () => {
1076
+ try {
1077
+ await this.waitInputReady()
1078
+ if (this.dryRun) {
1079
+ rec.dryRun = true; rec.outcome = 'not-dispatched'
1080
+ return { dryRun: true, op }
817
1081
  }
818
- }
819
- if (err.code === 'HUMAN_REOBSERVE') {
820
- try { this.inputContext.getStore()?.assertOwnership?.() }
821
- catch (ownerError) {
822
- err = workerCallError(ownerError.message, ownerError.code || 'COMPUTER_OWNER_CHANGED', op,
823
- err.requestId ?? null, err.outcome)
1082
+ const now = Date.now()
1083
+ this.rateWindow = this.rateWindow.filter(t => now - t < this.rateLimit.windowMs)
1084
+ if (this.rateWindow.length >= this.rateLimit.max) {
1085
+ throw workerCallError(`rate limit: ${this.rateLimit.max} actuations/${this.rateLimit.windowMs}ms exceeded`,
1086
+ 'RATE_LIMITED', op, null, 'not-dispatched')
1087
+ }
1088
+ this.rateWindow.push(now)
1089
+ const data = await this.call(op, args)
1090
+ rec.result = auditReceipt(data); rec.outcome = 'delivered'
1091
+ return data
1092
+ } catch (initial) {
1093
+ let err = initial
1094
+ if (err.code === 'HUMAN_YIELD') {
1095
+ try {
1096
+ await this.call('ping', {})
1097
+ if (rec.workerGeneration && rec.workerGeneration !== this.workerGeneration) {
1098
+ throw workerReobserveError(this.workerGeneration, { ...err, op })
1099
+ }
1100
+ await this.waitForHuman()
1101
+ this.inputContext.getStore()?.assertOwnership?.()
1102
+ err = humanReobserveError({ ...err, op })
1103
+ } catch (waitError) {
1104
+ err = waitError.code === 'WORKER_REOBSERVE' ? waitError : Object.assign(workerCallError(waitError.detail || waitError.message || String(waitError), waitError.code || 'handoff-wait-failed',
1105
+ op, err.requestId ?? null, err.outcome === 'not-dispatched' ? 'not-dispatched' : 'unknown'),
1106
+ { ...(waitError.handoffWait ? { handoffWait: waitError.handoffWait } : {}),
1107
+ ...(waitError.cancelSource ? { cancelSource: waitError.cancelSource } : {}),
1108
+ ...(waitError.cleanup ? { cleanup: waitError.cleanup } : {}) })
1109
+ }
824
1110
  }
1111
+ if (err.code === 'HUMAN_REOBSERVE' || err.code === 'WORKER_REOBSERVE') {
1112
+ try { this.inputContext.getStore()?.assertOwnership?.() }
1113
+ catch (ownerError) {
1114
+ err = workerCallError(ownerError.message, ownerError.code || 'COMPUTER_OWNER_CHANGED', op,
1115
+ err.requestId ?? null, err.outcome)
1116
+ }
1117
+ }
1118
+ // UIA/native errors may echo names, values or typed text. The caller gets
1119
+ // the original exception; the persistent audit records a bounded code.
1120
+ rec.code = auditCode(err.code); rec.error = rec.code
1121
+ rec.outcome = err.outcome || 'unknown'
1122
+ if (err.requestId != null) rec.requestId = err.requestId
1123
+ if (err.cleanup) rec.cleanup = err.cleanup
1124
+ if (err.handoffWait) rec.handoffWait = err.handoffWait
1125
+ throw err
1126
+ } finally {
1127
+ rec.phase = 'settled'; rec.settledAt = Date.now(); rec.elapsedMs = rec.settledAt - rec.ts
1128
+ try { this.emit('action:settled', { ...rec }) }
1129
+ finally { this.activeActions.delete(rec.attemptId); settleAction() }
825
1130
  }
826
- rec.error = String(err.message || err)
827
- throw err
828
- } finally {
829
- this.emit('action:settled', rec)
830
- }
1131
+ })
831
1132
  }
1133
+
832
1134
  }
833
1135
 
834
1136
  function staleMark (reason) {
@@ -837,16 +1139,36 @@ function staleMark (reason) {
837
1139
  }
838
1140
 
839
1141
  function sameMarkFrame (a, b) {
840
- return !!a?.sig && a.sig === b?.sig && a.hwnd === b.hwnd && a.pid === b.pid &&
841
- ['x', 'y', 'width', 'height'].every(k => Number.isFinite(a.region?.[k]) && a.region[k] === b.region?.[k])
1142
+ const identity = a?.hwnd === b?.hwnd && a?.pid === b?.pid &&
1143
+ ['x', 'y', 'width', 'height'].every(k => Number.isFinite(a.region?.[k]) && a.region[k] === b.region[k])
1144
+ if (!identity) return false
1145
+ // A uiaOnly landmark frame (sig === null, test report 2026-11-04) deliberately carries no
1146
+ // pixel signature: identity is geometry alone, and the click path's occlusion check remains.
1147
+ if (a.sig === null) return true
1148
+ return !!a?.sig && a.sig === b?.sig
842
1149
  }
843
1150
 
844
1151
  /** Keep secrets out of the audit log. */
845
1152
  function redact (args) {
846
- if (args && typeof args.text === 'string' && args.text.length > 0) {
847
- return { ...args, text: `<${args.text.length} chars>` }
848
- }
849
- return args
1153
+ if (!args || typeof args !== 'object') return {}
1154
+ const safeStrings = new Set(['combo', 'button', 'direction', 'action', 'mode', 'preset', 'property'])
1155
+ return Object.fromEntries(Object.entries(args).map(([key, value]) => [key,
1156
+ typeof value === 'string' ? safeStrings.has(key) ? value.slice(0, 128) : `<${value.length} chars>` :
1157
+ typeof value === 'number' || typeof value === 'boolean' || value === null ? value : '<structured value>']))
1158
+ }
1159
+
1160
+ function auditCode (code) {
1161
+ return typeof code === 'string' && /^[A-Za-z0-9_-]{1,64}$/.test(code) ? code : 'action-error'
1162
+ }
1163
+
1164
+ // Explicit receipt allowlist: never persist screenshots, focus/UIA objects,
1165
+ // clipboard content, passwords, arbitrary strings or provider result trees.
1166
+ function auditReceipt (data) {
1167
+ if (!data || typeof data !== 'object') return {}
1168
+ const keys = ['ok', 'delivered', 'count', 'chars', 'typed', 'clicked', 'moved', 'settled',
1169
+ 'elapsedMs', 'durationMs', 'hwnd', 'x', 'y', 'width', 'height', 'dryRun']
1170
+ return Object.fromEntries(keys.filter(key => typeof data[key] === 'boolean' ||
1171
+ (typeof data[key] === 'number' && Number.isFinite(data[key]))).map(key => [key, data[key]]))
850
1172
  }
851
1173
 
852
1174
  /**
@@ -859,6 +1181,7 @@ function buildMarks (flat, maxMarks = 60, minSize = 8) {
859
1181
  const interactive = []
860
1182
  for (const e of flat) {
861
1183
  if (interactive.length >= maxMarks) break
1184
+ if (e.isOffscreen === true || e.intersectsVirtualScreen === false) continue
862
1185
  const r = e.rect || {}
863
1186
  const name = (e.name || '').replace(/\s*\r\n\s*/g, ' | ').trim()
864
1187
  const key = `${e.role}|${name}|${r.x},${r.y}`