dsh-codex-style-computer-use 1.3.0 → 2.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +53 -22
- package/README.zh.md +46 -21
- package/icon.svg +29 -0
- package/lib/audit.js +7 -0
- package/lib/batch.js +14 -19
- package/lib/bin/attention-worker.exe +0 -0
- package/lib/bin/attention-worker.exe.json +1 -1
- package/lib/compact-feedback.js +175 -0
- package/lib/continuation.js +1 -1
- package/lib/core/bin/dsh-computer-use-worker.exe +0 -0
- package/lib/core/bin/dsh-computer-use-worker.exe.json +2 -2
- package/lib/core/compile-worker.ps1 +5 -1
- package/lib/core/index.js +452 -129
- package/lib/core/native-runtime.js +23 -1
- package/lib/core/winmd/Windows.winmd +0 -0
- package/lib/core/worker.cs +2415 -356
- package/lib/feedback-routes.json +11 -0
- package/lib/feedback.js +49 -16
- package/lib/image-output.js +59 -11
- package/lib/index.js +42 -7
- package/lib/key-command.js +37 -0
- package/lib/media-contract.js +43 -0
- package/lib/observations.js +267 -13
- package/lib/policy.js +30 -10
- package/lib/tool-contract.js +56 -0
- package/lib/tools.js +174 -51
- package/lib/verification-region.js +17 -0
- package/mcp/package.json +2 -1
- package/mcp/src/execution.js +8 -5
- package/mcp/src/index.js +16 -267
- package/mcp/src/tools.js +271 -0
- package/package.json +5 -2
- package/scripts/check-behavior.mjs +83 -0
- package/scripts/measure-tool-tokens.mjs +21 -0
- package/scripts/render-promo.py +81 -0
- package/skills/computer-use/SKILL.md +35 -8
- package/skills/computer-use/references/lifecycle.md +11 -7
- package/skills/computer-use/references/maintenance.md +9 -5
- package/skills/computer-use/references/precision.md +39 -14
- package/skills/computer-use/references/progressive-control.md +10 -4
- package/skills/computer-use/references/zero-uia-apps.md +54 -0
package/lib/core/index.js
CHANGED
|
@@ -16,12 +16,19 @@ import { AsyncLocalStorage } from 'node:async_hooks'
|
|
|
16
16
|
import { existsSync, readFileSync, statSync } from 'node:fs'
|
|
17
17
|
import { join } from 'node:path'
|
|
18
18
|
import { fileURLToPath } from 'node:url'
|
|
19
|
-
import { prepareNativeBinary, workerReferences } from './native-runtime.js'
|
|
19
|
+
import { prepareNativeBinary, workerReferences, nativeRuntimeIdentity } from './native-runtime.js'
|
|
20
|
+
import { parseKeyCommand } from '../key-command.js'
|
|
21
|
+
import { verificationRegion } from '../verification-region.js'
|
|
20
22
|
|
|
21
23
|
const __dirname = fileURLToPath(new URL('.', import.meta.url))
|
|
22
24
|
|
|
23
25
|
const DEFAULT_TIMEOUT_MS = 15_000
|
|
24
26
|
const CAPTURE_TIMEOUT_MS = 30_000
|
|
27
|
+
const INPUT_CLEANUP_GRACE_MS = 1500
|
|
28
|
+
// A UIA tree walk can block the resident's synchronous COM thread. Bound the
|
|
29
|
+
// caller's wait separately from recovery of a worker that never returns.
|
|
30
|
+
const UIA_READS = new Set(['uia', 'uiaFromPoint', 'uiaFocused'])
|
|
31
|
+
const URGENT_CONTROL = new Set(['cancelInput', 'abort', 'panic', 'exit', 'endAsk'])
|
|
25
32
|
const INPUT_OPS = new Set(['click', 'shiftClick', 'move', 'drag', 'scroll', 'selectRange',
|
|
26
33
|
'type', 'key', 'uiaAct', 'windowOp', 'activate', 'clipWrite', 'calibrate', 'selftest'])
|
|
27
34
|
const isInputOp = (op, args = {}) => INPUT_OPS.has(op) || (op === 'focus' && args.refocus === true)
|
|
@@ -46,7 +53,7 @@ const ASK_DEADLINE_MS = 15_000
|
|
|
46
53
|
const END_ASK_DEADLINE_MS = 5_000
|
|
47
54
|
export function workerCallError (message, code, op, requestId, outcome = 'unknown') {
|
|
48
55
|
return Object.assign(new Error(`${message} [${code}; outcome=${outcome}]. ${outcome === 'unknown' ? 'Inspect current state before retrying a write.' : 'No input was dispatched by this operation.'}`),
|
|
49
|
-
{ code, op, requestId, outcome })
|
|
56
|
+
{ code, op, requestId, outcome, detail: message })
|
|
50
57
|
}
|
|
51
58
|
|
|
52
59
|
/** A handoff interrupts an input operation, never the authorized task. */
|
|
@@ -71,6 +78,19 @@ function humanReobserveError (error = {}) {
|
|
|
71
78
|
{ transient: true, taskStatus: 'continuing', continuation })
|
|
72
79
|
}
|
|
73
80
|
|
|
81
|
+
function workerReobserveError (workerGeneration, interrupted = {}) {
|
|
82
|
+
const outcome = interrupted.outcome === 'unknown' ? 'unknown' : 'not-dispatched'
|
|
83
|
+
const continuation = {
|
|
84
|
+
required: true, taskStatus: 'continuing', reason: 'worker-restarted',
|
|
85
|
+
nextAction: 'observe-current-target-then-continue-original-task',
|
|
86
|
+
replayInterruptedInput: false, outcome, workerGeneration,
|
|
87
|
+
interruptedOperation: interrupted.op || null, requestId: interrupted.requestId ?? null,
|
|
88
|
+
instruction: 'The native worker restarted and previous desktop evidence is invalid. Obtain fresh target evidence, inspect any interrupted operation, and continue the original authorized task. This is worker recovery, not a human handback or permission to replay a write.',
|
|
89
|
+
}
|
|
90
|
+
return Object.assign(workerCallError(continuation.instruction, 'WORKER_REOBSERVE', continuation.interruptedOperation, continuation.requestId, outcome),
|
|
91
|
+
{ transient: true, taskStatus: 'continuing', continuation })
|
|
92
|
+
}
|
|
93
|
+
|
|
74
94
|
/** Interactive roles that receive Set-of-Marks labels. */
|
|
75
95
|
const SOM_ROLES = new Set([
|
|
76
96
|
'Button', 'MenuItem', 'ListItem', 'TreeItem', 'DataItem', 'TabItem',
|
|
@@ -106,6 +126,13 @@ export class ComputerUse extends EventEmitter {
|
|
|
106
126
|
this.stopped = false // true while the human's brake is engaged (see 'panic' event)
|
|
107
127
|
this.exited = false // true once the human ended the session (see 'exit' event)
|
|
108
128
|
this.cooperation = { phase: 'idle', active: false, revision: 0 }
|
|
129
|
+
this.workerGeneration = null
|
|
130
|
+
this.nativeStateSeq = -1
|
|
131
|
+
this.workerStateInvalid = false
|
|
132
|
+
this.runtime = null
|
|
133
|
+
this.inputCleanupGraceMs = INPUT_CLEANUP_GRACE_MS
|
|
134
|
+
this.shutdownGraceMs = INPUT_CLEANUP_GRACE_MS
|
|
135
|
+
this.activeActions = new Map()
|
|
109
136
|
this.inputContext = new AsyncLocalStorage()
|
|
110
137
|
// Monotonic counter of UNSOLICITED state transitions (panic/ask/resume/exit). Anything that
|
|
111
138
|
// learns the machine's state asynchronously — above all the `ask` op, which can take seconds and
|
|
@@ -141,13 +168,17 @@ export class ComputerUse extends EventEmitter {
|
|
|
141
168
|
// app (main vs renderer/GPU), so the worker is also given the executable path, which
|
|
142
169
|
// recognises every process of the app at once. The worker refuses to drive any window of
|
|
143
170
|
// either; one-shot CLI runs (no env) keep the guard disabled.
|
|
144
|
-
|
|
171
|
+
this.spawnRuntime = nativeRuntimeIdentity(this.workerExe)
|
|
172
|
+
const env = { ...process.env, DSH_CU_HOST_PID: String(process.pid), DSH_CU_HOST_EXE: process.execPath || '',
|
|
173
|
+
DSH_CU_PACKAGE_VERSION: this.spawnRuntime.packageVersion, DSH_CU_BUILD_ID: this.spawnRuntime.buildId,
|
|
174
|
+
DSH_CU_SOURCE_HASH: this.spawnRuntime.sourceHash || '', DSH_CU_BINARY_HASH: this.spawnRuntime.binaryHash }
|
|
145
175
|
this.log(`[core] host guard armed for pid ${process.pid} (${process.execPath})`)
|
|
146
176
|
this.proc = spawn(this.workerExe, [], { env, windowsHide: true, stdio: ['pipe', 'pipe', 'pipe'] })
|
|
147
177
|
// A CHILD THAT FAILS TO START MUST NAME ITSELF (2026-09-13). Every computer_* tool died with a
|
|
148
178
|
// bare `Error: spawn UNKNOWN` — no path, no syscall, no Windows code — because NOTHING listened
|
|
149
179
|
// for this child's 'error' event: an unhandled 'error' on an EventEmitter is thrown, so the rich
|
|
150
180
|
// error object (which names the file Windows refused) was discarded on the way out.
|
|
181
|
+
const child = this.proc
|
|
151
182
|
this.proc.on('error', (e) => {
|
|
152
183
|
let file = '?'
|
|
153
184
|
try { file = `${existsSync(this.workerExe)} exists, ${statSync(this.workerExe).size} bytes` } catch { file = 'stat failed' }
|
|
@@ -165,27 +196,40 @@ export class ComputerUse extends EventEmitter {
|
|
|
165
196
|
const line = buf.slice(0, nl).trim()
|
|
166
197
|
buf = buf.slice(nl + 1)
|
|
167
198
|
if (!line) continue
|
|
168
|
-
this.onLine(line)
|
|
199
|
+
if (this.proc === child) this.onLine(line)
|
|
169
200
|
}
|
|
170
201
|
})
|
|
171
202
|
this.proc.stderr.on('data', (d) => this.log(`[worker:stderr] ${d}`))
|
|
172
|
-
this.proc.on('exit', this.onWorkerExit)
|
|
203
|
+
this.proc.on('exit', (code) => { if (this.proc === child) this.onWorkerExit(code) })
|
|
173
204
|
return new Promise((resolve, reject) => {
|
|
174
|
-
const
|
|
175
|
-
// surface the real reason instead of letting the raw emitter error escape
|
|
176
|
-
this.proc.once('error', (e) => {
|
|
205
|
+
const finish = (error) => {
|
|
177
206
|
clearTimeout(t)
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
207
|
+
this.off('ready', ready); this.off('dead', dead); this.off('worker-incompatible', incompatible)
|
|
208
|
+
child.off('error', failed)
|
|
209
|
+
if (error) reject(error); else resolve()
|
|
210
|
+
}
|
|
211
|
+
const ready = () => finish()
|
|
212
|
+
const dead = code => finish(new Error(`worker died at startup (code ${code})`))
|
|
213
|
+
const incompatible = error => finish(error)
|
|
214
|
+
const failed = e => finish(new Error(`worker could not be started: code=${e.code} syscall=${e.syscall} path=${e.path || this.workerExe}`))
|
|
215
|
+
const t = setTimeout(() => {
|
|
216
|
+
finish(new Error('worker ready timeout'))
|
|
217
|
+
try { if (this.proc === child) child.kill() } catch { /* already gone */ }
|
|
218
|
+
}, 10_000)
|
|
219
|
+
// surface the real reason instead of letting the raw emitter error escape
|
|
220
|
+
child.once('error', failed)
|
|
221
|
+
this.once('ready', ready); this.once('dead', dead); this.once('worker-incompatible', incompatible)
|
|
182
222
|
})
|
|
183
223
|
}
|
|
184
224
|
|
|
185
225
|
onWorkerExit (code) {
|
|
186
226
|
this.ready = false
|
|
187
227
|
this.proc = null
|
|
188
|
-
|
|
228
|
+
this.workerStateInvalid = true
|
|
229
|
+
this.cooperation = { phase: 'unknown', active: false, revision: 0, valid: false, workerGeneration: this.workerGeneration }
|
|
230
|
+
this.maps.clear(); this.lastMarks = []; this.lastMarkFrame = null
|
|
231
|
+
this.finishUiaRead(false)
|
|
232
|
+
for (const [id, p] of this.pending) { clearTimeout(p.timer); p.cleanup?.(); p.reject(Object.assign(workerCallError('worker exited', 'worker-exited', p.op, id), { workerGeneration: p.workerGeneration, cleanup: { cleanupConfirmed: false, cleanupState: 'worker-exited' } })) }
|
|
189
233
|
this.pending.clear()
|
|
190
234
|
this.emit('dead', code)
|
|
191
235
|
}
|
|
@@ -194,6 +238,17 @@ export class ComputerUse extends EventEmitter {
|
|
|
194
238
|
let msg
|
|
195
239
|
try { msg = JSON.parse(line) } catch { this.log(`[worker:badline] ${line.slice(0, 200)}`); return }
|
|
196
240
|
if (msg.event === 'ready') {
|
|
241
|
+
// Production spawns must speak the ordered protocol. An old custom binary
|
|
242
|
+
// cannot safely participate in cancellation or generation-aware recovery.
|
|
243
|
+
if (this.spawnRuntime && (msg.protocolVersion !== 2 || typeof msg.workerGeneration !== 'string' ||
|
|
244
|
+
!msg.workerGeneration || !Number.isSafeInteger(msg.cooperation?.stateSeq) ||
|
|
245
|
+
msg.cooperation.workerGeneration !== msg.workerGeneration)) {
|
|
246
|
+
this.ready = false; this.workerStateInvalid = true
|
|
247
|
+
this.emit('worker-incompatible', workerCallError('native worker protocol is incompatible; reinstall or rebuild the current helper',
|
|
248
|
+
'WORKER_PROTOCOL_INCOMPATIBLE', null, null, 'not-dispatched'))
|
|
249
|
+
try { this.proc?.kill() } catch { /* already gone */ }
|
|
250
|
+
return
|
|
251
|
+
}
|
|
197
252
|
this.ready = true
|
|
198
253
|
// COLD-START STATE SYNC (cold-start bug, 2026-09-13): the worker has already adopted any STOP
|
|
199
254
|
// or EXITED record that predates this process, and it reports that state with `ready`. Without
|
|
@@ -203,14 +258,32 @@ export class ComputerUse extends EventEmitter {
|
|
|
203
258
|
// released — the mirror simply starts out agreeing with the machine.
|
|
204
259
|
if (typeof msg.stopped === 'boolean') this.stopped = msg.stopped
|
|
205
260
|
if (typeof msg.exited === 'boolean') this.exited = msg.exited
|
|
206
|
-
this.
|
|
261
|
+
this.workerGeneration = msg.workerGeneration || msg.runtime?.workerGeneration || msg.cooperation?.workerGeneration || null
|
|
262
|
+
this.nativeStateSeq = -1
|
|
263
|
+
this.runtime = this.spawnRuntime ? { ...this.spawnRuntime, ...msg.runtime, workerGeneration: this.workerGeneration } : msg.runtime || null
|
|
264
|
+
this.cooperation = { phase: 'idle', active: false, revision: 0 }
|
|
265
|
+
this.workerStateInvalid = false
|
|
266
|
+
if (msg.cooperation) this.syncCooperation(msg.cooperation, false)
|
|
267
|
+
this.maps.clear(); this.lastMarks = []; this.lastMarkFrame = null
|
|
207
268
|
this.emit('ready', msg)
|
|
208
269
|
return
|
|
209
270
|
}
|
|
271
|
+
// stdout from a replaced child is already filtered; generation also guards
|
|
272
|
+
// protocol-level delayed events and records restored by an observer.
|
|
273
|
+
const generation = msg.workerGeneration || msg.cooperation?.workerGeneration || msg.data?.cooperation?.workerGeneration
|
|
274
|
+
if (generation && this.workerGeneration && generation !== this.workerGeneration) return
|
|
210
275
|
if (msg.event === 'handoff') {
|
|
211
276
|
this.syncCooperation(msg.cooperation)
|
|
212
277
|
return
|
|
213
278
|
}
|
|
279
|
+
// Secure-desktop transitions (UAC prompt / lock screen, human request 2026-11-04): the
|
|
280
|
+
// worker refuses to dispatch while it stands and clears it the moment the normal desktop
|
|
281
|
+
// returns. Mirror the state so callers can show it, and let the plugin notify the human.
|
|
282
|
+
if (msg.event === 'secure') {
|
|
283
|
+
this.secureDesktop = msg.why === 'active'
|
|
284
|
+
this.emit('secure', msg)
|
|
285
|
+
return
|
|
286
|
+
}
|
|
214
287
|
// Durable brake events are separate from automatic physical-input handoff.
|
|
215
288
|
// panic — ESC: the human ABORTED. The machine is stopped; wrap up briefly, then wait.
|
|
216
289
|
// resume — Ctrl+Alt+R: work may continue (a PAUSE ends; it never ends the cycle).
|
|
@@ -236,6 +309,7 @@ export class ComputerUse extends EventEmitter {
|
|
|
236
309
|
this.emit(msg.event, msg)
|
|
237
310
|
return
|
|
238
311
|
}
|
|
312
|
+
if (this.uiaRead?.id === msg.id) this.finishUiaRead(true)
|
|
239
313
|
const p = this.pending.get(msg.id)
|
|
240
314
|
if (!p) return
|
|
241
315
|
this.pending.delete(msg.id)
|
|
@@ -246,6 +320,8 @@ export class ComputerUse extends EventEmitter {
|
|
|
246
320
|
else {
|
|
247
321
|
const error = workerCallError(msg.error || 'worker error', msg.code || 'worker-refusal-or-error', p.op, msg.id,
|
|
248
322
|
msg.outcome === 'not-dispatched' ? 'not-dispatched' : 'unknown')
|
|
323
|
+
if (msg.glowHealth) error.glowHealth = msg.glowHealth
|
|
324
|
+
for (const key of ['dragStarted', 'expectedCursor', 'actualCursor', 'cursorCheck']) if (msg[key] !== undefined) error[key] = msg[key]
|
|
249
325
|
p.reject(error.code === 'HUMAN_REOBSERVE' ? humanReobserveError(error) : error)
|
|
250
326
|
}
|
|
251
327
|
}
|
|
@@ -257,96 +333,260 @@ export class ComputerUse extends EventEmitter {
|
|
|
257
333
|
* @param {boolean} [killOnTimeout] kill a wedged worker when that cap expires (default true)
|
|
258
334
|
* @returns {Promise<any>} worker `data` payload
|
|
259
335
|
*/
|
|
336
|
+
// Cleanup runs under its own explicit authority. A cancelled user context must
|
|
337
|
+
// not block key release or the closing of a question; ordinary tools never use it.
|
|
338
|
+
callInternal (op, args = {}, timeoutMs = DEFAULT_TIMEOUT_MS, killOnTimeout = true, beforeDispatch) {
|
|
339
|
+
if (!['cancelInput', 'calm', 'endAsk', 'abort'].includes(op)) throw new Error('unsupported internal cleanup operation')
|
|
340
|
+
return this.inputContext.run({ internalCleanup: true }, () => this.call(op, args, timeoutMs, killOnTimeout, beforeDispatch))
|
|
341
|
+
}
|
|
342
|
+
|
|
343
|
+
async cancelRequest (requestId, child, reason, killOnFailure = true) {
|
|
344
|
+
const cleanup = { requestId, workerGeneration: this.workerGeneration, requested: true,
|
|
345
|
+
cleanupConfirmed: false, cleanupState: 'unconfirmed', workerKilled: false }
|
|
346
|
+
const sameWorker = () => {
|
|
347
|
+
if (this.proc !== child) throw Object.assign(new Error('original worker exited before cleanup'), { code: 'worker-replaced' })
|
|
348
|
+
}
|
|
349
|
+
try {
|
|
350
|
+
sameWorker()
|
|
351
|
+
const ack = await this.callInternal('cancelInput', { requestId }, this.inputCleanupGraceMs, false, sameWorker)
|
|
352
|
+
if (ack?.requestId !== undefined && ack.requestId !== requestId) throw new Error('cancellation acknowledgement named another request')
|
|
353
|
+
if (ack?.workerGeneration && ack.workerGeneration !== cleanup.workerGeneration) throw new Error('cancellation acknowledgement named another worker')
|
|
354
|
+
cleanup.cancelled = ack?.cancelled === true
|
|
355
|
+
cleanup.cleanupConfirmed = ack?.cleanupConfirmed === true && ack?.ownedInputCount === 0
|
|
356
|
+
cleanup.cleanupState = ack?.cleanupState || (cleanup.cleanupConfirmed ? 'released' : 'unconfirmed')
|
|
357
|
+
if (Number.isInteger(ack?.ownedInputCount)) cleanup.ownedInputCount = ack.ownedInputCount
|
|
358
|
+
} catch (error) {
|
|
359
|
+
cleanup.cleanupState = this.proc !== child ? 'worker-exited' : 'ack-failed'
|
|
360
|
+
cleanup.errorCode = error.code || 'cleanup-error'
|
|
361
|
+
}
|
|
362
|
+
if (!cleanup.cleanupConfirmed && killOnFailure && this.proc === child) {
|
|
363
|
+
cleanup.killReason = reason
|
|
364
|
+
try { cleanup.workerKilled = child.kill() !== false } catch { cleanup.workerKilled = false }
|
|
365
|
+
}
|
|
366
|
+
return cleanup
|
|
367
|
+
}
|
|
368
|
+
|
|
260
369
|
async call (op, args = {}, timeoutMs = DEFAULT_TIMEOUT_MS, killOnTimeout = true, beforeDispatch) {
|
|
261
370
|
const input = isInputOp(op, args)
|
|
262
|
-
const
|
|
371
|
+
const priorGeneration = this.workerGeneration
|
|
372
|
+
const invalidAtStart = this.workerStateInvalid
|
|
373
|
+
const context = this.inputContext.getStore()
|
|
263
374
|
const signal = context?.signal
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
|
|
375
|
+
const check = () => {
|
|
376
|
+
if (signal?.aborted) throw workerCallError('owning tool cancelled', 'INPUT_CANCELLED', op, null, 'not-dispatched')
|
|
377
|
+
beforeDispatch?.()
|
|
378
|
+
context?.assertOwnership?.()
|
|
379
|
+
}
|
|
380
|
+
check()
|
|
267
381
|
try { await this.ensureWorker() }
|
|
268
382
|
catch (error) { throw workerCallError(error.message || String(error), 'worker-unavailable', op, null, 'not-dispatched') }
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
|
|
383
|
+
const child = this.proc
|
|
384
|
+
if (!child) throw workerCallError('worker exited before dispatch', 'worker-exited', op, null, 'not-dispatched')
|
|
385
|
+
const queueDeadline = Date.now() + timeoutMs
|
|
386
|
+
while (this.uiaRead && !URGENT_CONTROL.has(op)) {
|
|
387
|
+
await this.waitForUiaRead(op, input, signal, queueDeadline)
|
|
388
|
+
if (this.proc !== child) throw workerCallError('worker exited before dispatch', 'worker-exited', op, null, 'not-dispatched')
|
|
389
|
+
if (Date.now() >= queueDeadline) throw workerCallError('deadline expired waiting for the UIA read', 'worker-uia-busy', op, null, 'not-dispatched')
|
|
390
|
+
}
|
|
391
|
+
check()
|
|
392
|
+
if (input && (invalidAtStart || (priorGeneration && priorGeneration !== this.workerGeneration))) {
|
|
393
|
+
if (this.stopped || this.exited) throw workerCallError('manual pause/exit remains authoritative', 'ABORTED', op, null, 'not-dispatched')
|
|
394
|
+
throw workerReobserveError(this.workerGeneration, { op })
|
|
395
|
+
}
|
|
272
396
|
const id = ++this.seq
|
|
273
|
-
|
|
274
|
-
|
|
275
|
-
|
|
276
|
-
|
|
277
|
-
//
|
|
278
|
-
const signed = { ...args, __cycle: true }
|
|
279
|
-
signed.
|
|
280
|
-
// One warning per external call/batch, with a fresh identity for each context.
|
|
281
|
-
// Never let a caller-supplied argument reuse another tool's announcement.
|
|
282
|
-
if (input) signed.__inputSeries = this.inputContext.getStore()?.inputSeries || randomUUID()
|
|
397
|
+
const generation = this.workerGeneration
|
|
398
|
+
if (input && context?.actionRecord) Object.assign(context.actionRecord, {
|
|
399
|
+
requestId: id, workerGeneration: generation, runtime: this.runtime,
|
|
400
|
+
})
|
|
401
|
+
// __cycle is signed last. A caller cannot forge another input series or turn.
|
|
402
|
+
const signed = { ...args, __cycle: context?.internalCleanup !== true, __kickoff: this.kickoffRequired }
|
|
403
|
+
if (input) signed.__inputSeries = context?.inputSeries || randomUUID()
|
|
283
404
|
return new Promise((resolve, reject) => {
|
|
284
405
|
let written = false
|
|
406
|
+
const read = UIA_READS.has(op) && killOnTimeout ? (this.uiaRead = { id, child, timedOut: false }) : null
|
|
285
407
|
const cleanup = () => signal?.removeEventListener('abort', abort)
|
|
286
|
-
const
|
|
408
|
+
const deferReadRecovery = () => {
|
|
409
|
+
if (!read || this.uiaRead !== read) return
|
|
410
|
+
read.timedOut = true
|
|
411
|
+
this.emit('uia-read-settled', false)
|
|
412
|
+
const grace = Math.min(timeoutMs, DEFAULT_TIMEOUT_MS)
|
|
413
|
+
read.watchdog = setTimeout(() => {
|
|
414
|
+
if (this.proc !== child || this.uiaRead !== read) return
|
|
415
|
+
this.log(`[core] UIA read ${id} (${op}) still blocked after ${grace}ms recovery grace; stopping worker pid=${child.pid}`)
|
|
416
|
+
try { child.kill() } catch { /* already gone */ }
|
|
417
|
+
}, grace)
|
|
418
|
+
}
|
|
419
|
+
const fail = async (code, message, timeout) => {
|
|
287
420
|
if (!this.pending.delete(id)) return
|
|
288
421
|
clearTimeout(timer); cleanup()
|
|
289
|
-
|
|
290
|
-
|
|
291
|
-
|
|
422
|
+
const error = workerCallError(message, code, op, id, input && written ? 'unknown' : 'not-dispatched')
|
|
423
|
+
error.operationDispatched = written
|
|
424
|
+
error.workerGeneration = generation
|
|
425
|
+
if (input && written) {
|
|
426
|
+
// Stop the original request out of band, then allow a bounded cleanup
|
|
427
|
+
// acknowledgement before killing a worker that may still own a key down.
|
|
428
|
+
error.cleanup = await this.cancelRequest(id, child, timeout ? 'input-timeout' : 'caller-cancelled', killOnTimeout)
|
|
429
|
+
} else if (read) {
|
|
430
|
+
deferReadRecovery()
|
|
431
|
+
} else if (timeout && killOnTimeout) {
|
|
432
|
+
try { if (this.proc === child) child.kill() } catch { /* already gone */ }
|
|
433
|
+
}
|
|
434
|
+
reject(error)
|
|
435
|
+
}
|
|
436
|
+
const abort = () => {
|
|
437
|
+
// beginAsk is a receipt-producing transaction: after dispatch we need
|
|
438
|
+
// its askId to release the exact brake. ask() checks cancellation before
|
|
439
|
+
// showing the card and always closes that receipt in its finally.
|
|
440
|
+
if (op === 'beginAsk' && written) return
|
|
441
|
+
void fail('INPUT_CANCELLED', 'owning tool cancelled; inspect any accepted input before retrying', false)
|
|
292
442
|
}
|
|
293
443
|
const timer = setTimeout(() => {
|
|
294
|
-
|
|
295
|
-
|
|
296
|
-
|
|
297
|
-
|
|
298
|
-
|
|
299
|
-
|
|
300
|
-
// retry, or the next op's SyncFromFile, can still finish the release.
|
|
301
|
-
//
|
|
302
|
-
// Everything else keeps today's behaviour: a timed-out op means the worker is wedged (a hung
|
|
303
|
-
// UIA/COM provider call blocks its single-threaded loop forever), so kill it and let the NEXT
|
|
304
|
-
// call respawn fresh instead of queueing behind a corpse; pending calls are rejected via
|
|
305
|
-
// onWorkerExit.
|
|
306
|
-
if (killOnTimeout) { try { this.proc && this.proc.kill() } catch { /* already gone */ } }
|
|
307
|
-
reject(workerCallError(`worker timeout: ${op} (${timeoutMs}ms)` +
|
|
308
|
-
(killOnTimeout
|
|
309
|
-
? ' — worker killed, next call respawns'
|
|
310
|
-
: ' — worker left alive so the brake can still be released'),
|
|
311
|
-
'worker-timeout', op, id))
|
|
312
|
-
}, timeoutMs)
|
|
313
|
-
this.pending.set(id, { resolve, reject, timer, op, input, cleanup, abort })
|
|
444
|
+
const message = read
|
|
445
|
+
? `UIA observation timed out: ${op} (${timeoutMs}ms); worker still finishing this read. Narrow the tree or use a point query once it returns.`
|
|
446
|
+
: `worker timeout: ${op} (${timeoutMs}ms)` + (input ? ' — cancellation and owned-input cleanup requested before worker recovery' : killOnTimeout ? ' — worker recovery requested' : ' — worker left alive so cleanup can finish')
|
|
447
|
+
void fail('worker-timeout', message, true)
|
|
448
|
+
}, Math.max(0, queueDeadline - Date.now()))
|
|
449
|
+
this.pending.set(id, { resolve, reject, timer, op, input, cleanup, abort, child, workerGeneration: generation })
|
|
314
450
|
signal?.addEventListener('abort', abort, { once: true })
|
|
315
451
|
if (signal?.aborted) { abort(); return }
|
|
316
|
-
|
|
317
|
-
|
|
452
|
+
try {
|
|
453
|
+
written = true // a synchronous pipe failure still leaves dispatch uncertain
|
|
454
|
+
child.stdin.write(JSON.stringify({ id, op, args: signed }) + '\r\n')
|
|
455
|
+
} catch (error) {
|
|
456
|
+
void fail('worker-write-failed', 'worker input pipe rejected the request: ' + error.message, false)
|
|
457
|
+
}
|
|
458
|
+
})
|
|
459
|
+
}
|
|
460
|
+
|
|
461
|
+
finishUiaRead (completed) {
|
|
462
|
+
const read = this.uiaRead
|
|
463
|
+
if (!read) return
|
|
464
|
+
this.uiaRead = null
|
|
465
|
+
clearTimeout(read.watchdog)
|
|
466
|
+
this.emit('uia-read-settled', completed && !read.timedOut)
|
|
467
|
+
}
|
|
468
|
+
|
|
469
|
+
// Only the stdin reader's cancellation command is out of band. Preserve
|
|
470
|
+
// emergency/release requests, but never queue ordinary work behind a slow
|
|
471
|
+
// observation after its caller has given up. Do not replay any request.
|
|
472
|
+
waitForUiaRead (op, input, signal, deadline) {
|
|
473
|
+
const busy = () => workerCallError('native UIA observation still busy; this request was not sent',
|
|
474
|
+
'worker-uia-busy', op, null, 'not-dispatched')
|
|
475
|
+
if (this.uiaRead.timedOut || Date.now() >= deadline) return Promise.reject(busy())
|
|
476
|
+
return new Promise((resolve, reject) => {
|
|
477
|
+
const finish = (error) => {
|
|
478
|
+
clearTimeout(timer)
|
|
479
|
+
this.off('uia-read-settled', settled)
|
|
480
|
+
this.off('input-cancel', cancel)
|
|
481
|
+
signal?.removeEventListener('abort', cancel)
|
|
482
|
+
if (error) reject(error); else resolve()
|
|
483
|
+
}
|
|
484
|
+
const settled = (completed) => finish(completed ? null : busy())
|
|
485
|
+
const cancel = () => finish(workerCallError('cancelled before dispatch', 'INPUT_CANCELLED', op, null, 'not-dispatched'))
|
|
486
|
+
const timer = setTimeout(() => finish(busy()), Math.max(0, deadline - Date.now()))
|
|
487
|
+
this.once('uia-read-settled', settled)
|
|
488
|
+
if (input) this.once('input-cancel', cancel)
|
|
489
|
+
signal?.addEventListener('abort', cancel, { once: true })
|
|
490
|
+
if (signal?.aborted) cancel()
|
|
318
491
|
})
|
|
319
492
|
}
|
|
320
493
|
|
|
321
494
|
async kill () {
|
|
495
|
+
const ownAttempt = this.inputContext.getStore()?.actionRecord?.attemptId
|
|
496
|
+
const actions = [...this.activeActions.entries()].filter(([id]) => id !== ownAttempt).map(([, settled]) => settled)
|
|
322
497
|
this.emit('input-cancel')
|
|
323
|
-
|
|
324
|
-
|
|
325
|
-
|
|
498
|
+
const child = this.proc
|
|
499
|
+
const bounded = async promise => {
|
|
500
|
+
let timer
|
|
501
|
+
try { return await Promise.race([promise.then(() => true), new Promise(resolve => { timer = setTimeout(() => resolve(false), this.shutdownGraceMs) })]) }
|
|
502
|
+
finally { clearTimeout(timer) }
|
|
503
|
+
}
|
|
504
|
+
if (child) {
|
|
505
|
+
let exited, detach
|
|
506
|
+
const death = new Promise(resolve => {
|
|
507
|
+
exited = resolve
|
|
508
|
+
if (typeof child.once === 'function') {
|
|
509
|
+
child.once('exit', exited); detach = () => child.off('exit', exited)
|
|
510
|
+
} else {
|
|
511
|
+
this.once('dead', exited); detach = () => this.off('dead', exited)
|
|
512
|
+
}
|
|
513
|
+
})
|
|
514
|
+
try {
|
|
515
|
+
const sameWorker = () => { if (this.proc !== child) throw new Error('original worker already exited') }
|
|
516
|
+
try { await this.callInternal('abort', {}, 1000, false, sameWorker) } catch { /* best effort */ }
|
|
517
|
+
if (this.proc === child) {
|
|
518
|
+
try { child.kill() } catch { /* failure is reported as unknown below */ }
|
|
519
|
+
if (!await bounded(death)) {
|
|
520
|
+
// Termination itself is uncertain. Settle local callers honestly so
|
|
521
|
+
// disposal cannot silently discard their final audit records.
|
|
522
|
+
for (const [id, pending] of this.pending) if (pending.child === child) {
|
|
523
|
+
this.pending.delete(id); clearTimeout(pending.timer); pending.cleanup?.()
|
|
524
|
+
pending.reject(Object.assign(workerCallError('worker shutdown did not confirm process exit', 'worker-shutdown-unconfirmed', pending.op, id),
|
|
525
|
+
{ cleanup: { cleanupConfirmed: false, cleanupState: 'shutdown-unconfirmed' } }))
|
|
526
|
+
}
|
|
527
|
+
}
|
|
528
|
+
}
|
|
529
|
+
} finally { detach() }
|
|
530
|
+
}
|
|
531
|
+
// In-flight actuate() finally blocks emit the persistent settlement. Waiting
|
|
532
|
+
// for process exit alone is too early; never wait for the action invoking us.
|
|
533
|
+
await bounded(Promise.all(actions))
|
|
326
534
|
}
|
|
327
535
|
|
|
328
536
|
// ---------------- screen observation ----------------
|
|
329
|
-
syncCooperation (state) {
|
|
330
|
-
if (!state ||
|
|
537
|
+
syncCooperation (state, notify = true) {
|
|
538
|
+
if (!state || this.workerStateInvalid) return false
|
|
539
|
+
if (state.workerGeneration && this.workerGeneration && state.workerGeneration !== this.workerGeneration) return false
|
|
540
|
+
if (Number.isSafeInteger(state.stateSeq)) {
|
|
541
|
+
if (state.stateSeq <= this.nativeStateSeq) return false
|
|
542
|
+
} else if (this.nativeStateSeq >= 0) return false // never downgrade an ordered worker
|
|
543
|
+
if (Number(state.revision) < Number(this.cooperation.revision)) return false
|
|
331
544
|
if (Number(state.revision) === Number(this.cooperation.revision) && Number.isFinite(state.inputEpoch) &&
|
|
332
|
-
state.inputEpoch < (this.cooperation.inputEpoch || 0)) return
|
|
545
|
+
state.inputEpoch < (this.cooperation.inputEpoch || 0)) return false
|
|
546
|
+
if (Number.isSafeInteger(state.stateSeq)) this.nativeStateSeq = state.stateSeq
|
|
333
547
|
const inputChanged = Number.isFinite(state.inputEpoch) && state.inputEpoch !== (this.cooperation.inputEpoch || 0)
|
|
334
548
|
this.cooperation = state
|
|
335
549
|
if (state.active || inputChanged) { this.maps.clear(); this.lastMarks = []; this.lastMarkFrame = null }
|
|
336
|
-
this.emit('handoff', state)
|
|
550
|
+
if (notify) this.emit('handoff', state)
|
|
551
|
+
return true
|
|
337
552
|
}
|
|
338
553
|
|
|
339
554
|
withInputContext (context, fn) {
|
|
340
|
-
return this.inputContext.run({ signal: context?.signal, assertOwnership: context?.assertOwnership,
|
|
555
|
+
return this.inputContext.run({ signal: context?.signal, assertOwnership: context?.assertOwnership,
|
|
556
|
+
cycleId: context?.cycleId ?? null, sessionKey: context?.sessionKey ?? null, agentKey: context?.agentKey ?? null,
|
|
557
|
+
inputSeries: randomUUID() }, fn)
|
|
341
558
|
}
|
|
342
559
|
|
|
343
560
|
waitForHuman () {
|
|
344
561
|
const signal = this.inputContext.getStore()?.signal
|
|
562
|
+
const started = performance.now()
|
|
563
|
+
const state = () => Object.fromEntries(['phase', 'active', 'revision', 'heldCount', 'lastKind', 'quietRemainingMs', 'inputEpoch']
|
|
564
|
+
.filter(key => this.cooperation[key] !== undefined).map(key => [key, this.cooperation[key]]))
|
|
565
|
+
const report = (phase, extra = {}) => {
|
|
566
|
+
const record = { ts: Date.now(), op: 'handoff-wait', phase, waitedMs: Math.round(performance.now() - started), cooperation: state(), ...extra }
|
|
567
|
+
try { this.emit('handoff-wait', record) } catch { /* diagnostics never own the wait */ }
|
|
568
|
+
return record
|
|
569
|
+
}
|
|
345
570
|
return new Promise((resolve, reject) => {
|
|
571
|
+
let finished = false
|
|
346
572
|
const events = ['handoff', 'panic', 'ask', 'exit', 'dead', 'input-cancel']
|
|
347
573
|
const cleanup = () => { for (const event of events) this.off(event, listeners[event]); signal?.removeEventListener('abort', cancelled) }
|
|
348
|
-
const finish = (error) => {
|
|
349
|
-
|
|
574
|
+
const finish = (error) => {
|
|
575
|
+
if (finished) return
|
|
576
|
+
finished = true
|
|
577
|
+
cleanup()
|
|
578
|
+
const record = report(error ? 'ended-with-error' : 'handback', error ? { code: error.code, cancelSource: error.cancelSource || null } : {})
|
|
579
|
+
if (error) { error.handoffWait = record; reject(error) } else resolve()
|
|
580
|
+
}
|
|
581
|
+
const cancelled = () => {
|
|
582
|
+
const source = signal?.aborted ? 'host-signal' : 'core-input-cancel'
|
|
583
|
+
const code = signal?.reason?.code || signal?.reason?.name
|
|
584
|
+
const reasonCode = typeof code === 'string' && /^[A-Za-z0-9_-]{1,64}$/.test(code) ? code : 'unspecified'
|
|
585
|
+
const snapshot = state()
|
|
586
|
+
const error = workerCallError(`waiting for the human was cancelled; source=${source}; reasonCode=${reasonCode}; waitedMs=${Math.round(performance.now() - started)}; phase=${snapshot.phase}; heldCount=${snapshot.heldCount ?? 'unknown'}. Cancellation ended the local wait; it does not explain why handback was delayed`, 'INPUT_CANCELLED', null, null, 'not-dispatched')
|
|
587
|
+
error.cancelSource = source
|
|
588
|
+
finish(error)
|
|
589
|
+
}
|
|
350
590
|
const check = () => {
|
|
351
591
|
if (signal?.aborted) return cancelled()
|
|
352
592
|
if (this.stopped || this.exited) return finish(workerCallError('manual pause/exit remains authoritative', 'ABORTED', null, null, 'not-dispatched'))
|
|
@@ -357,11 +597,23 @@ export class ComputerUse extends EventEmitter {
|
|
|
357
597
|
event === 'input-cancel' ? cancelled : check]))
|
|
358
598
|
for (const event of events) this.on(event, listeners[event])
|
|
359
599
|
signal?.addEventListener('abort', cancelled, { once: true })
|
|
600
|
+
report('started')
|
|
360
601
|
check()
|
|
361
602
|
})
|
|
362
603
|
}
|
|
363
604
|
|
|
364
605
|
async waitInputReady () {
|
|
606
|
+
if (this.workerStateInvalid) {
|
|
607
|
+
const context = this.inputContext.getStore()
|
|
608
|
+
if (context?.signal?.aborted) throw workerCallError('owning tool cancelled', 'INPUT_CANCELLED', null, null, 'not-dispatched')
|
|
609
|
+
context?.assertOwnership?.()
|
|
610
|
+
await this.ensureWorker()
|
|
611
|
+
if (context?.signal?.aborted) throw workerCallError('owning tool cancelled', 'INPUT_CANCELLED', null, null, 'not-dispatched')
|
|
612
|
+
context?.assertOwnership?.()
|
|
613
|
+
if (this.workerStateInvalid) throw workerCallError('native worker has not supplied a fresh ready snapshot', 'WORKER_STATE_UNAVAILABLE', null, null, 'not-dispatched')
|
|
614
|
+
if (this.stopped || this.exited) throw workerCallError('manual pause/exit remains authoritative', 'ABORTED', null, null, 'not-dispatched')
|
|
615
|
+
throw workerReobserveError(this.workerGeneration)
|
|
616
|
+
}
|
|
365
617
|
if (!this.cooperation.active) return
|
|
366
618
|
await this.waitForHuman()
|
|
367
619
|
this.inputContext.getStore()?.assertOwnership?.()
|
|
@@ -409,8 +661,10 @@ export class ComputerUse extends EventEmitter {
|
|
|
409
661
|
const active = (windows.windows || []).find(w => w.active) || null
|
|
410
662
|
return {
|
|
411
663
|
active, windows: windows.windows, cursor,
|
|
664
|
+
cooperation: windows.cooperation, indicator: windows.indicator, virtualScreen: windows.virtualScreen,
|
|
665
|
+
runtime: this.runtime, workerGeneration: this.workerGeneration,
|
|
412
666
|
uiaRoot: uia && uia.root, uiaFlat: (uia && uia.flat) || [],
|
|
413
|
-
uiaCount: uia && uia.count, uiaTruncated: !uia || uia.truncated !== false, ts: Date.now(),
|
|
667
|
+
uiaCount: uia && uia.count, uiaTruncated: !uia || uia.truncated !== false, uiaBudget: uia?.budget, ts: Date.now(),
|
|
414
668
|
}
|
|
415
669
|
}
|
|
416
670
|
|
|
@@ -420,7 +674,8 @@ export class ComputerUse extends EventEmitter {
|
|
|
420
674
|
* mark ids refer to the same observed map as the text table.
|
|
421
675
|
*/
|
|
422
676
|
async marks (opts = {}) {
|
|
423
|
-
const map = opts.landmarkMap || await this.landmarks({ hwnd: opts.hwnd, depth: opts.uiaDepth ??
|
|
677
|
+
const map = opts.landmarkMap || await this.landmarks({ hwnd: opts.hwnd, depth: opts.uiaDepth ?? 6,
|
|
678
|
+
maxNodes: opts.uiaMaxNodes ?? 300, maxMarks: opts.maxMarks })
|
|
424
679
|
await this.validateMarkFrame()
|
|
425
680
|
const state = { active: { hwnd: map.hwnd }, cursor: await this.call('cursor', {}) }
|
|
426
681
|
// capture FIRST so the visible region drives the ghost-mark filter:
|
|
@@ -446,17 +701,24 @@ export class ComputerUse extends EventEmitter {
|
|
|
446
701
|
}, CAPTURE_TIMEOUT_MS)
|
|
447
702
|
annotated = Buffer.from(res.image, 'base64')
|
|
448
703
|
}
|
|
449
|
-
return { marks: markList, annotated, raw: opts.raw ? shot.image : undefined,
|
|
704
|
+
return { marks: markList, annotated, raw: opts.raw || !annotated ? shot.image : undefined,
|
|
705
|
+
mime: annotated ? 'image/jpeg' : shot.mime, rawMime: shot.mime,
|
|
706
|
+
region: shot.region, imageSize: shot.imageSize, capturedAt: shot.capturedAt,
|
|
707
|
+
cursorInImage: shot.cursorInImage, scaledTo: shot.scaledTo,
|
|
708
|
+
settled: shot.settled, settleMs: shot.settleMs,
|
|
709
|
+
state: { active: state.active, cursor: state.cursor }, ts: shot.ts ?? Date.now() }
|
|
450
710
|
}
|
|
451
711
|
|
|
452
712
|
// ---------------- actions (all audited) ----------------
|
|
453
713
|
async move (x, y) { return this.actuate('move', { x, y }) }
|
|
454
714
|
|
|
455
715
|
async click (target, opts = {}) {
|
|
716
|
+
const witness = verificationRegion(opts.verificationRegion)
|
|
456
717
|
const pt = await this.resolveTarget(target) // {x,y} | {mark:'M3'} | [x,y]
|
|
457
718
|
return this.actuate('click', {
|
|
458
719
|
x: pt.x, y: pt.y, button: opts.button || 'left', clicks: opts.clicks || 1,
|
|
459
720
|
settleMs: opts.settleMs ?? 40,
|
|
721
|
+
...(witness ? { verificationRegion: witness } : {}),
|
|
460
722
|
...((pt.expectHwnd || opts.expectHwnd) ? { expectHwnd: pt.expectHwnd || opts.expectHwnd } : {}),
|
|
461
723
|
})
|
|
462
724
|
}
|
|
@@ -464,7 +726,8 @@ export class ComputerUse extends EventEmitter {
|
|
|
464
726
|
async drag (from, to, opts = {}) {
|
|
465
727
|
const a = await this.resolveTarget(from)
|
|
466
728
|
const b = await this.resolveTarget(to)
|
|
467
|
-
return this.actuate('drag', { fromX: a.x, fromY: a.y, toX: b.x, toY: b.y, ...opts
|
|
729
|
+
return this.actuate('drag', { fromX: a.x, fromY: a.y, toX: b.x, toY: b.y, ...opts,
|
|
730
|
+
...((a.expectHwnd || opts.expectHwnd) ? { expectHwnd: a.expectHwnd || opts.expectHwnd } : {}) })
|
|
468
731
|
}
|
|
469
732
|
|
|
470
733
|
async scroll (direction, lines, opts = {}) {
|
|
@@ -481,7 +744,9 @@ export class ComputerUse extends EventEmitter {
|
|
|
481
744
|
// Anchor-based selection: click the START, scroll freely, then shift+click the END.
|
|
482
745
|
// Replaces drag-select, whose end point depends on drag duration and auto-scroll speed.
|
|
483
746
|
async shiftClick (x, y, opts = {}) {
|
|
747
|
+
const witness = verificationRegion(opts.verificationRegion)
|
|
484
748
|
return this.actuate('shiftClick', { x, y, clicks: opts.clicks || 1, settleMs: opts.settleMs ?? 90,
|
|
749
|
+
...(witness ? { verificationRegion: witness } : {}),
|
|
485
750
|
...(opts.expectHwnd ? { expectHwnd: opts.expectHwnd } : {}) })
|
|
486
751
|
}
|
|
487
752
|
|
|
@@ -503,7 +768,7 @@ export class ComputerUse extends EventEmitter {
|
|
|
503
768
|
* emergency brake. Called when the agent's turn ends, so a lingering blue border can never
|
|
504
769
|
* claim the machine is still under control after control actually stopped.
|
|
505
770
|
*/
|
|
506
|
-
async calm ({ beforeDispatch } = {}) { return this.
|
|
771
|
+
async calm ({ beforeDispatch } = {}) { return this.callInternal('calm', {}, DEFAULT_TIMEOUT_MS, true, beforeDispatch) }
|
|
507
772
|
|
|
508
773
|
/// **Ask the human one question, machine paused — the ONE question path.**
|
|
509
774
|
///
|
|
@@ -548,10 +813,17 @@ export class ComputerUse extends EventEmitter {
|
|
|
548
813
|
this._askOpen = null
|
|
549
814
|
}
|
|
550
815
|
const askId = b && b.askId
|
|
816
|
+
const askWorker = this.proc
|
|
817
|
+
const sameAskWorker = () => {
|
|
818
|
+
if (this.proc !== askWorker) throw Object.assign(new Error('question worker exited; its askId cannot release a replacement worker'), { code: 'worker-replaced' })
|
|
819
|
+
}
|
|
551
820
|
let answer = null
|
|
552
821
|
let rel = null
|
|
553
822
|
let via = 'ok'
|
|
554
823
|
try {
|
|
824
|
+
const context = this.inputContext.getStore()
|
|
825
|
+
if (context?.signal?.aborted) throw workerCallError('owning tool cancelled after the ask brake was engaged', 'INPUT_CANCELLED', 'beginAsk', null, 'unknown')
|
|
826
|
+
context?.assertOwnership?.()
|
|
555
827
|
answer = await opts.onAsk({ askId, hostRaised: !!(b && b.hostRaised), caretPlaced: !!(b && b.caretPlaced) })
|
|
556
828
|
// `via` is an AUDIT LABEL ONLY — it decides nothing. The pause decision is `keepPause`, and it
|
|
557
829
|
// comes from the human: ①让你停 keeps the brake, and every other ending (②放你走, a typed custom
|
|
@@ -576,14 +848,14 @@ export class ComputerUse extends EventEmitter {
|
|
|
576
848
|
// reply describes a machine that no longer exists and must not be applied on top of it.
|
|
577
849
|
const beforeRelease = this.stateSeq
|
|
578
850
|
try {
|
|
579
|
-
rel = await this.
|
|
851
|
+
rel = await this.callInternal('endAsk', { askId, via, keepPause }, END_ASK_DEADLINE_MS, /* killOnTimeout */ false, sameAskWorker)
|
|
580
852
|
} catch (e) {
|
|
581
853
|
// ONE retry: the first attempt may have timed out against a briefly busy worker. A second
|
|
582
854
|
// failure is not swallowed — the audit pair (ENGAGE-ASK with no ASK-RELEASE) is the signal,
|
|
583
855
|
// and the orphan sweep is the backstop. NEVER a kill here (killOnTimeout:false above): the
|
|
584
856
|
// dead worker's STOP file would outlive every waiter, which is the defect being removed.
|
|
585
857
|
try {
|
|
586
|
-
rel = await this.
|
|
858
|
+
rel = await this.callInternal('endAsk', { askId, via: via + '+retry', keepPause }, END_ASK_DEADLINE_MS, false, sameAskWorker)
|
|
587
859
|
} catch (e2) {
|
|
588
860
|
this.log(`[ask] endAsk failed twice: ${e2.message} — the brake may need the orphan sweep (the next worker call sweeps it, or the human's Ctrl+Alt+R)`)
|
|
589
861
|
}
|
|
@@ -643,25 +915,32 @@ export class ComputerUse extends EventEmitter {
|
|
|
643
915
|
async landmarks (opts = {}) {
|
|
644
916
|
const step = opts.step ?? 16
|
|
645
917
|
const insetPct = opts.insetPct ?? 10
|
|
918
|
+
// UIA-ONLY MODE (test report 2026-11-04): an animated neighbour (DSH Pet) keeps flipping the
|
|
919
|
+
// sampled pixel signature, so every refresh came back STALE_MARK. uiaOnly pins identity by
|
|
920
|
+
// window handle / process / geometry alone — the sampled signature is read (for the report)
|
|
921
|
+
// but never compared; dispatch-time occlusion checking still guards the actual click.
|
|
922
|
+
const uiaOnly = opts.uiaOnly === true
|
|
646
923
|
const fs = await this.frameSig({ hwnd: opts.hwnd, step, insetPct })
|
|
924
|
+
if (uiaOnly) fs.sig = null
|
|
647
925
|
const hwnd = fs.hwnd
|
|
648
|
-
if (!hwnd || !fs.sig) throw staleMark('window snapshot unavailable')
|
|
926
|
+
if (!hwnd || (!fs.sig && !uiaOnly)) throw staleMark('window snapshot unavailable')
|
|
649
927
|
const cached = this.maps.get(String(hwnd))
|
|
650
|
-
const key = JSON.stringify([opts.depth ??
|
|
928
|
+
const key = JSON.stringify([opts.depth ?? 6, opts.maxNodes ?? 300, opts.maxMarks ?? 80, opts.minSize ?? 8, opts.role, opts.nameContains, uiaOnly])
|
|
651
929
|
if (cached && sameMarkFrame(cached.frame, fs) && cached.key === key && !opts.refresh) {
|
|
652
930
|
this.lastMarks = cached.marks
|
|
653
931
|
this.lastMarkFrame = { ...fs, step, insetPct }
|
|
654
|
-
return { hwnd, marks: cached.marks, count: cached.marks.length, reused: true, ageMs: Date.now() - cached.ts, sig: fs.sig }
|
|
932
|
+
return { hwnd, marks: cached.marks, count: cached.marks.length, reused: true, ageMs: Date.now() - cached.ts, sig: fs.sig, uiaOnly }
|
|
655
933
|
}
|
|
656
934
|
const uia = await this.call('uia', {
|
|
657
935
|
hwnd: hwnd || undefined,
|
|
658
|
-
depth: opts.depth ??
|
|
659
|
-
maxNodes: opts.maxNodes ??
|
|
936
|
+
depth: opts.depth ?? 6,
|
|
937
|
+
maxNodes: opts.maxNodes ?? 300,
|
|
660
938
|
role: opts.role,
|
|
661
939
|
nameContains: opts.nameContains,
|
|
662
940
|
})
|
|
663
941
|
const marks = buildMarks(uia.flat || [], opts.maxMarks ?? 80, opts.minSize ?? 8)
|
|
664
942
|
const after = await this.frameSig({ hwnd, step, insetPct })
|
|
943
|
+
if (uiaOnly) after.sig = null
|
|
665
944
|
if (!sameMarkFrame(fs, after)) {
|
|
666
945
|
const changed = ['hwnd', 'pid'].filter(k => fs[k] !== after[k])
|
|
667
946
|
for (const k of ['x', 'y', 'width', 'height']) if (fs.region?.[k] !== after.region?.[k]) changed.push(`region.${k}`)
|
|
@@ -674,7 +953,7 @@ export class ComputerUse extends EventEmitter {
|
|
|
674
953
|
this.lastMarkFrame = { ...after, step, insetPct }
|
|
675
954
|
this.maps.set(String(hwnd), { marks, frame: after, sig: after.sig, key, ts: Date.now() })
|
|
676
955
|
if (this.maps.size > 16) this.maps.delete(this.maps.keys().next().value)
|
|
677
|
-
return { hwnd, marks, count: marks.length, reused: false, sig: fs ? fs.sig : null, elements: (uia.flat || []).length }
|
|
956
|
+
return { hwnd, marks, count: marks.length, reused: false, sig: fs ? fs.sig : null, uiaOnly, elements: (uia.flat || []).length }
|
|
678
957
|
}
|
|
679
958
|
|
|
680
959
|
// The human's emergency brake, reachable from the model side too (ESC / Ctrl+Alt+Q are
|
|
@@ -716,7 +995,7 @@ export class ComputerUse extends EventEmitter {
|
|
|
716
995
|
async calibrate (opts = {}) { return this.actuate('calibrate', opts) }
|
|
717
996
|
|
|
718
997
|
async key (combo, opts = {}) {
|
|
719
|
-
return this.actuate('key', { combo, holdMs: opts.holdMs ?? 60, ...(opts.expectHwnd ? { expectHwnd: opts.expectHwnd } : {}) })
|
|
998
|
+
return this.actuate('key', { combo: parseKeyCommand(combo).combo, holdMs: opts.holdMs ?? 60, ...(opts.expectHwnd ? { expectHwnd: opts.expectHwnd } : {}) })
|
|
720
999
|
}
|
|
721
1000
|
|
|
722
1001
|
async type (text, opts = {}) {
|
|
@@ -748,6 +1027,8 @@ export class ComputerUse extends EventEmitter {
|
|
|
748
1027
|
}
|
|
749
1028
|
|
|
750
1029
|
async clipRead () { return this.call('clipRead', {}) }
|
|
1030
|
+
/** Worker-side OCR: text + per-word boxes in ABSOLUTE screen pixels (2026-11-04). */
|
|
1031
|
+
async ocr (region) { return this.call('ocr', region ? { ...region } : {}) }
|
|
751
1032
|
async clipWrite (text) { return this.actuate('clipWrite', { text }) }
|
|
752
1033
|
|
|
753
1034
|
async activateWindow (sel) { return this.actuate('activate', sel) }
|
|
@@ -783,52 +1064,73 @@ export class ComputerUse extends EventEmitter {
|
|
|
783
1064
|
|
|
784
1065
|
/** Every state-changing op funnels through here for the audit trail. */
|
|
785
1066
|
async actuate (op, args) {
|
|
786
|
-
|
|
787
|
-
|
|
788
|
-
|
|
789
|
-
|
|
790
|
-
|
|
791
|
-
}
|
|
792
|
-
|
|
793
|
-
this.
|
|
794
|
-
|
|
795
|
-
|
|
796
|
-
|
|
797
|
-
|
|
798
|
-
|
|
799
|
-
|
|
800
|
-
try {
|
|
801
|
-
const data = await this.call(op, args)
|
|
802
|
-
rec.result = data
|
|
803
|
-
return data
|
|
804
|
-
} catch (err) {
|
|
805
|
-
if (err.code === 'HUMAN_YIELD') {
|
|
806
|
-
// The hook's event can arrive just after its operation refusal. One cheap local
|
|
807
|
-
// status read closes that race; waiting spends no model requests and never replays input.
|
|
808
|
-
try {
|
|
809
|
-
await this.call('ping', {})
|
|
810
|
-
await this.waitForHuman()
|
|
811
|
-
this.inputContext.getStore()?.assertOwnership?.()
|
|
812
|
-
err = humanReobserveError({ ...err, op })
|
|
813
|
-
} catch (waitError) {
|
|
814
|
-
// Cancelling the local wait cannot erase input already accepted before handoff.
|
|
815
|
-
err = workerCallError(waitError.message || String(waitError), waitError.code || 'handoff-wait-failed',
|
|
816
|
-
op, err.requestId ?? null, err.outcome === 'not-dispatched' ? 'not-dispatched' : 'unknown')
|
|
1067
|
+
const parent = this.inputContext.getStore() || {}
|
|
1068
|
+
const rec = { attemptId: randomUUID(), requestId: null, op, args: redact(args),
|
|
1069
|
+
phase: 'started', ts: Date.now(), cycleId: parent.cycleId ?? null,
|
|
1070
|
+
sessionKey: parent.sessionKey ?? null, agentKey: parent.agentKey ?? null,
|
|
1071
|
+
workerGeneration: this.workerGeneration, runtime: this.runtime }
|
|
1072
|
+
this.emit('action', { ...rec })
|
|
1073
|
+
let settleAction
|
|
1074
|
+
this.activeActions.set(rec.attemptId, new Promise(resolve => { settleAction = resolve }))
|
|
1075
|
+
return this.inputContext.run({ ...parent, inputSeries: parent.inputSeries || randomUUID(), actionRecord: rec }, async () => {
|
|
1076
|
+
try {
|
|
1077
|
+
await this.waitInputReady()
|
|
1078
|
+
if (this.dryRun) {
|
|
1079
|
+
rec.dryRun = true; rec.outcome = 'not-dispatched'
|
|
1080
|
+
return { dryRun: true, op }
|
|
817
1081
|
}
|
|
818
|
-
|
|
819
|
-
|
|
820
|
-
|
|
821
|
-
|
|
822
|
-
|
|
823
|
-
|
|
1082
|
+
const now = Date.now()
|
|
1083
|
+
this.rateWindow = this.rateWindow.filter(t => now - t < this.rateLimit.windowMs)
|
|
1084
|
+
if (this.rateWindow.length >= this.rateLimit.max) {
|
|
1085
|
+
throw workerCallError(`rate limit: ${this.rateLimit.max} actuations/${this.rateLimit.windowMs}ms exceeded`,
|
|
1086
|
+
'RATE_LIMITED', op, null, 'not-dispatched')
|
|
1087
|
+
}
|
|
1088
|
+
this.rateWindow.push(now)
|
|
1089
|
+
const data = await this.call(op, args)
|
|
1090
|
+
rec.result = auditReceipt(data); rec.outcome = 'delivered'
|
|
1091
|
+
return data
|
|
1092
|
+
} catch (initial) {
|
|
1093
|
+
let err = initial
|
|
1094
|
+
if (err.code === 'HUMAN_YIELD') {
|
|
1095
|
+
try {
|
|
1096
|
+
await this.call('ping', {})
|
|
1097
|
+
if (rec.workerGeneration && rec.workerGeneration !== this.workerGeneration) {
|
|
1098
|
+
throw workerReobserveError(this.workerGeneration, { ...err, op })
|
|
1099
|
+
}
|
|
1100
|
+
await this.waitForHuman()
|
|
1101
|
+
this.inputContext.getStore()?.assertOwnership?.()
|
|
1102
|
+
err = humanReobserveError({ ...err, op })
|
|
1103
|
+
} catch (waitError) {
|
|
1104
|
+
err = waitError.code === 'WORKER_REOBSERVE' ? waitError : Object.assign(workerCallError(waitError.detail || waitError.message || String(waitError), waitError.code || 'handoff-wait-failed',
|
|
1105
|
+
op, err.requestId ?? null, err.outcome === 'not-dispatched' ? 'not-dispatched' : 'unknown'),
|
|
1106
|
+
{ ...(waitError.handoffWait ? { handoffWait: waitError.handoffWait } : {}),
|
|
1107
|
+
...(waitError.cancelSource ? { cancelSource: waitError.cancelSource } : {}),
|
|
1108
|
+
...(waitError.cleanup ? { cleanup: waitError.cleanup } : {}) })
|
|
1109
|
+
}
|
|
824
1110
|
}
|
|
1111
|
+
if (err.code === 'HUMAN_REOBSERVE' || err.code === 'WORKER_REOBSERVE') {
|
|
1112
|
+
try { this.inputContext.getStore()?.assertOwnership?.() }
|
|
1113
|
+
catch (ownerError) {
|
|
1114
|
+
err = workerCallError(ownerError.message, ownerError.code || 'COMPUTER_OWNER_CHANGED', op,
|
|
1115
|
+
err.requestId ?? null, err.outcome)
|
|
1116
|
+
}
|
|
1117
|
+
}
|
|
1118
|
+
// UIA/native errors may echo names, values or typed text. The caller gets
|
|
1119
|
+
// the original exception; the persistent audit records a bounded code.
|
|
1120
|
+
rec.code = auditCode(err.code); rec.error = rec.code
|
|
1121
|
+
rec.outcome = err.outcome || 'unknown'
|
|
1122
|
+
if (err.requestId != null) rec.requestId = err.requestId
|
|
1123
|
+
if (err.cleanup) rec.cleanup = err.cleanup
|
|
1124
|
+
if (err.handoffWait) rec.handoffWait = err.handoffWait
|
|
1125
|
+
throw err
|
|
1126
|
+
} finally {
|
|
1127
|
+
rec.phase = 'settled'; rec.settledAt = Date.now(); rec.elapsedMs = rec.settledAt - rec.ts
|
|
1128
|
+
try { this.emit('action:settled', { ...rec }) }
|
|
1129
|
+
finally { this.activeActions.delete(rec.attemptId); settleAction() }
|
|
825
1130
|
}
|
|
826
|
-
|
|
827
|
-
throw err
|
|
828
|
-
} finally {
|
|
829
|
-
this.emit('action:settled', rec)
|
|
830
|
-
}
|
|
1131
|
+
})
|
|
831
1132
|
}
|
|
1133
|
+
|
|
832
1134
|
}
|
|
833
1135
|
|
|
834
1136
|
function staleMark (reason) {
|
|
@@ -837,16 +1139,36 @@ function staleMark (reason) {
|
|
|
837
1139
|
}
|
|
838
1140
|
|
|
839
1141
|
function sameMarkFrame (a, b) {
|
|
840
|
-
|
|
841
|
-
['x', 'y', 'width', 'height'].every(k => Number.isFinite(a.region?.[k]) && a.region[k] === b.region
|
|
1142
|
+
const identity = a?.hwnd === b?.hwnd && a?.pid === b?.pid &&
|
|
1143
|
+
['x', 'y', 'width', 'height'].every(k => Number.isFinite(a.region?.[k]) && a.region[k] === b.region[k])
|
|
1144
|
+
if (!identity) return false
|
|
1145
|
+
// A uiaOnly landmark frame (sig === null, test report 2026-11-04) deliberately carries no
|
|
1146
|
+
// pixel signature: identity is geometry alone, and the click path's occlusion check remains.
|
|
1147
|
+
if (a.sig === null) return true
|
|
1148
|
+
return !!a?.sig && a.sig === b?.sig
|
|
842
1149
|
}
|
|
843
1150
|
|
|
844
1151
|
/** Keep secrets out of the audit log. */
|
|
845
1152
|
function redact (args) {
|
|
846
|
-
if (args
|
|
847
|
-
|
|
848
|
-
|
|
849
|
-
|
|
1153
|
+
if (!args || typeof args !== 'object') return {}
|
|
1154
|
+
const safeStrings = new Set(['combo', 'button', 'direction', 'action', 'mode', 'preset', 'property'])
|
|
1155
|
+
return Object.fromEntries(Object.entries(args).map(([key, value]) => [key,
|
|
1156
|
+
typeof value === 'string' ? safeStrings.has(key) ? value.slice(0, 128) : `<${value.length} chars>` :
|
|
1157
|
+
typeof value === 'number' || typeof value === 'boolean' || value === null ? value : '<structured value>']))
|
|
1158
|
+
}
|
|
1159
|
+
|
|
1160
|
+
function auditCode (code) {
|
|
1161
|
+
return typeof code === 'string' && /^[A-Za-z0-9_-]{1,64}$/.test(code) ? code : 'action-error'
|
|
1162
|
+
}
|
|
1163
|
+
|
|
1164
|
+
// Explicit receipt allowlist: never persist screenshots, focus/UIA objects,
|
|
1165
|
+
// clipboard content, passwords, arbitrary strings or provider result trees.
|
|
1166
|
+
function auditReceipt (data) {
|
|
1167
|
+
if (!data || typeof data !== 'object') return {}
|
|
1168
|
+
const keys = ['ok', 'delivered', 'count', 'chars', 'typed', 'clicked', 'moved', 'settled',
|
|
1169
|
+
'elapsedMs', 'durationMs', 'hwnd', 'x', 'y', 'width', 'height', 'dryRun']
|
|
1170
|
+
return Object.fromEntries(keys.filter(key => typeof data[key] === 'boolean' ||
|
|
1171
|
+
(typeof data[key] === 'number' && Number.isFinite(data[key]))).map(key => [key, data[key]]))
|
|
850
1172
|
}
|
|
851
1173
|
|
|
852
1174
|
/**
|
|
@@ -859,6 +1181,7 @@ function buildMarks (flat, maxMarks = 60, minSize = 8) {
|
|
|
859
1181
|
const interactive = []
|
|
860
1182
|
for (const e of flat) {
|
|
861
1183
|
if (interactive.length >= maxMarks) break
|
|
1184
|
+
if (e.isOffscreen === true || e.intersectsVirtualScreen === false) continue
|
|
862
1185
|
const r = e.rect || {}
|
|
863
1186
|
const name = (e.name || '').replace(/\s*\r\n\s*/g, ' | ').trim()
|
|
864
1187
|
const key = `${e.role}|${name}|${r.x},${r.y}`
|