dsh-codex-style-computer-use 1.2.3 → 1.3.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +27 -14
- package/README.zh.md +28 -14
- package/icon.svg +29 -0
- package/lib/batch.js +21 -5
- package/lib/bin/attention-worker.exe +0 -0
- package/lib/bin/attention-worker.exe.json +1 -1
- package/lib/continuation.js +14 -0
- package/lib/core/bin/dsh-computer-use-worker.exe +0 -0
- package/lib/core/bin/dsh-computer-use-worker.exe.json +2 -2
- package/lib/core/compile-worker.ps1 +5 -1
- package/lib/core/index.js +170 -28
- package/lib/core/native-runtime.js +6 -1
- package/lib/core/winmd/Windows.winmd +0 -0
- package/lib/core/worker.cs +1116 -369
- package/lib/cycle-lifecycle.js +66 -0
- package/lib/cycle.js +99 -9
- package/lib/index.js +68 -618
- package/lib/policy.js +24 -3
- package/lib/skill-gate.js +1 -1
- package/lib/terminal-cleanup.js +57 -0
- package/lib/tools.js +58 -6
- package/mcp/src/execution.js +22 -0
- package/mcp/src/index.js +36 -13
- package/package.json +4 -2
- package/scripts/render-promo.py +81 -0
- package/skills/computer-use/SKILL.md +23 -3
- package/skills/computer-use/references/lifecycle.md +23 -12
- package/skills/computer-use/references/zero-uia-apps.md +54 -0
package/lib/core/index.js
CHANGED
|
@@ -10,7 +10,7 @@
|
|
|
10
10
|
// {"event":"ready"} announcement.
|
|
11
11
|
|
|
12
12
|
import { spawn, execFileSync } from 'node:child_process'
|
|
13
|
-
import { createHash } from 'node:crypto'
|
|
13
|
+
import { createHash, randomUUID } from 'node:crypto'
|
|
14
14
|
import { EventEmitter } from 'node:events'
|
|
15
15
|
import { AsyncLocalStorage } from 'node:async_hooks'
|
|
16
16
|
import { existsSync, readFileSync, statSync } from 'node:fs'
|
|
@@ -22,8 +22,13 @@ const __dirname = fileURLToPath(new URL('.', import.meta.url))
|
|
|
22
22
|
|
|
23
23
|
const DEFAULT_TIMEOUT_MS = 15_000
|
|
24
24
|
const CAPTURE_TIMEOUT_MS = 30_000
|
|
25
|
+
// A UIA tree walk can block the resident's synchronous COM thread. Bound the
|
|
26
|
+
// caller's wait separately from recovery of a worker that never returns.
|
|
27
|
+
const UIA_READS = new Set(['uia', 'uiaFromPoint', 'uiaFocused'])
|
|
28
|
+
const URGENT_CONTROL = new Set(['cancelInput', 'abort', 'panic', 'exit', 'endAsk'])
|
|
25
29
|
const INPUT_OPS = new Set(['click', 'shiftClick', 'move', 'drag', 'scroll', 'selectRange',
|
|
26
30
|
'type', 'key', 'uiaAct', 'windowOp', 'activate', 'clipWrite', 'calibrate', 'selftest'])
|
|
31
|
+
const isInputOp = (op, args = {}) => INPUT_OPS.has(op) || (op === 'focus' && args.refocus === true)
|
|
27
32
|
// THE CORE'S ASK DEADLINE, AND IT IS A BARE LITERAL ON PURPOSE (merge-design.md §3, §3.2).
|
|
28
33
|
//
|
|
29
34
|
// It bounds ONE thing: a wedged worker inside `beginAsk`'s raise sequence (a hung UIA provider), or
|
|
@@ -48,6 +53,28 @@ export function workerCallError (message, code, op, requestId, outcome = 'unknow
|
|
|
48
53
|
{ code, op, requestId, outcome })
|
|
49
54
|
}
|
|
50
55
|
|
|
56
|
+
/** A handoff interrupts an input operation, never the authorized task. */
|
|
57
|
+
export function humanContinuation (error = {}) {
|
|
58
|
+
return {
|
|
59
|
+
required: true,
|
|
60
|
+
taskStatus: 'continuing',
|
|
61
|
+
handback: 'received',
|
|
62
|
+
nextAction: 'observe-current-target-then-continue-original-task',
|
|
63
|
+
replayInterruptedInput: false,
|
|
64
|
+
interruptedOperation: error.op || null,
|
|
65
|
+
requestId: error.requestId ?? null,
|
|
66
|
+
outcome: error.outcome === 'not-dispatched' ? 'not-dispatched' : 'unknown',
|
|
67
|
+
instruction: 'Temporary input interruption only. Control has been handed back. You MUST immediately obtain fresh target evidence, verify any partial effect, and continue the original authorized task from its unfinished step until complete. Do not abandon the task, issue a final wrap-up, or ask for permission to continue because of this handoff. Never blindly replay an interrupted write. Explicit user cancellation, manual pause, or exit still takes precedence.',
|
|
68
|
+
}
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
function humanReobserveError (error = {}) {
|
|
72
|
+
const continuation = humanContinuation(error)
|
|
73
|
+
return Object.assign(workerCallError(continuation.instruction, 'HUMAN_REOBSERVE',
|
|
74
|
+
continuation.interruptedOperation, continuation.requestId, continuation.outcome),
|
|
75
|
+
{ transient: true, taskStatus: 'continuing', continuation })
|
|
76
|
+
}
|
|
77
|
+
|
|
51
78
|
/** Interactive roles that receive Set-of-Marks labels. */
|
|
52
79
|
const SOM_ROLES = new Set([
|
|
53
80
|
'Button', 'MenuItem', 'ListItem', 'TreeItem', 'DataItem', 'TabItem',
|
|
@@ -125,6 +152,7 @@ export class ComputerUse extends EventEmitter {
|
|
|
125
152
|
// bare `Error: spawn UNKNOWN` — no path, no syscall, no Windows code — because NOTHING listened
|
|
126
153
|
// for this child's 'error' event: an unhandled 'error' on an EventEmitter is thrown, so the rich
|
|
127
154
|
// error object (which names the file Windows refused) was discarded on the way out.
|
|
155
|
+
const child = this.proc
|
|
128
156
|
this.proc.on('error', (e) => {
|
|
129
157
|
let file = '?'
|
|
130
158
|
try { file = `${existsSync(this.workerExe)} exists, ${statSync(this.workerExe).size} bytes` } catch { file = 'stat failed' }
|
|
@@ -142,11 +170,11 @@ export class ComputerUse extends EventEmitter {
|
|
|
142
170
|
const line = buf.slice(0, nl).trim()
|
|
143
171
|
buf = buf.slice(nl + 1)
|
|
144
172
|
if (!line) continue
|
|
145
|
-
this.onLine(line)
|
|
173
|
+
if (this.proc === child) this.onLine(line)
|
|
146
174
|
}
|
|
147
175
|
})
|
|
148
176
|
this.proc.stderr.on('data', (d) => this.log(`[worker:stderr] ${d}`))
|
|
149
|
-
this.proc.on('exit', this.onWorkerExit)
|
|
177
|
+
this.proc.on('exit', (code) => { if (this.proc === child) this.onWorkerExit(code) })
|
|
150
178
|
return new Promise((resolve, reject) => {
|
|
151
179
|
const t = setTimeout(() => reject(new Error('worker ready timeout')), 10_000)
|
|
152
180
|
// surface the real reason instead of letting the raw emitter error escape
|
|
@@ -162,6 +190,7 @@ export class ComputerUse extends EventEmitter {
|
|
|
162
190
|
onWorkerExit (code) {
|
|
163
191
|
this.ready = false
|
|
164
192
|
this.proc = null
|
|
193
|
+
this.finishUiaRead(false)
|
|
165
194
|
for (const [id, p] of this.pending) { clearTimeout(p.timer); p.cleanup?.(); p.reject(workerCallError('worker exited', 'worker-exited', p.op, id)) }
|
|
166
195
|
this.pending.clear()
|
|
167
196
|
this.emit('dead', code)
|
|
@@ -188,6 +217,14 @@ export class ComputerUse extends EventEmitter {
|
|
|
188
217
|
this.syncCooperation(msg.cooperation)
|
|
189
218
|
return
|
|
190
219
|
}
|
|
220
|
+
// Secure-desktop transitions (UAC prompt / lock screen, human request 2026-11-04): the
|
|
221
|
+
// worker refuses to dispatch while it stands and clears it the moment the normal desktop
|
|
222
|
+
// returns. Mirror the state so callers can show it, and let the plugin notify the human.
|
|
223
|
+
if (msg.event === 'secure') {
|
|
224
|
+
this.secureDesktop = msg.why === 'active'
|
|
225
|
+
this.emit('secure', msg)
|
|
226
|
+
return
|
|
227
|
+
}
|
|
191
228
|
// Durable brake events are separate from automatic physical-input handoff.
|
|
192
229
|
// panic — ESC: the human ABORTED. The machine is stopped; wrap up briefly, then wait.
|
|
193
230
|
// resume — Ctrl+Alt+R: work may continue (a PAUSE ends; it never ends the cycle).
|
|
@@ -213,6 +250,7 @@ export class ComputerUse extends EventEmitter {
|
|
|
213
250
|
this.emit(msg.event, msg)
|
|
214
251
|
return
|
|
215
252
|
}
|
|
253
|
+
if (this.uiaRead?.id === msg.id) this.finishUiaRead(true)
|
|
216
254
|
const p = this.pending.get(msg.id)
|
|
217
255
|
if (!p) return
|
|
218
256
|
this.pending.delete(msg.id)
|
|
@@ -220,8 +258,12 @@ export class ComputerUse extends EventEmitter {
|
|
|
220
258
|
p.cleanup?.()
|
|
221
259
|
if (msg.data?.cooperation) this.syncCooperation(msg.data.cooperation)
|
|
222
260
|
if (msg.ok) p.resolve(msg.data)
|
|
223
|
-
else
|
|
224
|
-
msg.
|
|
261
|
+
else {
|
|
262
|
+
const error = workerCallError(msg.error || 'worker error', msg.code || 'worker-refusal-or-error', p.op, msg.id,
|
|
263
|
+
msg.outcome === 'not-dispatched' ? 'not-dispatched' : 'unknown')
|
|
264
|
+
if (msg.glowHealth) error.glowHealth = msg.glowHealth
|
|
265
|
+
p.reject(error.code === 'HUMAN_REOBSERVE' ? humanReobserveError(error) : error)
|
|
266
|
+
}
|
|
225
267
|
}
|
|
226
268
|
|
|
227
269
|
/**
|
|
@@ -231,12 +273,26 @@ export class ComputerUse extends EventEmitter {
|
|
|
231
273
|
* @param {boolean} [killOnTimeout] kill a wedged worker when that cap expires (default true)
|
|
232
274
|
* @returns {Promise<any>} worker `data` payload
|
|
233
275
|
*/
|
|
234
|
-
async call (op, args = {}, timeoutMs = DEFAULT_TIMEOUT_MS, killOnTimeout = true) {
|
|
235
|
-
const
|
|
276
|
+
async call (op, args = {}, timeoutMs = DEFAULT_TIMEOUT_MS, killOnTimeout = true, beforeDispatch) {
|
|
277
|
+
const input = isInputOp(op, args)
|
|
278
|
+
const context = input ? this.inputContext.getStore() : undefined
|
|
279
|
+
const signal = context?.signal
|
|
236
280
|
if (signal?.aborted) throw workerCallError('owning tool cancelled', 'INPUT_CANCELLED', op, null, 'not-dispatched')
|
|
281
|
+
beforeDispatch?.()
|
|
282
|
+
context?.assertOwnership?.()
|
|
237
283
|
try { await this.ensureWorker() }
|
|
238
284
|
catch (error) { throw workerCallError(error.message || String(error), 'worker-unavailable', op, null, 'not-dispatched') }
|
|
285
|
+
const child = this.proc
|
|
286
|
+
if (!child) throw workerCallError('worker exited before dispatch', 'worker-exited', op, null, 'not-dispatched')
|
|
287
|
+
const queueDeadline = Date.now() + timeoutMs
|
|
288
|
+
while (this.uiaRead && !URGENT_CONTROL.has(op)) {
|
|
289
|
+
await this.waitForUiaRead(op, input, signal, queueDeadline)
|
|
290
|
+
if (this.proc !== child) throw workerCallError('worker exited before dispatch', 'worker-exited', op, null, 'not-dispatched')
|
|
291
|
+
if (Date.now() >= queueDeadline) throw workerCallError('deadline expired waiting for the UIA read', 'worker-uia-busy', op, null, 'not-dispatched')
|
|
292
|
+
}
|
|
239
293
|
if (signal?.aborted) throw workerCallError('owning tool cancelled', 'INPUT_CANCELLED', op, null, 'not-dispatched')
|
|
294
|
+
beforeDispatch?.()
|
|
295
|
+
context?.assertOwnership?.()
|
|
240
296
|
const id = ++this.seq
|
|
241
297
|
// `__cycle` is this layer's SIGNATURE on a call made on the agent's behalf, and the worker
|
|
242
298
|
// opens (or keeps lit) a computer-use CYCLE only for signed calls. A one-shot
|
|
@@ -245,8 +301,12 @@ export class ComputerUse extends EventEmitter {
|
|
|
245
301
|
// LAST so an op's own args can never forge it.
|
|
246
302
|
const signed = { ...args, __cycle: true }
|
|
247
303
|
signed.__kickoff = this.kickoffRequired
|
|
304
|
+
// One warning per external call/batch, with a fresh identity for each context.
|
|
305
|
+
// Never let a caller-supplied argument reuse another tool's announcement.
|
|
306
|
+
if (input) signed.__inputSeries = this.inputContext.getStore()?.inputSeries || randomUUID()
|
|
248
307
|
return new Promise((resolve, reject) => {
|
|
249
308
|
let written = false
|
|
309
|
+
const read = UIA_READS.has(op) && killOnTimeout ? (this.uiaRead = { id, child, timedOut: false }) : null
|
|
250
310
|
const cleanup = () => signal?.removeEventListener('abort', abort)
|
|
251
311
|
const abort = () => {
|
|
252
312
|
if (!this.pending.delete(id)) return
|
|
@@ -258,6 +318,21 @@ export class ComputerUse extends EventEmitter {
|
|
|
258
318
|
const timer = setTimeout(() => {
|
|
259
319
|
this.pending.delete(id)
|
|
260
320
|
cleanup()
|
|
321
|
+
if (read && this.uiaRead === read) {
|
|
322
|
+
read.timedOut = true
|
|
323
|
+
this.emit('uia-read-settled', false)
|
|
324
|
+
// Drop the response by id when it eventually arrives. A later reply for
|
|
325
|
+
// this id cancels only this watchdog, never a newer worker's deadline.
|
|
326
|
+
const grace = Math.min(timeoutMs, DEFAULT_TIMEOUT_MS)
|
|
327
|
+
read.watchdog = setTimeout(() => {
|
|
328
|
+
if (this.proc !== child || this.uiaRead !== read) return
|
|
329
|
+
this.log(`[core] UIA read ${id} (${op}) still blocked after ${grace}ms recovery grace; stopping worker pid=${child.pid}`)
|
|
330
|
+
try { child.kill() } catch { /* already gone */ }
|
|
331
|
+
}, grace)
|
|
332
|
+
reject(workerCallError(`UIA observation timed out: ${op} (${timeoutMs}ms); worker still finishing this read. Narrow the tree or use a point query once it returns.`,
|
|
333
|
+
'worker-timeout', op, id, 'not-dispatched'))
|
|
334
|
+
return
|
|
335
|
+
}
|
|
261
336
|
// A CALL THAT RELEASES THE BRAKE MUST NOT BE KILLED FOR BEING SLOW. `proc.kill()` deletes no
|
|
262
337
|
// STOP file (there is no ProcessExit handler in worker.cs; `abort` is Environment.Exit(0) at
|
|
263
338
|
// worker.cs:3045), so killing here would end a question with the brake still up — the exact
|
|
@@ -268,18 +343,51 @@ export class ComputerUse extends EventEmitter {
|
|
|
268
343
|
// UIA/COM provider call blocks its single-threaded loop forever), so kill it and let the NEXT
|
|
269
344
|
// call respawn fresh instead of queueing behind a corpse; pending calls are rejected via
|
|
270
345
|
// onWorkerExit.
|
|
271
|
-
if (killOnTimeout) { try { this.proc
|
|
346
|
+
if (killOnTimeout) { try { if (this.proc === child) child.kill() } catch { /* already gone */ } }
|
|
272
347
|
reject(workerCallError(`worker timeout: ${op} (${timeoutMs}ms)` +
|
|
273
348
|
(killOnTimeout
|
|
274
349
|
? ' — worker killed, next call respawns'
|
|
275
350
|
: ' — worker left alive so the brake can still be released'),
|
|
276
351
|
'worker-timeout', op, id))
|
|
277
|
-
},
|
|
278
|
-
this.pending.set(id, { resolve, reject, timer, op, cleanup, abort })
|
|
352
|
+
}, Math.max(0, queueDeadline - Date.now()))
|
|
353
|
+
this.pending.set(id, { resolve, reject, timer, op, input, cleanup, abort })
|
|
279
354
|
signal?.addEventListener('abort', abort, { once: true })
|
|
280
355
|
if (signal?.aborted) { abort(); return }
|
|
281
356
|
written = true
|
|
282
|
-
|
|
357
|
+
child.stdin.write(JSON.stringify({ id, op, args: signed }) + '\r\n')
|
|
358
|
+
})
|
|
359
|
+
}
|
|
360
|
+
|
|
361
|
+
finishUiaRead (completed) {
|
|
362
|
+
const read = this.uiaRead
|
|
363
|
+
if (!read) return
|
|
364
|
+
this.uiaRead = null
|
|
365
|
+
clearTimeout(read.watchdog)
|
|
366
|
+
this.emit('uia-read-settled', completed && !read.timedOut)
|
|
367
|
+
}
|
|
368
|
+
|
|
369
|
+
// Only the stdin reader's cancellation command is out of band. Preserve
|
|
370
|
+
// emergency/release requests, but never queue ordinary work behind a slow
|
|
371
|
+
// observation after its caller has given up. Do not replay any request.
|
|
372
|
+
waitForUiaRead (op, input, signal, deadline) {
|
|
373
|
+
const busy = () => workerCallError('native UIA observation still busy; this request was not sent',
|
|
374
|
+
'worker-uia-busy', op, null, 'not-dispatched')
|
|
375
|
+
if (this.uiaRead.timedOut || Date.now() >= deadline) return Promise.reject(busy())
|
|
376
|
+
return new Promise((resolve, reject) => {
|
|
377
|
+
const finish = (error) => {
|
|
378
|
+
clearTimeout(timer)
|
|
379
|
+
this.off('uia-read-settled', settled)
|
|
380
|
+
this.off('input-cancel', cancel)
|
|
381
|
+
signal?.removeEventListener('abort', cancel)
|
|
382
|
+
if (error) reject(error); else resolve()
|
|
383
|
+
}
|
|
384
|
+
const settled = (completed) => finish(completed ? null : busy())
|
|
385
|
+
const cancel = () => finish(workerCallError('cancelled before dispatch', 'INPUT_CANCELLED', op, null, 'not-dispatched'))
|
|
386
|
+
const timer = setTimeout(() => finish(busy()), Math.max(0, deadline - Date.now()))
|
|
387
|
+
this.once('uia-read-settled', settled)
|
|
388
|
+
if (input) this.once('input-cancel', cancel)
|
|
389
|
+
signal?.addEventListener('abort', cancel, { once: true })
|
|
390
|
+
if (signal?.aborted) cancel()
|
|
283
391
|
})
|
|
284
392
|
}
|
|
285
393
|
|
|
@@ -293,12 +401,17 @@ export class ComputerUse extends EventEmitter {
|
|
|
293
401
|
// ---------------- screen observation ----------------
|
|
294
402
|
syncCooperation (state) {
|
|
295
403
|
if (!state || Number(state.revision) < Number(this.cooperation.revision)) return
|
|
404
|
+
if (Number(state.revision) === Number(this.cooperation.revision) && Number.isFinite(state.inputEpoch) &&
|
|
405
|
+
state.inputEpoch < (this.cooperation.inputEpoch || 0)) return
|
|
406
|
+
const inputChanged = Number.isFinite(state.inputEpoch) && state.inputEpoch !== (this.cooperation.inputEpoch || 0)
|
|
296
407
|
this.cooperation = state
|
|
297
|
-
if (state.active) { this.maps.clear(); this.lastMarks = []; this.lastMarkFrame = null }
|
|
408
|
+
if (state.active || inputChanged) { this.maps.clear(); this.lastMarks = []; this.lastMarkFrame = null }
|
|
298
409
|
this.emit('handoff', state)
|
|
299
410
|
}
|
|
300
411
|
|
|
301
|
-
withInputContext (context, fn) {
|
|
412
|
+
withInputContext (context, fn) {
|
|
413
|
+
return this.inputContext.run({ signal: context?.signal, assertOwnership: context?.assertOwnership, inputSeries: randomUUID() }, fn)
|
|
414
|
+
}
|
|
302
415
|
|
|
303
416
|
waitForHuman () {
|
|
304
417
|
const signal = this.inputContext.getStore()?.signal
|
|
@@ -324,13 +437,13 @@ export class ComputerUse extends EventEmitter {
|
|
|
324
437
|
async waitInputReady () {
|
|
325
438
|
if (!this.cooperation.active) return
|
|
326
439
|
await this.waitForHuman()
|
|
327
|
-
|
|
328
|
-
|
|
440
|
+
this.inputContext.getStore()?.assertOwnership?.()
|
|
441
|
+
throw humanReobserveError({ outcome: 'not-dispatched' })
|
|
329
442
|
}
|
|
330
443
|
|
|
331
444
|
cancelInputs () {
|
|
332
445
|
this.emit('input-cancel')
|
|
333
|
-
for (const pending of [...this.pending.values()]) if (INPUT_OPS.has(pending.op)) pending.abort?.()
|
|
446
|
+
for (const pending of [...this.pending.values()]) if (pending.input || INPUT_OPS.has(pending.op)) pending.abort?.()
|
|
334
447
|
}
|
|
335
448
|
|
|
336
449
|
/** Raw capture; returns { image: Buffer, mime, region, cursor, settled, settleMs, ts, scaledTo }. */
|
|
@@ -463,7 +576,7 @@ export class ComputerUse extends EventEmitter {
|
|
|
463
576
|
* emergency brake. Called when the agent's turn ends, so a lingering blue border can never
|
|
464
577
|
* claim the machine is still under control after control actually stopped.
|
|
465
578
|
*/
|
|
466
|
-
async calm () { return this.call('calm', {}) }
|
|
579
|
+
async calm ({ beforeDispatch } = {}) { return this.call('calm', {}, DEFAULT_TIMEOUT_MS, true, beforeDispatch) }
|
|
467
580
|
|
|
468
581
|
/// **Ask the human one question, machine paused — the ONE question path.**
|
|
469
582
|
///
|
|
@@ -584,7 +697,9 @@ export class ComputerUse extends EventEmitter {
|
|
|
584
697
|
* the intended target, whether it had to be re-asserted, and an integrity-level warning when
|
|
585
698
|
* Windows (UIPI) would silently discard our input. Text-only — no screenshot needed.
|
|
586
699
|
*/
|
|
587
|
-
async focus (opts = {}) {
|
|
700
|
+
async focus (opts = {}) {
|
|
701
|
+
return opts.refocus === true ? this.actuate('focus', opts) : this.call('focus', opts)
|
|
702
|
+
}
|
|
588
703
|
|
|
589
704
|
/** Sampled fingerprint plus window identity/geometry; a change invalidates cached marks. */
|
|
590
705
|
async frameSig (opts = {}) { return this.call('frameSig', opts) }
|
|
@@ -601,15 +716,21 @@ export class ComputerUse extends EventEmitter {
|
|
|
601
716
|
async landmarks (opts = {}) {
|
|
602
717
|
const step = opts.step ?? 16
|
|
603
718
|
const insetPct = opts.insetPct ?? 10
|
|
719
|
+
// UIA-ONLY MODE (test report 2026-11-04): an animated neighbour (DSH Pet) keeps flipping the
|
|
720
|
+
// sampled pixel signature, so every refresh came back STALE_MARK. uiaOnly pins identity by
|
|
721
|
+
// window handle / process / geometry alone — the sampled signature is read (for the report)
|
|
722
|
+
// but never compared; dispatch-time occlusion checking still guards the actual click.
|
|
723
|
+
const uiaOnly = opts.uiaOnly === true
|
|
604
724
|
const fs = await this.frameSig({ hwnd: opts.hwnd, step, insetPct })
|
|
725
|
+
if (uiaOnly) fs.sig = null
|
|
605
726
|
const hwnd = fs.hwnd
|
|
606
|
-
if (!hwnd || !fs.sig) throw staleMark('window snapshot unavailable')
|
|
727
|
+
if (!hwnd || (!fs.sig && !uiaOnly)) throw staleMark('window snapshot unavailable')
|
|
607
728
|
const cached = this.maps.get(String(hwnd))
|
|
608
|
-
const key = JSON.stringify([opts.depth ?? 14, opts.maxNodes ?? 1500, opts.maxMarks ?? 80, opts.minSize ?? 8, opts.role, opts.nameContains])
|
|
729
|
+
const key = JSON.stringify([opts.depth ?? 14, opts.maxNodes ?? 1500, opts.maxMarks ?? 80, opts.minSize ?? 8, opts.role, opts.nameContains, uiaOnly])
|
|
609
730
|
if (cached && sameMarkFrame(cached.frame, fs) && cached.key === key && !opts.refresh) {
|
|
610
731
|
this.lastMarks = cached.marks
|
|
611
732
|
this.lastMarkFrame = { ...fs, step, insetPct }
|
|
612
|
-
return { hwnd, marks: cached.marks, count: cached.marks.length, reused: true, ageMs: Date.now() - cached.ts, sig: fs.sig }
|
|
733
|
+
return { hwnd, marks: cached.marks, count: cached.marks.length, reused: true, ageMs: Date.now() - cached.ts, sig: fs.sig, uiaOnly }
|
|
613
734
|
}
|
|
614
735
|
const uia = await this.call('uia', {
|
|
615
736
|
hwnd: hwnd || undefined,
|
|
@@ -620,6 +741,7 @@ export class ComputerUse extends EventEmitter {
|
|
|
620
741
|
})
|
|
621
742
|
const marks = buildMarks(uia.flat || [], opts.maxMarks ?? 80, opts.minSize ?? 8)
|
|
622
743
|
const after = await this.frameSig({ hwnd, step, insetPct })
|
|
744
|
+
if (uiaOnly) after.sig = null
|
|
623
745
|
if (!sameMarkFrame(fs, after)) {
|
|
624
746
|
const changed = ['hwnd', 'pid'].filter(k => fs[k] !== after[k])
|
|
625
747
|
for (const k of ['x', 'y', 'width', 'height']) if (fs.region?.[k] !== after.region?.[k]) changed.push(`region.${k}`)
|
|
@@ -632,7 +754,7 @@ export class ComputerUse extends EventEmitter {
|
|
|
632
754
|
this.lastMarkFrame = { ...after, step, insetPct }
|
|
633
755
|
this.maps.set(String(hwnd), { marks, frame: after, sig: after.sig, key, ts: Date.now() })
|
|
634
756
|
if (this.maps.size > 16) this.maps.delete(this.maps.keys().next().value)
|
|
635
|
-
return { hwnd, marks, count: marks.length, reused: false, sig: fs ? fs.sig : null, elements: (uia.flat || []).length }
|
|
757
|
+
return { hwnd, marks, count: marks.length, reused: false, sig: fs ? fs.sig : null, uiaOnly, elements: (uia.flat || []).length }
|
|
636
758
|
}
|
|
637
759
|
|
|
638
760
|
// The human's emergency brake, reachable from the model side too (ESC / Ctrl+Alt+Q are
|
|
@@ -706,6 +828,8 @@ export class ComputerUse extends EventEmitter {
|
|
|
706
828
|
}
|
|
707
829
|
|
|
708
830
|
async clipRead () { return this.call('clipRead', {}) }
|
|
831
|
+
/** Worker-side OCR: text + per-word boxes in ABSOLUTE screen pixels (2026-11-04). */
|
|
832
|
+
async ocr (region) { return this.call('ocr', region ? { ...region } : {}) }
|
|
709
833
|
async clipWrite (text) { return this.actuate('clipWrite', { text }) }
|
|
710
834
|
|
|
711
835
|
async activateWindow (sel) { return this.actuate('activate', sel) }
|
|
@@ -763,10 +887,23 @@ export class ComputerUse extends EventEmitter {
|
|
|
763
887
|
if (err.code === 'HUMAN_YIELD') {
|
|
764
888
|
// The hook's event can arrive just after its operation refusal. One cheap local
|
|
765
889
|
// status read closes that race; waiting spends no model requests and never replays input.
|
|
766
|
-
|
|
767
|
-
|
|
768
|
-
|
|
769
|
-
|
|
890
|
+
try {
|
|
891
|
+
await this.call('ping', {})
|
|
892
|
+
await this.waitForHuman()
|
|
893
|
+
this.inputContext.getStore()?.assertOwnership?.()
|
|
894
|
+
err = humanReobserveError({ ...err, op })
|
|
895
|
+
} catch (waitError) {
|
|
896
|
+
// Cancelling the local wait cannot erase input already accepted before handoff.
|
|
897
|
+
err = workerCallError(waitError.message || String(waitError), waitError.code || 'handoff-wait-failed',
|
|
898
|
+
op, err.requestId ?? null, err.outcome === 'not-dispatched' ? 'not-dispatched' : 'unknown')
|
|
899
|
+
}
|
|
900
|
+
}
|
|
901
|
+
if (err.code === 'HUMAN_REOBSERVE') {
|
|
902
|
+
try { this.inputContext.getStore()?.assertOwnership?.() }
|
|
903
|
+
catch (ownerError) {
|
|
904
|
+
err = workerCallError(ownerError.message, ownerError.code || 'COMPUTER_OWNER_CHANGED', op,
|
|
905
|
+
err.requestId ?? null, err.outcome)
|
|
906
|
+
}
|
|
770
907
|
}
|
|
771
908
|
rec.error = String(err.message || err)
|
|
772
909
|
throw err
|
|
@@ -782,8 +919,13 @@ function staleMark (reason) {
|
|
|
782
919
|
}
|
|
783
920
|
|
|
784
921
|
function sameMarkFrame (a, b) {
|
|
785
|
-
|
|
786
|
-
['x', 'y', 'width', 'height'].every(k => Number.isFinite(a.region?.[k]) && a.region[k] === b.region
|
|
922
|
+
const identity = a?.hwnd === b?.hwnd && a?.pid === b?.pid &&
|
|
923
|
+
['x', 'y', 'width', 'height'].every(k => Number.isFinite(a.region?.[k]) && a.region[k] === b.region[k])
|
|
924
|
+
if (!identity) return false
|
|
925
|
+
// A uiaOnly landmark frame (sig === null, test report 2026-11-04) deliberately carries no
|
|
926
|
+
// pixel signature: identity is geometry alone, and the click path's occlusion check remains.
|
|
927
|
+
if (a.sig === null) return true
|
|
928
|
+
return !!a?.sig && a.sig === b?.sig
|
|
787
929
|
}
|
|
788
930
|
|
|
789
931
|
/** Keep secrets out of the audit log. */
|
|
@@ -13,7 +13,12 @@ const framework = env => path.join(env.SystemRoot || env.windir || 'C:\\Windows'
|
|
|
13
13
|
export function workerReferences (env = process.env) {
|
|
14
14
|
const gac = path.join(env.SystemRoot || env.windir || 'C:\\Windows', 'Microsoft.NET', 'assembly', 'GAC_MSIL')
|
|
15
15
|
return ['System.dll', 'System.Core.dll', 'System.Drawing.dll', 'System.Web.Extensions.dll', 'System.Windows.Forms.dll',
|
|
16
|
-
...['UIAutomationClient', 'UIAutomationTypes', 'WindowsBase'].map(name => path.join(gac, name, 'v4.0_4.0.0.0__31bf3856ad364e35', `${name}.dll`))
|
|
16
|
+
...['UIAutomationClient', 'UIAutomationTypes', 'WindowsBase'].map(name => path.join(gac, name, 'v4.0_4.0.0.0__31bf3856ad364e35', `${name}.dll`)),
|
|
17
|
+
// Inbox OCR (2026-11-04): the legacy csc cannot compose the split WinMetadata winmds, so the
|
|
18
|
+
// union Windows.winmd ships inside the package and the WinRT interop pieces come from the OS.
|
|
19
|
+
path.join(framework(env), 'System.Runtime.dll'),
|
|
20
|
+
path.join(gac, 'System.Runtime.WindowsRuntime', 'v4.0_4.0.0.0__b77a5c561934e089', 'System.Runtime.WindowsRuntime.dll'),
|
|
21
|
+
path.join(import.meta.dirname, 'winmd', 'Windows.winmd')]
|
|
17
22
|
}
|
|
18
23
|
|
|
19
24
|
export function compileNativeBinary (source, output, references, { env = process.env } = {}) {
|
|
Binary file
|