dsh-codex-style-computer-use 1.2.0-rc.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +144 -0
- package/README.zh.md +132 -0
- package/cordis.patch.yml +17 -0
- package/lib/artifacts.js +35 -0
- package/lib/attention-native.js +93 -0
- package/lib/attention-worker.cs +194 -0
- package/lib/attention.js +140 -0
- package/lib/audit.js +30 -0
- package/lib/batch.js +52 -0
- package/lib/bin/attention-worker.exe +0 -0
- package/lib/bin/attention-worker.exe.json +7 -0
- package/lib/core/bin/dsh-computer-use-worker.exe +0 -0
- package/lib/core/bin/dsh-computer-use-worker.exe.json +7 -0
- package/lib/core/compile-worker.ps1 +58 -0
- package/lib/core/index.js +844 -0
- package/lib/core/native-runtime.js +97 -0
- package/lib/core/worker.cs +6375 -0
- package/lib/cycle.js +314 -0
- package/lib/exposure.js +91 -0
- package/lib/feedback.js +35 -0
- package/lib/image-output.js +92 -0
- package/lib/index.js +1715 -0
- package/lib/observations.js +78 -0
- package/lib/policy.js +109 -0
- package/lib/recovery.js +8 -0
- package/lib/skill-gate.js +68 -0
- package/lib/tools.js +717 -0
- package/mcp/package.json +12 -0
- package/mcp/src/index.js +335 -0
- package/package.json +88 -0
- package/scripts/build-native.mjs +7 -0
- package/scripts/smoke-native.mjs +38 -0
- package/scripts/verify-package.mjs +28 -0
- package/skills/computer-use/SKILL.md +74 -0
- package/skills/computer-use/references/lifecycle.md +180 -0
- package/skills/computer-use/references/maintenance.md +56 -0
- package/skills/computer-use/references/precision.md +297 -0
- package/skills/computer-use/references/progressive-control.md +30 -0
- package/skills/computer-use/references/shared-input.md +27 -0
package/lib/cycle.js
ADDED
|
@@ -0,0 +1,314 @@
|
|
|
1
|
+
// lib/cycle.js — the computer-use CYCLE: identity, ownership, and the per-turn opening qualification.
|
|
2
|
+
//
|
|
3
|
+
// WHY THIS FILE EXISTS (agreed design, 2026-09-13; fixes A + B from the Codex review):
|
|
4
|
+
//
|
|
5
|
+
// * A cycle is a fact about the MACHINE. It has an id, an OWNER (the session + agent that is
|
|
6
|
+
// driving) and a state. The owner's identity does NOT include a turn: the same session driving
|
|
7
|
+
// again next turn is still the owner, and a pause keeps its owner.
|
|
8
|
+
// * A TURN is not an opener and not a reset. It only renews that session's *qualification* to open
|
|
9
|
+
// one. The qualification is per (session, agent, turn) and is kept PER IDENTITY, so another
|
|
10
|
+
// session's turn can neither renew it nor overwrite an already-consumed one.
|
|
11
|
+
// * The qualification comes from the CALL ITSELF (`exec.session` / `exec.agent`), never from a
|
|
12
|
+
// remembered `driverSession`: a brand-new session's first real call must be able to open a cycle
|
|
13
|
+
// with no history at all, and another session's lifecycle events must not touch it.
|
|
14
|
+
// * The first call of a turn CONSUMES the qualification even when nothing needs opening, so a late
|
|
15
|
+
// call in the same turn can never try again — and once an exit has been seen in a turn, that turn
|
|
16
|
+
// cannot open a cycle at all. The attempt itself is ONE shared settled-once promise per
|
|
17
|
+
// qualification, so concurrent calls in a turn cannot slip an op into the worker's stdin ahead
|
|
18
|
+
// of the `resume` that opens the cycle (build/test-cycle-gate.mjs measures that ordering).
|
|
19
|
+
// * `ended` is terminal for that id. A new cycle needs a NEW qualification (a later turn's first
|
|
20
|
+
// real call) and gets a NEW id. Nothing about Ctrl+Alt+R, a mouse event or a late callback can
|
|
21
|
+
// revive an ended cycle.
|
|
22
|
+
//
|
|
23
|
+
// NO MEMORY BOUND LIVES IN THIS FILE, AND THAT IS DELIBERATE (#5, 2026-09-13). A cap that evicts a
|
|
24
|
+
// consumed qualification is not a memory policy, it is a RE-AUTHORIZATION channel: after 64
|
|
25
|
+
// identities had come and gone, the oldest one's next call found no record of its consumed turn,
|
|
26
|
+
// looked brand new, ran the opener again and opened a cycle over a session that had already ended.
|
|
27
|
+
// Each identity keeps exactly ONE small entry — its current turn's qualification — which is the
|
|
28
|
+
// minimum this machine must remember to know whether a call may open a cycle. Entries are destroyed
|
|
29
|
+
// only on EVIDENCE that the identity is over (`forgetIdentity`, wired to agent/session disposal),
|
|
30
|
+
// and a forgotten identity is marked DEAD so a late call from it is refused rather than trusted.
|
|
31
|
+
//
|
|
32
|
+
// NO TIME-BASED POLICY LIVES HERE EITHER. `openedAt`/`lastAt` are diagnostics only; a paused cycle is
|
|
33
|
+
// never transferred to another session because time passed ("暂停只能由人的恢复/终结解除,时间不是授权").
|
|
34
|
+
// Ownership is released by FACTS: the owner's turn completing, its agent going idle, its identity
|
|
35
|
+
// being disposed, or an exit.
|
|
36
|
+
|
|
37
|
+
/** Session identity without assuming the host's shape (id may be a string, or a field on an object). */
|
|
38
|
+
export function sessionKeyOf (s) {
|
|
39
|
+
if (!s) return null
|
|
40
|
+
if (typeof s === 'string') return s
|
|
41
|
+
return s.id || s.sessionId || s.sessionKey || s.key || null
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
/** Agent identity: the scope routing key is the agent object itself, so prefer its own id. */
|
|
45
|
+
export function agentKeyOf (a) {
|
|
46
|
+
if (!a) return null
|
|
47
|
+
if (typeof a === 'string') return a
|
|
48
|
+
return a.id || a.agentId || a.key || null
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
export function createCycleGate ({
|
|
52
|
+
sessionKey = sessionKeyOf,
|
|
53
|
+
agentKey = agentKeyOf,
|
|
54
|
+
now = () => Date.now(),
|
|
55
|
+
} = {}) {
|
|
56
|
+
let nextId = 0
|
|
57
|
+
let exitVersion = 0 // bumped by every `ended`: a pending open compares it before/after its await
|
|
58
|
+
/** @type {{id:number, ownerIdent:string|null, ownerLabel:string, ownerSessionKey:string|null, ownerAgentKey:string|null, state:'open'|'paused'|'asking'|'ended', openedAt:number, lastAt:number, endedWhy:string}|null} */
|
|
59
|
+
let cycle = null
|
|
60
|
+
/** qualification PER IDENTITY: ident -> {turn, promise, dead}. One entry per identity, never evicted. */
|
|
61
|
+
const idents = new Map()
|
|
62
|
+
/** sessionKey -> turn generation, bumped ONLY by that session's own turn/start (never evicted) */
|
|
63
|
+
const turns = new Map()
|
|
64
|
+
/** ident -> the turn generation in which an exit was seen: that turn may not open a cycle */
|
|
65
|
+
const endedTurns = new Map()
|
|
66
|
+
const events = [] // bounded transition log for the guards and diagnostics
|
|
67
|
+
|
|
68
|
+
const note = (what, detail) => {
|
|
69
|
+
events.push({ at: now(), what, ...(detail || {}) })
|
|
70
|
+
if (events.length > 96) events.shift()
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
const identOf = (sessionKeyVal, agentKeyVal) => `${sessionKeyVal || '-'}|${agentKeyVal || '-'}`
|
|
74
|
+
|
|
75
|
+
/** A claim is what the CALL brings: whose call it is, and which of that session's turns it is in. */
|
|
76
|
+
function claim ({ session, agent } = {}) {
|
|
77
|
+
const sk = sessionKey(session)
|
|
78
|
+
const ak = agentKey(agent)
|
|
79
|
+
const turn = (sk && turns.get(sk)) || 0
|
|
80
|
+
const ident = identOf(sk, ak)
|
|
81
|
+
return {
|
|
82
|
+
sessionKey: sk,
|
|
83
|
+
agentKey: ak,
|
|
84
|
+
turn,
|
|
85
|
+
ident,
|
|
86
|
+
// Ownership identity (no turn) and qualification identity (with turn) are DIFFERENT things.
|
|
87
|
+
label: `${sk || 'unknown-session'}/${ak || 'unknown-agent'}`,
|
|
88
|
+
qualKey: `${ident}#t${turn}`,
|
|
89
|
+
}
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
/** Renewed by THAT session's turn/start only. Another session's event must not touch it (B). */
|
|
93
|
+
function noteTurnStart (session) {
|
|
94
|
+
const sk = sessionKey(session)
|
|
95
|
+
if (!sk) return 0
|
|
96
|
+
const n = (turns.get(sk) || 0) + 1
|
|
97
|
+
turns.set(sk, n)
|
|
98
|
+
note('turn-start', { session: sk, turn: n })
|
|
99
|
+
return n
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
const isLive = (c) => !!c && c.state !== 'ended'
|
|
103
|
+
const ownedBy = (c, cl) => !!c && c.ownerIdent === cl.ident
|
|
104
|
+
const ownerLabel = () => (cycle ? `${cycle.ownerLabel}#${cycle.id}` : 'none')
|
|
105
|
+
|
|
106
|
+
function rememberEndedTurn (cl) {
|
|
107
|
+
if (!cl) return
|
|
108
|
+
endedTurns.set(cl.ident, cl.turn)
|
|
109
|
+
note('ended-in-turn', { ident: cl.ident, turn: cl.turn })
|
|
110
|
+
}
|
|
111
|
+
|
|
112
|
+
/**
|
|
113
|
+
* An identity is OVER (its agent/session was disposed). This is the only way an entry is destroyed,
|
|
114
|
+
* and the entry is left as a TOMBSTONE rather than deleted: forgetting it would make a late call
|
|
115
|
+
* from a dead execution look brand new — and "brand new" is exactly what may open a cycle.
|
|
116
|
+
*/
|
|
117
|
+
function forgetIdentity ({ session, agent } = {}) {
|
|
118
|
+
const cl = claim({ session, agent })
|
|
119
|
+
const e = idents.get(cl.ident) || { turn: cl.turn, promise: null }
|
|
120
|
+
e.dead = true
|
|
121
|
+
idents.set(cl.ident, e)
|
|
122
|
+
let released = false
|
|
123
|
+
if (isLive(cycle) && cycle.ownerIdent === cl.ident) released = mark('ended', { why: 'owner identity disposed' })
|
|
124
|
+
note('identity-forgotten', { ident: cl.ident, released })
|
|
125
|
+
return { ident: cl.ident, released }
|
|
126
|
+
}
|
|
127
|
+
|
|
128
|
+
/**
|
|
129
|
+
* The ONE entry point every computer_* call goes through.
|
|
130
|
+
* Throws (tagged `dshCycleRefusal`) when another LIVE cycle owns this machine: that is a refusal,
|
|
131
|
+
* not a warning — lib/tools.js rethrows it so the call fails loudly instead of driving anyway.
|
|
132
|
+
*/
|
|
133
|
+
function open (cl, openFn) {
|
|
134
|
+
// A live cycle with NO owner is a pause/ask that arrived before any call claimed the machine.
|
|
135
|
+
// Ownership is free: the next real call adopts it. Only a cycle owned by a DIFFERENT session is
|
|
136
|
+
// a refusal.
|
|
137
|
+
if (isLive(cycle) && cycle.ownerIdent === null) {
|
|
138
|
+
cycle.ownerIdent = cl.ident
|
|
139
|
+
cycle.ownerLabel = cl.label
|
|
140
|
+
cycle.ownerSessionKey = cl.sessionKey
|
|
141
|
+
cycle.ownerAgentKey = cl.agentKey
|
|
142
|
+
note('adopted-ownerless', { id: cycle.id, by: cl.label, state: cycle.state })
|
|
143
|
+
}
|
|
144
|
+
if (isLive(cycle) && !ownedBy(cycle, cl)) {
|
|
145
|
+
const err = new Error(
|
|
146
|
+
`this machine is owned by another computer-use cycle (owner ${cycle.ownerLabel}, cycle ` +
|
|
147
|
+
`#${cycle.id}, state ${cycle.state}) — a session that is not the owner must not drive it. ` +
|
|
148
|
+
'Ask the human to end that session (Ctrl+Alt+Q) or wait for its turn to finish, then retry.')
|
|
149
|
+
err.dshCycleRefusal = true
|
|
150
|
+
note('refused-foreign', { owner: cycle.ownerLabel, id: cycle.id, by: cl.label })
|
|
151
|
+
throw err
|
|
152
|
+
}
|
|
153
|
+
|
|
154
|
+
// ONE qualification per (session, agent, turn), kept per identity and NEVER evicted.
|
|
155
|
+
let e = idents.get(cl.ident)
|
|
156
|
+
if (e && e.dead) {
|
|
157
|
+
const err = new Error(
|
|
158
|
+
`this computer-use execution identity is over (${cl.label}) — its agent or session was ` +
|
|
159
|
+
'disposed, so a late call from it may not open a cycle. Start the work in a live session.')
|
|
160
|
+
err.dshCycleRefusal = true
|
|
161
|
+
note('refused-dead-identity', { ident: cl.ident })
|
|
162
|
+
throw err
|
|
163
|
+
}
|
|
164
|
+
if (!e || e.turn !== cl.turn) {
|
|
165
|
+
e = { turn: cl.turn, promise: null, dead: false }
|
|
166
|
+
idents.set(cl.ident, e)
|
|
167
|
+
}
|
|
168
|
+
if (e.promise) return e.promise // already consumed this turn: nothing new is created
|
|
169
|
+
|
|
170
|
+
// FIRST call of this (session, agent, turn).
|
|
171
|
+
if (endedTurns.get(cl.ident) === cl.turn) {
|
|
172
|
+
// An exit was seen inside THIS turn: a late call may not open a new cycle. The call itself is
|
|
173
|
+
// not refused — observation still works, and a driving op is refused by the worker, whose own
|
|
174
|
+
// record is still on disk and whose message names the cycle.
|
|
175
|
+
e.promise = Promise.resolve({ opened: false, reason: 'the cycle was ended in this turn' })
|
|
176
|
+
note('blocked-after-end', { ident: cl.ident, turn: cl.turn })
|
|
177
|
+
return e.promise
|
|
178
|
+
}
|
|
179
|
+
|
|
180
|
+
// Ownership is RESERVED synchronously, before any await: otherwise two concurrent calls could
|
|
181
|
+
// both see "no cycle" and both believe they opened one. A NEW id is created only here — on the
|
|
182
|
+
// first call of a turn — never by a late call, and never for a live cycle that just changes hands.
|
|
183
|
+
if (!isLive(cycle)) {
|
|
184
|
+
cycle = {
|
|
185
|
+
id: ++nextId, ownerIdent: cl.ident, ownerLabel: cl.label,
|
|
186
|
+
ownerSessionKey: cl.sessionKey, ownerAgentKey: cl.agentKey,
|
|
187
|
+
state: 'open', openedAt: now(), lastAt: now(), endedWhy: '',
|
|
188
|
+
}
|
|
189
|
+
note('opened', { id: cycle.id, owner: cl.label })
|
|
190
|
+
} else {
|
|
191
|
+
cycle.lastAt = now() // diagnostics only: no policy reads this
|
|
192
|
+
}
|
|
193
|
+
|
|
194
|
+
// THE OPENING IS VALIDATED ON BOTH SIDES OF ITS AWAIT (#6, 2026-09-13). Capturing the exit
|
|
195
|
+
// version before the opener runs is not enough: the opener can take arbitrarily long (a worker
|
|
196
|
+
// spawn, a slow stdin write), and an exit that lands IN THAT WINDOW must void the opening even
|
|
197
|
+
// though `openFn` — and its `resume` op — already went out. A pending open whose cycle ended
|
|
198
|
+
// underneath it reports `voided`, never "opened", and the caller's next call in this turn still
|
|
199
|
+
// awaits the same settled promise rather than trying again.
|
|
200
|
+
const openedVersion = exitVersion
|
|
201
|
+
const openedId = cycle.id
|
|
202
|
+
// CHECK ON BOTH SIDES OF THE OPENER (#6). The first check runs BEFORE `openFn` is called at all:
|
|
203
|
+
// an exit that landed while this call was queued must mean no opener is even attempted — no
|
|
204
|
+
// `resume` op, no worker call, no external side effect to undo. The second check runs after the
|
|
205
|
+
// opener's await, because the opener itself can take arbitrarily long (a worker spawn, a slow
|
|
206
|
+
// stdin write) and an exit inside THAT window has to void the opening too.
|
|
207
|
+
const voided = (when) => ({ opened: false, voided: true, reason: `an exit landed ${when}` })
|
|
208
|
+
// THE MINIMAL FIX (#7, Codex review 2026-09-13): the pre-check and the CALL to openFn are one
|
|
209
|
+
// synchronous block — no `.then`, no `await` between them. They used to be separated by
|
|
210
|
+
// `return Promise.resolve().then(() => openFn())`, and that microtask gap was the bug: an exit
|
|
211
|
+
// that had ALREADY happened before openFn ran still got an opener started (its side effect went
|
|
212
|
+
// out, and only the result was voided afterwards).
|
|
213
|
+
//
|
|
214
|
+
// What is NOT required, and deliberately not done: making a caller's later microtask win the
|
|
215
|
+
// race. Microtasks are FIFO, so an end signalled after `open()` returns may legitimately land
|
|
216
|
+
// after the opener has started — "opener runs, then the exit voids its result" is a legal order,
|
|
217
|
+
// and forcing it the other way would need a macrotask delay on the cycle-opening path. The two
|
|
218
|
+
// checks below are exactly the two things that must hold: an ALREADY-ended cycle starts nothing
|
|
219
|
+
// (pre-check), and an opener that is still in flight when the exit lands is voided (post-await).
|
|
220
|
+
e.promise = Promise.resolve()
|
|
221
|
+
.then(() => {
|
|
222
|
+
if (exitVersion !== openedVersion || !cycle || cycle.id !== openedId || cycle.state === 'ended') {
|
|
223
|
+
note('open-cancelled-before-run', { key: cl.qualKey, exitVersion, was: openedVersion })
|
|
224
|
+
return voided('before this cycle was opened')
|
|
225
|
+
}
|
|
226
|
+
const started = (typeof openFn === 'function') ? openFn() : undefined
|
|
227
|
+
return Promise.resolve(started).then((outcome) => {
|
|
228
|
+
if (exitVersion !== openedVersion || !cycle || cycle.id !== openedId || cycle.state === 'ended') {
|
|
229
|
+
note('open-voided', { key: cl.qualKey, exitVersion, was: openedVersion })
|
|
230
|
+
return voided('while this cycle was being opened')
|
|
231
|
+
}
|
|
232
|
+
note('qualification-used', { key: cl.qualKey, opened: !!(outcome && outcome.opened) })
|
|
233
|
+
return outcome
|
|
234
|
+
})
|
|
235
|
+
})
|
|
236
|
+
// Keep an inert handler: a rejection nobody awaits must never surface as an unhandled rejection
|
|
237
|
+
// inside the host. Callers still see the rejection — that is the point.
|
|
238
|
+
e.promise.catch(() => {})
|
|
239
|
+
return e.promise
|
|
240
|
+
}
|
|
241
|
+
|
|
242
|
+
/**
|
|
243
|
+
* Worker/host-derived state. Only these transitions move `state`; `open()` never does.
|
|
244
|
+
* `attribution` names whose turn this transition belongs to (used to block that turn from opening).
|
|
245
|
+
*/
|
|
246
|
+
function mark (state, detail) {
|
|
247
|
+
if (!['open', 'paused', 'asking', 'ended'].includes(state)) return false
|
|
248
|
+
const attribution = detail && (detail.session || detail.agent)
|
|
249
|
+
? claim({ session: detail.session, agent: detail.agent })
|
|
250
|
+
: null
|
|
251
|
+
if (!cycle) {
|
|
252
|
+
// A brake/ask/exit with no cycle yet: the record exists with no owner, so ownership is free
|
|
253
|
+
// until the next real call claims it — which is what "no session owns this machine" means.
|
|
254
|
+
cycle = {
|
|
255
|
+
id: ++nextId, ownerIdent: null, ownerLabel: 'none',
|
|
256
|
+
ownerSessionKey: null, ownerAgentKey: null,
|
|
257
|
+
state, openedAt: now(), lastAt: now(), endedWhy: '',
|
|
258
|
+
}
|
|
259
|
+
} else {
|
|
260
|
+
if (cycle.state === 'ended' && state !== 'ended') return false // ended is terminal for this id
|
|
261
|
+
cycle.state = state
|
|
262
|
+
cycle.lastAt = now()
|
|
263
|
+
}
|
|
264
|
+
if (state === 'ended') {
|
|
265
|
+
exitVersion++ // every exit invalidates any opening that is still in flight (#6)
|
|
266
|
+
cycle.endedWhy = (detail && detail.why) || ''
|
|
267
|
+
// WHICH TURN SAW THE EXIT: the attributing session's current turn when we were told, else the
|
|
268
|
+
// owner's. That turn's late calls are blocked from opening (above) — no timer involved.
|
|
269
|
+
const sk = (attribution && attribution.sessionKey) || cycle.ownerSessionKey
|
|
270
|
+
const ak = (attribution && attribution.agentKey) || cycle.ownerAgentKey
|
|
271
|
+
if (sk || ak) rememberEndedTurn(claim({ session: sk, agent: ak ? { id: ak } : null }))
|
|
272
|
+
}
|
|
273
|
+
note('state', { id: cycle.id, state, why: (detail && detail.why) || '' })
|
|
274
|
+
return true
|
|
275
|
+
}
|
|
276
|
+
|
|
277
|
+
return {
|
|
278
|
+
claim,
|
|
279
|
+
noteTurnStart,
|
|
280
|
+
open,
|
|
281
|
+
mark,
|
|
282
|
+
forgetIdentity,
|
|
283
|
+
snapshot: () => ({
|
|
284
|
+
id: cycle ? cycle.id : 0,
|
|
285
|
+
state: cycle ? cycle.state : 'none',
|
|
286
|
+
owner: cycle ? cycle.ownerLabel : 'none',
|
|
287
|
+
ownerIdent: cycle ? cycle.ownerIdent : null,
|
|
288
|
+
ownerSessionKey: cycle ? cycle.ownerSessionKey : null,
|
|
289
|
+
ownerAgentKey: cycle ? cycle.ownerAgentKey : null,
|
|
290
|
+
endedWhy: cycle ? cycle.endedWhy : '',
|
|
291
|
+
exitVersion,
|
|
292
|
+
qualifications: [...idents.entries()].map(([ident, e]) => ({ ident, turn: e.turn, used: !!e.promise, dead: !!e.dead })),
|
|
293
|
+
endedTurns: Object.fromEntries(endedTurns),
|
|
294
|
+
turns: Object.fromEntries(turns),
|
|
295
|
+
events: events.slice(),
|
|
296
|
+
}),
|
|
297
|
+
/** Is this claim the owner of the live cycle? (event handlers use it to stay out of others' way) */
|
|
298
|
+
owns: (cl) => ownedBy(cycle, cl),
|
|
299
|
+
isLive: () => isLive(cycle),
|
|
300
|
+
state: () => (cycle ? cycle.state : 'none'),
|
|
301
|
+
/** Bumped by every `ended`: a caller may capture it and re-check before acting on a stale plan. */
|
|
302
|
+
exitVersion: () => exitVersion,
|
|
303
|
+
/** The owning SESSION/AGENT, so host-wide events can be attributed without re-deriving a claim. */
|
|
304
|
+
ownerSessionKey: () => (cycle ? cycle.ownerSessionKey : null),
|
|
305
|
+
ownsAgent: (ak) => !!cycle && !!ak && cycle.ownerAgentKey === ak,
|
|
306
|
+
ownsSession: (sk) => !!cycle && !!sk && cycle.ownerSessionKey === sk,
|
|
307
|
+
/** Release ownership on a FACT (session/agent disposed) — never on elapsed time. */
|
|
308
|
+
release (why) {
|
|
309
|
+
if (!isLive(cycle)) return false
|
|
310
|
+
return mark('ended', { why: why || 'owner released the machine' })
|
|
311
|
+
},
|
|
312
|
+
ownerLabel,
|
|
313
|
+
}
|
|
314
|
+
}
|
package/lib/exposure.js
ADDED
|
@@ -0,0 +1,91 @@
|
|
|
1
|
+
// Keep desktop execution schemas out of unrelated turns. Brakes stay globally callable.
|
|
2
|
+
export const ACTIVATION_TOOL = 'computer_use_activate'
|
|
3
|
+
const ALWAYS = new Set(['computer_ctrl', 'computer_ask'])
|
|
4
|
+
|
|
5
|
+
export function installToolExposure ({ ctx, definitions, defineTool, scopeOf, readSkill, skillPath, log, enabled = true }) {
|
|
6
|
+
const active = new Map()
|
|
7
|
+
const boundaries = new Map()
|
|
8
|
+
const agents = new Set()
|
|
9
|
+
const globalDisposers = []
|
|
10
|
+
const execution = definitions.filter(d => !ALWAYS.has(d.name))
|
|
11
|
+
const maskInherited = (agent) => {
|
|
12
|
+
if (active.has(agent) || boundaries.has(agent) || !agent?.ctx || scopeOf(agent.ctx) !== agent) return
|
|
13
|
+
const inherited = execution.filter(d => ctx.tools.get(d.name, agent)).map(d => d.name)
|
|
14
|
+
if (inherited.length) boundaries.set(agent, agent.ctx.tools.restrict({ deny: inherited }))
|
|
15
|
+
}
|
|
16
|
+
let disposed = false
|
|
17
|
+
const activate = (agent) => {
|
|
18
|
+
if (!enabled) return { activated: true, reused: true }
|
|
19
|
+
if (disposed) throw new Error('computer-use plugin was disposed')
|
|
20
|
+
if (!agent?.ctx?.tools || scopeOf(agent.ctx) !== agent) {
|
|
21
|
+
throw new Error('computer-use activation needs this Agent\'s scoped tools registry; this host can use progressiveTools:false for legacy registration')
|
|
22
|
+
}
|
|
23
|
+
if (active.has(agent)) return { activated: true, reused: true }
|
|
24
|
+
const disposers = []
|
|
25
|
+
try {
|
|
26
|
+
for (const definition of execution) disposers.push(agent.ctx.tools.register(definition))
|
|
27
|
+
active.set(agent, disposers)
|
|
28
|
+
agents.add(agent)
|
|
29
|
+
for (const other of agents) maskInherited(other)
|
|
30
|
+
return { activated: true, reused: false }
|
|
31
|
+
} catch (error) {
|
|
32
|
+
for (const dispose of disposers.reverse()) dispose()
|
|
33
|
+
throw error
|
|
34
|
+
}
|
|
35
|
+
}
|
|
36
|
+
const detach = (agent) => {
|
|
37
|
+
const disposers = active.get(agent)
|
|
38
|
+
active.delete(agent)
|
|
39
|
+
for (const dispose of (disposers || []).reverse()) dispose()
|
|
40
|
+
boundaries.get(agent)?.()
|
|
41
|
+
boundaries.delete(agent)
|
|
42
|
+
agents.delete(agent)
|
|
43
|
+
}
|
|
44
|
+
for (const definition of definitions) {
|
|
45
|
+
if (!enabled || ALWAYS.has(definition.name)) globalDisposers.push(ctx.tools.register(definition))
|
|
46
|
+
}
|
|
47
|
+
if (enabled) {
|
|
48
|
+
globalDisposers.push(ctx.tools.register(defineTool({
|
|
49
|
+
name: ACTIVATION_TOOL,
|
|
50
|
+
description: 'Load the Windows computer-use manual and activate desktop tools for this Agent. Use when the task needs mouse, keyboard, screenshots or UIA. Does not start control or release a human brake.',
|
|
51
|
+
parameters: {},
|
|
52
|
+
output: {
|
|
53
|
+
schema: { type: 'object', additionalProperties: true },
|
|
54
|
+
render: (_args, value) => [{ type: 'text', text: JSON.stringify(value) }],
|
|
55
|
+
},
|
|
56
|
+
isConcurrencySafe: () => false,
|
|
57
|
+
execute: (_args, exec) => {
|
|
58
|
+
if (active.has(exec.agent)) return { activated: true, reused: true }
|
|
59
|
+
const manual = readSkill()
|
|
60
|
+
return { ...activate(exec.agent), tools: definitions.map(d => d.name), manual, ...(skillPath ? { manualPath: skillPath } : {}) }
|
|
61
|
+
},
|
|
62
|
+
})))
|
|
63
|
+
// A successful normal Skill load/receipt also activates tools, avoiding an extra bootstrap call.
|
|
64
|
+
globalDisposers.push(ctx.on('tools/post-execute', async (exec, result, next) => {
|
|
65
|
+
const decision = await next()
|
|
66
|
+
const skillLoaded = exec.name === 'skill' && exec.arguments?.name === 'computer-use'
|
|
67
|
+
const acknowledged = exec.name === 'computer_ctrl' && exec.arguments?.action === 'acknowledge'
|
|
68
|
+
if (!result.isError && decision.kind === 'accept' && (skillLoaded || acknowledged) && exec.agent) activate(exec.agent)
|
|
69
|
+
return decision
|
|
70
|
+
}))
|
|
71
|
+
globalDisposers.push(ctx.on('agent/disposed', (event) => detach(event.agent || event)))
|
|
72
|
+
globalDisposers.push(ctx.on('agent/created', (event) => {
|
|
73
|
+
const agent = event.agent || event
|
|
74
|
+
agents.add(agent)
|
|
75
|
+
maskInherited(agent)
|
|
76
|
+
}))
|
|
77
|
+
globalDisposers.push(ctx.tools.guard(exec => execution.some(d => d.name === exec.name) && !active.has(exec.agent)
|
|
78
|
+
? 'Load computer-use or call computer_use_activate in this Agent before dispatching desktop tools' : undefined))
|
|
79
|
+
}
|
|
80
|
+
const dispose = () => {
|
|
81
|
+
if (disposed) return
|
|
82
|
+
disposed = true
|
|
83
|
+
for (const agent of active.keys()) detach(agent)
|
|
84
|
+
for (const agent of boundaries.keys()) detach(agent)
|
|
85
|
+
agents.clear()
|
|
86
|
+
for (const undo of globalDisposers.reverse()) undo()
|
|
87
|
+
}
|
|
88
|
+
ctx.effect(() => dispose)
|
|
89
|
+
log?.info?.(`tool exposure: ${enabled ? '3 bootstrap/safety tools; 19 after Agent activation' : 'legacy 18 global tools'}`)
|
|
90
|
+
return { activate, dispose }
|
|
91
|
+
}
|
package/lib/feedback.js
ADDED
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
// Route-specific observations, verified against the Desktop adapters on 2026-10-02.
|
|
2
|
+
// A model catalog's image flag is transport permission, not a grounding test.
|
|
3
|
+
export function feedbackRoute (exec) {
|
|
4
|
+
const routed = exec?.agent?.session?.requestHeader?.()?.config
|
|
5
|
+
return {
|
|
6
|
+
provider: routed?.provider ?? exec?.agent?.options?.provider,
|
|
7
|
+
model: routed?.model ?? exec?.agent?.options?.model,
|
|
8
|
+
}
|
|
9
|
+
}
|
|
10
|
+
|
|
11
|
+
export function unverifiedVisionRoute (exec) {
|
|
12
|
+
const { provider, model } = feedbackRoute(exec)
|
|
13
|
+
return provider === 'bigmodel-anthropic' && model === 'glm-5.3'
|
|
14
|
+
}
|
|
15
|
+
|
|
16
|
+
export async function feedbackMode (ctx, exec, enabled = true) {
|
|
17
|
+
if (!enabled || exec?.signal?.aborted) return null
|
|
18
|
+
if (unverifiedVisionRoute(exec)) return 'state'
|
|
19
|
+
const { provider, model } = feedbackRoute(exec)
|
|
20
|
+
const tested = (provider === 'bigmodel-anthropic' && model === 'glm-5.3-flash') ||
|
|
21
|
+
(provider === 'deepseek-official' && model === 'deepseek-flash')
|
|
22
|
+
if (!tested) return null
|
|
23
|
+
try {
|
|
24
|
+
if (!ctx.get?.('attachments')?.saveImage) return null
|
|
25
|
+
const active = await ctx.get?.('llm')?.resolveModelInfo?.(provider, model, exec?.signal)
|
|
26
|
+
return !exec?.signal?.aborted && active?.inputModalities?.includes('image') ? 'image' : null
|
|
27
|
+
} catch { return null }
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
export const feedbackActions = new Set([
|
|
31
|
+
'computer_click', 'computer_type', 'computer_key', 'computer_move',
|
|
32
|
+
'computer_drag', 'computer_scroll', 'computer_select',
|
|
33
|
+
])
|
|
34
|
+
|
|
35
|
+
export const feedbackVerification = 'Inspect this post-action observation before claiming the task effect. An input receipt alone is not verification. If observation failed, inspect before repeating the input.'
|
|
@@ -0,0 +1,92 @@
|
|
|
1
|
+
import { basename } from 'node:path'
|
|
2
|
+
import { unverifiedVisionRoute } from './feedback.js'
|
|
3
|
+
|
|
4
|
+
// DSH image blocks reference the host's durable attachment store. Paths and raw
|
|
5
|
+
// base64 are not image blocks. Keep a usable file when a route cannot accept one.
|
|
6
|
+
export async function attachScreenshot (ctx, exec, value, data) {
|
|
7
|
+
const fallback = (reason) => {
|
|
8
|
+
value.imageStatus = 'path-only'
|
|
9
|
+
value.imageReason = reason
|
|
10
|
+
value.view = reason === 'model-text-only'
|
|
11
|
+
? 'Image not attached: this model does not declare image input. Use UIA/text or switch to an image-capable model.'
|
|
12
|
+
: 'Image not attached. Use read_image on path if visual inspection is needed.'
|
|
13
|
+
if (value.settled === false) value.view = 'NOT settled — re-shoot. ' + value.view
|
|
14
|
+
return value
|
|
15
|
+
}
|
|
16
|
+
try {
|
|
17
|
+
const attachments = ctx.get?.('attachments')
|
|
18
|
+
const llm = ctx.get?.('llm')
|
|
19
|
+
if (!attachments?.saveImage) return fallback('attachments-unavailable')
|
|
20
|
+
const routed = exec?.agent?.session?.requestHeader?.()?.config
|
|
21
|
+
const provider = routed?.provider ?? exec?.agent?.options?.provider
|
|
22
|
+
const model = routed?.model ?? exec?.agent?.options?.model
|
|
23
|
+
if (!llm?.resolveModelInfo || !provider || !model) return fallback('route-unavailable')
|
|
24
|
+
const active = await llm.resolveModelInfo(provider, model, exec?.signal)
|
|
25
|
+
if (!active.inputModalities?.includes('image')) return fallback('model-text-only')
|
|
26
|
+
if (exec?.signal?.aborted) return fallback('cancelled')
|
|
27
|
+
if (!attachments.imageLimits?.mediaTypes?.includes(value.mime)) return fallback('media-type-unavailable')
|
|
28
|
+
const image = await attachments.saveImage({ data, mediaType: value.mime, name: basename(value.path) })
|
|
29
|
+
if (exec?.signal?.aborted) return fallback('cancelled')
|
|
30
|
+
value.image = image
|
|
31
|
+
value.imageStatus = 'attached'
|
|
32
|
+
const r = value.region
|
|
33
|
+
if (r && [r.x, r.y, r.width, r.height].every(Number.isFinite) && image.width > 0 && image.height > 0) {
|
|
34
|
+
value.coordinates = {
|
|
35
|
+
screenOrigin: [r.x, r.y], imageSize: [image.width, image.height],
|
|
36
|
+
screenPerImagePixel: [r.width / image.width, r.height / image.height],
|
|
37
|
+
}
|
|
38
|
+
}
|
|
39
|
+
value.view = value.settled === false
|
|
40
|
+
? 'NOT settled — may be a partial render; re-shoot.'
|
|
41
|
+
: 'Image attached. Inspect it directly; no read_image needed. Coordinates for clicks are screen pixels.'
|
|
42
|
+
if (unverifiedVisionRoute(exec)) {
|
|
43
|
+
value.visionStatus = 'unverified-route'
|
|
44
|
+
value.view = 'Image transport succeeded, but this GLM 5.3 route failed grounding tests. Use UIA/text readback; do not infer coordinates or success from unseen pixels. The image and path are retained for inspection.'
|
|
45
|
+
}
|
|
46
|
+
return value
|
|
47
|
+
} catch (error) {
|
|
48
|
+
// Capturing succeeded; an optional image channel must not discard its path.
|
|
49
|
+
return fallback(exec?.signal?.aborted ? 'cancelled' : 'attachment-failed')
|
|
50
|
+
}
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
export function renderScreenshot (value) {
|
|
54
|
+
const { image, ...text } = value
|
|
55
|
+
return [
|
|
56
|
+
{ type: 'text', text: JSON.stringify(text) },
|
|
57
|
+
...(image ? [{ type: 'image', attachment: image }] : []),
|
|
58
|
+
]
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
// A batch exposes one latest image as its checkpoint. Earlier explicit captures
|
|
62
|
+
// keep their paths for deliberate later reads, without multiplying vision input.
|
|
63
|
+
export function selectBatchScreenshot (result, automatic, kind = 'image') {
|
|
64
|
+
let latest = automatic
|
|
65
|
+
let index
|
|
66
|
+
if (!latest) {
|
|
67
|
+
for (const [i, action] of result.actions.entries()) {
|
|
68
|
+
if (action.tool === 'computer_shot' && action.result?.path) {
|
|
69
|
+
latest = action.result
|
|
70
|
+
index = i
|
|
71
|
+
}
|
|
72
|
+
}
|
|
73
|
+
}
|
|
74
|
+
for (const action of result.actions) {
|
|
75
|
+
const shot = action.result
|
|
76
|
+
if (shot?.image && shot !== latest) {
|
|
77
|
+
delete shot.image
|
|
78
|
+
shot.imageStatus = 'path-only'
|
|
79
|
+
shot.imageReason = 'earlier-batch-image'
|
|
80
|
+
shot.view = kind === 'state'
|
|
81
|
+
? 'A newer UIA state is returned. This earlier image remains at its path.'
|
|
82
|
+
: 'Only the latest batch screenshot is attached. Use read_image on this path if this earlier frame is needed.'
|
|
83
|
+
}
|
|
84
|
+
}
|
|
85
|
+
if (latest) {
|
|
86
|
+
result.observation = latest
|
|
87
|
+
if (kind === 'state') result.observationSource = 'automatic final UIA state'
|
|
88
|
+
else result.imageSource = automatic ? 'automatic final screenshot' : `action ${index + 1}; captured before any later actions`
|
|
89
|
+
result.view = latest.view
|
|
90
|
+
}
|
|
91
|
+
return result
|
|
92
|
+
}
|