dsh-codex-style-computer-use 1.2.0-rc.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (40) hide show
  1. package/LICENSE +21 -0
  2. package/README.md +144 -0
  3. package/README.zh.md +132 -0
  4. package/cordis.patch.yml +17 -0
  5. package/lib/artifacts.js +35 -0
  6. package/lib/attention-native.js +93 -0
  7. package/lib/attention-worker.cs +194 -0
  8. package/lib/attention.js +140 -0
  9. package/lib/audit.js +30 -0
  10. package/lib/batch.js +52 -0
  11. package/lib/bin/attention-worker.exe +0 -0
  12. package/lib/bin/attention-worker.exe.json +7 -0
  13. package/lib/core/bin/dsh-computer-use-worker.exe +0 -0
  14. package/lib/core/bin/dsh-computer-use-worker.exe.json +7 -0
  15. package/lib/core/compile-worker.ps1 +58 -0
  16. package/lib/core/index.js +844 -0
  17. package/lib/core/native-runtime.js +97 -0
  18. package/lib/core/worker.cs +6375 -0
  19. package/lib/cycle.js +314 -0
  20. package/lib/exposure.js +91 -0
  21. package/lib/feedback.js +35 -0
  22. package/lib/image-output.js +92 -0
  23. package/lib/index.js +1715 -0
  24. package/lib/observations.js +78 -0
  25. package/lib/policy.js +109 -0
  26. package/lib/recovery.js +8 -0
  27. package/lib/skill-gate.js +68 -0
  28. package/lib/tools.js +717 -0
  29. package/mcp/package.json +12 -0
  30. package/mcp/src/index.js +335 -0
  31. package/package.json +88 -0
  32. package/scripts/build-native.mjs +7 -0
  33. package/scripts/smoke-native.mjs +38 -0
  34. package/scripts/verify-package.mjs +28 -0
  35. package/skills/computer-use/SKILL.md +74 -0
  36. package/skills/computer-use/references/lifecycle.md +180 -0
  37. package/skills/computer-use/references/maintenance.md +56 -0
  38. package/skills/computer-use/references/precision.md +297 -0
  39. package/skills/computer-use/references/progressive-control.md +30 -0
  40. package/skills/computer-use/references/shared-input.md +27 -0
package/lib/cycle.js ADDED
@@ -0,0 +1,314 @@
1
+ // lib/cycle.js — the computer-use CYCLE: identity, ownership, and the per-turn opening qualification.
2
+ //
3
+ // WHY THIS FILE EXISTS (agreed design, 2026-09-13; fixes A + B from the Codex review):
4
+ //
5
+ // * A cycle is a fact about the MACHINE. It has an id, an OWNER (the session + agent that is
6
+ // driving) and a state. The owner's identity does NOT include a turn: the same session driving
7
+ // again next turn is still the owner, and a pause keeps its owner.
8
+ // * A TURN is not an opener and not a reset. It only renews that session's *qualification* to open
9
+ // one. The qualification is per (session, agent, turn) and is kept PER IDENTITY, so another
10
+ // session's turn can neither renew it nor overwrite an already-consumed one.
11
+ // * The qualification comes from the CALL ITSELF (`exec.session` / `exec.agent`), never from a
12
+ // remembered `driverSession`: a brand-new session's first real call must be able to open a cycle
13
+ // with no history at all, and another session's lifecycle events must not touch it.
14
+ // * The first call of a turn CONSUMES the qualification even when nothing needs opening, so a late
15
+ // call in the same turn can never try again — and once an exit has been seen in a turn, that turn
16
+ // cannot open a cycle at all. The attempt itself is ONE shared settled-once promise per
17
+ // qualification, so concurrent calls in a turn cannot slip an op into the worker's stdin ahead
18
+ // of the `resume` that opens the cycle (build/test-cycle-gate.mjs measures that ordering).
19
+ // * `ended` is terminal for that id. A new cycle needs a NEW qualification (a later turn's first
20
+ // real call) and gets a NEW id. Nothing about Ctrl+Alt+R, a mouse event or a late callback can
21
+ // revive an ended cycle.
22
+ //
23
+ // NO MEMORY BOUND LIVES IN THIS FILE, AND THAT IS DELIBERATE (#5, 2026-09-13). A cap that evicts a
24
+ // consumed qualification is not a memory policy, it is a RE-AUTHORIZATION channel: after 64
25
+ // identities had come and gone, the oldest one's next call found no record of its consumed turn,
26
+ // looked brand new, ran the opener again and opened a cycle over a session that had already ended.
27
+ // Each identity keeps exactly ONE small entry — its current turn's qualification — which is the
28
+ // minimum this machine must remember to know whether a call may open a cycle. Entries are destroyed
29
+ // only on EVIDENCE that the identity is over (`forgetIdentity`, wired to agent/session disposal),
30
+ // and a forgotten identity is marked DEAD so a late call from it is refused rather than trusted.
31
+ //
32
+ // NO TIME-BASED POLICY LIVES HERE EITHER. `openedAt`/`lastAt` are diagnostics only; a paused cycle is
33
+ // never transferred to another session because time passed ("暂停只能由人的恢复/终结解除,时间不是授权").
34
+ // Ownership is released by FACTS: the owner's turn completing, its agent going idle, its identity
35
+ // being disposed, or an exit.
36
+
37
+ /** Session identity without assuming the host's shape (id may be a string, or a field on an object). */
38
+ export function sessionKeyOf (s) {
39
+ if (!s) return null
40
+ if (typeof s === 'string') return s
41
+ return s.id || s.sessionId || s.sessionKey || s.key || null
42
+ }
43
+
44
+ /** Agent identity: the scope routing key is the agent object itself, so prefer its own id. */
45
+ export function agentKeyOf (a) {
46
+ if (!a) return null
47
+ if (typeof a === 'string') return a
48
+ return a.id || a.agentId || a.key || null
49
+ }
50
+
51
+ export function createCycleGate ({
52
+ sessionKey = sessionKeyOf,
53
+ agentKey = agentKeyOf,
54
+ now = () => Date.now(),
55
+ } = {}) {
56
+ let nextId = 0
57
+ let exitVersion = 0 // bumped by every `ended`: a pending open compares it before/after its await
58
+ /** @type {{id:number, ownerIdent:string|null, ownerLabel:string, ownerSessionKey:string|null, ownerAgentKey:string|null, state:'open'|'paused'|'asking'|'ended', openedAt:number, lastAt:number, endedWhy:string}|null} */
59
+ let cycle = null
60
+ /** qualification PER IDENTITY: ident -> {turn, promise, dead}. One entry per identity, never evicted. */
61
+ const idents = new Map()
62
+ /** sessionKey -> turn generation, bumped ONLY by that session's own turn/start (never evicted) */
63
+ const turns = new Map()
64
+ /** ident -> the turn generation in which an exit was seen: that turn may not open a cycle */
65
+ const endedTurns = new Map()
66
+ const events = [] // bounded transition log for the guards and diagnostics
67
+
68
+ const note = (what, detail) => {
69
+ events.push({ at: now(), what, ...(detail || {}) })
70
+ if (events.length > 96) events.shift()
71
+ }
72
+
73
+ const identOf = (sessionKeyVal, agentKeyVal) => `${sessionKeyVal || '-'}|${agentKeyVal || '-'}`
74
+
75
+ /** A claim is what the CALL brings: whose call it is, and which of that session's turns it is in. */
76
+ function claim ({ session, agent } = {}) {
77
+ const sk = sessionKey(session)
78
+ const ak = agentKey(agent)
79
+ const turn = (sk && turns.get(sk)) || 0
80
+ const ident = identOf(sk, ak)
81
+ return {
82
+ sessionKey: sk,
83
+ agentKey: ak,
84
+ turn,
85
+ ident,
86
+ // Ownership identity (no turn) and qualification identity (with turn) are DIFFERENT things.
87
+ label: `${sk || 'unknown-session'}/${ak || 'unknown-agent'}`,
88
+ qualKey: `${ident}#t${turn}`,
89
+ }
90
+ }
91
+
92
+ /** Renewed by THAT session's turn/start only. Another session's event must not touch it (B). */
93
+ function noteTurnStart (session) {
94
+ const sk = sessionKey(session)
95
+ if (!sk) return 0
96
+ const n = (turns.get(sk) || 0) + 1
97
+ turns.set(sk, n)
98
+ note('turn-start', { session: sk, turn: n })
99
+ return n
100
+ }
101
+
102
+ const isLive = (c) => !!c && c.state !== 'ended'
103
+ const ownedBy = (c, cl) => !!c && c.ownerIdent === cl.ident
104
+ const ownerLabel = () => (cycle ? `${cycle.ownerLabel}#${cycle.id}` : 'none')
105
+
106
+ function rememberEndedTurn (cl) {
107
+ if (!cl) return
108
+ endedTurns.set(cl.ident, cl.turn)
109
+ note('ended-in-turn', { ident: cl.ident, turn: cl.turn })
110
+ }
111
+
112
+ /**
113
+ * An identity is OVER (its agent/session was disposed). This is the only way an entry is destroyed,
114
+ * and the entry is left as a TOMBSTONE rather than deleted: forgetting it would make a late call
115
+ * from a dead execution look brand new — and "brand new" is exactly what may open a cycle.
116
+ */
117
+ function forgetIdentity ({ session, agent } = {}) {
118
+ const cl = claim({ session, agent })
119
+ const e = idents.get(cl.ident) || { turn: cl.turn, promise: null }
120
+ e.dead = true
121
+ idents.set(cl.ident, e)
122
+ let released = false
123
+ if (isLive(cycle) && cycle.ownerIdent === cl.ident) released = mark('ended', { why: 'owner identity disposed' })
124
+ note('identity-forgotten', { ident: cl.ident, released })
125
+ return { ident: cl.ident, released }
126
+ }
127
+
128
+ /**
129
+ * The ONE entry point every computer_* call goes through.
130
+ * Throws (tagged `dshCycleRefusal`) when another LIVE cycle owns this machine: that is a refusal,
131
+ * not a warning — lib/tools.js rethrows it so the call fails loudly instead of driving anyway.
132
+ */
133
+ function open (cl, openFn) {
134
+ // A live cycle with NO owner is a pause/ask that arrived before any call claimed the machine.
135
+ // Ownership is free: the next real call adopts it. Only a cycle owned by a DIFFERENT session is
136
+ // a refusal.
137
+ if (isLive(cycle) && cycle.ownerIdent === null) {
138
+ cycle.ownerIdent = cl.ident
139
+ cycle.ownerLabel = cl.label
140
+ cycle.ownerSessionKey = cl.sessionKey
141
+ cycle.ownerAgentKey = cl.agentKey
142
+ note('adopted-ownerless', { id: cycle.id, by: cl.label, state: cycle.state })
143
+ }
144
+ if (isLive(cycle) && !ownedBy(cycle, cl)) {
145
+ const err = new Error(
146
+ `this machine is owned by another computer-use cycle (owner ${cycle.ownerLabel}, cycle ` +
147
+ `#${cycle.id}, state ${cycle.state}) — a session that is not the owner must not drive it. ` +
148
+ 'Ask the human to end that session (Ctrl+Alt+Q) or wait for its turn to finish, then retry.')
149
+ err.dshCycleRefusal = true
150
+ note('refused-foreign', { owner: cycle.ownerLabel, id: cycle.id, by: cl.label })
151
+ throw err
152
+ }
153
+
154
+ // ONE qualification per (session, agent, turn), kept per identity and NEVER evicted.
155
+ let e = idents.get(cl.ident)
156
+ if (e && e.dead) {
157
+ const err = new Error(
158
+ `this computer-use execution identity is over (${cl.label}) — its agent or session was ` +
159
+ 'disposed, so a late call from it may not open a cycle. Start the work in a live session.')
160
+ err.dshCycleRefusal = true
161
+ note('refused-dead-identity', { ident: cl.ident })
162
+ throw err
163
+ }
164
+ if (!e || e.turn !== cl.turn) {
165
+ e = { turn: cl.turn, promise: null, dead: false }
166
+ idents.set(cl.ident, e)
167
+ }
168
+ if (e.promise) return e.promise // already consumed this turn: nothing new is created
169
+
170
+ // FIRST call of this (session, agent, turn).
171
+ if (endedTurns.get(cl.ident) === cl.turn) {
172
+ // An exit was seen inside THIS turn: a late call may not open a new cycle. The call itself is
173
+ // not refused — observation still works, and a driving op is refused by the worker, whose own
174
+ // record is still on disk and whose message names the cycle.
175
+ e.promise = Promise.resolve({ opened: false, reason: 'the cycle was ended in this turn' })
176
+ note('blocked-after-end', { ident: cl.ident, turn: cl.turn })
177
+ return e.promise
178
+ }
179
+
180
+ // Ownership is RESERVED synchronously, before any await: otherwise two concurrent calls could
181
+ // both see "no cycle" and both believe they opened one. A NEW id is created only here — on the
182
+ // first call of a turn — never by a late call, and never for a live cycle that just changes hands.
183
+ if (!isLive(cycle)) {
184
+ cycle = {
185
+ id: ++nextId, ownerIdent: cl.ident, ownerLabel: cl.label,
186
+ ownerSessionKey: cl.sessionKey, ownerAgentKey: cl.agentKey,
187
+ state: 'open', openedAt: now(), lastAt: now(), endedWhy: '',
188
+ }
189
+ note('opened', { id: cycle.id, owner: cl.label })
190
+ } else {
191
+ cycle.lastAt = now() // diagnostics only: no policy reads this
192
+ }
193
+
194
+ // THE OPENING IS VALIDATED ON BOTH SIDES OF ITS AWAIT (#6, 2026-09-13). Capturing the exit
195
+ // version before the opener runs is not enough: the opener can take arbitrarily long (a worker
196
+ // spawn, a slow stdin write), and an exit that lands IN THAT WINDOW must void the opening even
197
+ // though `openFn` — and its `resume` op — already went out. A pending open whose cycle ended
198
+ // underneath it reports `voided`, never "opened", and the caller's next call in this turn still
199
+ // awaits the same settled promise rather than trying again.
200
+ const openedVersion = exitVersion
201
+ const openedId = cycle.id
202
+ // CHECK ON BOTH SIDES OF THE OPENER (#6). The first check runs BEFORE `openFn` is called at all:
203
+ // an exit that landed while this call was queued must mean no opener is even attempted — no
204
+ // `resume` op, no worker call, no external side effect to undo. The second check runs after the
205
+ // opener's await, because the opener itself can take arbitrarily long (a worker spawn, a slow
206
+ // stdin write) and an exit inside THAT window has to void the opening too.
207
+ const voided = (when) => ({ opened: false, voided: true, reason: `an exit landed ${when}` })
208
+ // THE MINIMAL FIX (#7, Codex review 2026-09-13): the pre-check and the CALL to openFn are one
209
+ // synchronous block — no `.then`, no `await` between them. They used to be separated by
210
+ // `return Promise.resolve().then(() => openFn())`, and that microtask gap was the bug: an exit
211
+ // that had ALREADY happened before openFn ran still got an opener started (its side effect went
212
+ // out, and only the result was voided afterwards).
213
+ //
214
+ // What is NOT required, and deliberately not done: making a caller's later microtask win the
215
+ // race. Microtasks are FIFO, so an end signalled after `open()` returns may legitimately land
216
+ // after the opener has started — "opener runs, then the exit voids its result" is a legal order,
217
+ // and forcing it the other way would need a macrotask delay on the cycle-opening path. The two
218
+ // checks below are exactly the two things that must hold: an ALREADY-ended cycle starts nothing
219
+ // (pre-check), and an opener that is still in flight when the exit lands is voided (post-await).
220
+ e.promise = Promise.resolve()
221
+ .then(() => {
222
+ if (exitVersion !== openedVersion || !cycle || cycle.id !== openedId || cycle.state === 'ended') {
223
+ note('open-cancelled-before-run', { key: cl.qualKey, exitVersion, was: openedVersion })
224
+ return voided('before this cycle was opened')
225
+ }
226
+ const started = (typeof openFn === 'function') ? openFn() : undefined
227
+ return Promise.resolve(started).then((outcome) => {
228
+ if (exitVersion !== openedVersion || !cycle || cycle.id !== openedId || cycle.state === 'ended') {
229
+ note('open-voided', { key: cl.qualKey, exitVersion, was: openedVersion })
230
+ return voided('while this cycle was being opened')
231
+ }
232
+ note('qualification-used', { key: cl.qualKey, opened: !!(outcome && outcome.opened) })
233
+ return outcome
234
+ })
235
+ })
236
+ // Keep an inert handler: a rejection nobody awaits must never surface as an unhandled rejection
237
+ // inside the host. Callers still see the rejection — that is the point.
238
+ e.promise.catch(() => {})
239
+ return e.promise
240
+ }
241
+
242
+ /**
243
+ * Worker/host-derived state. Only these transitions move `state`; `open()` never does.
244
+ * `attribution` names whose turn this transition belongs to (used to block that turn from opening).
245
+ */
246
+ function mark (state, detail) {
247
+ if (!['open', 'paused', 'asking', 'ended'].includes(state)) return false
248
+ const attribution = detail && (detail.session || detail.agent)
249
+ ? claim({ session: detail.session, agent: detail.agent })
250
+ : null
251
+ if (!cycle) {
252
+ // A brake/ask/exit with no cycle yet: the record exists with no owner, so ownership is free
253
+ // until the next real call claims it — which is what "no session owns this machine" means.
254
+ cycle = {
255
+ id: ++nextId, ownerIdent: null, ownerLabel: 'none',
256
+ ownerSessionKey: null, ownerAgentKey: null,
257
+ state, openedAt: now(), lastAt: now(), endedWhy: '',
258
+ }
259
+ } else {
260
+ if (cycle.state === 'ended' && state !== 'ended') return false // ended is terminal for this id
261
+ cycle.state = state
262
+ cycle.lastAt = now()
263
+ }
264
+ if (state === 'ended') {
265
+ exitVersion++ // every exit invalidates any opening that is still in flight (#6)
266
+ cycle.endedWhy = (detail && detail.why) || ''
267
+ // WHICH TURN SAW THE EXIT: the attributing session's current turn when we were told, else the
268
+ // owner's. That turn's late calls are blocked from opening (above) — no timer involved.
269
+ const sk = (attribution && attribution.sessionKey) || cycle.ownerSessionKey
270
+ const ak = (attribution && attribution.agentKey) || cycle.ownerAgentKey
271
+ if (sk || ak) rememberEndedTurn(claim({ session: sk, agent: ak ? { id: ak } : null }))
272
+ }
273
+ note('state', { id: cycle.id, state, why: (detail && detail.why) || '' })
274
+ return true
275
+ }
276
+
277
+ return {
278
+ claim,
279
+ noteTurnStart,
280
+ open,
281
+ mark,
282
+ forgetIdentity,
283
+ snapshot: () => ({
284
+ id: cycle ? cycle.id : 0,
285
+ state: cycle ? cycle.state : 'none',
286
+ owner: cycle ? cycle.ownerLabel : 'none',
287
+ ownerIdent: cycle ? cycle.ownerIdent : null,
288
+ ownerSessionKey: cycle ? cycle.ownerSessionKey : null,
289
+ ownerAgentKey: cycle ? cycle.ownerAgentKey : null,
290
+ endedWhy: cycle ? cycle.endedWhy : '',
291
+ exitVersion,
292
+ qualifications: [...idents.entries()].map(([ident, e]) => ({ ident, turn: e.turn, used: !!e.promise, dead: !!e.dead })),
293
+ endedTurns: Object.fromEntries(endedTurns),
294
+ turns: Object.fromEntries(turns),
295
+ events: events.slice(),
296
+ }),
297
+ /** Is this claim the owner of the live cycle? (event handlers use it to stay out of others' way) */
298
+ owns: (cl) => ownedBy(cycle, cl),
299
+ isLive: () => isLive(cycle),
300
+ state: () => (cycle ? cycle.state : 'none'),
301
+ /** Bumped by every `ended`: a caller may capture it and re-check before acting on a stale plan. */
302
+ exitVersion: () => exitVersion,
303
+ /** The owning SESSION/AGENT, so host-wide events can be attributed without re-deriving a claim. */
304
+ ownerSessionKey: () => (cycle ? cycle.ownerSessionKey : null),
305
+ ownsAgent: (ak) => !!cycle && !!ak && cycle.ownerAgentKey === ak,
306
+ ownsSession: (sk) => !!cycle && !!sk && cycle.ownerSessionKey === sk,
307
+ /** Release ownership on a FACT (session/agent disposed) — never on elapsed time. */
308
+ release (why) {
309
+ if (!isLive(cycle)) return false
310
+ return mark('ended', { why: why || 'owner released the machine' })
311
+ },
312
+ ownerLabel,
313
+ }
314
+ }
@@ -0,0 +1,91 @@
1
+ // Keep desktop execution schemas out of unrelated turns. Brakes stay globally callable.
2
+ export const ACTIVATION_TOOL = 'computer_use_activate'
3
+ const ALWAYS = new Set(['computer_ctrl', 'computer_ask'])
4
+
5
+ export function installToolExposure ({ ctx, definitions, defineTool, scopeOf, readSkill, skillPath, log, enabled = true }) {
6
+ const active = new Map()
7
+ const boundaries = new Map()
8
+ const agents = new Set()
9
+ const globalDisposers = []
10
+ const execution = definitions.filter(d => !ALWAYS.has(d.name))
11
+ const maskInherited = (agent) => {
12
+ if (active.has(agent) || boundaries.has(agent) || !agent?.ctx || scopeOf(agent.ctx) !== agent) return
13
+ const inherited = execution.filter(d => ctx.tools.get(d.name, agent)).map(d => d.name)
14
+ if (inherited.length) boundaries.set(agent, agent.ctx.tools.restrict({ deny: inherited }))
15
+ }
16
+ let disposed = false
17
+ const activate = (agent) => {
18
+ if (!enabled) return { activated: true, reused: true }
19
+ if (disposed) throw new Error('computer-use plugin was disposed')
20
+ if (!agent?.ctx?.tools || scopeOf(agent.ctx) !== agent) {
21
+ throw new Error('computer-use activation needs this Agent\'s scoped tools registry; this host can use progressiveTools:false for legacy registration')
22
+ }
23
+ if (active.has(agent)) return { activated: true, reused: true }
24
+ const disposers = []
25
+ try {
26
+ for (const definition of execution) disposers.push(agent.ctx.tools.register(definition))
27
+ active.set(agent, disposers)
28
+ agents.add(agent)
29
+ for (const other of agents) maskInherited(other)
30
+ return { activated: true, reused: false }
31
+ } catch (error) {
32
+ for (const dispose of disposers.reverse()) dispose()
33
+ throw error
34
+ }
35
+ }
36
+ const detach = (agent) => {
37
+ const disposers = active.get(agent)
38
+ active.delete(agent)
39
+ for (const dispose of (disposers || []).reverse()) dispose()
40
+ boundaries.get(agent)?.()
41
+ boundaries.delete(agent)
42
+ agents.delete(agent)
43
+ }
44
+ for (const definition of definitions) {
45
+ if (!enabled || ALWAYS.has(definition.name)) globalDisposers.push(ctx.tools.register(definition))
46
+ }
47
+ if (enabled) {
48
+ globalDisposers.push(ctx.tools.register(defineTool({
49
+ name: ACTIVATION_TOOL,
50
+ description: 'Load the Windows computer-use manual and activate desktop tools for this Agent. Use when the task needs mouse, keyboard, screenshots or UIA. Does not start control or release a human brake.',
51
+ parameters: {},
52
+ output: {
53
+ schema: { type: 'object', additionalProperties: true },
54
+ render: (_args, value) => [{ type: 'text', text: JSON.stringify(value) }],
55
+ },
56
+ isConcurrencySafe: () => false,
57
+ execute: (_args, exec) => {
58
+ if (active.has(exec.agent)) return { activated: true, reused: true }
59
+ const manual = readSkill()
60
+ return { ...activate(exec.agent), tools: definitions.map(d => d.name), manual, ...(skillPath ? { manualPath: skillPath } : {}) }
61
+ },
62
+ })))
63
+ // A successful normal Skill load/receipt also activates tools, avoiding an extra bootstrap call.
64
+ globalDisposers.push(ctx.on('tools/post-execute', async (exec, result, next) => {
65
+ const decision = await next()
66
+ const skillLoaded = exec.name === 'skill' && exec.arguments?.name === 'computer-use'
67
+ const acknowledged = exec.name === 'computer_ctrl' && exec.arguments?.action === 'acknowledge'
68
+ if (!result.isError && decision.kind === 'accept' && (skillLoaded || acknowledged) && exec.agent) activate(exec.agent)
69
+ return decision
70
+ }))
71
+ globalDisposers.push(ctx.on('agent/disposed', (event) => detach(event.agent || event)))
72
+ globalDisposers.push(ctx.on('agent/created', (event) => {
73
+ const agent = event.agent || event
74
+ agents.add(agent)
75
+ maskInherited(agent)
76
+ }))
77
+ globalDisposers.push(ctx.tools.guard(exec => execution.some(d => d.name === exec.name) && !active.has(exec.agent)
78
+ ? 'Load computer-use or call computer_use_activate in this Agent before dispatching desktop tools' : undefined))
79
+ }
80
+ const dispose = () => {
81
+ if (disposed) return
82
+ disposed = true
83
+ for (const agent of active.keys()) detach(agent)
84
+ for (const agent of boundaries.keys()) detach(agent)
85
+ agents.clear()
86
+ for (const undo of globalDisposers.reverse()) undo()
87
+ }
88
+ ctx.effect(() => dispose)
89
+ log?.info?.(`tool exposure: ${enabled ? '3 bootstrap/safety tools; 19 after Agent activation' : 'legacy 18 global tools'}`)
90
+ return { activate, dispose }
91
+ }
@@ -0,0 +1,35 @@
1
+ // Route-specific observations, verified against the Desktop adapters on 2026-10-02.
2
+ // A model catalog's image flag is transport permission, not a grounding test.
3
+ export function feedbackRoute (exec) {
4
+ const routed = exec?.agent?.session?.requestHeader?.()?.config
5
+ return {
6
+ provider: routed?.provider ?? exec?.agent?.options?.provider,
7
+ model: routed?.model ?? exec?.agent?.options?.model,
8
+ }
9
+ }
10
+
11
+ export function unverifiedVisionRoute (exec) {
12
+ const { provider, model } = feedbackRoute(exec)
13
+ return provider === 'bigmodel-anthropic' && model === 'glm-5.3'
14
+ }
15
+
16
+ export async function feedbackMode (ctx, exec, enabled = true) {
17
+ if (!enabled || exec?.signal?.aborted) return null
18
+ if (unverifiedVisionRoute(exec)) return 'state'
19
+ const { provider, model } = feedbackRoute(exec)
20
+ const tested = (provider === 'bigmodel-anthropic' && model === 'glm-5.3-flash') ||
21
+ (provider === 'deepseek-official' && model === 'deepseek-flash')
22
+ if (!tested) return null
23
+ try {
24
+ if (!ctx.get?.('attachments')?.saveImage) return null
25
+ const active = await ctx.get?.('llm')?.resolveModelInfo?.(provider, model, exec?.signal)
26
+ return !exec?.signal?.aborted && active?.inputModalities?.includes('image') ? 'image' : null
27
+ } catch { return null }
28
+ }
29
+
30
+ export const feedbackActions = new Set([
31
+ 'computer_click', 'computer_type', 'computer_key', 'computer_move',
32
+ 'computer_drag', 'computer_scroll', 'computer_select',
33
+ ])
34
+
35
+ export const feedbackVerification = 'Inspect this post-action observation before claiming the task effect. An input receipt alone is not verification. If observation failed, inspect before repeating the input.'
@@ -0,0 +1,92 @@
1
+ import { basename } from 'node:path'
2
+ import { unverifiedVisionRoute } from './feedback.js'
3
+
4
+ // DSH image blocks reference the host's durable attachment store. Paths and raw
5
+ // base64 are not image blocks. Keep a usable file when a route cannot accept one.
6
+ export async function attachScreenshot (ctx, exec, value, data) {
7
+ const fallback = (reason) => {
8
+ value.imageStatus = 'path-only'
9
+ value.imageReason = reason
10
+ value.view = reason === 'model-text-only'
11
+ ? 'Image not attached: this model does not declare image input. Use UIA/text or switch to an image-capable model.'
12
+ : 'Image not attached. Use read_image on path if visual inspection is needed.'
13
+ if (value.settled === false) value.view = 'NOT settled — re-shoot. ' + value.view
14
+ return value
15
+ }
16
+ try {
17
+ const attachments = ctx.get?.('attachments')
18
+ const llm = ctx.get?.('llm')
19
+ if (!attachments?.saveImage) return fallback('attachments-unavailable')
20
+ const routed = exec?.agent?.session?.requestHeader?.()?.config
21
+ const provider = routed?.provider ?? exec?.agent?.options?.provider
22
+ const model = routed?.model ?? exec?.agent?.options?.model
23
+ if (!llm?.resolveModelInfo || !provider || !model) return fallback('route-unavailable')
24
+ const active = await llm.resolveModelInfo(provider, model, exec?.signal)
25
+ if (!active.inputModalities?.includes('image')) return fallback('model-text-only')
26
+ if (exec?.signal?.aborted) return fallback('cancelled')
27
+ if (!attachments.imageLimits?.mediaTypes?.includes(value.mime)) return fallback('media-type-unavailable')
28
+ const image = await attachments.saveImage({ data, mediaType: value.mime, name: basename(value.path) })
29
+ if (exec?.signal?.aborted) return fallback('cancelled')
30
+ value.image = image
31
+ value.imageStatus = 'attached'
32
+ const r = value.region
33
+ if (r && [r.x, r.y, r.width, r.height].every(Number.isFinite) && image.width > 0 && image.height > 0) {
34
+ value.coordinates = {
35
+ screenOrigin: [r.x, r.y], imageSize: [image.width, image.height],
36
+ screenPerImagePixel: [r.width / image.width, r.height / image.height],
37
+ }
38
+ }
39
+ value.view = value.settled === false
40
+ ? 'NOT settled — may be a partial render; re-shoot.'
41
+ : 'Image attached. Inspect it directly; no read_image needed. Coordinates for clicks are screen pixels.'
42
+ if (unverifiedVisionRoute(exec)) {
43
+ value.visionStatus = 'unverified-route'
44
+ value.view = 'Image transport succeeded, but this GLM 5.3 route failed grounding tests. Use UIA/text readback; do not infer coordinates or success from unseen pixels. The image and path are retained for inspection.'
45
+ }
46
+ return value
47
+ } catch (error) {
48
+ // Capturing succeeded; an optional image channel must not discard its path.
49
+ return fallback(exec?.signal?.aborted ? 'cancelled' : 'attachment-failed')
50
+ }
51
+ }
52
+
53
+ export function renderScreenshot (value) {
54
+ const { image, ...text } = value
55
+ return [
56
+ { type: 'text', text: JSON.stringify(text) },
57
+ ...(image ? [{ type: 'image', attachment: image }] : []),
58
+ ]
59
+ }
60
+
61
+ // A batch exposes one latest image as its checkpoint. Earlier explicit captures
62
+ // keep their paths for deliberate later reads, without multiplying vision input.
63
+ export function selectBatchScreenshot (result, automatic, kind = 'image') {
64
+ let latest = automatic
65
+ let index
66
+ if (!latest) {
67
+ for (const [i, action] of result.actions.entries()) {
68
+ if (action.tool === 'computer_shot' && action.result?.path) {
69
+ latest = action.result
70
+ index = i
71
+ }
72
+ }
73
+ }
74
+ for (const action of result.actions) {
75
+ const shot = action.result
76
+ if (shot?.image && shot !== latest) {
77
+ delete shot.image
78
+ shot.imageStatus = 'path-only'
79
+ shot.imageReason = 'earlier-batch-image'
80
+ shot.view = kind === 'state'
81
+ ? 'A newer UIA state is returned. This earlier image remains at its path.'
82
+ : 'Only the latest batch screenshot is attached. Use read_image on this path if this earlier frame is needed.'
83
+ }
84
+ }
85
+ if (latest) {
86
+ result.observation = latest
87
+ if (kind === 'state') result.observationSource = 'automatic final UIA state'
88
+ else result.imageSource = automatic ? 'automatic final screenshot' : `action ${index + 1}; captured before any later actions`
89
+ result.view = latest.view
90
+ }
91
+ return result
92
+ }