dsh-codex-style-computer-use 1.3.1 → 2.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (46) hide show
  1. package/README.ar.md +234 -0
  2. package/README.de.md +234 -0
  3. package/README.es.md +264 -0
  4. package/README.fa.md +234 -0
  5. package/README.fr.md +243 -0
  6. package/README.md +72 -17
  7. package/README.pt.md +234 -0
  8. package/README.ru.md +259 -0
  9. package/README.zh.md +66 -16
  10. package/icon.svg +20 -4
  11. package/lib/audit.js +7 -0
  12. package/lib/batch.js +14 -19
  13. package/lib/bin/attention-worker.exe +0 -0
  14. package/lib/bin/attention-worker.exe.json +1 -1
  15. package/lib/compact-feedback.js +175 -0
  16. package/lib/continuation.js +1 -1
  17. package/lib/core/bin/dsh-computer-use-worker.exe +0 -0
  18. package/lib/core/bin/dsh-computer-use-worker.exe.json +2 -2
  19. package/lib/core/index.js +403 -136
  20. package/lib/core/native-runtime.js +17 -0
  21. package/lib/core/worker.cs +2160 -229
  22. package/lib/feedback-routes.json +11 -0
  23. package/lib/feedback.js +49 -16
  24. package/lib/image-output.js +87 -17
  25. package/lib/index.js +25 -7
  26. package/lib/key-command.js +37 -0
  27. package/lib/launch.js +67 -0
  28. package/lib/media-contract.js +43 -0
  29. package/lib/observations.js +267 -13
  30. package/lib/policy.js +29 -9
  31. package/lib/tool-contract.js +56 -0
  32. package/lib/tools.js +203 -52
  33. package/lib/verification-region.js +17 -0
  34. package/mcp/package.json +2 -1
  35. package/mcp/src/execution.js +8 -5
  36. package/mcp/src/index.js +16 -267
  37. package/mcp/src/tools.js +271 -0
  38. package/package.json +6 -1
  39. package/scripts/check-behavior.mjs +83 -0
  40. package/scripts/measure-tool-tokens.mjs +21 -0
  41. package/scripts/verify-package.mjs +9 -0
  42. package/skills/computer-use/SKILL.md +55 -43
  43. package/skills/computer-use/references/lifecycle.md +11 -7
  44. package/skills/computer-use/references/maintenance.md +9 -5
  45. package/skills/computer-use/references/precision.md +39 -14
  46. package/skills/computer-use/references/progressive-control.md +10 -4
package/mcp/src/index.js CHANGED
@@ -2,20 +2,14 @@
2
2
  // @dsh/computer-use-mcp — MCP stdio server (stage-0)
3
3
  //
4
4
  // Zero-dependency MCP server (JSON-RPC 2.0 over stdio, one message per line)
5
- // exposing the computer-use core. Implements the subset of the 2026-07-28
5
+ // exposing the computer-use core. Implements the subset of the 2025-06-18
6
6
  // spec a tools-only server needs: initialize / notifications/initialized /
7
7
  // ping / tools/list / tools/call. Screenshots come back as MCP image content
8
8
  // blocks; errors use isError:true so the model can self-correct.
9
9
  //
10
- // Safety gates (DESIGN §8), in order:
11
- // DSH_CU_KILL=1 refuse to start (hard kill switch)
12
- // DSH_CU_ALLOW_INPUT=1 register mutation tools without approval window
13
- // (use ONLY when the host already gates calls,
14
- // e.g. the DSH plugin wires its own approval)
15
- // otherwise mutation tools exist but every call requires a
16
- // fresh cu_approve window (≤10 min), mirroring
17
- // sandraschi's approve_automation pattern
18
- // DSH_CU_DRY_RUN=1 actuations are simulated and logged only
10
+ // The launching host owns user authorization. Input is disabled unless it starts
11
+ // this server with DSH_CU_ALLOW_INPUT=1; no model-callable tool grants consent.
12
+ // DSH_CU_KILL=1 disables startup; DSH_CU_DRY_RUN=1 simulates permitted input.
19
13
 
20
14
  // resolve the core package across layouts: installed dependency, sibling in
21
15
  // node_modules (@dsh scope), or repo monorepo checkout (packages/*)
@@ -28,10 +22,14 @@ async function loadCore () {
28
22
  throw new Error(`cannot locate @dsh/computer-use-core (tried ${candidates.join(', ')}): ${lastErr && lastErr.message}`)
29
23
  }
30
24
  import { createToolQueue, continuationResult } from './execution.js'
25
+ import { createMcpToolSurface, renderMcpResult } from './tools.js'
26
+ import { actionError } from '../../lib/tool-contract.js'
27
+ import { readFileSync } from 'node:fs'
31
28
  const { ComputerUse, humanContinuation } = await loadCore()
32
29
 
33
30
  const PROTOCOL_VERSION = '2025-06-18'
34
- const SERVER_INFO = { name: 'dsh-computer-use', version: '0.1.0' }
31
+ const packageInfo = JSON.parse(readFileSync(new URL('../../package.json', import.meta.url), 'utf8'))
32
+ const SERVER_INFO = { name: packageInfo.name + '-mcp', version: packageInfo.version }
35
33
 
36
34
  if (process.env.DSH_CU_KILL === '1') {
37
35
  process.stderr.write('[dsh-computer-use] DSH_CU_KILL=1 — refusing to start\n')
@@ -43,256 +41,8 @@ const dryRun = process.env.DSH_CU_DRY_RUN === '1'
43
41
  const cu = new ComputerUse({ dryRun })
44
42
  const enqueueTool = createToolQueue()
45
43
  const requests = new Map()
46
- let approvalUntil = 0
47
44
 
48
- // ---------------- tool definitions ----------------
49
- // Naming follows the Anthropic computer_toolset_20260801 members where one
50
- // exists (screenshot/key/type/scroll/click...) plus Windows-specific and
51
- // SoM extras. Screenshots are pre-scaled to ≤1280px by the core.
52
- const MUTATION_NOTE = ' MUTATING: requires an active approval window (cu_approve) or DSH_CU_ALLOW_INPUT=1.'
53
-
54
- const TOOLS = [
55
- {
56
- name: 'cu_screenshot',
57
- description: 'Capture the screen (or a region). Returns a JPEG/PNG image plus cursor position and capture region metadata. Coordinates everywhere are absolute screen pixels unless a mark id is used.',
58
- inputSchema: {
59
- type: 'object', additionalProperties: false,
60
- properties: {
61
- x: { type: 'integer' }, y: { type: 'integer' },
62
- width: { type: 'integer' }, height: { type: 'integer' },
63
- format: { type: 'string', enum: ['jpeg', 'png'], description: 'png for text-precise scenes' },
64
- quality: { type: 'integer', minimum: 40, maximum: 95 },
65
- maxWidth: { type: 'integer', description: 'downscale width; default 1280' },
66
- cursor: { type: 'boolean', description: 'draw the mouse cursor into the shot' },
67
- },
68
- },
69
- },
70
- {
71
- name: 'cu_screen_state',
72
- description: 'Text-first screen state: active window, visible top-level windows, cursor, and a shallow UIA (accessibility) tree of the active window. Works without vision. Use this as your primary observation; screenshots verify.',
73
- inputSchema: {
74
- type: 'object', additionalProperties: false,
75
- properties: {
76
- uiaDepth: { type: 'integer', minimum: 1, maximum: 24 },
77
- uiaMaxNodes: { type: 'integer', minimum: 10, maximum: 1200 },
78
- },
79
- },
80
- },
81
- {
82
- name: 'cu_marks',
83
- description: 'Set-of-Marks: labels clickable UI elements M1..Mn (from UIA) and returns an annotated screenshot + a mark table. Prefer clicking by mark id — it is immune to coordinate drift.',
84
- inputSchema: {
85
- type: 'object', additionalProperties: false,
86
- properties: {
87
- maxMarks: { type: 'integer', minimum: 5, maximum: 120 },
88
- uiaDepth: { type: 'integer', minimum: 1, maximum: 12 },
89
- },
90
- },
91
- },
92
- {
93
- name: 'cu_windows',
94
- description: 'List visible top-level windows (title, class, process, rect, active, minimized).',
95
- inputSchema: { type: 'object', additionalProperties: false, properties: {} },
96
- },
97
- {
98
- name: 'cu_uia_query',
99
- description: 'Query the UIA accessibility tree of a window (or the whole desktop) for elements by role/name. Returns flat elements with screen rectangles — precise, vision-free grounding.',
100
- inputSchema: {
101
- type: 'object', additionalProperties: false,
102
- properties: {
103
- hwnd: { type: 'integer' }, titleContains: { type: 'string' },
104
- depth: { type: 'integer', minimum: 1, maximum: 24 },
105
- maxNodes: { type: 'integer', minimum: 10, maximum: 1200 },
106
- role: { type: 'string', description: 'e.g. Button, Edit, Hyperlink, TabItem' },
107
- nameContains: { type: 'string' },
108
- },
109
- },
110
- },
111
- {
112
- name: 'cu_uia_at_point',
113
- description: 'What UI element is at screen point (x,y)? Returns the element plus ancestors — use to verify a click target before clicking.',
114
- inputSchema: {
115
- type: 'object', additionalProperties: false, required: ['x', 'y'],
116
- properties: { x: { type: 'integer' }, y: { type: 'integer' } },
117
- },
118
- },
119
- {
120
- name: 'cu_cursor',
121
- description: 'Current mouse cursor position.',
122
- inputSchema: { type: 'object', additionalProperties: false, properties: {} },
123
- },
124
- {
125
- name: 'cu_clipboard_read',
126
- description: 'Read the text clipboard.',
127
- inputSchema: { type: 'object', additionalProperties: false, properties: {} },
128
- },
129
- {
130
- name: 'cu_wait',
131
- description: 'Wait until the UI goes idle (foreground window + focused element stable) or timeout.',
132
- inputSchema: {
133
- type: 'object', additionalProperties: false,
134
- properties: {
135
- timeoutMs: { type: 'integer', maximum: 30000 },
136
- stableMs: { type: 'integer', maximum: 5000 },
137
- },
138
- },
139
- },
140
- // ---- mutation tools ----
141
- tool('cu_release', 'Release computer-use ownership as soon as desktop work and verification are finished, even while the conversation continues. Preserves a manual pause or pending question. Later desktop work can start a fresh cycle.', {
142
- type: 'object', additionalProperties: false, properties: {},
143
- }, false),
144
- tool('cu_approve', 'Open a mutation approval window (human-in-the-loop gate). minutes ≤ 10.', {
145
- type: 'object', additionalProperties: false, required: ['minutes'],
146
- properties: { minutes: { type: 'integer', minimum: 1, maximum: 10 } },
147
- }, false),
148
- tool('cu_move', 'Move the mouse cursor.', {
149
- type: 'object', additionalProperties: false, required: ['x', 'y'],
150
- properties: { x: { type: 'integer' }, y: { type: 'integer' } },
151
- }),
152
- tool('cu_click', 'Click the mouse (left/right/middle; single/double/triple) at a mark or point.', {
153
- type: 'object', additionalProperties: false,
154
- properties: {
155
- mark: { type: 'string', description: 'mark id from cu_marks, e.g. "M7"' },
156
- x: { type: 'integer' }, y: { type: 'integer' },
157
- button: { type: 'string', enum: ['left', 'right', 'middle'] },
158
- clicks: { type: 'integer', minimum: 1, maximum: 3, description: '2=double 3=triple' },
159
- },
160
- }),
161
- tool('cu_drag', 'Drag from one mark/point to another with a humanized ease curve.', {
162
- type: 'object', additionalProperties: false,
163
- properties: {
164
- fromMark: { type: 'string' }, toMark: { type: 'string' },
165
- fromX: { type: 'integer' }, fromY: { type: 'integer' },
166
- toX: { type: 'integer' }, toY: { type: 'integer' },
167
- durationMs: { type: 'integer', maximum: 3000 },
168
- },
169
- }),
170
- tool('cu_scroll', 'Scroll (up/down/left/right) lines of wheel at a position.', {
171
- type: 'object', additionalProperties: false, required: ['direction'],
172
- properties: {
173
- direction: { type: 'string', enum: ['up', 'down', 'left', 'right'] },
174
- lines: { type: 'integer', minimum: 1, maximum: 30 },
175
- mark: { type: 'string' }, x: { type: 'integer' }, y: { type: 'integer' },
176
- },
177
- }),
178
- tool('cu_key', 'Press a key combo, e.g. "ctrl+shift+t", "enter", "alt+f4", "win".', {
179
- type: 'object', additionalProperties: false, required: ['combo'],
180
- properties: { combo: { type: 'string' }, holdMs: { type: 'integer', maximum: 5000 } },
181
- }),
182
- tool('cu_type', 'Type text. CJK/long text is auto-pasted via clipboard (IME-proof); ASCII short text uses SendInput unicode events. Refuses password fields unless allowPassword is set (ask the human first).', {
183
- type: 'object', additionalProperties: false, required: ['text'],
184
- properties: {
185
- text: { type: 'string' },
186
- mode: { type: 'string', enum: ['unicode', 'paste'] },
187
- allowPassword: { type: 'boolean' },
188
- },
189
- }),
190
- tool('cu_clipboard_write', 'Write text to the clipboard.', {
191
- type: 'object', additionalProperties: false, required: ['text'],
192
- properties: { text: { type: 'string' } },
193
- }),
194
- tool('cu_activate_window', 'Bring a window to the foreground (hwnd or title substring).', {
195
- type: 'object', additionalProperties: false,
196
- properties: { hwnd: { type: 'integer' }, titleContains: { type: 'string' } },
197
- }),
198
- tool('cu_window_op', 'minimize | maximize | restore | close a window.', {
199
- type: 'object', additionalProperties: false, required: ['op'],
200
- properties: {
201
- op: { type: 'string', enum: ['minimize', 'maximize', 'restore', 'close'] },
202
- hwnd: { type: 'integer' }, titleContains: { type: 'string' },
203
- },
204
- }),
205
- tool('cu_batch', 'Execute computer actions serially. A failure stops later steps. A human handoff returns a continuing-task checkpoint: inspect the interrupted effect and resume remaining work. End with a screenshot when verification needs one.',
206
- {
207
- type: 'object', additionalProperties: false, required: ['actions'],
208
- properties: {
209
- actions: {
210
- type: 'array', minItems: 1, maxItems: 20,
211
- items: {
212
- type: 'object', additionalProperties: false, required: ['tool'],
213
- properties: { tool: { type: 'string' }, args: { type: 'object' } },
214
- },
215
- },
216
- },
217
- }),
218
- ]
219
-
220
- function tool (name, description, inputSchema, mutating = true) {
221
- return { name, description: description + (mutating && !allowInputDirect ? MUTATION_NOTE : ''), inputSchema, mutating }
222
- }
223
-
224
- // ---------------- dispatch ----------------
225
- const HANDLERS = {
226
- cu_screenshot: (a) => cu.screenshot(a),
227
- cu_screen_state: (a) => cu.screenState(a),
228
- cu_marks: (a) => cu.marks(a),
229
- cu_windows: () => cu.call('windows', {}),
230
- cu_uia_query: async (a) => {
231
- const res = await cu.call('uia', a)
232
- if (a.role || a.nameContains) {
233
- const pred = (e) => (!a.role || e.role === a.role) && (!a.nameContains || (e.name || '').toLowerCase().includes(a.nameContains.toLowerCase()))
234
- res.flat = (res.flat || []).filter(pred).slice(0, 60)
235
- } else res.flat = (res.flat || []).slice(0, 60)
236
- return res
237
- },
238
- cu_uia_at_point: (a) => cu.call('uiaFromPoint', a),
239
- cu_cursor: () => cu.call('cursor', {}),
240
- cu_clipboard_read: () => cu.clipRead(),
241
- cu_wait: (a) => cu.waitForIdle(a),
242
- cu_release: async () => ({ ...await cu.calm(), conversationContinues: true }),
243
- cu_approve: async (a) => {
244
- const mins = Math.min(10, Math.max(1, a.minutes || 1))
245
- approvalUntil = Date.now() + mins * 60_000
246
- return { approved: true, until: new Date(approvalUntil).toISOString() }
247
- },
248
- cu_move: (a) => cu.move(a.x, a.y),
249
- cu_click: (a) => cu.click(targetOf(a), a),
250
- cu_drag: async (a) => cu.drag(targetOf({ mark: a.fromMark, x: a.fromX, y: a.fromY }), targetOf({ mark: a.toMark, x: a.toX, y: a.toY }), { durationMs: a.durationMs }),
251
- cu_scroll: (a) => cu.scroll(a.direction, a.lines || 3, a.mark || (a.x !== undefined ? { x: a.x, y: a.y } : null)),
252
- cu_key: (a) => cu.key(a.combo, { holdMs: a.holdMs }),
253
- cu_type: (a) => cu.type(a.text, { mode: a.mode, allowPassword: a.allowPassword }),
254
- cu_clipboard_write: (a) => cu.clipWrite(a.text),
255
- cu_activate_window: (a) => cu.activateWindow(a),
256
- cu_window_op: (a) => cu.windowOp({ hwnd: a.hwnd, titleContains: a.titleContains }, a.op),
257
- cu_batch: async (a) => {
258
- const out = []
259
- let failed = false
260
- let checkpoint = null
261
- for (const step of a.actions) {
262
- if (failed) { out.push({ tool: step.tool, ok: false, outcome: 'not-dispatched',
263
- error: checkpoint ? 'Deferred until fresh target evidence; continue the original task.' : 'Not executed: an earlier computer action in this turn failed.' }); continue }
264
- try { out.push({ tool: step.tool, ok: true, result: await runTool(step.tool, step.args || {}) }) }
265
- catch (e) {
266
- failed = true
267
- const interrupted = continuationResult(e, humanContinuation)
268
- if (interrupted) checkpoint = { ...interrupted, resumeFromActionIndex: out.length }
269
- out.push({ tool: step.tool, ok: false, error: String(e.message || e),
270
- ...(interrupted || { outcome: e.outcome || 'unknown' }) })
271
- }
272
- }
273
- return { actions: out, ...(checkpoint || {}) }
274
- },
275
- }
276
-
277
- function targetOf (a) {
278
- if (a.mark) return { mark: a.mark }
279
- if (a.x !== undefined && a.y !== undefined) return { x: a.x, y: a.y }
280
- throw new Error('provide mark or x,y')
281
- }
282
-
283
- async function runTool (name, args) {
284
- const def = TOOLS.find(t => t.name === name)
285
- if (!def) throw new Error(`unknown tool: ${name}`)
286
- if (def.mutating && !allowInputDirect && name !== 'cu_approve' && Date.now() > approvalUntil) {
287
- throw new Error('no active approval window: call cu_approve {minutes} first (human consent gate)')
288
- }
289
- const res = await HANDLERS[name](args || {})
290
- // image-bearing results get an MCP image content block
291
- if (res && (res.image instanceof Buffer || res.annotated instanceof Buffer)) {
292
- return { value: res, image: res.annotated instanceof Buffer ? res.annotated : res.image }
293
- }
294
- return { value: res }
295
- }
45
+ const { tools: TOOLS, run: runTool } = createMcpToolSurface(cu, { allowInput: allowInputDirect })
296
46
 
297
47
  // ---------------- MCP plumbing ----------------
298
48
  let buf = ''
@@ -335,15 +85,14 @@ async function handleLine (line) {
335
85
  const controller = new AbortController()
336
86
  requests.set(msg.id, controller)
337
87
  try {
338
- const { value, image } = await enqueueTool(() =>
339
- cu.withInputContext({ signal: controller.signal }, () => runTool(name, args)), controller.signal)
340
- const content = [{ type: 'text', text: JSON.stringify(value, null, 1).slice(0, 60_000) }]
341
- if (image) content.push({ type: 'image', data: image.toString('base64'), mimeType: 'image/jpeg' })
342
- send({ jsonrpc: '2.0', id: msg.id, result: { content, isError: false } })
88
+ const value = await enqueueTool(() =>
89
+ cu.withInputContext({ signal: controller.signal }, () => runTool(name, args, { signal: controller.signal })), controller.signal)
90
+ send({ jsonrpc: '2.0', id: msg.id, result: renderMcpResult(name, args, value) })
343
91
  } catch (e) {
344
- const checkpoint = continuationResult(e, humanContinuation)
92
+ const checkpoint = !controller.signal.aborted && !cu.stopped && !cu.exited
93
+ ? continuationResult(e, humanContinuation) : null
345
94
  send({ jsonrpc: '2.0', id: msg.id, result: { content: [{ type: 'text',
346
- text: checkpoint ? JSON.stringify(checkpoint) : String(e.message || e) }], isError: !checkpoint } })
95
+ text: JSON.stringify(checkpoint || actionError(e)) }], isError: !checkpoint } })
347
96
  } finally { requests.delete(msg.id) }
348
97
  break
349
98
  }
@@ -0,0 +1,271 @@
1
+ import { runBatch, actionRefusal } from '../../lib/batch.js'
2
+ import { parseKeyCommand } from '../../lib/key-command.js'
3
+ import { splitMediaResult } from '../../lib/media-contract.js'
4
+
5
+ // This adapter owns transport names only. Serial failure/cancellation/result
6
+ // semantics are shared with the DSH adapter through runBatch.
7
+ export function createMcpToolSurface (cu, { allowInput = false } = {}) {
8
+ const MUTATION_NOTE = ' Input requires host authorization configured at startup with DSH_CU_ALLOW_INPUT=1. Tool calls cannot enable it.'
9
+ const tools = [
10
+ {
11
+ name: 'cu_screenshot',
12
+ description: 'Capture the screen (or a region). Returns a JPEG/PNG image plus cursor position and capture region metadata. Coordinates everywhere are absolute screen pixels unless a mark id is used.',
13
+ inputSchema: {
14
+ type: 'object', additionalProperties: false,
15
+ properties: {
16
+ x: { type: 'integer' }, y: { type: 'integer' },
17
+ width: { type: 'integer' }, height: { type: 'integer' },
18
+ format: { type: 'string', enum: ['jpeg', 'png'], description: 'png for text-precise scenes' },
19
+ quality: { type: 'integer', minimum: 40, maximum: 95 },
20
+ maxWidth: { type: 'integer', description: 'downscale width; default 1280' },
21
+ cursor: { type: 'boolean', description: 'draw the mouse cursor into the shot' },
22
+ },
23
+ },
24
+ },
25
+ {
26
+ name: 'cu_screen_state',
27
+ description: 'Text-first screen state: active window, visible top-level windows, cursor, and a shallow UIA (accessibility) tree of the active window. Works without vision. Use this as your primary observation; screenshots verify.',
28
+ inputSchema: {
29
+ type: 'object', additionalProperties: false,
30
+ properties: {
31
+ uiaDepth: { type: 'integer', minimum: 1, maximum: 24 },
32
+ uiaMaxNodes: { type: 'integer', minimum: 10, maximum: 1200 },
33
+ },
34
+ },
35
+ },
36
+ {
37
+ name: 'cu_marks',
38
+ description: 'Set-of-Marks: labels clickable UI elements M1..Mn (from UIA) and returns an annotated screenshot + a mark table. Marks are snapshots: refresh after scrolling or changes; stale evidence refuses input.',
39
+ inputSchema: {
40
+ type: 'object', additionalProperties: false,
41
+ properties: {
42
+ maxMarks: { type: 'integer', minimum: 5, maximum: 120 },
43
+ uiaDepth: { type: 'integer', minimum: 1, maximum: 12 },
44
+ },
45
+ },
46
+ },
47
+ {
48
+ name: 'cu_windows',
49
+ description: 'List visible top-level windows (title, class, process, rect, active, minimized).',
50
+ inputSchema: { type: 'object', additionalProperties: false, properties: {} },
51
+ },
52
+ {
53
+ name: 'cu_uia_query',
54
+ description: 'Bounded UIA query inside an explicit window. Default depth6/120 nodes/1200ms soft budget. Broad searches require an observed point anchor. Truncated results are partial evidence; a single provider call cannot be preempted.',
55
+ inputSchema: {
56
+ type: 'object', additionalProperties: false,
57
+ properties: {
58
+ hwnd: { type: 'integer' }, titleContains: { type: 'string' },
59
+ depth: { type: 'integer', minimum: 1, maximum: 24 },
60
+ maxNodes: { type: 'integer', minimum: 10, maximum: 1200 },
61
+ role: { type: 'string', description: 'e.g. Button, Edit, Hyperlink, TabItem' },
62
+ nameContains: { type: 'string' },
63
+ anchor: { type: 'string', description: 'Observed screen point "x,y" inside hwnd; query its subtree' },
64
+ budgetMs: { type: 'integer', minimum: 100, maximum: 5000 },
65
+ },
66
+ },
67
+ },
68
+ {
69
+ name: 'cu_uia_at_point',
70
+ description: 'What UI element is at screen point (x,y)? Returns the element plus ancestors — use to verify a click target before clicking.',
71
+ inputSchema: {
72
+ type: 'object', additionalProperties: false, required: ['x', 'y'],
73
+ properties: { x: { type: 'integer' }, y: { type: 'integer' } },
74
+ },
75
+ },
76
+ {
77
+ name: 'cu_cursor',
78
+ description: 'Current mouse cursor position.',
79
+ inputSchema: { type: 'object', additionalProperties: false, properties: {} },
80
+ },
81
+ {
82
+ name: 'cu_clipboard_read',
83
+ description: 'Read the text clipboard.',
84
+ inputSchema: { type: 'object', additionalProperties: false, properties: {} },
85
+ },
86
+ {
87
+ name: 'cu_wait',
88
+ description: 'Wait until the UI goes idle (foreground window + focused element stable) or timeout.',
89
+ inputSchema: {
90
+ type: 'object', additionalProperties: false,
91
+ properties: {
92
+ timeoutMs: { type: 'integer', maximum: 30000 },
93
+ stableMs: { type: 'integer', maximum: 5000 },
94
+ },
95
+ },
96
+ },
97
+ // ---- mutation tools ----
98
+ tool('cu_release', 'Release computer-use ownership as soon as desktop work and verification are finished, even while the conversation continues. Preserves a manual pause or pending question. Later desktop work can start a fresh cycle.', {
99
+ type: 'object', additionalProperties: false, properties: {},
100
+ }, false),
101
+ tool('cu_hold', 'Reserve the current scene between actions (breathing blue); human takeover remains authoritative.', { type: 'object', additionalProperties: false, properties: {} }, false),
102
+ tool('cu_free', 'Release scene reservation while keeping observation available (cyan).', { type: 'object', additionalProperties: false, properties: {} }, false),
103
+ tool('cu_approve', 'Explain host-managed authorization. This compatibility tool cannot grant approval or enable input.', {
104
+ type: 'object', additionalProperties: false,
105
+ properties: { minutes: { type: 'integer', minimum: 1, maximum: 10, description: 'Ignored; accepted only for old clients.' } },
106
+ }, false),
107
+ tool('cu_move', 'Hover without clicking; returns Windows standard cursor evidence. Browser/custom cursors may be unrecognized; unknown proves neither a handle hit nor a miss.', {
108
+ type: 'object', additionalProperties: false, required: ['x', 'y'],
109
+ properties: { x: { type: 'integer' }, y: { type: 'integer' } },
110
+ }),
111
+ tool('cu_click', 'Click the mouse (left/right/middle; single/double/triple) at a mark or point.', {
112
+ type: 'object', additionalProperties: false,
113
+ properties: {
114
+ mark: { type: 'string', description: 'mark id from cu_marks, e.g. "M7"' },
115
+ x: { type: 'integer' }, y: { type: 'integer' },
116
+ button: { type: 'string', enum: ['left', 'right', 'middle'] },
117
+ clicks: { type: 'integer', minimum: 1, maximum: 3, description: '2=double 3=triple' },
118
+ verificationRegion: { type: 'string', description: 'Expected effect rectangle "x,y,w,h" in screen pixels, independent of click location. Pixel change does not prove semantic success.' },
119
+ },
120
+ }),
121
+ tool('cu_drag', 'Drag measured screen points. Optional expectCursor checks a recognized Windows standard shape before button-down; browser/custom cursors may be unverifiable. Ground those handles independently. cursorCheck says matched or not-requested; completion is delivery, not proof of layout success.', {
122
+ type: 'object', additionalProperties: false,
123
+ properties: {
124
+ fromMark: { type: 'string' }, toMark: { type: 'string' },
125
+ fromX: { type: 'integer' }, fromY: { type: 'integer' },
126
+ toX: { type: 'integer' }, toY: { type: 'integer' },
127
+ durationMs: { type: 'integer', maximum: 3000 },
128
+ expectCursor: { type: 'string', enum: ['resize', 'resize-ew', 'resize-ns', 'resize-nesw', 'resize-nwse', 'move', 'hand'], description: 'Optional standard shape check. Unknown refuses it; omit only with independently verified target geometry/hit-test.' },
129
+ hwnd: { type: 'integer', description: 'expected window at the drag start' },
130
+ },
131
+ }),
132
+ tool('cu_scroll', 'Scroll (up/down/left/right) lines of wheel at a position.', {
133
+ type: 'object', additionalProperties: false, required: ['direction'],
134
+ properties: {
135
+ direction: { type: 'string', enum: ['up', 'down', 'left', 'right'] },
136
+ lines: { type: 'integer', minimum: 1, maximum: 30 },
137
+ mark: { type: 'string' }, x: { type: 'integer' }, y: { type: 'integer' },
138
+ },
139
+ }),
140
+ tool('cu_key', 'Press a key combo, e.g. "ctrl+shift+t", "enter", "alt+f4", "win".', {
141
+ type: 'object', additionalProperties: false, required: ['combo'],
142
+ properties: { combo: { type: 'string' }, holdMs: { type: 'integer', maximum: 5000 } },
143
+ }),
144
+ tool('cu_type', 'Type text. CJK/long text is auto-pasted via clipboard (IME-proof); ASCII short text uses SendInput unicode events. Refuses password fields unless allowPassword is set (ask the human first).', {
145
+ type: 'object', additionalProperties: false, required: ['text'],
146
+ properties: {
147
+ text: { type: 'string' },
148
+ mode: { type: 'string', enum: ['unicode', 'paste'] },
149
+ allowPassword: { type: 'boolean' },
150
+ },
151
+ }),
152
+ tool('cu_clipboard_write', 'Write text to the clipboard.', {
153
+ type: 'object', additionalProperties: false, required: ['text'],
154
+ properties: { text: { type: 'string' } },
155
+ }),
156
+ tool('cu_activate_window', 'Bring a window to the foreground (hwnd or title substring).', {
157
+ type: 'object', additionalProperties: false,
158
+ properties: { hwnd: { type: 'integer' }, titleContains: { type: 'string' } },
159
+ }),
160
+ tool('cu_window_op', 'minimize | maximize | restore | close a window.', {
161
+ type: 'object', additionalProperties: false, required: ['op'],
162
+ properties: {
163
+ op: { type: 'string', enum: ['minimize', 'maximize', 'restore', 'close'] },
164
+ hwnd: { type: 'integer' }, titleContains: { type: 'string' },
165
+ },
166
+ }),
167
+ tool('cu_batch', 'Execute computer actions serially. A failure stops later steps. A human handoff returns a continuing-task checkpoint: inspect the interrupted effect and resume remaining work. End with a screenshot when verification needs one.',
168
+ {
169
+ type: 'object', additionalProperties: false, required: ['actions'],
170
+ properties: {
171
+ actions: {
172
+ type: 'array', minItems: 1, maxItems: 20,
173
+ items: {
174
+ type: 'object', additionalProperties: false, required: ['tool'],
175
+ properties: { tool: { type: 'string' }, args: { type: 'object' } },
176
+ },
177
+ },
178
+ },
179
+ }),
180
+ ]
181
+
182
+ function tool (name, description, inputSchema, mutating = true) {
183
+ return { name, description: description + (mutating && !allowInput ? MUTATION_NOTE : ''), inputSchema, mutating }
184
+ }
185
+
186
+
187
+ const handlers = {
188
+ cu_screenshot: a => cu.screenshot(a),
189
+ cu_screen_state: a => cu.screenState(a),
190
+ cu_marks: a => cu.marks({ ...a, raw: true }),
191
+ cu_windows: () => cu.call('windows', {}),
192
+ cu_uia_query: async a => {
193
+ const result = await cu.call('uia', { ...a, query: true, depth: a.depth ?? 6, maxNodes: a.maxNodes ?? 120, budgetMs: a.budgetMs ?? 1200 })
194
+ const matches = (result.flat || []).filter(e => (!a.role || e.role === a.role) &&
195
+ (!a.nameContains || (e.name || '').toLowerCase().includes(a.nameContains.toLowerCase())))
196
+ return { ...result, flat: matches.slice(0, 60), truncated: result.truncated === true || matches.length > 60,
197
+ returnedCount: Math.min(60, matches.length) }
198
+ },
199
+ cu_uia_at_point: a => cu.call('uiaFromPoint', a),
200
+ cu_cursor: () => cu.call('cursor', {}),
201
+ cu_clipboard_read: () => cu.clipRead(),
202
+ cu_wait: a => cu.waitForIdle(a),
203
+ cu_release: async () => ({ ...await cu.calm(), conversationContinues: true }),
204
+ cu_hold: () => cu.indicator({ hold: true }),
205
+ cu_free: () => cu.indicator({ hold: false }),
206
+ cu_approve: () => ({ approved: false, inputEnabled: allowInput, approvalSource: 'host-configuration',
207
+ reason: 'This tool does not grant human consent. Input is enabled only by the launching host configuration; the host is responsible for action authorization.' }),
208
+ cu_move: a => cu.move(a.x, a.y),
209
+ cu_click: a => cu.click(targetOf(a), a),
210
+ cu_drag: a => cu.drag(targetOf({ mark: a.fromMark, x: a.fromX, y: a.fromY }),
211
+ targetOf({ mark: a.toMark, x: a.toX, y: a.toY }), { durationMs: a.durationMs, expectCursor: a.expectCursor, expectHwnd: a.hwnd }),
212
+ cu_scroll: a => cu.scroll(a.direction, a.lines ?? 3, a.mark || a.x !== undefined ? { at: targetOf(a) } : {}),
213
+ cu_key: a => cu.key(parseKeyCommand(a.combo).combo, { holdMs: a.holdMs }),
214
+ cu_type: a => cu.type(a.text, { mode: a.mode, allowPassword: a.allowPassword }),
215
+ cu_clipboard_write: a => cu.clipWrite(a.text),
216
+ cu_activate_window: a => cu.activateWindow(a),
217
+ cu_window_op: a => cu.windowOp({ hwnd: a.hwnd, titleContains: a.titleContains }, a.op),
218
+ }
219
+ async function run (name, args = {}, exec = {}, depth = 0) {
220
+ const definition = tools.find(t => t.name === name)
221
+ if (!definition) throw new Error('unknown tool: ' + name)
222
+ validateArguments(args, definition.inputSchema)
223
+ if (exec.signal?.aborted) throw Object.assign(new Error('MCP request cancelled before dispatch'), { code: 'INPUT_CANCELLED', outcome: 'not-dispatched' })
224
+ if (name === 'cu_batch') {
225
+ if (depth >= 4) throw new Error('batch nesting limit exceeded')
226
+ const batchHandlers = Object.fromEntries(tools.map(t => [t.name, (a, context) => run(t.name, a, context, depth + 1)]))
227
+ return runBatch({ steps: args.actions, handlers: batchHandlers, signal: exec.signal, exec,
228
+ stopped: () => cu.stopped || cu.exited, shot: 'never', screenshot: async () => null })
229
+ }
230
+ if (definition.mutating && !allowInput) throw Object.assign(new Error(
231
+ 'MCP_INPUT_DISABLED: the launching host has not enabled input. Configure DSH_CU_ALLOW_INPUT=1 only when that host owns action authorization; cu_approve cannot grant it.'),
232
+ { code: 'MCP_INPUT_DISABLED', outcome: 'not-dispatched' })
233
+ return handlers[name](args, exec)
234
+ }
235
+ return { tools, run }
236
+ }
237
+
238
+ function targetOf (args) {
239
+ if (typeof args.mark === 'string' && args.mark) return { mark: args.mark }
240
+ if (Number.isFinite(args.x) && Number.isFinite(args.y)) return { x: args.x, y: args.y }
241
+ throw new Error('provide mark or finite x,y')
242
+ }
243
+
244
+ // MCP's schema is discovery metadata; enforce it at this boundary too, including
245
+ // each nested batch step, so a client skipping validation cannot bypass the contract.
246
+ function validateArguments (args, schema) {
247
+ if (!args || typeof args !== 'object' || Array.isArray(args)) throw new Error('arguments must be an object')
248
+ for (const key of schema.required || []) if (args[key] === undefined) throw new Error('missing argument: ' + key)
249
+ for (const [key, value] of Object.entries(args)) {
250
+ const spec = schema.properties?.[key]
251
+ if (!spec) { if (schema.additionalProperties === false) throw new Error('unknown argument: ' + key); continue }
252
+ if (spec.type === 'integer' ? !Number.isInteger(value) : spec.type === 'array' ? !Array.isArray(value) : typeof value !== spec.type)
253
+ throw new Error('invalid argument type: ' + key)
254
+ if (spec.enum && !spec.enum.includes(value)) throw new Error('invalid argument value: ' + key)
255
+ if (spec.minimum !== undefined && value < spec.minimum || spec.maximum !== undefined && value > spec.maximum)
256
+ throw new Error('argument outside range: ' + key)
257
+ if (spec.type === 'array') {
258
+ if (spec.minItems !== undefined && value.length < spec.minItems || spec.maxItems !== undefined && value.length > spec.maxItems)
259
+ throw new Error('array outside size range: ' + key)
260
+ if (spec.items?.type === 'object') for (const entry of value) validateArguments(entry, spec.items)
261
+ }
262
+ }
263
+ }
264
+
265
+ export function renderMcpResult (name, args, result) {
266
+ const media = splitMediaResult(result)
267
+ const value = media.image ? { ...media.value, imageSource: media.imageSource } : media.value
268
+ const content = [{ type: 'text', text: JSON.stringify(value, null, 1) }]
269
+ if (media.image) content.push({ type: 'image', data: media.image.toString('base64'), mimeType: media.mime })
270
+ return { content, isError: result?.taskStatus === 'continuing' ? false : !!actionRefusal({ tool: name, args }, result) }
271
+ }
package/package.json CHANGED
@@ -1,9 +1,13 @@
1
1
  {
2
2
  "name": "dsh-codex-style-computer-use",
3
- "version": "1.3.1",
3
+ "version": "2.1.0",
4
4
  "description": "Codex-style Computer Use for DeepSeek Harness: Windows desktop control, screenshots, UI Automation, human input handoff, approval gates and prebuilt native workers. Install and update from dsh-market without PowerShell 7 or build steps.",
5
5
  "type": "module",
6
6
  "main": "lib/index.js",
7
+ "bin": {
8
+ "dsh-codex-style-computer-use": "mcp/src/index.js",
9
+ "dsh-computer-use-mcp": "mcp/src/index.js"
10
+ },
7
11
  "exports": {
8
12
  ".": "./lib/index.js",
9
13
  "./package.json": "./package.json",
@@ -79,6 +83,7 @@
79
83
  "registry": "https://registry.npmjs.org/"
80
84
  },
81
85
  "scripts": {
86
+ "test": "node scripts/check-behavior.mjs",
82
87
  "build:native": "node scripts/build-native.mjs",
83
88
  "verify:package": "node scripts/verify-package.mjs"
84
89
  },