dsh-codex-style-computer-use 2.0.0 → 2.2.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.ar.md +234 -0
- package/README.de.md +234 -0
- package/README.es.md +264 -0
- package/README.fa.md +234 -0
- package/README.fr.md +243 -0
- package/README.md +45 -8
- package/README.pt.md +234 -0
- package/README.ru.md +259 -0
- package/README.zh.md +45 -8
- package/client.js +130 -0
- package/icon.svg +20 -4
- package/lib/bin/attention-worker.exe +0 -0
- package/lib/bin/attention-worker.exe.json +1 -1
- package/lib/core/bin/dsh-computer-use-worker.exe +0 -0
- package/lib/core/bin/dsh-computer-use-worker.exe.json +2 -2
- package/lib/core/index.js +51 -3
- package/lib/core/worker.cs +467 -18
- package/lib/host-debug.js +88 -0
- package/lib/image-output.js +28 -6
- package/lib/index.js +104 -0
- package/lib/launch.js +67 -0
- package/lib/policy.js +22 -2
- package/lib/tools.js +74 -8
- package/package.json +101 -91
- package/scripts/check-behavior.mjs +1 -1
- package/scripts/verify-package.mjs +9 -0
- package/skills/computer-use/SKILL.md +59 -50
|
@@ -0,0 +1,88 @@
|
|
|
1
|
+
// host-debug.js — the two gates that decide whether the agent may drive its OWN Harness window.
|
|
2
|
+
//
|
|
3
|
+
// WHY THIS EXISTS
|
|
4
|
+
// The host guard refuses every driving op aimed at the DSH window. That is right for ordinary
|
|
5
|
+
// work (the agent must never approve its own approvals, type as the human, or press Stop), but it
|
|
6
|
+
// also makes self-iteration — debugging another plugin inside the same instance — impossible.
|
|
7
|
+
// The answer is not to weaken the default; it is an explicit DEBUG MODE with two gates:
|
|
8
|
+
//
|
|
9
|
+
// 1. the CALLING CONVERSATION has full permission: its sandbox mode is danger-full-access;
|
|
10
|
+
// 2. debug mode was switched on BY HAND in Settings → 电脑操控(调试).
|
|
11
|
+
//
|
|
12
|
+
// Neither gate can be turned on from inside a tool call: gate 1 is the session's own policy (the
|
|
13
|
+
// human sets it in the composer) and gate 2 is a host-side file that only the settings page
|
|
14
|
+
// writes. Tools can read both; they cannot write either.
|
|
15
|
+
//
|
|
16
|
+
// SETTINGS PAGE CONTRACT (host side):
|
|
17
|
+
// GET /api/computer-use/host-debug -> { enabled, since, session:{id,sandbox,fullAccess}, lastBlockedAt }
|
|
18
|
+
// POST /api/computer-use/host-debug -> { enabled: boolean } (writes the file, returns the same shape)
|
|
19
|
+
import { existsSync, mkdirSync, readFileSync, statSync, writeFileSync } from 'node:fs'
|
|
20
|
+
import path from 'node:path'
|
|
21
|
+
|
|
22
|
+
const dshHome = () => process.env.DSH_HOME || path.join(process.env.USERPROFILE || process.env.HOME || '.', '.dsh')
|
|
23
|
+
|
|
24
|
+
/** Where the debug switch lives. Deliberately NOT inside any profile: one machine, one switch. */
|
|
25
|
+
export const HOST_DEBUG_FILE = () => path.join(dshHome(), 'data', 'computer-use', 'host-debug.json')
|
|
26
|
+
|
|
27
|
+
function readJson (file) {
|
|
28
|
+
try { return JSON.parse(readFileSync(file, 'utf8')) } catch { return null }
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
/** Hot read; the worker also polls this file (500 ms cache) so a toggle needs no restart. */
|
|
32
|
+
export function readHostDebug () {
|
|
33
|
+
const j = readJson(HOST_DEBUG_FILE())
|
|
34
|
+
if (!j || typeof j !== 'object') return { enabled: false, since: null, lastBlockedAt: null }
|
|
35
|
+
return {
|
|
36
|
+
enabled: j.enabled === true,
|
|
37
|
+
since: typeof j.since === 'string' ? j.since : null,
|
|
38
|
+
lastBlockedAt: typeof j.lastBlockedAt === 'string' ? j.lastBlockedAt : null,
|
|
39
|
+
}
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
export function writeHostDebug (enabled) {
|
|
43
|
+
const file = HOST_DEBUG_FILE()
|
|
44
|
+
mkdirSync(path.dirname(file), { recursive: true })
|
|
45
|
+
const previous = readHostDebug()
|
|
46
|
+
const next = {
|
|
47
|
+
enabled: enabled === true,
|
|
48
|
+
since: enabled === true ? (previous.enabled ? previous.since : new Date().toISOString()) : null,
|
|
49
|
+
lastBlockedAt: previous.lastBlockedAt,
|
|
50
|
+
}
|
|
51
|
+
writeFileSync(file, JSON.stringify(next, null, 2) + '\n', 'utf8')
|
|
52
|
+
return next
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
/** Record that a host-driving call was refused by a gate, so the settings page can say so. */
|
|
56
|
+
export function noteHostBlocked () {
|
|
57
|
+
try {
|
|
58
|
+
const file = HOST_DEBUG_FILE()
|
|
59
|
+
const current = readHostDebug()
|
|
60
|
+
if (!existsSync(file)) return
|
|
61
|
+
writeFileSync(file, JSON.stringify({ ...current, lastBlockedAt: new Date().toISOString() }, null, 2) + '\n', 'utf8')
|
|
62
|
+
} catch { /* a failed note must never break the refusal itself */ }
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
const SESSION_ROOTS = () => [
|
|
66
|
+
path.join(dshHome(), 'storages', 'session_projcache', 'sessions'),
|
|
67
|
+
path.join(dshHome(), 'storages', 'sessions'),
|
|
68
|
+
path.join(dshHome(), 'sessions'),
|
|
69
|
+
]
|
|
70
|
+
|
|
71
|
+
/** The session record is a live cache under DSH_HOME; find the one for this id (cheap, per call). */
|
|
72
|
+
function sessionRecord (sessionId) {
|
|
73
|
+
if (!sessionId || typeof sessionId !== 'string') return null
|
|
74
|
+
for (const root of SESSION_ROOTS()) {
|
|
75
|
+
const file = path.join(root, `${sessionId}.json`)
|
|
76
|
+
if (existsSync(file)) return readJson(file)
|
|
77
|
+
}
|
|
78
|
+
return null
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
/** Gate 1: does the CALLING conversation hold full permission? Unknown => false (fail closed). */
|
|
82
|
+
export function sessionPermission (sessionId) {
|
|
83
|
+
const rec = sessionRecord(sessionId)
|
|
84
|
+
const rows = rec?.record?.rows
|
|
85
|
+
const sandbox = rows?.permissions?.val?.sandbox || rows?.sandboxMode?.val || null
|
|
86
|
+
const approval = rows?.permissions?.val?.approval || rows?.approvalState?.val?.policy || null
|
|
87
|
+
return { id: sessionId || null, sandbox, approval, fullAccess: sandbox === 'danger-full-access' }
|
|
88
|
+
}
|
package/lib/image-output.js
CHANGED
|
@@ -43,10 +43,16 @@ export async function attachScreenshot (ctx, exec, value, data, options = {}) {
|
|
|
43
43
|
value.imageReason = reason
|
|
44
44
|
value.view = reason === 'model-text-only'
|
|
45
45
|
? 'Image not attached: this model does not declare image input. Use UIA/text or switch to an image-capable model.'
|
|
46
|
-
:
|
|
46
|
+
: reason === 'delivery-path'
|
|
47
|
+
? 'Path delivery requested: no image block was attached. The file is on path; call read_image only if visual inspection is actually needed.'
|
|
48
|
+
: 'Image not attached. Use read_image on path if visual inspection is needed.'
|
|
47
49
|
if (value.settled === false) value.view = 'NOT settled — re-shoot. ' + value.view
|
|
48
50
|
return value
|
|
49
51
|
}
|
|
52
|
+
// delivery:"path" — the caller has another way to verify (UIA/OCR/window text), so the pixels
|
|
53
|
+
// are not evidence it will read. The file is still written and the path still returned; only
|
|
54
|
+
// the image block is withheld, which is what keeps it out of the re-sent conversation history.
|
|
55
|
+
if (options.delivery === 'path') return fallback('delivery-path')
|
|
50
56
|
try {
|
|
51
57
|
if (!validImageSize(value.imageSize)) return fallback('invalid-source-image-size')
|
|
52
58
|
const sourceSize = [...value.imageSize]
|
|
@@ -107,7 +113,12 @@ export function renderScreenshot (value) {
|
|
|
107
113
|
|
|
108
114
|
// A batch exposes one latest image as its checkpoint. Earlier explicit captures
|
|
109
115
|
// keep their paths for deliberate later reads, without multiplying vision input.
|
|
110
|
-
|
|
116
|
+
//
|
|
117
|
+
// `delivery` is the batch-level request, and it has to reach further than the checkpoint: when
|
|
118
|
+
// no checkpoint happens (shot:"never", a readback tail, or a later failure) the observation
|
|
119
|
+
// falls back to the last explicit step image. A batch that asked for paths must not hand one
|
|
120
|
+
// back as a picture just because the checkpoint was skipped — same pixels, quieter route.
|
|
121
|
+
export function selectBatchScreenshot (result, automatic, kind = 'image', delivery = 'auto') {
|
|
111
122
|
let latest = automatic
|
|
112
123
|
let index
|
|
113
124
|
if (!latest) {
|
|
@@ -118,19 +129,30 @@ export function selectBatchScreenshot (result, automatic, kind = 'image') {
|
|
|
118
129
|
}
|
|
119
130
|
}
|
|
120
131
|
}
|
|
132
|
+
const pathDelivery = delivery === 'path'
|
|
133
|
+
const pathNote = 'Path delivery requested: no image block was attached. Use read_image on this path only if these pixels are the evidence.'
|
|
121
134
|
for (const action of result.actions) {
|
|
122
135
|
const shot = action.result
|
|
123
136
|
if (shot?.image && shot !== latest) {
|
|
124
137
|
delete shot.image
|
|
125
138
|
restoreFileGeometry(shot)
|
|
126
139
|
shot.imageStatus = 'path-only'
|
|
127
|
-
shot.imageReason = 'earlier-batch-image'
|
|
128
|
-
shot.view =
|
|
129
|
-
?
|
|
130
|
-
:
|
|
140
|
+
shot.imageReason = pathDelivery ? 'delivery-path' : 'earlier-batch-image'
|
|
141
|
+
shot.view = pathDelivery
|
|
142
|
+
? pathNote
|
|
143
|
+
: kind === 'state'
|
|
144
|
+
? 'A newer UIA state is returned. This earlier image remains at its path.'
|
|
145
|
+
: 'Only the latest batch screenshot is attached. Use read_image on this path if this earlier frame is needed.'
|
|
131
146
|
}
|
|
132
147
|
}
|
|
133
148
|
if (latest) {
|
|
149
|
+
if (pathDelivery && latest.image) {
|
|
150
|
+
delete latest.image
|
|
151
|
+
restoreFileGeometry(latest)
|
|
152
|
+
latest.imageStatus = 'path-only'
|
|
153
|
+
latest.imageReason = 'delivery-path'
|
|
154
|
+
latest.view = pathNote
|
|
155
|
+
}
|
|
134
156
|
result.observation = latest
|
|
135
157
|
if (kind === 'state') result.observationSource = 'automatic final UIA state'
|
|
136
158
|
else result.imageSource = automatic ? 'automatic final screenshot' : `action ${index + 1}; captured before any later actions`
|
package/lib/index.js
CHANGED
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { installApprovalAttribution } from './approval-attribution.js'
|
|
2
|
+
import { HOST_DEBUG_FILE, noteHostBlocked, readHostDebug, sessionPermission, writeHostDebug } from './host-debug.js'
|
|
2
3
|
import { installHumanAttention } from './attention.js'
|
|
3
4
|
// dsh-computer-use — DSH Cordis plugin
|
|
4
5
|
//
|
|
@@ -410,6 +411,87 @@ export const Config = Schema.object({
|
|
|
410
411
|
]).default('stop').description('legacy setting; ignored — ordinary completion releases computer use automatically'),
|
|
411
412
|
})
|
|
412
413
|
|
|
414
|
+
// ---- host-driving debug gates: the settings page talks to this same-origin route ----
|
|
415
|
+
// GET returns the two gates' state for a session; POST writes ONLY the debug switch. Neither gate
|
|
416
|
+
// can be flipped from a tool call: the route is host-side and the switch is a file a tool cannot
|
|
417
|
+
// write (the agent has no filesystem tool that could reach it without the user seeing it in
|
|
418
|
+
// tool output; see lib/host-debug.js for the contract).
|
|
419
|
+
function installHostDebugRoute (ctx, log) {
|
|
420
|
+
if (typeof ctx?.inject !== 'function') return
|
|
421
|
+
ctx.inject(['webServer'], (sctx) => {
|
|
422
|
+
try {
|
|
423
|
+
sctx.effect(() => sctx.webServer.register({
|
|
424
|
+
kind: 'prefix',
|
|
425
|
+
path: '/api/computer-use/host-debug',
|
|
426
|
+
handler: (req, res) => handleHostDebug(req, res, sctx),
|
|
427
|
+
}), 'dsh-computer-use: host-debug settings route')
|
|
428
|
+
} catch (e) { log.info('[dsh-computer-use] host-debug route not mounted: ' + (e?.message ?? e)) }
|
|
429
|
+
})
|
|
430
|
+
}
|
|
431
|
+
|
|
432
|
+
function handleHostDebug (req, res, sctx) {
|
|
433
|
+
const send = (code, body) => { try { res.statusCode = code; res.setHeader('content-type', 'application/json; charset=utf-8'); res.end(JSON.stringify(body)) } catch { } }
|
|
434
|
+
try {
|
|
435
|
+
let connection = null
|
|
436
|
+
try { connection = sctx.get('connection') } catch { connection = null }
|
|
437
|
+
const rejection = connection && typeof connection.requestRejection === 'function' ? connection.requestRejection(req) : null
|
|
438
|
+
if (rejection) { send(403, { error: 'rejected by host', detail: String(rejection) }); return }
|
|
439
|
+
const url = new URL(req.url, 'http://127.0.0.1')
|
|
440
|
+
const sid = url.searchParams.get('session') || process.env.DSH_SESSION_ID || null
|
|
441
|
+
const status = () => {
|
|
442
|
+
const debug = readHostDebug()
|
|
443
|
+
const perm = sessionPermission(sid)
|
|
444
|
+
return { enabled: debug.enabled, since: debug.since, lastBlockedAt: debug.lastBlockedAt, session: perm, ready: debug.enabled && perm.fullAccess }
|
|
445
|
+
}
|
|
446
|
+
if (req.method === 'GET') { send(200, status()); return }
|
|
447
|
+
if (req.method === 'POST') {
|
|
448
|
+
const chunks = []
|
|
449
|
+
req.on('data', (c) => { if (chunks.length < 64) chunks.push(c) })
|
|
450
|
+
req.on('end', () => {
|
|
451
|
+
try {
|
|
452
|
+
const body = JSON.parse(Buffer.concat(chunks).toString('utf8') || '{}')
|
|
453
|
+
if (typeof body.enabled !== 'boolean') { send(400, { error: 'enabled must be a boolean' }); return }
|
|
454
|
+
writeHostDebug(body.enabled)
|
|
455
|
+
send(200, status())
|
|
456
|
+
} catch (e) { send(400, { error: String(e?.message ?? e) }) }
|
|
457
|
+
})
|
|
458
|
+
return
|
|
459
|
+
}
|
|
460
|
+
send(405, { error: 'GET or POST' })
|
|
461
|
+
} catch (e) { send(500, { error: String(e?.message ?? e) }) }
|
|
462
|
+
}
|
|
463
|
+
|
|
464
|
+
// ---- gate notice: one card, not a stream of them --------------------------------------------
|
|
465
|
+
// A refused host-driving call is the moment the human can act, and the tool error alone is easy to
|
|
466
|
+
// miss inside a long turn. This pops ONE ask card (10-minute dedupe, never blocks the failed tool
|
|
467
|
+
// call) that names both gates and where to open them. It is a NOTICE: choosing "去开启" only tells
|
|
468
|
+
// the human where the switch is — the plugin must never flip either gate itself.
|
|
469
|
+
let gateNoticeAt = 0
|
|
470
|
+
let gateNoticePending = false
|
|
471
|
+
function notifyHostGate (log) {
|
|
472
|
+
try {
|
|
473
|
+
if (gateNoticePending) return
|
|
474
|
+
if (Date.now() - gateNoticeAt < 10 * 60_000) return
|
|
475
|
+
if (!asker || typeof asker.ask !== 'function') return
|
|
476
|
+
gateNoticePending = true
|
|
477
|
+
gateNoticeAt = Date.now()
|
|
478
|
+
Promise.resolve()
|
|
479
|
+
.then(() => asker.ask({
|
|
480
|
+
questions: [{
|
|
481
|
+
id: 'computer_use_host_gate',
|
|
482
|
+
header: '电脑操控 · 调试门禁',
|
|
483
|
+
question: '检测到自迭代调试被门禁拦住:需要 ① 本对话为「完全权限」,② 在 设置 → 电脑操控(调试) 中手动打开调试模式。两项都满足后,AI 才能点击/输入 Harness 自身的窗口。',
|
|
484
|
+
options: [
|
|
485
|
+
{ label: '我去开启调试模式', description: '打开 设置 → 电脑操控(调试) 并切换开关;本对话还需保持「完全权限」。' },
|
|
486
|
+
{ label: '先不开启,做别的', description: 'AI 继续只读观察 Harness,不驱动它;关闭/最小化宿主始终禁止。' },
|
|
487
|
+
],
|
|
488
|
+
}],
|
|
489
|
+
}))
|
|
490
|
+
.catch(() => { })
|
|
491
|
+
.finally(() => { gateNoticePending = false })
|
|
492
|
+
} catch { gateNoticePending = false }
|
|
493
|
+
}
|
|
494
|
+
|
|
413
495
|
export function apply (ctx, config) {
|
|
414
496
|
const log = ctx.logger('dsh-computer-use')
|
|
415
497
|
if (process.env.DSH_COMPUTER_USE_KILL === '1') {
|
|
@@ -418,6 +500,11 @@ export function apply (ctx, config) {
|
|
|
418
500
|
}
|
|
419
501
|
if (!config.enabled) return
|
|
420
502
|
installApprovalAttribution(ctx, log)
|
|
503
|
+
// Harness compatibility: several fixtures evaluate this module with the `import` lines stripped
|
|
504
|
+
// and only a fixed argument list (test-cycle-no-timeout and friends). `typeof <binding>` is safe
|
|
505
|
+
// even when the binding was stripped, so the debug-gate wiring degrades to "not mounted" there
|
|
506
|
+
// instead of throwing a ReferenceError — the production module always has the real imports.
|
|
507
|
+
try { if (typeof installHostDebugRoute === 'function') installHostDebugRoute(ctx, log) } catch { }
|
|
421
508
|
registerBundledSkill(ctx, log)
|
|
422
509
|
enableHumanAttention(ctx, log)
|
|
423
510
|
// THE QUESTION SERVICE, INJECTED LAZILY (merge-design.md §5.3 item 16 — defence in depth).
|
|
@@ -436,12 +523,29 @@ export function apply (ctx, config) {
|
|
|
436
523
|
})
|
|
437
524
|
|
|
438
525
|
const dataDir = mkDataDir(config.snapshotDir)
|
|
526
|
+
// The two host-driving gates are resolved PER CALL: the debug switch is hot (a settings toggle
|
|
527
|
+
// needs no restart), and gate 1 is the CALLING conversation's own policy, not the host's.
|
|
528
|
+
const gateSessionId = (context) => {
|
|
529
|
+
try {
|
|
530
|
+
return context?.sessionKey || (typeof context?.session === 'string' ? context.session : context?.session?.id) ||
|
|
531
|
+
context?.sessionId || process.env.DSH_SESSION_ID || null
|
|
532
|
+
} catch { return process.env.DSH_SESSION_ID || null }
|
|
533
|
+
}
|
|
439
534
|
const cu = new ComputerUse({
|
|
440
535
|
workerExe: config.workerExe || undefined,
|
|
441
536
|
dryRun: config.dryRun,
|
|
442
537
|
kickoffRequired: config.automationMode !== 'read-only',
|
|
443
538
|
rateLimit: { max: config.maxActionsPerMinute, windowMs: 60_000 },
|
|
444
539
|
log: (m) => log.info(m),
|
|
540
|
+
hostDebugFile: typeof HOST_DEBUG_FILE === 'function' ? HOST_DEBUG_FILE() : '',
|
|
541
|
+
hostGate: typeof readHostDebug === 'function' && typeof sessionPermission === 'function'
|
|
542
|
+
? (context) => {
|
|
543
|
+
try { return { debug: readHostDebug().enabled, full: sessionPermission(gateSessionId(context)).fullAccess } } catch { return null }
|
|
544
|
+
}
|
|
545
|
+
: null,
|
|
546
|
+
onHostBlocked: typeof noteHostBlocked === 'function'
|
|
547
|
+
? () => { try { noteHostBlocked() } catch { } notifyHostGate(log) }
|
|
548
|
+
: null,
|
|
445
549
|
})
|
|
446
550
|
const detachAudit = installAudit(cu, dataDir)
|
|
447
551
|
|
package/lib/launch.js
ADDED
|
@@ -0,0 +1,67 @@
|
|
|
1
|
+
// launch.js — validation for computer_launch. EXECUTION lives in the worker (lib/core/worker.cs
|
|
2
|
+
// `launch` op) on purpose: a launch must go through the SAME funnel as every other actuation —
|
|
3
|
+
// brake/pause checks, the status halo, the audit JSONL and the rate window — and the human has
|
|
4
|
+
// to SEE that the machine was just acted on. A Node-side spawn skipped all of that (measured
|
|
5
|
+
// 2026-11-07: the halo never lit, so nothing told the human an app was opening).
|
|
6
|
+
//
|
|
7
|
+
// Validation still lives here so an impossible target is refused BEFORE dispatch: the receipt
|
|
8
|
+
// says outcome:"not-dispatched", and that has to be true. The policy layer also needs the string.
|
|
9
|
+
//
|
|
10
|
+
// RESOLUTION IS ShellExecute's (worker: ProcessStartInfo.UseShellExecute). Measured on this
|
|
11
|
+
// machine: `explorer.exe calc` opens a FILE EXPLORER window (a bare word is a location to
|
|
12
|
+
// explorer), while ShellExecute("calc") starts Calculator — so the launcher, not a shell
|
|
13
|
+
// wrapper, is the correct resolver for App Paths aliases, shell: verbs, URLs and associations.
|
|
14
|
+
//
|
|
15
|
+
// The target crosses as ONE value: no command line is composed here or in the worker, and no
|
|
16
|
+
// shell interpreter is involved. The deny-list below is defence in depth for anything
|
|
17
|
+
// downstream that might interpolate the string — the boundary is the single-value call.
|
|
18
|
+
const MAX_LEN = 2000
|
|
19
|
+
|
|
20
|
+
// Control characters: C0 (includes \r \n \t), DEL, C1, and the Unicode line/paragraph
|
|
21
|
+
// separators (U+2028/U+2029 still break line discipline in several parsers). None has an
|
|
22
|
+
// honest use in a path, URL or alias.
|
|
23
|
+
const CONTROL = /[\u0000-\u001f\u007f-\u009f\u2028\u2029]/
|
|
24
|
+
|
|
25
|
+
// Metacharacters for a shell or a string interpolator. Percent is deliberately NOT here:
|
|
26
|
+
// "%20"-style percent-encoding is a legitimate, unavoidable part of URLs, and no shell is
|
|
27
|
+
// involved, so there is no %VAR% expansion to fear.
|
|
28
|
+
const INJECTIVE = /["'&|;<>$`^]/
|
|
29
|
+
|
|
30
|
+
const badTarget = (why) => {
|
|
31
|
+
throw Object.assign(new Error(`bad launch target: ${why}`), { code: 'LAUNCH_BAD_TARGET', outcome: 'not-dispatched' })
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
/** Trim and validate a launch target; throws LAUNCH_BAD_TARGET (not-dispatched) when impossible.
|
|
35
|
+
* Everything is checked on the TRIMMED string, i.e. on exactly what the worker will receive. */
|
|
36
|
+
export function validateLaunchTarget (target) {
|
|
37
|
+
if (typeof target !== 'string') badTarget('must be a string')
|
|
38
|
+
const t = target.trim()
|
|
39
|
+
if (!t) badTarget('empty (or only whitespace)')
|
|
40
|
+
if (t.length > MAX_LEN) badTarget(`longer than ${MAX_LEN} characters`)
|
|
41
|
+
const c = CONTROL.exec(t)
|
|
42
|
+
if (c) badTarget(`control character U+${c[0].codePointAt(0).toString(16).toUpperCase().padStart(4, '0')}`)
|
|
43
|
+
const m = INJECTIVE.exec(t)
|
|
44
|
+
if (m) badTarget(`forbidden character ${JSON.stringify(m[0])}`)
|
|
45
|
+
return t
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
const badArgs = (why) => {
|
|
49
|
+
throw Object.assign(new Error(`bad launch arguments: ${why}`), { code: 'LAUNCH_BAD_ARGS', outcome: 'not-dispatched' })
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
/** Validate the optional argument array. Each entry is ONE argument VALUE — never a command
|
|
53
|
+
* line, never shell syntax: the worker quotes each entry exactly once. Spaces, backslashes and
|
|
54
|
+
* `--flag=value` are ordinary; only NUL, control characters and empties are impossible. */
|
|
55
|
+
export function validateLaunchArgs (args) {
|
|
56
|
+
if (args === undefined || args === null) return undefined
|
|
57
|
+
if (!Array.isArray(args)) badArgs('must be an array of strings')
|
|
58
|
+
if (args.length > 64) badArgs('more than 64 arguments')
|
|
59
|
+
return args.map((value, index) => {
|
|
60
|
+
if (typeof value !== 'string') badArgs(`argument ${index} must be a string`)
|
|
61
|
+
if (value.length === 0) badArgs(`argument ${index} is empty`)
|
|
62
|
+
if (value.length > MAX_LEN) badArgs(`argument ${index} longer than ${MAX_LEN} characters`)
|
|
63
|
+
const c = CONTROL.exec(value)
|
|
64
|
+
if (c) badArgs(`argument ${index} contains control character U+${c[0].codePointAt(0).toString(16).toUpperCase().padStart(4, '0')}`)
|
|
65
|
+
return value
|
|
66
|
+
})
|
|
67
|
+
}
|
package/lib/policy.js
CHANGED
|
@@ -27,9 +27,18 @@ export const OBSERVATION_TOOLS = new Set([
|
|
|
27
27
|
export const ACTUATION_TOOLS = new Set([
|
|
28
28
|
'computer_click', 'computer_move', 'computer_drag', 'computer_scroll', 'computer_select',
|
|
29
29
|
'computer_key', 'computer_type', 'computer_uia_act', 'computer_window', 'computer_batch',
|
|
30
|
+
'computer_launch', 'computer_focus',
|
|
30
31
|
])
|
|
31
32
|
|
|
32
33
|
export const HIGH_RISK_NOTE = 'high-risk action (destructive key / closing a window / password field) — always requires human approval'
|
|
34
|
+
export const LAUNCH_HIGH_RISK_NOTE = 'high-risk launch (executable / script / installer / UNC / file: target) — explorer will run what it resolves; always requires human approval'
|
|
35
|
+
|
|
36
|
+
// A launch target that IS code (or points at it) escalates everywhere it can: the launcher
|
|
37
|
+
// executes what it resolves, so argv discipline cannot make an .exe/.bat/UNC target "just a
|
|
38
|
+
// file". Extensions only by design: bare App-Paths aliases (calc, cmd) stay allowed — the
|
|
39
|
+
// measured trap here was the opposite direction (explorer opened a FOLDER for an alias), and
|
|
40
|
+
// a path spelling of the same binary asks while an alias does not carry arguments either way.
|
|
41
|
+
const LAUNCH_CODE_TARGET = /\.(exe|scr|msi|hta|bat|cmd|com|pif|cpl|msc|reg|jse|vbe|vbs|ws|wsf|wsh|ps1|lnk|url|jar|application|appref-ms|appx|msix|gadget|sct)$/i
|
|
33
42
|
|
|
34
43
|
/** @returns {{kind:'allow'|'deny'|'ask', reason?: string}} */
|
|
35
44
|
export function policyDecision (name, args, mode) {
|
|
@@ -76,14 +85,15 @@ export function policyDecision (name, args, mode) {
|
|
|
76
85
|
function decideActuation (name, a, mode) {
|
|
77
86
|
let hr
|
|
78
87
|
try { hr = highRisk(name, a) } catch (error) { return { kind: 'deny', reason: error.message } }
|
|
88
|
+
const note = name === 'computer_launch' ? LAUNCH_HIGH_RISK_NOTE : HIGH_RISK_NOTE
|
|
79
89
|
switch (mode) {
|
|
80
90
|
case 'read-only':
|
|
81
91
|
return { kind: 'deny', reason: `automationMode=read-only: ${name} drives the machine` }
|
|
82
92
|
case 'standard':
|
|
83
|
-
return hr ? { kind: 'ask', reason:
|
|
93
|
+
return hr ? { kind: 'ask', reason: note + riskSuffix(name, a) }
|
|
84
94
|
: { kind: 'ask', reason: `automationMode=standard: actuation ${name}` }
|
|
85
95
|
case 'autonomous':
|
|
86
|
-
return hr ? { kind: 'ask', reason:
|
|
96
|
+
return hr ? { kind: 'ask', reason: note + riskSuffix(name, a) } : { kind: 'allow' }
|
|
87
97
|
case 'unrestricted':
|
|
88
98
|
return { kind: 'allow' }
|
|
89
99
|
default:
|
|
@@ -98,6 +108,7 @@ function riskSuffix (name, a) {
|
|
|
98
108
|
if (name === 'computer_key') return ` [combo=${a.combo}]`
|
|
99
109
|
if (name === 'computer_window') return ` [op=${a.op}]`
|
|
100
110
|
if (name === 'computer_type') return ' [allowPassword]'
|
|
111
|
+
if (name === 'computer_launch') return ` [target=${String(a.target || '').slice(0, 80)}]`
|
|
101
112
|
if (name === 'computer_batch') {
|
|
102
113
|
const actions = Array.isArray(a.actions) ? a.actions : []
|
|
103
114
|
for (let i = 0; i < actions.length; i++) {
|
|
@@ -118,6 +129,15 @@ export function highRisk (name, args) {
|
|
|
118
129
|
}
|
|
119
130
|
if (name === 'computer_window' && String(a.op || 'activate') === 'close') return true
|
|
120
131
|
if (name === 'computer_type' && a.allowPassword) return true
|
|
132
|
+
if (name === 'computer_launch') {
|
|
133
|
+
// Win32 strips trailing dots/spaces/ADS-colons when resolving, so "x.exe." RUNS as x.exe —
|
|
134
|
+
// normalize before matching or the $ anchor is bypassable. http(s) targets lose ?query and
|
|
135
|
+
// #fragment first so "x.exe?dl=1" is judged exactly like "x.exe". Forward slashes map to
|
|
136
|
+
// backslashes at resolve time, so //server/share is the UNC it will become.
|
|
137
|
+
let target = String(a.target || '').trim().replace(/[.\s:]+$/, '')
|
|
138
|
+
if (/^\w+:\/\//i.test(target)) target = target.replace(/[?#].*$/, '')
|
|
139
|
+
if (LAUNCH_CODE_TARGET.test(target) || /^\\\\|^\/\//.test(target) || /^file:/i.test(target)) return true
|
|
140
|
+
}
|
|
121
141
|
if (name === 'computer_batch') {
|
|
122
142
|
const actions = Array.isArray(a.actions) ? a.actions : []
|
|
123
143
|
for (const step of actions) {
|
package/lib/tools.js
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
// tools.js — defineTool registrations for dsh-computer-use.
|
|
2
2
|
//
|
|
3
|
-
//
|
|
3
|
+
// 20 TOOLS. The previous 28 were merged, not deleted; every capability still exists and the
|
|
4
4
|
// behavioural detail lives in the skill (loaded only for computer-use work, while these
|
|
5
5
|
// definitions are paid for on EVERY request).
|
|
6
6
|
//
|
|
@@ -22,11 +22,13 @@ import { ObservationStore } from './observations.js'
|
|
|
22
22
|
import { feedbackPolicy, feedbackActions, feedbackVerification, selectFeedbackDetail } from './feedback.js'
|
|
23
23
|
import { collectCompactState, limitCompactFeedback, feedbackViewport } from './compact-feedback.js'
|
|
24
24
|
import { handoffResult } from './continuation.js'
|
|
25
|
+
import { validateLaunchArgs, validateLaunchTarget } from './launch.js'
|
|
25
26
|
import { join } from 'node:path'
|
|
26
27
|
|
|
27
28
|
const P = (type, description, extra = {}) => ({ type, description, ...extra })
|
|
28
29
|
const cooperativeTools = new Set(['computer_click', 'computer_move', 'computer_drag', 'computer_scroll',
|
|
29
|
-
'computer_select', 'computer_key', 'computer_type', 'computer_window', 'computer_uia_act', 'computer_batch', 'computer_clip'
|
|
30
|
+
'computer_select', 'computer_key', 'computer_type', 'computer_window', 'computer_uia_act', 'computer_batch', 'computer_clip',
|
|
31
|
+
'computer_focus'])
|
|
30
32
|
const usesInput = (name, args = {}) => name === 'computer_clip' ? args.text !== undefined :
|
|
31
33
|
name === 'computer_batch' ? (args.actions || []).some(step => usesInput(step.tool, step.args)) :
|
|
32
34
|
cooperativeTools.has(name)
|
|
@@ -73,8 +75,21 @@ export function registerTools ({ ctx, cu, config, dataDir, log, snapshotPath, de
|
|
|
73
75
|
return saveOwnedShot(dataDir, p, buf)
|
|
74
76
|
}
|
|
75
77
|
|
|
78
|
+
// Per-call image delivery. `auto` is the existing behaviour and stays the default; `path`
|
|
79
|
+
// withholds the image block without withholding the file. An unknown value refuses here
|
|
80
|
+
// rather than silently behaving like auto — a typo must not quietly reintroduce the cost the
|
|
81
|
+
// caller asked to avoid.
|
|
82
|
+
const deliveryMode = (value) => {
|
|
83
|
+
if (value === undefined || value === 'auto') return 'auto'
|
|
84
|
+
if (value === 'path') return 'path'
|
|
85
|
+
throw Object.assign(new Error('delivery must be "auto" or "path"'), { code: 'BAD_DELIVERY', outcome: 'not-dispatched' })
|
|
86
|
+
}
|
|
87
|
+
|
|
76
88
|
// Shared by direct calls, explicit batch steps, and the batch checkpoint.
|
|
77
89
|
const takeScreenshot = async (args, exec) => {
|
|
90
|
+
// Refuse a bad value BEFORE the capture: a refused request that already wrote a file and
|
|
91
|
+
// reports outcome:"not-dispatched" is a false receipt.
|
|
92
|
+
const delivery = deliveryMode(args.delivery)
|
|
78
93
|
const r = rect(args.region)
|
|
79
94
|
const shot = await cu.screenshot({
|
|
80
95
|
x: r && r.x, y: r && r.y, width: r && r.width, height: r && r.height,
|
|
@@ -83,7 +98,7 @@ export function registerTools ({ ctx, cu, config, dataDir, log, snapshotPath, de
|
|
|
83
98
|
})
|
|
84
99
|
const path = saveShot(shot.image, shot.mime === 'image/png' ? 'png' : 'jpg')
|
|
85
100
|
const res = { path, mime: shot.mime, region: shot.region, ...screenshotMetadata(shot) }
|
|
86
|
-
return attachScreenshot(ctx, exec, res, shot.image, { rules: config.feedbackRules })
|
|
101
|
+
return attachScreenshot(ctx, exec, res, shot.image, { rules: config.feedbackRules, delivery })
|
|
87
102
|
}
|
|
88
103
|
|
|
89
104
|
const clickTarget = async (args) => {
|
|
@@ -149,7 +164,7 @@ export function registerTools ({ ctx, cu, config, dataDir, log, snapshotPath, de
|
|
|
149
164
|
if (exec?.signal?.aborted || cu.stopped || cu.exited || cu.cooperation?.active) return true
|
|
150
165
|
try { return !!hooks?.askBlocked?.('computer_shot') } catch { return true }
|
|
151
166
|
}
|
|
152
|
-
const captureFeedback = async (exec, policy, context = {}) => {
|
|
167
|
+
const captureFeedback = async (exec, policy, context = {}, delivery = 'auto') => {
|
|
153
168
|
// Route resolution is async: the human can brake while it is pending.
|
|
154
169
|
if (feedbackBlocked(exec)) return { observationSkipped: 'cancelled-or-paused' }
|
|
155
170
|
const compact = policy.detail === 'compact'
|
|
@@ -160,7 +175,7 @@ export function registerTools ({ ctx, cu, config, dataDir, log, snapshotPath, de
|
|
|
160
175
|
const observation = policy.kind === 'state'
|
|
161
176
|
? compact ? await collectCompactState(cu, { ...context, checkAllowed,
|
|
162
177
|
archive: raw => hooks?.archiveFeedback ? hooks.archiveFeedback(exec, raw) : archiveFeedback(exec, raw) }) : await observeState({}, exec)
|
|
163
|
-
: await takeScreenshot(compact ? { region: feedbackViewport(context.args, context.receipt).region } : {}, exec)
|
|
178
|
+
: await takeScreenshot(compact ? { region: feedbackViewport(context.args, context.receipt).region, delivery } : { delivery }, exec)
|
|
164
179
|
const envelope = { observation, feedback: policy, verification: feedbackVerification }
|
|
165
180
|
if (compact && policy.kind === 'state' && observation.inlineEvidence) {
|
|
166
181
|
// Resource fallback deliberately favors recoverable evidence over the
|
|
@@ -354,6 +369,7 @@ export function registerTools ({ ctx, cu, config, dataDir, log, snapshotPath, de
|
|
|
354
369
|
preset: P('string', 'taskbar'),
|
|
355
370
|
png: P('boolean', 'PNG instead of JPEG'),
|
|
356
371
|
wait: P('string', 'auto (default) | none'),
|
|
372
|
+
delivery: P('string', 'auto (default) | path — path writes the file and returns its absolute path with no image block'),
|
|
357
373
|
},
|
|
358
374
|
output: out((a, v) => renderScreenshot(v)),
|
|
359
375
|
isConcurrencySafe: () => false,
|
|
@@ -496,6 +512,7 @@ export function registerTools ({ ctx, cu, config, dataDir, log, snapshotPath, de
|
|
|
496
512
|
clicks: P('number', '1 | 2 (double) | 3 (triple)'),
|
|
497
513
|
shift: P('boolean', 'hold SHIFT (range selection)'),
|
|
498
514
|
verificationRegion: P('string', 'expected effect region "x,y,w,h" in screen pixels; may differ from click point; a change alone does not prove task success'),
|
|
515
|
+
allowCovered: P('boolean', 'click a pixel owned by the target app\'s OWN embedded renderer (a descendant process of the target, e.g. a WeChat article panel); an unrelated window on top is still refused and the receipt says COVERED-UNRELATED'),
|
|
499
516
|
},
|
|
500
517
|
output: outJSON(),
|
|
501
518
|
isConcurrencySafe: () => false,
|
|
@@ -626,6 +643,49 @@ export function registerTools ({ ctx, cu, config, dataDir, log, snapshotPath, de
|
|
|
626
643
|
async execute (args) { return args.text === undefined ? cu.clipRead() : cu.clipWrite(args.text) },
|
|
627
644
|
})
|
|
628
645
|
|
|
646
|
+
// ---------------- launch ----------------
|
|
647
|
+
// ONE call replaces the Win+R route (activate, combo, type, enter, verify): the worker resolves
|
|
648
|
+
// the target with ShellExecute, so App Paths aliases (calc), shell: folders, URLs, folders and
|
|
649
|
+
// document associations all work. The launch is a real actuation — brake checks, the status
|
|
650
|
+
// halo (with a short self-expiring tail, because a launch returns in milliseconds) and the
|
|
651
|
+
// audit trail all come from the worker op, not from this file.
|
|
652
|
+
reg({
|
|
653
|
+
name: 'computer_launch',
|
|
654
|
+
description: 'Open an app/URL/folder/document in one call — replaces the Win+R route (activate, combo, type, enter). Aliases (calc), shell: folders, http/https URLs, absolute paths. Pass `args` to start an executable with arguments (a dedicated browser profile: target "…\\msedge.exe", args ["--user-data-dir=…", "--new-window", "<url>"]); each entry is one argument value, quoted once in the worker. Launched means the target reached the launcher, not that the app is ready; verify with computer_state.',
|
|
655
|
+
parameters: {
|
|
656
|
+
target: P('string', 'app alias (calc) | shell: folder | http/https URL | absolute path', { required: true }),
|
|
657
|
+
args: P('array', 'optional argument values for an executable target, one array entry per argument (e.g. ["--user-data-dir=D:\\\\profiles\\\\job", "--new-window", "https://…"]); no shell quoting — spaces and backslashes are ordinary'),
|
|
658
|
+
tailMs: P('number', 'how long to keep the status halo lit after the launch, 0–10000 ms (default 2500)'),
|
|
659
|
+
},
|
|
660
|
+
output: outJSON(),
|
|
661
|
+
isConcurrencySafe: () => false,
|
|
662
|
+
timeoutMs: 15_000,
|
|
663
|
+
async execute (args) { return cu.launch(validateLaunchTarget(args.target), { args: validateLaunchArgs(args.args) }) },
|
|
664
|
+
})
|
|
665
|
+
|
|
666
|
+
// ---------------- focus before typing ----------------
|
|
667
|
+
reg({
|
|
668
|
+
name: 'computer_focus',
|
|
669
|
+
description: 'Put keyboard focus on a FIELD and PROVE it before typing: clicks x,y, then reports the focused UIA control and whether a caret is provable there (OS caret, else a two-frame blink check). Typing follows the focused control, not the window — activating a dialog can leave focus in the document, where text is silently swallowed. Type only when verified:true.',
|
|
670
|
+
parameters: {
|
|
671
|
+
x: P('number', 'field centre x in screen pixels', { required: true }),
|
|
672
|
+
y: P('number', 'field centre y in screen pixels', { required: true }),
|
|
673
|
+
hwnd: P('number', 'target window; a modal dialog of the same app is accepted'),
|
|
674
|
+
verifyCaret: P('boolean', 'when the OS reports no caret, fall back to a two-frame blink check (default true)'),
|
|
675
|
+
},
|
|
676
|
+
output: outJSON(),
|
|
677
|
+
isConcurrencySafe: () => false,
|
|
678
|
+
timeoutMs: 15_000,
|
|
679
|
+
async execute (args) {
|
|
680
|
+
return cu.focusField({
|
|
681
|
+
x: args.x,
|
|
682
|
+
y: args.y,
|
|
683
|
+
...(args.hwnd !== undefined ? { hwnd: args.hwnd } : {}),
|
|
684
|
+
...(args.verifyCaret !== undefined ? { verifyCaret: args.verifyCaret } : {}),
|
|
685
|
+
})
|
|
686
|
+
},
|
|
687
|
+
})
|
|
688
|
+
|
|
629
689
|
// ---------------- the agent asks ----------------
|
|
630
690
|
reg({
|
|
631
691
|
name: 'computer_ask',
|
|
@@ -686,6 +746,7 @@ export function registerTools ({ ctx, cu, config, dataDir, log, snapshotPath, de
|
|
|
686
746
|
parameters: {
|
|
687
747
|
actions: P('array', '[{tool:"computer_click", args:{mark:"M3"}}, ...]', { required: true }),
|
|
688
748
|
shot: P('string', 'auto (default) | always | never; no automatic image after failure, cancel or pause'),
|
|
749
|
+
delivery: P('string', 'auto (default) | path — the checkpoint keeps its absolute path but attaches no image'),
|
|
689
750
|
},
|
|
690
751
|
output: out((a, v) => {
|
|
691
752
|
const { observation, ...receipt } = v
|
|
@@ -699,6 +760,9 @@ export function registerTools ({ ctx, cu, config, dataDir, log, snapshotPath, de
|
|
|
699
760
|
isConcurrencySafe: () => false,
|
|
700
761
|
timeoutMs: 120_000,
|
|
701
762
|
async execute (args, exec) {
|
|
763
|
+
// Validate before any step runs: a bad delivery value must not be discovered only at the
|
|
764
|
+
// checkpoint, after the actions have already been dispatched.
|
|
765
|
+
const delivery = deliveryMode(args.delivery)
|
|
702
766
|
let automatic, feedback, lastContext = {}
|
|
703
767
|
const detail = selectFeedbackDetail(args.feedback,
|
|
704
768
|
config.actionFeedback === false ? 'none' : config.actionFeedbackDetail ?? 'compact')
|
|
@@ -719,15 +783,15 @@ export function registerTools ({ ctx, cu, config, dataDir, log, snapshotPath, de
|
|
|
719
783
|
if (feedbackBlocked(exec)) { feedback = { observationSkipped: 'cancelled-or-paused' }; return null }
|
|
720
784
|
if (!policy.kind) { feedback = { feedback: policy, observationSkipped: policy.reason || 'disabled' }; return null }
|
|
721
785
|
hooks?.assertOwnership?.(exec)
|
|
722
|
-
if (policy.kind === 'state') { feedback = await captureFeedback(exec, policy, lastContext); return null }
|
|
723
|
-
const captured = await captureFeedback(exec, policy, lastContext)
|
|
786
|
+
if (policy.kind === 'state') { feedback = await captureFeedback(exec, policy, lastContext, delivery); return null }
|
|
787
|
+
const captured = await captureFeedback(exec, policy, lastContext, delivery)
|
|
724
788
|
const { observation, ...details } = captured
|
|
725
789
|
automatic = observation
|
|
726
790
|
feedback = details
|
|
727
791
|
if (!automatic) return null
|
|
728
792
|
return automatic.path
|
|
729
793
|
} })
|
|
730
|
-
return { ...selectBatchScreenshot(result, feedback?.observation ?? automatic, feedback?.feedback?.kind), ...feedback }
|
|
794
|
+
return { ...selectBatchScreenshot(result, feedback?.observation ?? automatic, feedback?.feedback?.kind, delivery), ...feedback }
|
|
731
795
|
},
|
|
732
796
|
})
|
|
733
797
|
|
|
@@ -798,6 +862,8 @@ export function registerTools ({ ctx, cu, config, dataDir, log, snapshotPath, de
|
|
|
798
862
|
computer_type: (a) => cu.type(a.text, { mode: a.mode, allowPassword: a.allowPassword }),
|
|
799
863
|
computer_window: (a) => ((a.op || 'activate') === 'activate' ? cu.activateWindow({ hwnd: a.hwnd }) : cu.windowOp({ hwnd: a.hwnd }, a.op, rect(a.rect) || {})),
|
|
800
864
|
computer_clip: (a) => (a.text === undefined ? cu.clipRead() : cu.clipWrite(a.text)),
|
|
865
|
+
computer_launch: (a) => cu.launch(validateLaunchTarget(a.target), { args: validateLaunchArgs(a.args) }),
|
|
866
|
+
computer_focus: (a) => cu.focusField({ x: a.x, y: a.y, ...(a.hwnd !== undefined ? { hwnd: a.hwnd } : {}), ...(a.verifyCaret !== undefined ? { verifyCaret: a.verifyCaret } : {}) }),
|
|
801
867
|
computer_uia_act: actUia,
|
|
802
868
|
computer_ask: (a, exec) => askQuestion(a, exec),
|
|
803
869
|
computer_wait: (a) => a.mode === 'stable' ? cu.waitStable({ timeoutMs: a.timeoutMs, hwnd: a.hwnd })
|