dsh-codex-style-computer-use 1.3.1 → 2.1.0

package/lib/tools.js

@@ -1,6 +1,6 @@
 // tools.js — defineTool registrations for dsh-computer-use.
 //
-// 18 TOOLS. The previous 28 were merged, not deleted; every capability still exists and the
+// 20 TOOLS. The previous 28 were merged, not deleted; every capability still exists and the
 // behavioural detail lives in the skill (loaded only for computer-use work, while these
 // definitions are paid for on EVERY request).
 //
@@ -17,18 +17,27 @@
 
 import { runBatch, actionRefusal } from './batch.js'
 import { saveOwnedShot } from './artifacts.js'
-import { attachScreenshot, renderScreenshot, selectBatchScreenshot } from './image-output.js'
+import { attachScreenshot, renderScreenshot, selectBatchScreenshot, screenshotMetadata } from './image-output.js'
 import { ObservationStore } from './observations.js'
-import { feedbackMode, feedbackActions, feedbackVerification } from './feedback.js'
+import { feedbackPolicy, feedbackActions, feedbackVerification, selectFeedbackDetail } from './feedback.js'
+import { collectCompactState, limitCompactFeedback, feedbackViewport } from './compact-feedback.js'
 import { handoffResult } from './continuation.js'
+import { validateLaunchArgs, validateLaunchTarget } from './launch.js'
+import { join } from 'node:path'
 
 const P = (type, description, extra = {}) => ({ type, description, ...extra })
 const cooperativeTools = new Set(['computer_click', 'computer_move', 'computer_drag', 'computer_scroll',
-  'computer_select', 'computer_key', 'computer_type', 'computer_window', 'computer_uia_act', 'computer_batch', 'computer_clip'])
+  'computer_select', 'computer_key', 'computer_type', 'computer_window', 'computer_uia_act', 'computer_batch', 'computer_clip',
+  'computer_focus'])
 const usesInput = (name, args = {}) => name === 'computer_clip' ? args.text !== undefined :
   name === 'computer_batch' ? (args.actions || []).some(step => usesInput(step.tool, step.args)) :
   cooperativeTools.has(name)
 
+export const isCycleStateOnly = (name, args = {}) => name === 'computer_ctrl'
+  ? ['hold', 'free', 'release', 'clear_cache'].includes(args?.action)
+  : name === 'computer_batch' && Array.isArray(args?.actions) && args.actions.length > 0 &&
+    args.actions.every(step => isCycleStateOnly(step?.tool, step?.args))
+
 /** "M3" | "x,y" | "x y" -> {mark:'M3'} | {x,y}. One parameter instead of four. */
 const pt = (v) => {
   if (v === undefined || v === null) return null
@@ -49,14 +58,38 @@ const rect = (v) => {
 }
 
 export function registerTools ({ ctx, cu, config, dataDir, log, snapshotPath, defineTool, hooks, ensureCycle, register }) {
-  const observations = new ObservationStore()
+  const observations = new ObservationStore({ archiveDir: typeof dataDir?.root === 'string' ? join(dataDir.root, 'observation-cache') : undefined })
+  let observationEpoch
+  const syncObservationEpoch = () => {
+    const epoch = JSON.stringify([cu.workerGeneration ?? null, cu.cooperation?.revision ?? null, cu.stateSeq ?? 0])
+    if (observationEpoch !== undefined && observationEpoch !== epoch) observations.invalidate('worker-or-human-state-changed')
+    observationEpoch = epoch
+  }
+  const archiveFeedback = (exec, raw) => {
+    syncObservationEpoch()
+    const saved = observations.publish(exec, 'state', ['post-action', raw.point?.x, raw.point?.y], raw, undefined, { archive: true })
+    return { snapshot: saved.observation, archive: saved.archive }
+  }
   const saveShot = (buf, ext = 'jpg') => {
     const p = snapshotPath(dataDir).replace(/\.jpg$/, `.${ext}`)
     return saveOwnedShot(dataDir, p, buf)
   }
 
+  // Per-call image delivery. `auto` is the existing behaviour and stays the default; `path`
+  // withholds the image block without withholding the file. An unknown value refuses here
+  // rather than silently behaving like auto — a typo must not quietly reintroduce the cost the
+  // caller asked to avoid.
+  const deliveryMode = (value) => {
+    if (value === undefined || value === 'auto') return 'auto'
+    if (value === 'path') return 'path'
+    throw Object.assign(new Error('delivery must be "auto" or "path"'), { code: 'BAD_DELIVERY', outcome: 'not-dispatched' })
+  }
+
   // Shared by direct calls, explicit batch steps, and the batch checkpoint.
   const takeScreenshot = async (args, exec) => {
+    // Refuse a bad value BEFORE the capture: a refused request that already wrote a file and
+    // reports outcome:"not-dispatched" is a false receipt.
+    const delivery = deliveryMode(args.delivery)
     const r = rect(args.region)
     const shot = await cu.screenshot({
       x: r && r.x, y: r && r.y, width: r && r.width, height: r && r.height,
@@ -64,9 +97,8 @@ export function registerTools ({ ctx, cu, config, dataDir, log, snapshotPath, de
       maxWidth: r && r.width <= 900 ? r.width : 1280,
     })
     const path = saveShot(shot.image, shot.mime === 'image/png' ? 'png' : 'jpg')
-    const res = { path, mime: shot.mime, region: shot.region }
-    if (typeof shot.settled === 'boolean') res.settled = shot.settled
-    return attachScreenshot(ctx, exec, res, shot.image)
+    const res = { path, mime: shot.mime, region: shot.region, ...screenshotMetadata(shot) }
+    return attachScreenshot(ctx, exec, res, shot.image, { rules: config.feedbackRules, delivery })
   }
 
   const clickTarget = async (args) => {
@@ -81,26 +113,30 @@ export function registerTools ({ ctx, cu, config, dataDir, log, snapshotPath, de
   }
 
   const queryUia = async (args, exec) => {
-    if (args.snapshot) return observations.get(exec, args.snapshot, 'uia')
+    if (args.snapshot) return observations.get(exec, args.snapshot, 'uia', { offset: args.offset, limit: args.limit })
     const p = pt(args.at)
     if (p && p.x !== undefined) return cu.call('uiaFromPoint', { x: p.x, y: p.y })
     // Identity witnesses help an intended action, but are expensive for a broad inventory.
     const targets = !!args.hwnd && (args.targets ?? !!(args.id || args.name))
-    const res = await cu.call('uia', { hwnd: args.hwnd, depth: args.depth ?? 16, maxNodes: 3000,
+    const res = await cu.call('uia', { hwnd: args.hwnd, depth: args.depth ?? 6, maxNodes: args.maxNodes ?? 120,
+      budgetMs: args.budgetMs ?? 1200, anchor: args.anchor,
       query: true, targets, role: args.role, nameContains: args.name, automationId: args.id, limit: args.limit ?? 80 })
-    return observations.publish(exec, 'uia', [args.hwnd ?? null, args.depth ?? 16, args.role ?? null, args.name ?? null, args.id ?? null, args.limit ?? 80, targets],
-      { count: res.count, elements: res.flat || [], scanned: res.scanned, truncated: res.truncated, hwnd: res.hwnd ?? args.hwnd, pid: res.pid }, args.since)
+    syncObservationEpoch()
+    return observations.publish(exec, 'uia', [args.hwnd ?? null, args.depth ?? 6, args.maxNodes ?? 120, args.budgetMs ?? 1200, args.anchor ?? null, args.role ?? null, args.name ?? null, args.id ?? null, args.limit ?? 80, targets],
+      { count: res.count, elements: res.flat || [], scanned: res.scanned, truncated: res.truncated, budget: res.budget, hwnd: res.hwnd ?? args.hwnd, pid: res.pid }, args.since)
   }
   const actUia = (args, exec) => {
     if (args.target && !observations.ownsTarget(exec, args.target)) {
       throw Object.assign(new Error('TARGET_NOT_OBSERVED: obtain a fresh computer_uia target in this Agent before acting'), { code: 'TARGET_NOT_OBSERVED', outcome: 'not-dispatched' })
     }
     return cu.uiaAct({ hwnd: args.hwnd, nameContains: args.name, automationId: args.id, target: args.target, allowRebind: args.rebind === true,
-      role: args.role, action: args.action, value: args.value, requireUnique: true, depth: args.depth ?? 24, maxNodes: 3500 })
+      role: args.role, action: args.action, value: args.value, requireUnique: true,
+      depth: args.depth ?? 6, maxNodes: 300, anchor: args.anchor, budgetMs: args.budgetMs ?? 1200 })
   }
   const observeState = async (args, exec) => {
-    if (args.snapshot) return observations.get(exec, args.snapshot, 'state')
+    if (args.snapshot) return observations.get(exec, args.snapshot, 'state', { offset: args.offset, limit: args.limit })
     const result = args.windows ? await cu.call('windows', {}) : await cu.screenState({ uiaDepth: args.depth })
+    syncObservationEpoch()
     return observations.publish(exec, 'state', [!!args.windows, args.depth ?? 6], result, args.since)
   }
 
@@ -128,28 +164,46 @@ export function registerTools ({ ctx, cu, config, dataDir, log, snapshotPath, de
     if (exec?.signal?.aborted || cu.stopped || cu.exited || cu.cooperation?.active) return true
     try { return !!hooks?.askBlocked?.('computer_shot') } catch { return true }
   }
-  const captureFeedback = async (exec, mode) => {
+  const captureFeedback = async (exec, policy, context = {}, delivery = 'auto') => {
     // Route resolution is async: the human can brake while it is pending.
     if (feedbackBlocked(exec)) return { observationSkipped: 'cancelled-or-paused' }
-    const observation = mode === 'state' ? await observeState({}, exec) : await takeScreenshot({}, exec)
-    return { observation, feedback: { kind: mode }, verification: feedbackVerification }
+    const compact = policy.detail === 'compact'
+    const checkAllowed = () => {
+      if (feedbackBlocked(exec)) throw new Error('compact observation cancelled-or-paused')
+      hooks?.assertOwnership?.(exec)
+    }
+    const observation = policy.kind === 'state'
+      ? compact ? await collectCompactState(cu, { ...context, checkAllowed,
+        archive: raw => hooks?.archiveFeedback ? hooks.archiveFeedback(exec, raw) : archiveFeedback(exec, raw) }) : await observeState({}, exec)
+      : await takeScreenshot(compact ? { region: feedbackViewport(context.args, context.receipt).region, delivery } : { delivery }, exec)
+    const envelope = { observation, feedback: policy, verification: feedbackVerification }
+    if (compact && policy.kind === 'state' && observation.inlineEvidence) {
+      // Resource fallback deliberately favors recoverable evidence over the
+      // compact byte cap. This is the already-read local result, never a retry.
+      envelope.feedback = { ...policy, detail: 'full', reason: 'cache unavailable; complete local evidence returned inline' }
+      return envelope
+    }
+    // Images retain their existing attachment/coordinate contract. The compact
+    // text budget bounds the extra local UIA envelope, never the native receipt.
+    return compact && policy.kind === 'state' ? limitCompactFeedback(envelope) : envelope
   }
 
   const renderFeedback = (original, args, value) => {
     if (!value?.feedback && !value?.observationError && !value?.observationSkipped) return original(args, value)
-    const { observation, feedback, ...receipt } = value
+    const { observation, feedback, budget, ...receipt } = value
     return [
       ...original(args, receipt),
       ...(observation ? feedback?.kind === 'image' ? renderScreenshot(observation)
-        : [{ type: 'text', text: JSON.stringify({ feedback, observation }) }] : []),
+        : [{ type: 'text', text: JSON.stringify({ feedback, observation, ...(budget ? { budget } : {}) }) }] : []),
     ]
   }
 
   const reg = (def) => {
     const inner = def.execute
-    const hasFeedback = feedbackActions.has(def.name) && config.actionFeedback !== false
+    const hasFeedback = feedbackActions.has(def.name)
+    if (hasFeedback || def.name === 'computer_batch') def.parameters.feedback = P('string', 'compact (default) | full | none', { enum: ['compact', 'full', 'none'] })
     if (hasFeedback) {
-      def.description += ' Returns a route-specific checkpoint.'
+      def.description += ' Returns a route-specific checkpoint; input success does not verify the task effect.'
       const original = def.output.render
       def.output.render = (args, value) => renderFeedback(original, args, value)
       // Input plus the existing bounded screenshot/state call share one deadline.
@@ -159,7 +213,10 @@ export function registerTools ({ ctx, cu, config, dataDir, log, snapshotPath, de
     // human wait may be long; the owning tool's cancellation still releases it immediately.
     if (cooperativeTools.has(def.name)) def.timeoutMs = Math.max(def.timeoutMs, 3_600_000)
     def.execute = async (args, exec) => {
+      if (def.name === 'computer_ctrl' && args?.action === 'clear_cache')
+        return lossless({ ...observations.clear(exec), desktopInput: false })
       const releaseOnly = def.name === 'computer_ctrl' && args?.action === 'release'
+      const stateOnly = isCycleStateOnly(def.name, args)
       // hooks drive the "the agent stopped driving" signal: the activity border is cut the
       // instant the last computer-use call of a turn finishes.
       if (hooks && hooks.onStart) { try { hooks.onStart(exec) } catch { /* never break a tool */ } }
@@ -203,7 +260,7 @@ export function registerTools ({ ctx, cu, config, dataDir, log, snapshotPath, de
           err.dshCycleRefusal = true
           throw err
         }
-        if (ensureCycle && !releaseOnly) {
+        if (ensureCycle && !stateOnly) {
           let opened = null
           try { opened = await ensureCycle(exec) } catch (e) {
             // An OWNERSHIP refusal is a VERDICT, not a hiccup: another session owns this machine
@@ -244,27 +301,36 @@ export function registerTools ({ ctx, cu, config, dataDir, log, snapshotPath, de
         // registration-only probe does exactly that), and the earlier `hooks.onDispatch` form threw
         // `Cannot read properties of undefined` straight into the dispatch path.
         const invoke = async () => {
+          syncObservationEpoch()
           if (usesInput(def.name, args) && cu.waitInputReady) await cu.waitInputReady()
           if (!releaseOnly && hooks?.assertOwnership) hooks.assertOwnership(exec)
           if (hooks && hooks.onDispatch) { try { hooks.onDispatch(exec) } catch { /* never break a dispatch for a log */ } }
           return inner(args, exec)
         }
-        const inputContext = { ...exec, assertOwnership: () => hooks?.assertOwnership?.(exec) }
-        const out = cu.withInputContext ? await cu.withInputContext(inputContext, invoke) : await invoke()
-        if (out === undefined || out === null) return out
-        let observed = out
-        if (hasFeedback && !actionRefusal({ tool: def.name, args }, out) && !feedbackBlocked(exec)) {
-          try {
-            const mode = await feedbackMode(ctx, exec, true)
-            if (mode) observed = { ...out, ...await captureFeedback(exec, mode) }
-          } catch (error) {
-            // The input already returned. A failed optional observation must never
-            // hide that receipt or turn it into an apparent failed/retryable write.
-            observed = { ...out, observationError: String(error.message || error), verification: feedbackVerification }
+        const inputContext = { ...exec, ...hooks?.contextOf?.(exec), assertOwnership: () => hooks?.assertOwnership?.(exec) }
+        const executeWithFeedback = async () => {
+          const detail = hasFeedback ? selectFeedbackDetail(args?.feedback,
+            config.actionFeedback === false ? 'none' : config.actionFeedbackDetail ?? 'compact') : null
+          const out = await invoke()
+          if (out === undefined || out === null) return out
+          let observed = out
+          if (hasFeedback && detail !== 'none' && !actionRefusal({ tool: def.name, args }, out) && !feedbackBlocked(exec)) {
+            try {
+              const policy = await feedbackPolicy(ctx, exec, true, { rules: config.feedbackRules, detail })
+              if (policy.kind) {
+                hooks?.assertOwnership?.(exec)
+                const captured = await captureFeedback(exec, policy, { args, receipt: out, tool: def.name })
+                observed = { ...out, ...captured, verification: out.verification ?? captured.verification }
+              }
+            } catch (error) {
+              // Optional feedback cannot erase an input receipt or invite replay.
+              observed = { ...out, observationError: String(error.message || error), verification: out.verification ?? feedbackVerification }
+            }
           }
+          const safe = lossless(observed)
+          return safe === undefined ? null : safe
         }
-        const safe = lossless(observed)
-        return safe === undefined ? null : safe
+        return cu.withInputContext ? await cu.withInputContext(inputContext, executeWithFeedback) : await executeWithFeedback()
       } catch (error) {
         // The native/local wait has already finished. Return an explicit continuation
         // checkpoint through normal tool output so the host does not end the task as an error.
@@ -303,6 +369,7 @@ export function registerTools ({ ctx, cu, config, dataDir, log, snapshotPath, de
       preset: P('string', 'taskbar'),
       png: P('boolean', 'PNG instead of JPEG'),
       wait: P('string', 'auto (default) | none'),
+      delivery: P('string', 'auto (default) | path — path writes the file and returns its absolute path with no image block'),
     },
     output: out((a, v) => renderScreenshot(v)),
     isConcurrencySafe: () => false,
@@ -346,6 +413,8 @@ export function registerTools ({ ctx, cu, config, dataDir, log, snapshotPath, de
       depth: P('number', 'tree depth (default 6)'),
       since: P('string', 'base observation ID for changes; invalid base returns full'),
       snapshot: P('string', 'retrieve this full observation without capturing again'),
+      offset: P('number', 'snapshot only: first row, zero-based'),
+      limit: P('number', 'snapshot only: rows per array, 1–80'),
     },
     output: outObservation(),
     isConcurrencySafe: () => false,
@@ -388,15 +457,19 @@ export function registerTools ({ ctx, cu, config, dataDir, log, snapshotPath, de
     description: "Query hwnd plus role/name/exact id; returns controls and patterns. Scoped name/id queries also return target handles. since gives fresh changes; omit for full. snapshot retrieves a retained full result. at hit-tests. truncated:true is partial evidence.",
     parameters: {
       at: P('string', 'hit test at "x,y"'),
-      hwnd: P('number', 'window; omit to search desktop'),
+      hwnd: P('number', 'required window scope for a tree query; at needs no tree'),
       role: P('string', 'Button | Edit | Hyperlink | TabItem | ...'),
       name: P('string', 'name substring'),
       id: P('string', 'exact AutomationId from UIA'),
       targets: P('boolean', 'recovery handles; default on for hwnd + name/id, off for broad queries'),
       limit: P('number', 'maximum matches, 1–80 (default 80)'),
-      depth: P('number', 'tree depth, 1–24 (default 16)'),
+      depth: P('number', 'tree depth, default 6; broader traversal needs anchor'),
+      anchor: P('string', 'observed "x,y" inside hwnd; search only this element subtree'),
+      maxNodes: P('number', 'scanned node cap, default 120; above 300 needs anchor'),
+      budgetMs: P('number', 'soft traversal budget 100–5000 ms (default 1200); one provider call can exceed it'),
       since: P('string', 'base observation ID; repeat the same query'),
       snapshot: P('string', 'retrieve a retained full observation ID'),
+      offset: P('number', 'snapshot only: first row, zero-based; limit bounds the page'),
     },
     output: outObservation(),
     isConcurrencySafe: () => false,
@@ -408,13 +481,15 @@ export function registerTools ({ ctx, cu, config, dataDir, log, snapshotPath, de
     name: 'computer_uia_act',
     description: "Act through UIA. Prefer an observed target handle; rebind:true permits unique identity/semantic recovery inside the same window. Or use hwnd + exact id. Ambiguous, changed or incomplete evidence refuses; verify readback.",
     parameters: {
-      hwnd: P('number', 'window (default: desktop)'),
+      hwnd: P('number', 'required window scope unless using an observed target'),
       name: P('string', 'element name substring'),
       id: P('string', 'exact AutomationId from UIA; scope with hwnd'),
       role: P('string', 'TabItem | Button | ListItem | ComboBox | Edit | MenuItem'),
       action: P('string', 'invoke (default) | select | expand | collapse | toggle | setValue | focus | scrollIntoView'),
       value: P('string', 'value for setValue'),
-      depth: P('number', 'tree depth, 1–24 (default 24)'),
+      depth: P('number', 'tree depth, default 6; broader traversal needs anchor'),
+      anchor: P('string', 'observed "x,y" inside hwnd; narrow selector search'),
+      budgetMs: P('number', 'soft traversal budget 100–5000 ms (default 1200)'),
       target: P('string', 'opaque target from computer_uia; do not combine with selectors'),
       rebind: P('boolean', 'allow unique recovery if the observed native element was replaced'),
     },
@@ -436,6 +511,8 @@ export function registerTools ({ ctx, cu, config, dataDir, log, snapshotPath, de
       button: P('string', 'left (default) | right | middle'),
       clicks: P('number', '1 | 2 (double) | 3 (triple)'),
       shift: P('boolean', 'hold SHIFT (range selection)'),
+      verificationRegion: P('string', 'expected effect region "x,y,w,h" in screen pixels; may differ from click point; a change alone does not prove task success'),
+      allowCovered: P('boolean', 'click a pixel owned by the target app\'s OWN embedded renderer (a descendant process of the target, e.g. a WeChat article panel); an unrelated window on top is still refused and the receipt says COVERED-UNRELATED'),
     },
     output: outJSON(),
     isConcurrencySafe: () => false,
@@ -445,7 +522,7 @@ export function registerTools ({ ctx, cu, config, dataDir, log, snapshotPath, de
 
   reg({
     name: 'computer_move',
-    description: "Move without clicking to reveal hover controls.",
+    description: "Hover without clicking; returns Windows standard cursor evidence. Browser col-resize/row-resize or custom cursors may be unrecognized; unknown proves neither a handle hit nor a miss.",
     parameters: { at: P('string', '"x,y"', { required: true }) },
     output: outJSON(),
     isConcurrencySafe: () => false,
@@ -459,16 +536,18 @@ export function registerTools ({ ctx, cu, config, dataDir, log, snapshotPath, de
 
   reg({
     name: 'computer_drag',
-    description: "Drag between measured points. Prefer computer_select for text ranges.",
+    description: "Drag measured screen points. Optional expectCursor checks a recognized Windows standard shape before button-down; browser/custom cursors may be unverifiable. Ground those handles independently. cursorCheck says matched or not-requested; ok is delivery, not layout success. Verify application geometry. Prefer computer_select for text.",
     parameters: {
       from: P('string', '"M3" or "x,y"', { required: true }),
       to: P('string', '"M7" or "x,y"', { required: true }),
+      expectCursor: P('string', 'Optional standard shape: resize | resize-ew | resize-ns | resize-nesw | resize-nwse | move | hand. Unknown refuses this check; omit only with independently verified target geometry/hit-test.'),
+      hwnd: P('number', 'expected window at the drag start'),
     },
     output: outJSON(),
     isConcurrencySafe: () => false,
     timeoutMs: 15_000,
     async execute (args) {
-      return cu.drag(await cu.resolveTarget(pt(args.from)), await cu.resolveTarget(pt(args.to)))
+      return cu.drag(await cu.resolveTarget(pt(args.from)), await cu.resolveTarget(pt(args.to)), { expectCursor: args.expectCursor, expectHwnd: args.hwnd })
     },
   })
 
@@ -564,6 +643,49 @@ export function registerTools ({ ctx, cu, config, dataDir, log, snapshotPath, de
     async execute (args) { return args.text === undefined ? cu.clipRead() : cu.clipWrite(args.text) },
   })
 
+  // ---------------- launch ----------------
+  // ONE call replaces the Win+R route (activate, combo, type, enter, verify): the worker resolves
+  // the target with ShellExecute, so App Paths aliases (calc), shell: folders, URLs, folders and
+  // document associations all work. The launch is a real actuation — brake checks, the status
+  // halo (with a short self-expiring tail, because a launch returns in milliseconds) and the
+  // audit trail all come from the worker op, not from this file.
+  reg({
+    name: 'computer_launch',
+    description: 'Open an app/URL/folder/document in one call — replaces the Win+R route (activate, combo, type, enter). Aliases (calc), shell: folders, http/https URLs, absolute paths. Pass `args` to start an executable with arguments (a dedicated browser profile: target "…\\msedge.exe", args ["--user-data-dir=…", "--new-window", "<url>"]); each entry is one argument value, quoted once in the worker. Launched means the target reached the launcher, not that the app is ready; verify with computer_state.',
+    parameters: {
+      target: P('string', 'app alias (calc) | shell: folder | http/https URL | absolute path', { required: true }),
+      args: P('array', 'optional argument values for an executable target, one array entry per argument (e.g. ["--user-data-dir=D:\\\\profiles\\\\job", "--new-window", "https://…"]); no shell quoting — spaces and backslashes are ordinary'),
+      tailMs: P('number', 'how long to keep the status halo lit after the launch, 0–10000 ms (default 2500)'),
+    },
+    output: outJSON(),
+    isConcurrencySafe: () => false,
+    timeoutMs: 15_000,
+    async execute (args) { return cu.launch(validateLaunchTarget(args.target), { args: validateLaunchArgs(args.args) }) },
+  })
+
+  // ---------------- focus before typing ----------------
+  reg({
+    name: 'computer_focus',
+    description: 'Put keyboard focus on a FIELD and PROVE it before typing: clicks x,y, then reports the focused UIA control and whether a caret is provable there (OS caret, else a two-frame blink check). Typing follows the focused control, not the window — activating a dialog can leave focus in the document, where text is silently swallowed. Type only when verified:true.',
+    parameters: {
+      x: P('number', 'field centre x in screen pixels', { required: true }),
+      y: P('number', 'field centre y in screen pixels', { required: true }),
+      hwnd: P('number', 'target window; a modal dialog of the same app is accepted'),
+      verifyCaret: P('boolean', 'when the OS reports no caret, fall back to a two-frame blink check (default true)'),
+    },
+    output: outJSON(),
+    isConcurrencySafe: () => false,
+    timeoutMs: 15_000,
+    async execute (args) {
+      return cu.focusField({
+        x: args.x,
+        y: args.y,
+        ...(args.hwnd !== undefined ? { hwnd: args.hwnd } : {}),
+        ...(args.verifyCaret !== undefined ? { verifyCaret: args.verifyCaret } : {}),
+      })
+    },
+  })
+
   // ---------------- the agent asks ----------------
   reg({
     name: 'computer_ask',
@@ -624,6 +746,7 @@ export function registerTools ({ ctx, cu, config, dataDir, log, snapshotPath, de
     parameters: {
       actions: P('array', '[{tool:"computer_click", args:{mark:"M3"}}, ...]', { required: true }),
       shot: P('string', 'auto (default) | always | never; no automatic image after failure, cancel or pause'),
+      delivery: P('string', 'auto (default) | path — the checkpoint keeps its absolute path but attaches no image'),
     },
     output: out((a, v) => {
       const { observation, ...receipt } = v
@@ -637,27 +760,47 @@ export function registerTools ({ ctx, cu, config, dataDir, log, snapshotPath, de
     isConcurrencySafe: () => false,
     timeoutMs: 120_000,
     async execute (args, exec) {
-      let automatic, feedback
-      const result = await runBatch({ steps: args.actions, handlers: HANDLERS, signal: exec && exec.signal,
+      // Validate before any step runs: a bad delivery value must not be discovered only at the
+      // checkpoint, after the actions have already been dispatched.
+      const delivery = deliveryMode(args.delivery)
+      let automatic, feedback, lastContext = {}
+      const detail = selectFeedbackDetail(args.feedback,
+        config.actionFeedback === false ? 'none' : config.actionFeedbackDetail ?? 'compact')
+      // Batch still observes once at its final checkpoint. Capture the actual
+      // result coordinates without re-resolving potentially stale mark ids.
+      const feedbackHandlers = Object.fromEntries(Object.entries(HANDLERS).map(([name, handler]) => [name,
+        async (input, context) => {
+          const receipt = await handler(input, context)
+          lastContext = { args: input, receipt, tool: name }
+          return receipt
+        }]))
+      const result = await runBatch({ steps: args.actions, handlers: feedbackHandlers, signal: exec && exec.signal,
         stopped: () => cu.stopped || cu.exited, shot: args.shot, exec,
         assertOwnership: () => hooks?.assertOwnership?.(exec),
         screenshot: async () => {
-          const mode = args.shot === 'always' ? 'image' : await feedbackMode(ctx, exec, config.actionFeedback !== false)
+          const policy = args.shot === 'always' ? { kind: 'image', detail: 'full', reason: 'explicit batch screenshot' }
+            : await feedbackPolicy(ctx, exec, config.actionFeedback !== false || args.feedback !== undefined, { rules: config.feedbackRules, detail })
           if (feedbackBlocked(exec)) { feedback = { observationSkipped: 'cancelled-or-paused' }; return null }
-          if (mode === 'state') { feedback = await captureFeedback(exec, mode); return null }
-          automatic = await takeScreenshot({}, exec)
+          if (!policy.kind) { feedback = { feedback: policy, observationSkipped: policy.reason || 'disabled' }; return null }
+          hooks?.assertOwnership?.(exec)
+          if (policy.kind === 'state') { feedback = await captureFeedback(exec, policy, lastContext, delivery); return null }
+          const captured = await captureFeedback(exec, policy, lastContext, delivery)
+          const { observation, ...details } = captured
+          automatic = observation
+          feedback = details
+          if (!automatic) return null
           return automatic.path
         } })
-      return { ...selectBatchScreenshot(result, feedback?.observation ?? automatic, feedback?.feedback?.kind), ...feedback }
+      return { ...selectBatchScreenshot(result, feedback?.observation ?? automatic, feedback?.feedback?.kind, delivery), ...feedback }
     },
   })
 
   // ---------------- meta ----------------
   reg({
     name: 'computer_ctrl',
-    description: 'After final desktop verification, call release immediately to free control and monitoring; continue the conversation normally. A later required desktop call acquires a fresh cycle. stop raises a manual brake, released by human Ctrl+Alt+R; recover releases a verified temporary diagnostic pause. exit aborts the session. selftest/calibrate diagnose input; indicator controls the border; acknowledge accepts the skill receipt.',
+    description: 'hold requests stable page/pointer state between actions (breathing blue); free allows human use while observing (cyan). Neither blocks human takeover. After final verification, release ends desktop control while the conversation continues. stop raises a manual brake, released by human Ctrl+Alt+R; recover checks a temporary diagnostic pause; exit aborts. selftest/calibrate diagnose input; indicator controls the border; acknowledge accepts the skill receipt.',
     parameters: {
-      action: P('string', 'release | selftest | calibrate | stop | exit | indicator | acknowledge | recover', { required: true }),
+      action: P('string', 'hold | free | release | clear_cache | selftest | calibrate | stop | exit | indicator | acknowledge | recover', { required: true }),
       why: P('string', 'reason recorded with a stop'),
       temporary: P('boolean', 'stop: explicitly agent-owned diagnostic pause, eligible for checked recovery; ordinary waits use computer_wait'),
       pauseId: P('string', 'recover: credential returned by an explicit temporary stop'),
@@ -671,6 +814,9 @@ export function registerTools ({ ctx, cu, config, dataDir, log, snapshotPath, de
     timeoutMs: 30_000,
     async execute (args, exec) {
       switch (args.action) {
+        case 'clear_cache': return { ...observations.clear(exec), desktopInput: false }
+        case 'hold': return cu.indicator({ hold: true })
+        case 'free': return cu.indicator({ hold: false })
         case 'release': return hooks.releaseCycle(exec, args.why)
         case 'stop': return args.temporary ? hooks.temporaryStop(exec, args.why) : cu.panic(args.why)
         case 'recover': return hooks.recover(exec, args.pauseId)
@@ -706,7 +852,7 @@ export function registerTools ({ ctx, cu, config, dataDir, log, snapshotPath, de
   const HANDLERS = {
     computer_click: clickTarget,
     computer_move: (a) => { const p = pt(a.at); return cu.move(p.x, p.y) },
-    computer_drag: async (a) => cu.drag(await cu.resolveTarget(pt(a.from)), await cu.resolveTarget(pt(a.to))),
+    computer_drag: async (a) => cu.drag(await cu.resolveTarget(pt(a.from)), await cu.resolveTarget(pt(a.to)), { expectCursor: a.expectCursor, expectHwnd: a.hwnd }),
     computer_scroll: (a) => cu.scroll(a.dir, a.lines || 3, pt(a.at) ? { at: pt(a.at) } : {}),
     computer_select: async (a) => {
       const f = await cu.resolveTarget(pt(a.from)); const t = await cu.resolveTarget(pt(a.to))
@@ -716,6 +862,8 @@ export function registerTools ({ ctx, cu, config, dataDir, log, snapshotPath, de
     computer_type: (a) => cu.type(a.text, { mode: a.mode, allowPassword: a.allowPassword }),
     computer_window: (a) => ((a.op || 'activate') === 'activate' ? cu.activateWindow({ hwnd: a.hwnd }) : cu.windowOp({ hwnd: a.hwnd }, a.op, rect(a.rect) || {})),
     computer_clip: (a) => (a.text === undefined ? cu.clipRead() : cu.clipWrite(a.text)),
+    computer_launch: (a) => cu.launch(validateLaunchTarget(a.target), { args: validateLaunchArgs(a.args) }),
+  computer_focus: (a) => cu.focusField({ x: a.x, y: a.y, ...(a.hwnd !== undefined ? { hwnd: a.hwnd } : {}), ...(a.verifyCaret !== undefined ? { verifyCaret: a.verifyCaret } : {}) }),
     computer_uia_act: actUia,
     computer_ask: (a, exec) => askQuestion(a, exec),
     computer_wait: (a) => a.mode === 'stable' ? cu.waitStable({ timeoutMs: a.timeoutMs, hwnd: a.hwnd })
@@ -749,6 +897,9 @@ export function registerTools ({ ctx, cu, config, dataDir, log, snapshotPath, de
     computer_uia: queryUia,
     computer_ctrl: (a, exec) => {
       switch (a.action) {
+        case 'clear_cache': return { ...observations.clear(exec), desktopInput: false }
+        case 'hold': return cu.indicator({ hold: true })
+        case 'free': return cu.indicator({ hold: false })
         case 'release': return hooks.releaseCycle(exec, a.why)
         case 'stop': return a.temporary ? hooks.temporaryStop(exec, a.why) : cu.panic(a.why)
         case 'recover': return hooks.recover(exec, a.pauseId)

package/lib/verification-region.js

@@ -0,0 +1,17 @@
+// A witness rectangle describes where the caller expects an effect, independently
+// of the input point. Validation happens before focus, pointer or button effects.
+export function verificationRegion (value) {
+  if (value === undefined) return undefined
+  const tokens = typeof value === 'string' ? value.trim().split(/[,\s]+/) : null
+  const parts = tokens ? (tokens.some(token => !token) ? [] : tokens.map(Number))
+    : value && typeof value === 'object' && !Array.isArray(value)
+      ? [value.x, value.y, value.width, value.height] : []
+  const [x, y, width, height] = parts
+  if (parts.length !== 4 || !parts.every(n => Number.isInteger(n) && n > -2147483648 && n <= 2147483647) ||
+      width <= 0 || height <= 0 || width * height > 16777216 ||
+      x + width > 2147483647 || y + height > 2147483647) {
+    throw Object.assign(new Error('verificationRegion must be "x,y,width,height" in integer screen pixels with positive size (maximum 16M pixels). No input was dispatched.'),
+      { code: 'INVALID_VERIFICATION_REGION', outcome: 'not-dispatched' })
+  }
+  return { x, y, width, height }
+}

package/mcp/package.json

@@ -1,7 +1,8 @@
 {
   "name": "@dsh/computer-use-mcp",
   "version": "0.1.0",
-  "description": "MCP stdio server exposing dsh-computer-use to any MCP client (Claude Desktop, Codex, ...). Zero npm dependencies.",
+  "private": true,
+  "description": "Embedded MCP stdio adapter for the root dsh-computer-use package; not a standalone package. Zero npm dependencies.",
   "type": "module",
   "main": "src/index.js",
   "bin": { "dsh-computer-use-mcp": "src/index.js" },

package/mcp/src/execution.js

@@ -1,5 +1,7 @@
 // Serialize whole MCP tool calls, including batches and human handoff waits.
 // The native desktop lease also excludes other worker processes.
+import { handoffResult } from '../../lib/continuation.js'
+
 export function createToolQueue () {
   let tail = Promise.resolve()
   return (run, signal) => {
@@ -14,9 +16,10 @@ export function createToolQueue () {
 }
 
 export function continuationResult (error, describe) {
-  if (error?.code !== 'HUMAN_REOBSERVE') return null
-  return {
-    status: 'interrupted', code: error.code, taskStatus: 'continuing', transient: true,
-    outcome: error.outcome || 'unknown', continuation: error.continuation || describe(error),
-  }
+  const shared = handoffResult(error)
+  if (shared) return shared
+  // Compatibility for a bare human-handoff error from an older core. Worker
+  // recovery must retain its own description, never invent a human handback.
+  if (error?.code !== 'HUMAN_REOBSERVE' || !describe) return null
+  return handoffResult({ ...error, continuation: describe(error) })
 }