dsh-codex-style-computer-use 1.3.0 → 2.0.0

package/lib/tools.js

@@ -17,10 +17,12 @@
 
 import { runBatch, actionRefusal } from './batch.js'
 import { saveOwnedShot } from './artifacts.js'
-import { attachScreenshot, renderScreenshot, selectBatchScreenshot } from './image-output.js'
+import { attachScreenshot, renderScreenshot, selectBatchScreenshot, screenshotMetadata } from './image-output.js'
 import { ObservationStore } from './observations.js'
-import { feedbackMode, feedbackActions, feedbackVerification } from './feedback.js'
+import { feedbackPolicy, feedbackActions, feedbackVerification, selectFeedbackDetail } from './feedback.js'
+import { collectCompactState, limitCompactFeedback, feedbackViewport } from './compact-feedback.js'
 import { handoffResult } from './continuation.js'
+import { join } from 'node:path'
 
 const P = (type, description, extra = {}) => ({ type, description, ...extra })
 const cooperativeTools = new Set(['computer_click', 'computer_move', 'computer_drag', 'computer_scroll',
@@ -29,6 +31,11 @@ const usesInput = (name, args = {}) => name === 'computer_clip' ? args.text !==
   name === 'computer_batch' ? (args.actions || []).some(step => usesInput(step.tool, step.args)) :
   cooperativeTools.has(name)
 
+export const isCycleStateOnly = (name, args = {}) => name === 'computer_ctrl'
+  ? ['hold', 'free', 'release', 'clear_cache'].includes(args?.action)
+  : name === 'computer_batch' && Array.isArray(args?.actions) && args.actions.length > 0 &&
+    args.actions.every(step => isCycleStateOnly(step?.tool, step?.args))
+
 /** "M3" | "x,y" | "x y" -> {mark:'M3'} | {x,y}. One parameter instead of four. */
 const pt = (v) => {
   if (v === undefined || v === null) return null
@@ -49,7 +56,18 @@ const rect = (v) => {
 }
 
 export function registerTools ({ ctx, cu, config, dataDir, log, snapshotPath, defineTool, hooks, ensureCycle, register }) {
-  const observations = new ObservationStore()
+  const observations = new ObservationStore({ archiveDir: typeof dataDir?.root === 'string' ? join(dataDir.root, 'observation-cache') : undefined })
+  let observationEpoch
+  const syncObservationEpoch = () => {
+    const epoch = JSON.stringify([cu.workerGeneration ?? null, cu.cooperation?.revision ?? null, cu.stateSeq ?? 0])
+    if (observationEpoch !== undefined && observationEpoch !== epoch) observations.invalidate('worker-or-human-state-changed')
+    observationEpoch = epoch
+  }
+  const archiveFeedback = (exec, raw) => {
+    syncObservationEpoch()
+    const saved = observations.publish(exec, 'state', ['post-action', raw.point?.x, raw.point?.y], raw, undefined, { archive: true })
+    return { snapshot: saved.observation, archive: saved.archive }
+  }
   const saveShot = (buf, ext = 'jpg') => {
     const p = snapshotPath(dataDir).replace(/\.jpg$/, `.${ext}`)
     return saveOwnedShot(dataDir, p, buf)
@@ -64,9 +82,8 @@ export function registerTools ({ ctx, cu, config, dataDir, log, snapshotPath, de
       maxWidth: r && r.width <= 900 ? r.width : 1280,
     })
     const path = saveShot(shot.image, shot.mime === 'image/png' ? 'png' : 'jpg')
-    const res = { path, mime: shot.mime, region: shot.region }
-    if (typeof shot.settled === 'boolean') res.settled = shot.settled
-    return attachScreenshot(ctx, exec, res, shot.image)
+    const res = { path, mime: shot.mime, region: shot.region, ...screenshotMetadata(shot) }
+    return attachScreenshot(ctx, exec, res, shot.image, { rules: config.feedbackRules })
   }
 
   const clickTarget = async (args) => {
@@ -81,26 +98,30 @@ export function registerTools ({ ctx, cu, config, dataDir, log, snapshotPath, de
   }
 
   const queryUia = async (args, exec) => {
-    if (args.snapshot) return observations.get(exec, args.snapshot, 'uia')
+    if (args.snapshot) return observations.get(exec, args.snapshot, 'uia', { offset: args.offset, limit: args.limit })
     const p = pt(args.at)
     if (p && p.x !== undefined) return cu.call('uiaFromPoint', { x: p.x, y: p.y })
     // Identity witnesses help an intended action, but are expensive for a broad inventory.
     const targets = !!args.hwnd && (args.targets ?? !!(args.id || args.name))
-    const res = await cu.call('uia', { hwnd: args.hwnd, depth: args.depth ?? 16, maxNodes: 3000,
+    const res = await cu.call('uia', { hwnd: args.hwnd, depth: args.depth ?? 6, maxNodes: args.maxNodes ?? 120,
+      budgetMs: args.budgetMs ?? 1200, anchor: args.anchor,
       query: true, targets, role: args.role, nameContains: args.name, automationId: args.id, limit: args.limit ?? 80 })
-    return observations.publish(exec, 'uia', [args.hwnd ?? null, args.depth ?? 16, args.role ?? null, args.name ?? null, args.id ?? null, args.limit ?? 80, targets],
-      { count: res.count, elements: res.flat || [], scanned: res.scanned, truncated: res.truncated, hwnd: res.hwnd ?? args.hwnd, pid: res.pid }, args.since)
+    syncObservationEpoch()
+    return observations.publish(exec, 'uia', [args.hwnd ?? null, args.depth ?? 6, args.maxNodes ?? 120, args.budgetMs ?? 1200, args.anchor ?? null, args.role ?? null, args.name ?? null, args.id ?? null, args.limit ?? 80, targets],
+      { count: res.count, elements: res.flat || [], scanned: res.scanned, truncated: res.truncated, budget: res.budget, hwnd: res.hwnd ?? args.hwnd, pid: res.pid }, args.since)
   }
   const actUia = (args, exec) => {
     if (args.target && !observations.ownsTarget(exec, args.target)) {
       throw Object.assign(new Error('TARGET_NOT_OBSERVED: obtain a fresh computer_uia target in this Agent before acting'), { code: 'TARGET_NOT_OBSERVED', outcome: 'not-dispatched' })
     }
     return cu.uiaAct({ hwnd: args.hwnd, nameContains: args.name, automationId: args.id, target: args.target, allowRebind: args.rebind === true,
-      role: args.role, action: args.action, value: args.value, requireUnique: true, depth: args.depth ?? 24, maxNodes: 3500 })
+      role: args.role, action: args.action, value: args.value, requireUnique: true,
+      depth: args.depth ?? 6, maxNodes: 300, anchor: args.anchor, budgetMs: args.budgetMs ?? 1200 })
   }
   const observeState = async (args, exec) => {
-    if (args.snapshot) return observations.get(exec, args.snapshot, 'state')
+    if (args.snapshot) return observations.get(exec, args.snapshot, 'state', { offset: args.offset, limit: args.limit })
     const result = args.windows ? await cu.call('windows', {}) : await cu.screenState({ uiaDepth: args.depth })
+    syncObservationEpoch()
     return observations.publish(exec, 'state', [!!args.windows, args.depth ?? 6], result, args.since)
   }
 
@@ -128,28 +149,46 @@ export function registerTools ({ ctx, cu, config, dataDir, log, snapshotPath, de
     if (exec?.signal?.aborted || cu.stopped || cu.exited || cu.cooperation?.active) return true
     try { return !!hooks?.askBlocked?.('computer_shot') } catch { return true }
   }
-  const captureFeedback = async (exec, mode) => {
+  const captureFeedback = async (exec, policy, context = {}) => {
     // Route resolution is async: the human can brake while it is pending.
     if (feedbackBlocked(exec)) return { observationSkipped: 'cancelled-or-paused' }
-    const observation = mode === 'state' ? await observeState({}, exec) : await takeScreenshot({}, exec)
-    return { observation, feedback: { kind: mode }, verification: feedbackVerification }
+    const compact = policy.detail === 'compact'
+    const checkAllowed = () => {
+      if (feedbackBlocked(exec)) throw new Error('compact observation cancelled-or-paused')
+      hooks?.assertOwnership?.(exec)
+    }
+    const observation = policy.kind === 'state'
+      ? compact ? await collectCompactState(cu, { ...context, checkAllowed,
+        archive: raw => hooks?.archiveFeedback ? hooks.archiveFeedback(exec, raw) : archiveFeedback(exec, raw) }) : await observeState({}, exec)
+      : await takeScreenshot(compact ? { region: feedbackViewport(context.args, context.receipt).region } : {}, exec)
+    const envelope = { observation, feedback: policy, verification: feedbackVerification }
+    if (compact && policy.kind === 'state' && observation.inlineEvidence) {
+      // Resource fallback deliberately favors recoverable evidence over the
+      // compact byte cap. This is the already-read local result, never a retry.
+      envelope.feedback = { ...policy, detail: 'full', reason: 'cache unavailable; complete local evidence returned inline' }
+      return envelope
+    }
+    // Images retain their existing attachment/coordinate contract. The compact
+    // text budget bounds the extra local UIA envelope, never the native receipt.
+    return compact && policy.kind === 'state' ? limitCompactFeedback(envelope) : envelope
   }
 
   const renderFeedback = (original, args, value) => {
     if (!value?.feedback && !value?.observationError && !value?.observationSkipped) return original(args, value)
-    const { observation, feedback, ...receipt } = value
+    const { observation, feedback, budget, ...receipt } = value
     return [
       ...original(args, receipt),
       ...(observation ? feedback?.kind === 'image' ? renderScreenshot(observation)
-        : [{ type: 'text', text: JSON.stringify({ feedback, observation }) }] : []),
+        : [{ type: 'text', text: JSON.stringify({ feedback, observation, ...(budget ? { budget } : {}) }) }] : []),
     ]
   }
 
   const reg = (def) => {
     const inner = def.execute
-    const hasFeedback = feedbackActions.has(def.name) && config.actionFeedback !== false
+    const hasFeedback = feedbackActions.has(def.name)
+    if (hasFeedback || def.name === 'computer_batch') def.parameters.feedback = P('string', 'compact (default) | full | none', { enum: ['compact', 'full', 'none'] })
     if (hasFeedback) {
-      def.description += ' Returns a route-specific checkpoint.'
+      def.description += ' Returns a route-specific checkpoint; input success does not verify the task effect.'
       const original = def.output.render
       def.output.render = (args, value) => renderFeedback(original, args, value)
       // Input plus the existing bounded screenshot/state call share one deadline.
@@ -159,7 +198,10 @@ export function registerTools ({ ctx, cu, config, dataDir, log, snapshotPath, de
     // human wait may be long; the owning tool's cancellation still releases it immediately.
     if (cooperativeTools.has(def.name)) def.timeoutMs = Math.max(def.timeoutMs, 3_600_000)
     def.execute = async (args, exec) => {
+      if (def.name === 'computer_ctrl' && args?.action === 'clear_cache')
+        return lossless({ ...observations.clear(exec), desktopInput: false })
       const releaseOnly = def.name === 'computer_ctrl' && args?.action === 'release'
+      const stateOnly = isCycleStateOnly(def.name, args)
       // hooks drive the "the agent stopped driving" signal: the activity border is cut the
       // instant the last computer-use call of a turn finishes.
       if (hooks && hooks.onStart) { try { hooks.onStart(exec) } catch { /* never break a tool */ } }
@@ -203,7 +245,7 @@ export function registerTools ({ ctx, cu, config, dataDir, log, snapshotPath, de
           err.dshCycleRefusal = true
           throw err
         }
-        if (ensureCycle && !releaseOnly) {
+        if (ensureCycle && !stateOnly) {
           let opened = null
           try { opened = await ensureCycle(exec) } catch (e) {
             // An OWNERSHIP refusal is a VERDICT, not a hiccup: another session owns this machine
@@ -244,27 +286,36 @@ export function registerTools ({ ctx, cu, config, dataDir, log, snapshotPath, de
         // registration-only probe does exactly that), and the earlier `hooks.onDispatch` form threw
         // `Cannot read properties of undefined` straight into the dispatch path.
         const invoke = async () => {
+          syncObservationEpoch()
           if (usesInput(def.name, args) && cu.waitInputReady) await cu.waitInputReady()
           if (!releaseOnly && hooks?.assertOwnership) hooks.assertOwnership(exec)
           if (hooks && hooks.onDispatch) { try { hooks.onDispatch(exec) } catch { /* never break a dispatch for a log */ } }
           return inner(args, exec)
         }
-        const inputContext = { ...exec, assertOwnership: () => hooks?.assertOwnership?.(exec) }
-        const out = cu.withInputContext ? await cu.withInputContext(inputContext, invoke) : await invoke()
-        if (out === undefined || out === null) return out
-        let observed = out
-        if (hasFeedback && !actionRefusal({ tool: def.name, args }, out) && !feedbackBlocked(exec)) {
-          try {
-            const mode = await feedbackMode(ctx, exec, true)
-            if (mode) observed = { ...out, ...await captureFeedback(exec, mode) }
-          } catch (error) {
-            // The input already returned. A failed optional observation must never
-            // hide that receipt or turn it into an apparent failed/retryable write.
-            observed = { ...out, observationError: String(error.message || error), verification: feedbackVerification }
+        const inputContext = { ...exec, ...hooks?.contextOf?.(exec), assertOwnership: () => hooks?.assertOwnership?.(exec) }
+        const executeWithFeedback = async () => {
+          const detail = hasFeedback ? selectFeedbackDetail(args?.feedback,
+            config.actionFeedback === false ? 'none' : config.actionFeedbackDetail ?? 'compact') : null
+          const out = await invoke()
+          if (out === undefined || out === null) return out
+          let observed = out
+          if (hasFeedback && detail !== 'none' && !actionRefusal({ tool: def.name, args }, out) && !feedbackBlocked(exec)) {
+            try {
+              const policy = await feedbackPolicy(ctx, exec, true, { rules: config.feedbackRules, detail })
+              if (policy.kind) {
+                hooks?.assertOwnership?.(exec)
+                const captured = await captureFeedback(exec, policy, { args, receipt: out, tool: def.name })
+                observed = { ...out, ...captured, verification: out.verification ?? captured.verification }
+              }
+            } catch (error) {
+              // Optional feedback cannot erase an input receipt or invite replay.
+              observed = { ...out, observationError: String(error.message || error), verification: out.verification ?? feedbackVerification }
+            }
           }
+          const safe = lossless(observed)
+          return safe === undefined ? null : safe
         }
-        const safe = lossless(observed)
-        return safe === undefined ? null : safe
+        return cu.withInputContext ? await cu.withInputContext(inputContext, executeWithFeedback) : await executeWithFeedback()
       } catch (error) {
         // The native/local wait has already finished. Return an explicit continuation
         // checkpoint through normal tool output so the host does not end the task as an error.
@@ -310,6 +361,34 @@ export function registerTools ({ ctx, cu, config, dataDir, log, snapshotPath, de
     execute: takeScreenshot,
   })
 
+  reg({
+    name: 'computer_ocr',
+    description: "Read TEXT with coordinates from a screen region (inbox Windows OCR). Returns lines/words with absolute-pixel boxes — locate controls BY NAME in zero-UIA apps instead of spending a screenshot; words' x,y are click-ready centers offset by w/2,h/2. Crop tight (16M px cap); zh/en mixed supported.",
+    parameters: {
+      region: P('string', 'crop "x,y,w,h" (default: whole screen)'),
+    },
+    output: out((a, v) => {
+      if (!v || !v.lines) return [{ type: 'text', text: 'no text' }]
+      const head = `ocr ${v.language} region=${v.region ? v.region.join(',') : ''} lines=${v.lines.length}`
+      const body = v.lines.map(l => {
+        const w0 = l.words[0]
+        return `${w0 ? `@${w0.x},${w0.y} ` : ''}${l.text}`
+      }).join('\n')
+      return [{ type: 'text', text: head + '\n' + body }]
+    }),
+    isConcurrencySafe: () => false,
+    timeoutMs: 30_000,
+    async execute (args) {
+      let region
+      if (args.region) {
+        const parts = String(args.region).split(',').map(Number)
+        if (parts.length !== 4 || parts.some(n => !Number.isFinite(n))) throw new Error('region must be "x,y,w,h"')
+        region = { x: parts[0], y: parts[1], width: parts[2], height: parts[3] }
+      }
+      return cu.ocr(region)
+    },
+  })
+
   reg({
     name: 'computer_state',
     description: "Window/UIA state; start with windows:true, then activate the target. since returns fresh changes against that observation; omit for full. snapshot retrieves a retained full observation for this Agent.",
@@ -318,6 +397,8 @@ export function registerTools ({ ctx, cu, config, dataDir, log, snapshotPath, de
       depth: P('number', 'tree depth (default 6)'),
       since: P('string', 'base observation ID for changes; invalid base returns full'),
       snapshot: P('string', 'retrieve this full observation without capturing again'),
+      offset: P('number', 'snapshot only: first row, zero-based'),
+      limit: P('number', 'snapshot only: rows per array, 1–80'),
     },
     output: outObservation(),
     isConcurrencySafe: () => false,
@@ -327,11 +408,12 @@ export function registerTools ({ ctx, cu, config, dataDir, log, snapshotPath, de
 
   reg({
     name: 'computer_marks',
-    description: "Cached UIA map with snapshot IDs. Reuse while unchanged; mark actions check window identity, geometry and sampled pixels. STALE_MARK requires refresh and reselecting the intended element. Refreshed IDs replace old IDs.",
+    description: "Cached UIA map with snapshot IDs. Reuse while unchanged; mark actions check window identity, geometry and sampled pixels. STALE_MARK requires refresh and reselecting the intended element. Refreshed IDs replace old IDs. uiaOnly pins identity by window+geometry when an animated neighbour keeps invalidating the pixel sample (the click-time occlusion check still applies).",
     parameters: {
       hwnd: P('number', 'window (default: active)'),
       shot: P('boolean', 'also return a marked-up screenshot path'),
       refresh: P('boolean', 'rebuild instead of reusing the cache'),
+      uiaOnly: P('boolean', 'skip the sampled-pixel comparison; identity = window+geometry only'),
     },
     output: out((a, v) => {
       if (!v || !v.marks) return [{ type: 'text', text: 'no marks' }]
@@ -341,7 +423,7 @@ export function registerTools ({ ctx, cu, config, dataDir, log, snapshotPath, de
     isConcurrencySafe: () => false,
     timeoutMs: 45_000,
     async execute (args) {
-      const res = await cu.landmarks({ hwnd: args.hwnd, refresh: args.refresh })
+      const res = await cu.landmarks({ hwnd: args.hwnd, refresh: args.refresh, uiaOnly: args.uiaOnly })
       const res2 = {
         count: res.count, reused: !!res.reused, hwnd: res.hwnd,
         marks: res.marks.map(m => ({ id: m.id, role: m.role, name: m.name, center: [m.cx, m.cy] })),
@@ -359,15 +441,19 @@ export function registerTools ({ ctx, cu, config, dataDir, log, snapshotPath, de
     description: "Query hwnd plus role/name/exact id; returns controls and patterns. Scoped name/id queries also return target handles. since gives fresh changes; omit for full. snapshot retrieves a retained full result. at hit-tests. truncated:true is partial evidence.",
     parameters: {
       at: P('string', 'hit test at "x,y"'),
-      hwnd: P('number', 'window; omit to search desktop'),
+      hwnd: P('number', 'required window scope for a tree query; at needs no tree'),
       role: P('string', 'Button | Edit | Hyperlink | TabItem | ...'),
       name: P('string', 'name substring'),
       id: P('string', 'exact AutomationId from UIA'),
       targets: P('boolean', 'recovery handles; default on for hwnd + name/id, off for broad queries'),
       limit: P('number', 'maximum matches, 1–80 (default 80)'),
-      depth: P('number', 'tree depth, 1–24 (default 16)'),
+      depth: P('number', 'tree depth, default 6; broader traversal needs anchor'),
+      anchor: P('string', 'observed "x,y" inside hwnd; search only this element subtree'),
+      maxNodes: P('number', 'scanned node cap, default 120; above 300 needs anchor'),
+      budgetMs: P('number', 'soft traversal budget 100–5000 ms (default 1200); one provider call can exceed it'),
       since: P('string', 'base observation ID; repeat the same query'),
       snapshot: P('string', 'retrieve a retained full observation ID'),
+      offset: P('number', 'snapshot only: first row, zero-based; limit bounds the page'),
     },
     output: outObservation(),
     isConcurrencySafe: () => false,
@@ -379,13 +465,15 @@ export function registerTools ({ ctx, cu, config, dataDir, log, snapshotPath, de
     name: 'computer_uia_act',
     description: "Act through UIA. Prefer an observed target handle; rebind:true permits unique identity/semantic recovery inside the same window. Or use hwnd + exact id. Ambiguous, changed or incomplete evidence refuses; verify readback.",
     parameters: {
-      hwnd: P('number', 'window (default: desktop)'),
+      hwnd: P('number', 'required window scope unless using an observed target'),
       name: P('string', 'element name substring'),
       id: P('string', 'exact AutomationId from UIA; scope with hwnd'),
       role: P('string', 'TabItem | Button | ListItem | ComboBox | Edit | MenuItem'),
       action: P('string', 'invoke (default) | select | expand | collapse | toggle | setValue | focus | scrollIntoView'),
       value: P('string', 'value for setValue'),
-      depth: P('number', 'tree depth, 1–24 (default 24)'),
+      depth: P('number', 'tree depth, default 6; broader traversal needs anchor'),
+      anchor: P('string', 'observed "x,y" inside hwnd; narrow selector search'),
+      budgetMs: P('number', 'soft traversal budget 100–5000 ms (default 1200)'),
       target: P('string', 'opaque target from computer_uia; do not combine with selectors'),
       rebind: P('boolean', 'allow unique recovery if the observed native element was replaced'),
     },
@@ -407,6 +495,7 @@ export function registerTools ({ ctx, cu, config, dataDir, log, snapshotPath, de
       button: P('string', 'left (default) | right | middle'),
       clicks: P('number', '1 | 2 (double) | 3 (triple)'),
       shift: P('boolean', 'hold SHIFT (range selection)'),
+      verificationRegion: P('string', 'expected effect region "x,y,w,h" in screen pixels; may differ from click point; a change alone does not prove task success'),
     },
     output: outJSON(),
     isConcurrencySafe: () => false,
@@ -416,7 +505,7 @@ export function registerTools ({ ctx, cu, config, dataDir, log, snapshotPath, de
 
   reg({
     name: 'computer_move',
-    description: "Move without clicking to reveal hover controls.",
+    description: "Hover without clicking; returns Windows standard cursor evidence. Browser col-resize/row-resize or custom cursors may be unrecognized; unknown proves neither a handle hit nor a miss.",
     parameters: { at: P('string', '"x,y"', { required: true }) },
     output: outJSON(),
     isConcurrencySafe: () => false,
@@ -430,16 +519,18 @@ export function registerTools ({ ctx, cu, config, dataDir, log, snapshotPath, de
 
   reg({
     name: 'computer_drag',
-    description: "Drag between measured points. Prefer computer_select for text ranges.",
+    description: "Drag measured screen points. Optional expectCursor checks a recognized Windows standard shape before button-down; browser/custom cursors may be unverifiable. Ground those handles independently. cursorCheck says matched or not-requested; ok is delivery, not layout success. Verify application geometry. Prefer computer_select for text.",
     parameters: {
       from: P('string', '"M3" or "x,y"', { required: true }),
       to: P('string', '"M7" or "x,y"', { required: true }),
+      expectCursor: P('string', 'Optional standard shape: resize | resize-ew | resize-ns | resize-nesw | resize-nwse | move | hand. Unknown refuses this check; omit only with independently verified target geometry/hit-test.'),
+      hwnd: P('number', 'expected window at the drag start'),
     },
     output: outJSON(),
     isConcurrencySafe: () => false,
     timeoutMs: 15_000,
     async execute (args) {
-      return cu.drag(await cu.resolveTarget(pt(args.from)), await cu.resolveTarget(pt(args.to)))
+      return cu.drag(await cu.resolveTarget(pt(args.from)), await cu.resolveTarget(pt(args.to)), { expectCursor: args.expectCursor, expectHwnd: args.hwnd })
     },
   })
 
@@ -608,15 +699,32 @@ export function registerTools ({ ctx, cu, config, dataDir, log, snapshotPath, de
     isConcurrencySafe: () => false,
     timeoutMs: 120_000,
     async execute (args, exec) {
-      let automatic, feedback
-      const result = await runBatch({ steps: args.actions, handlers: HANDLERS, signal: exec && exec.signal,
+      let automatic, feedback, lastContext = {}
+      const detail = selectFeedbackDetail(args.feedback,
+        config.actionFeedback === false ? 'none' : config.actionFeedbackDetail ?? 'compact')
+      // Batch still observes once at its final checkpoint. Capture the actual
+      // result coordinates without re-resolving potentially stale mark ids.
+      const feedbackHandlers = Object.fromEntries(Object.entries(HANDLERS).map(([name, handler]) => [name,
+        async (input, context) => {
+          const receipt = await handler(input, context)
+          lastContext = { args: input, receipt, tool: name }
+          return receipt
+        }]))
+      const result = await runBatch({ steps: args.actions, handlers: feedbackHandlers, signal: exec && exec.signal,
         stopped: () => cu.stopped || cu.exited, shot: args.shot, exec,
         assertOwnership: () => hooks?.assertOwnership?.(exec),
         screenshot: async () => {
-          const mode = args.shot === 'always' ? 'image' : await feedbackMode(ctx, exec, config.actionFeedback !== false)
+          const policy = args.shot === 'always' ? { kind: 'image', detail: 'full', reason: 'explicit batch screenshot' }
+            : await feedbackPolicy(ctx, exec, config.actionFeedback !== false || args.feedback !== undefined, { rules: config.feedbackRules, detail })
           if (feedbackBlocked(exec)) { feedback = { observationSkipped: 'cancelled-or-paused' }; return null }
-          if (mode === 'state') { feedback = await captureFeedback(exec, mode); return null }
-          automatic = await takeScreenshot({}, exec)
+          if (!policy.kind) { feedback = { feedback: policy, observationSkipped: policy.reason || 'disabled' }; return null }
+          hooks?.assertOwnership?.(exec)
+          if (policy.kind === 'state') { feedback = await captureFeedback(exec, policy, lastContext); return null }
+          const captured = await captureFeedback(exec, policy, lastContext)
+          const { observation, ...details } = captured
+          automatic = observation
+          feedback = details
+          if (!automatic) return null
           return automatic.path
         } })
       return { ...selectBatchScreenshot(result, feedback?.observation ?? automatic, feedback?.feedback?.kind), ...feedback }
@@ -626,9 +734,9 @@ export function registerTools ({ ctx, cu, config, dataDir, log, snapshotPath, de
   // ---------------- meta ----------------
   reg({
     name: 'computer_ctrl',
-    description: 'After final desktop verification, call release immediately to free control and monitoring; continue the conversation normally. A later required desktop call acquires a fresh cycle. stop raises a manual brake, released by human Ctrl+Alt+R; recover releases a verified temporary diagnostic pause. exit aborts the session. selftest/calibrate diagnose input; indicator controls the border; acknowledge accepts the skill receipt.',
+    description: 'hold requests stable page/pointer state between actions (breathing blue); free allows human use while observing (cyan). Neither blocks human takeover. After final verification, release ends desktop control while the conversation continues. stop raises a manual brake, released by human Ctrl+Alt+R; recover checks a temporary diagnostic pause; exit aborts. selftest/calibrate diagnose input; indicator controls the border; acknowledge accepts the skill receipt.',
     parameters: {
-      action: P('string', 'release | selftest | calibrate | stop | exit | indicator | acknowledge | recover', { required: true }),
+      action: P('string', 'hold | free | release | clear_cache | selftest | calibrate | stop | exit | indicator | acknowledge | recover', { required: true }),
       why: P('string', 'reason recorded with a stop'),
       temporary: P('boolean', 'stop: explicitly agent-owned diagnostic pause, eligible for checked recovery; ordinary waits use computer_wait'),
       pauseId: P('string', 'recover: credential returned by an explicit temporary stop'),
@@ -642,6 +750,9 @@ export function registerTools ({ ctx, cu, config, dataDir, log, snapshotPath, de
     timeoutMs: 30_000,
     async execute (args, exec) {
       switch (args.action) {
+        case 'clear_cache': return { ...observations.clear(exec), desktopInput: false }
+        case 'hold': return cu.indicator({ hold: true })
+        case 'free': return cu.indicator({ hold: false })
         case 'release': return hooks.releaseCycle(exec, args.why)
         case 'stop': return args.temporary ? hooks.temporaryStop(exec, args.why) : cu.panic(args.why)
         case 'recover': return hooks.recover(exec, args.pauseId)
@@ -677,7 +788,7 @@ export function registerTools ({ ctx, cu, config, dataDir, log, snapshotPath, de
   const HANDLERS = {
     computer_click: clickTarget,
     computer_move: (a) => { const p = pt(a.at); return cu.move(p.x, p.y) },
-    computer_drag: async (a) => cu.drag(await cu.resolveTarget(pt(a.from)), await cu.resolveTarget(pt(a.to))),
+    computer_drag: async (a) => cu.drag(await cu.resolveTarget(pt(a.from)), await cu.resolveTarget(pt(a.to)), { expectCursor: a.expectCursor, expectHwnd: a.hwnd }),
     computer_scroll: (a) => cu.scroll(a.dir, a.lines || 3, pt(a.at) ? { at: pt(a.at) } : {}),
     computer_select: async (a) => {
       const f = await cu.resolveTarget(pt(a.from)); const t = await cu.resolveTarget(pt(a.to))
@@ -695,6 +806,15 @@ export function registerTools ({ ctx, cu, config, dataDir, log, snapshotPath, de
     // take a screenshot: "unknown tool computer_shot" (reported 2026-09-12). The map must cover
     // the WHOLE surface — a caller should never have to know which subset happens to be wired.
     computer_shot: takeScreenshot,
+    computer_ocr: async (a) => {
+      let region
+      if (a.region) {
+        const parts = String(a.region).split(',').map(Number)
+        if (parts.length !== 4 || parts.some(n => !Number.isFinite(n))) throw new Error('region must be "x,y,w,h"')
+        region = { x: parts[0], y: parts[1], width: parts[2], height: parts[3] }
+      }
+      return cu.ocr(region)
+    },
     computer_state: observeState,
     computer_marks: async (a) => {
       const res = await cu.landmarks({ hwnd: a.hwnd, refresh: a.refresh })
@@ -711,6 +831,9 @@ export function registerTools ({ ctx, cu, config, dataDir, log, snapshotPath, de
     computer_uia: queryUia,
     computer_ctrl: (a, exec) => {
       switch (a.action) {
+        case 'clear_cache': return { ...observations.clear(exec), desktopInput: false }
+        case 'hold': return cu.indicator({ hold: true })
+        case 'free': return cu.indicator({ hold: false })
         case 'release': return hooks.releaseCycle(exec, a.why)
         case 'stop': return a.temporary ? hooks.temporaryStop(exec, a.why) : cu.panic(a.why)
         case 'recover': return hooks.recover(exec, a.pauseId)

package/lib/verification-region.js

@@ -0,0 +1,17 @@
+// A witness rectangle describes where the caller expects an effect, independently
+// of the input point. Validation happens before focus, pointer or button effects.
+export function verificationRegion (value) {
+  if (value === undefined) return undefined
+  const tokens = typeof value === 'string' ? value.trim().split(/[,\s]+/) : null
+  const parts = tokens ? (tokens.some(token => !token) ? [] : tokens.map(Number))
+    : value && typeof value === 'object' && !Array.isArray(value)
+      ? [value.x, value.y, value.width, value.height] : []
+  const [x, y, width, height] = parts
+  if (parts.length !== 4 || !parts.every(n => Number.isInteger(n) && n > -2147483648 && n <= 2147483647) ||
+      width <= 0 || height <= 0 || width * height > 16777216 ||
+      x + width > 2147483647 || y + height > 2147483647) {
+    throw Object.assign(new Error('verificationRegion must be "x,y,width,height" in integer screen pixels with positive size (maximum 16M pixels). No input was dispatched.'),
+      { code: 'INVALID_VERIFICATION_REGION', outcome: 'not-dispatched' })
+  }
+  return { x, y, width, height }
+}

package/mcp/package.json

@@ -1,7 +1,8 @@
 {
   "name": "@dsh/computer-use-mcp",
   "version": "0.1.0",
-  "description": "MCP stdio server exposing dsh-computer-use to any MCP client (Claude Desktop, Codex, ...). Zero npm dependencies.",
+  "private": true,
+  "description": "Embedded MCP stdio adapter for the root dsh-computer-use package; not a standalone package. Zero npm dependencies.",
   "type": "module",
   "main": "src/index.js",
   "bin": { "dsh-computer-use-mcp": "src/index.js" },

package/mcp/src/execution.js

@@ -1,5 +1,7 @@
 // Serialize whole MCP tool calls, including batches and human handoff waits.
 // The native desktop lease also excludes other worker processes.
+import { handoffResult } from '../../lib/continuation.js'
+
 export function createToolQueue () {
   let tail = Promise.resolve()
   return (run, signal) => {
@@ -14,9 +16,10 @@ export function createToolQueue () {
 }
 
 export function continuationResult (error, describe) {
-  if (error?.code !== 'HUMAN_REOBSERVE') return null
-  return {
-    status: 'interrupted', code: error.code, taskStatus: 'continuing', transient: true,
-    outcome: error.outcome || 'unknown', continuation: error.continuation || describe(error),
-  }
+  const shared = handoffResult(error)
+  if (shared) return shared
+  // Compatibility for a bare human-handoff error from an older core. Worker
+  // recovery must retain its own description, never invent a human handback.
+  if (error?.code !== 'HUMAN_REOBSERVE' || !describe) return null
+  return handoffResult({ ...error, continuation: describe(error) })
 }