dsh-codex-style-computer-use 1.3.1 → 2.0.0

package/lib/feedback-routes.json

@@ -0,0 +1,11 @@
+[
+  {
+    "provider": "bigmodel-anthropic",
+    "model": "glm-5.3",
+    "adapterVersion": "unversioned",
+    "grounding": "failed",
+    "testedAt": "2026-10-02T00:00:00Z",
+    "expiresAt": "2026-12-01T00:00:00Z",
+    "reason": "Historical Desktop grounding test failed; adapter version was not recorded. Conservative text fallback expires after 60 days and must be revalidated."
+  }
+]

package/lib/feedback.js

@@ -1,5 +1,8 @@
-// Route-specific observations, verified against the Desktop adapters on 2026-10-02.
-// A model catalog's image flag is transport permission, not a grounding test.
+import { readFileSync } from 'node:fs'
+
+// Transport capability and grounding are separate evidence. Route exceptions
+// are dated data, can be replaced in settings, and expire.
+export const DEFAULT_FEEDBACK_RULES = Object.freeze(JSON.parse(readFileSync(new URL('./feedback-routes.json', import.meta.url), 'utf8')))
 export function feedbackRoute (exec) {
   const routed = exec?.agent?.session?.requestHeader?.()?.config
   return {
@@ -8,23 +11,53 @@ export function feedbackRoute (exec) {
   }
 }
 
-export function unverifiedVisionRoute (exec) {
-  const { provider, model } = feedbackRoute(exec)
-  return provider === 'bigmodel-anthropic' && model === 'glm-5.3'
+export function visionEvidence (exec, { rules = DEFAULT_FEEDBACK_RULES, adapterVersion = 'unversioned', now = Date.now() } = {}) {
+  const route = feedbackRoute(exec)
+  for (const rule of [...(Array.isArray(rules) ? rules : [])].reverse()) {
+    const tested = Date.parse(rule.testedAt), expires = Date.parse(rule.expiresAt)
+    if (rule.provider !== route.provider || rule.model !== route.model ||
+        rule.adapterVersion !== adapterVersion || !Number.isFinite(tested) || !Number.isFinite(expires) ||
+        tested > now || expires <= now || expires <= tested || !['failed', 'verified'].includes(rule.grounding)) continue
+    // Unversioned historical tests may select a conservative text fallback,
+    // but cannot certify the current adapter's visual grounding.
+    if (rule.grounding === 'verified' && adapterVersion === 'unversioned') continue
+    return { grounding: rule.grounding, adapterVersion, testedAt: rule.testedAt, expiresAt: rule.expiresAt,
+      reason: rule.reason || 'configured route evidence' }
+  }
+  return { grounding: 'unverified', adapterVersion, reason: 'no current matching grounding record' }
+}
+
+export function unverifiedVisionRoute (exec, options) {
+  return visionEvidence(exec, options).grounding === 'failed'
 }
 
-export async function feedbackMode (ctx, exec, enabled = true) {
-  if (!enabled || exec?.signal?.aborted) return null
-  if (unverifiedVisionRoute(exec)) return 'state'
-  const { provider, model } = feedbackRoute(exec)
-  const tested = (provider === 'bigmodel-anthropic' && model === 'glm-5.3-flash') ||
-    (provider === 'deepseek-official' && model === 'deepseek-flash')
-  if (!tested) return null
+export function selectFeedbackDetail (requested, fallback = 'compact') {
+  const detail = requested ?? fallback
+  if (!['compact', 'full', 'none'].includes(detail)) throw Object.assign(new Error('feedback must be compact, full or none'),
+    { code: 'INVALID_FEEDBACK', outcome: 'not-dispatched' })
+  return detail
+}
+
+export async function feedbackPolicy (ctx, exec, enabled = true, options = {}) {
+  const route = feedbackRoute(exec)
+  const detail = selectFeedbackDetail(options.detail, 'full')
+  if (!enabled || detail === 'none' || exec?.signal?.aborted) return { kind: null, detail, reason: exec?.signal?.aborted ? 'cancelled' : !enabled ? 'disabled' : 'feedback-none' }
+  let active
   try {
-    if (!ctx.get?.('attachments')?.saveImage) return null
-    const active = await ctx.get?.('llm')?.resolveModelInfo?.(provider, model, exec?.signal)
-    return !exec?.signal?.aborted && active?.inputModalities?.includes('image') ? 'image' : null
-  } catch { return null }
+    if (route.provider && route.model) active = await ctx.get?.('llm')?.resolveModelInfo?.(route.provider, route.model, exec?.signal)
+  } catch { /* missing capability evidence selects text, not silence */ }
+  if (exec?.signal?.aborted) return { kind: null, reason: 'cancelled' }
+  const evidence = visionEvidence(exec, { ...options, adapterVersion: active?.adapterVersion || 'unversioned' })
+  const base = { route, detail, ...evidence }
+  if (evidence.grounding === 'failed') return { ...base, kind: 'state', reason: 'current grounding record requires UIA/text readback' }
+  if (!active) return { ...base, kind: 'state', reason: 'model-capabilities-unavailable' }
+  if (!active.inputModalities?.includes('image')) return { ...base, kind: 'state', reason: 'model-text-only' }
+  if (!ctx.get?.('attachments')?.saveImage) return { ...base, kind: 'state', reason: 'attachments-unavailable' }
+  return { ...base, kind: 'image', reason: 'image transport declared; inspect returned evidence to verify effect' }
+}
+
+export async function feedbackMode (ctx, exec, enabled = true, options) {
+  return (await feedbackPolicy(ctx, exec, enabled, options)).kind
 }
 
 export const feedbackActions = new Set([

package/lib/image-output.js

@@ -1,10 +1,44 @@
 import { basename } from 'node:path'
-import { unverifiedVisionRoute } from './feedback.js'
+import { visionEvidence } from './feedback.js'
+import { captureCoordinates } from './media-contract.js'
+
+export function screenshotMetadata (shot) {
+  const value = {}
+  for (const key of ['capturedAt', 'ts', 'imageSize', 'settled', 'settleMs', 'settleDiffPct', 'settleNote',
+    'cursor', 'cursorRequested', 'cursorDrawn', 'cursorScreenPos', 'cursorInImage', 'cursorNote']) {
+    if (shot[key] !== undefined) value[key] = shot[key]
+  }
+  return value
+}
+
+function setCoordinates (value, size) {
+  const coordinates = captureCoordinates(value.region, size)
+  if (!coordinates) { delete value.coordinates; delete value.coordinateRule; return }
+  value.coordinates = coordinates
+  value.coordinateRule = 'screen = screenOrigin + imagePixel * screenPerImagePixel (per axis). Do not apply DPI or browser zoom again. After resize/move/zoom, obtain fresh evidence.'
+}
+
+const validImageSize = size => Array.isArray(size) && size.length === 2 && size.every(n => Number.isFinite(n) && n > 0)
+const validPoint = point => Number.isFinite(point?.x) && Number.isFinite(point?.y)
+
+function restoreFileGeometry (value) {
+  if (!value.sourceImageSize) return
+  value.imageSize = value.sourceImageSize
+  if (value.sourceCursorInImage) value.cursorInImage = value.sourceCursorInImage
+  else delete value.cursorInImage
+  delete value.sourceImageSize
+  delete value.sourceCursorInImage
+  setCoordinates(value, value.imageSize)
+}
 
 // DSH image blocks reference the host's durable attachment store. Paths and raw
 // base64 are not image blocks. Keep a usable file when a route cannot accept one.
-export async function attachScreenshot (ctx, exec, value, data) {
+export async function attachScreenshot (ctx, exec, value, data, options = {}) {
+  // The local file has geometry even when the model route cannot accept images.
+  setCoordinates(value, value.imageSize)
   const fallback = (reason) => {
+    delete value.image
+    restoreFileGeometry(value)
     value.imageStatus = 'path-only'
     value.imageReason = reason
     value.view = reason === 'model-text-only'
@@ -14,6 +48,8 @@ export async function attachScreenshot (ctx, exec, value, data) {
     return value
   }
   try {
+    if (!validImageSize(value.imageSize)) return fallback('invalid-source-image-size')
+    const sourceSize = [...value.imageSize]
     const attachments = ctx.get?.('attachments')
     const llm = ctx.get?.('llm')
     if (!attachments?.saveImage) return fallback('attachments-unavailable')
@@ -27,21 +63,32 @@ export async function attachScreenshot (ctx, exec, value, data) {
     if (!attachments.imageLimits?.mediaTypes?.includes(value.mime)) return fallback('media-type-unavailable')
     const image = await attachments.saveImage({ data, mediaType: value.mime, name: basename(value.path) })
     if (exec?.signal?.aborted) return fallback('cancelled')
+    const attachedSize = [image?.width, image?.height]
+    if (!validImageSize(attachedSize)) return fallback('invalid-attachment-image-size')
+    const evidence = visionEvidence(exec, { ...options, adapterVersion: active.adapterVersion || 'unversioned' })
+    const resized = attachedSize.some((n, axis) => n !== sourceSize[axis])
+    if (resized) {
+      value.sourceImageSize = sourceSize
+      if (validPoint(value.cursorInImage)) {
+        value.sourceCursorInImage = { ...value.cursorInImage }
+        value.cursorInImage = { x: value.cursorInImage.x * attachedSize[0] / sourceSize[0],
+          y: value.cursorInImage.y * attachedSize[1] / sourceSize[1] }
+      } else delete value.cursorInImage
+      value.imageSize = attachedSize
+    }
     value.image = image
     value.imageStatus = 'attached'
-    const r = value.region
-    if (r && [r.x, r.y, r.width, r.height].every(Number.isFinite) && image.width > 0 && image.height > 0) {
-      value.coordinates = {
-        screenOrigin: [r.x, r.y], imageSize: [image.width, image.height],
-        screenPerImagePixel: [r.width / image.width, r.height / image.height],
-      }
-    }
+    setCoordinates(value, attachedSize)
+    if (resized && value.coordinateRule) value.coordinateRule += ' imageSize, cursorInImage and coordinates describe the attached image. path retains the original capture: use sourceImageSize and sourceCursorInImage for that file; map its pixels using region.width/sourceImageSize[0] and region.height/sourceImageSize[1].'
     value.view = value.settled === false
       ? 'NOT settled — may be a partial render; re-shoot.'
       : 'Image attached. Inspect it directly; no read_image needed. Coordinates for clicks are screen pixels.'
-    if (unverifiedVisionRoute(exec)) {
+    value.visionEvidence = evidence
+    if (value.visionEvidence.grounding === 'failed') {
       value.visionStatus = 'unverified-route'
-      value.view = 'Image transport succeeded, but this GLM 5.3 route failed grounding tests. Use UIA/text readback; do not infer coordinates or success from unseen pixels. The image and path are retained for inspection.'
+      value.view = 'Image transport succeeded, but a current route record reports failed grounding. Use UIA/text readback; the image and path remain available. See visionEvidence for the scope and expiry.'
+    } else {
+      value.visionStatus = value.visionEvidence.grounding === 'verified' ? 'grounding-verified' : 'grounding-unverified'
     }
     return value
   } catch (error) {
@@ -75,6 +122,7 @@ export function selectBatchScreenshot (result, automatic, kind = 'image') {
     const shot = action.result
     if (shot?.image && shot !== latest) {
       delete shot.image
+      restoreFileGeometry(shot)
       shot.imageStatus = 'path-only'
       shot.imageReason = 'earlier-batch-image'
       shot.view = kind === 'state'

package/lib/index.js

@@ -31,8 +31,8 @@ import Schema from '@deepseek-ai/schemastery'
 import { ComputerUse } from './core/index.js'
 import { policyDecision, HIGH_RISK_NOTE } from './policy.js'
 import { acknowledge, isAcked, ACK_HINT, isDrivingCall } from './skill-gate.js'
-import { mkDataDir, snapshotPath, appendAudit } from './audit.js'
-import { registerTools } from './tools.js'
+import { mkDataDir, snapshotPath, installAudit } from './audit.js'
+import { registerTools, isCycleStateOnly } from './tools.js'
 import { installToolExposure, ACTIVATION_TOOL } from './exposure.js'
 import { scopeOf } from '@deepseek-ai/dsh-scope'
 import { createCycleGate, agentKeyOf } from './cycle.js'
@@ -382,7 +382,14 @@ export const inject = ['tools', 'userQuestions']
 export const Config = Schema.object({
   enabled: Schema.boolean().default(true).description('enable the computer-use toolset'),
   progressiveTools: Schema.boolean().default(true).description('expose desktop execution tools only to Agents that load/activate computer-use; brakes remain available'),
-  actionFeedback: Schema.boolean().default(true).description('post-input feedback for tested GLM 5.3 Flash / DeepSeek Flash routes; GLM 5.3 uses UIA state until its vision is verified'),
+  actionFeedback: Schema.boolean().default(true).description('automatic post-input observation; per-call feedback can explicitly select compact, full or none'),
+  actionFeedbackDetail: Schema.union([Schema.const('compact'), Schema.const('full'), Schema.const('none')]).default('compact').description('compact bounds local text readback to 6 KiB; vision keeps its image and crops only an explicitly requested verificationRegion. full retains complete route-specific feedback'),
+  feedbackRules: Schema.array(Schema.object({
+    provider: Schema.string().required(), model: Schema.string().required(),
+    adapterVersion: Schema.string().required().description('exact adapter version, or unversioned for a conservative failed-test fallback'),
+    grounding: Schema.union([Schema.const('failed'), Schema.const('verified')]).required(),
+    testedAt: Schema.string().required(), expiresAt: Schema.string().required(), reason: Schema.string(),
+  })).default(undefined).description('optional dated route grounding evidence; defaults to bundled expiring records; [] removes historical overrides'),
   automationMode: Schema.union([
     Schema.const('read-only').description('observation only'),
     Schema.const('standard').description('actuations require approval'),
@@ -436,7 +443,7 @@ export function apply (ctx, config) {
     rateLimit: { max: config.maxActionsPerMinute, windowMs: 60_000 },
     log: (m) => log.info(m),
   })
-  cu.on('action', (rec) => appendAudit(dataDir, rec))
+  const detachAudit = installAudit(cu, dataDir)
 
   // ---- the live agent of the session that drives this machine ------------------------------
   // Captured from the tool execution itself (`exec.agent` is the live Agent object); a session
@@ -560,7 +567,14 @@ export function apply (ctx, config) {
       const expect = exitRecordText()
       // Rejections ride the SHARED qualification promise: every later call in this turn awaits the
       // same outcome instead of retrying it into a resurrection.
-      await cu.resume(expect)
+      const cycleId = gate.snapshot().id
+      const openingContext = { signal: exec?.signal, cycleId, sessionKey: cl.sessionKey, agentKey: cl.agentKey,
+        assertOwnership: () => gate.assertOwnership(cl, cycleId) }
+      const resume = async () => {
+        await cu.resume(expect)
+      }
+      if (cu.withInputContext) await cu.withInputContext(openingContext, resume)
+      else await resume()
       if (!cu.exited && !exitedLatch()) {
         log.info('opened a new computer-use cycle for this turn (cleared the Ctrl+Alt+Q record left by an earlier cycle)')
         return { opened: true }
@@ -596,6 +610,10 @@ export function apply (ctx, config) {
     paused: pausedOrAsking, report: error => log.warn(`computer-use terminal cleanup failed: ${error.message}`),
   })
   const hooks = {
+    contextOf: (exec) => {
+      const owner = exec && typeof exec === 'object' ? callOwners.get(exec) : null
+      return owner ? { cycleId: owner.cycleId, sessionKey: owner.claim.sessionKey, agentKey: owner.claim.agentKey } : {}
+    },
     assertOwnership: (exec) => {
       const owner = exec && typeof exec === 'object' ? callOwners.get(exec) : null
       if (!owner) {
@@ -608,7 +626,7 @@ export function apply (ctx, config) {
     releaseCycle: (exec, why) => lifecycle.release(exec, why),
     onStart: (exec) => {
       inFlight++
-      if (exec?.name === 'computer_ctrl' && exec.arguments?.action === 'release') {
+      if (isCycleStateOnly(exec?.name, exec?.arguments)) {
         callOwners.set(exec, { claim: claimOf(exec), cycleId: gate.snapshot().id })
       }
     },
@@ -1154,7 +1172,7 @@ export function apply (ctx, config) {
   })
 
   ctx.effect(() => {
-    const disposer = () => { cu.kill().catch(() => {}) }
+    const disposer = () => { return cu.kill().catch(() => {}).finally(detachAudit) }
     return disposer
   })
 

package/lib/key-command.js

@@ -0,0 +1,37 @@
+// One command grammar for approval and dispatch. Native parsing implements the
+// same contract for direct worker clients; cross-language fixtures guard parity.
+export const KEY_ALIASES = Object.freeze({
+  control: 'ctrl', lwin: 'win', windows: 'win', meta: 'win',
+  return: 'enter', escape: 'esc', bksp: 'backspace', del: 'delete',
+  ins: 'insert', pageup: 'pgup', pagedown: 'pgdn', capslock: 'caps', printscreen: 'prtsc',
+})
+const modifierOrder = ['ctrl', 'alt', 'shift', 'win', 'rwin']
+const named = new Set([...modifierOrder, 'enter', 'esc', 'tab', 'space', 'backspace', 'delete',
+  'insert', 'home', 'end', 'pgup', 'pgdn', 'up', 'down', 'left', 'right', 'caps', 'apps',
+  'prtsc', 'numlock', 'plus', 'minus', 'comma', 'period'])
+
+export function parseKeyCommand (combo) {
+  if (typeof combo !== 'string' || !combo.trim()) throw new Error('KEY_COMMAND_INVALID: nonempty key combo required')
+  const unique = []
+  for (const part of combo.split('+')) {
+    const raw = part.trim().toLowerCase()
+    const key = KEY_ALIASES[raw] || raw
+    if (!key || !(named.has(key) || /^[a-z0-9]$/.test(key) || /^f(?:[1-9]|1\d|2[0-4])$/.test(key))) {
+      throw new Error(`KEY_COMMAND_INVALID: unknown or empty key ${JSON.stringify(raw)}`)
+    }
+    if (!unique.includes(key)) unique.push(key)
+  }
+  const modifiers = modifierOrder.filter(key => unique.includes(key))
+  const keys = unique.filter(key => !modifierOrder.includes(key))
+  const tokens = [...modifiers, ...keys]
+  return { combo: tokens.join('+'), tokens, modifiers, keys }
+}
+
+export function keyCommandRisk (combo) {
+  const { tokens } = parseKeyCommand(combo)
+  const has = key => tokens.includes(key)
+  const win = has('win') || has('rwin')
+  return has('delete') || has('backspace') ||
+    (win && ['r', 'l', 'd'].some(has)) || (has('alt') && has('f4')) ||
+    (has('ctrl') && has('shift') && has('esc'))
+}

package/lib/media-contract.js

@@ -0,0 +1,43 @@
+// Coordinate metadata is transport-independent; attachments and MCP content
+// must describe the same captured pixels without serializing binary bytes as text.
+export function captureCoordinates (region, imageSize) {
+  if (!region || ![region.x, region.y, region.width, region.height].every(Number.isFinite) ||
+      region.width <= 0 || region.height <= 0 || !Array.isArray(imageSize) || imageSize.length !== 2 ||
+      !imageSize.every(n => Number.isFinite(n) && n > 0)) return null
+  return { screenOrigin: [region.x, region.y], imageSize,
+    screenPerImagePixel: [region.width / imageSize[0], region.height / imageSize[1]] }
+}
+
+export function imageMime (bytes, declared) {
+  if (bytes.length >= 8 && bytes.subarray(0, 8).equals(Buffer.from([137, 80, 78, 71, 13, 10, 26, 10]))) return 'image/png'
+  if (bytes.length >= 3 && bytes[0] === 255 && bytes[1] === 216 && bytes[2] === 255) return 'image/jpeg'
+  if (declared === 'image/png' || declared === 'image/jpeg') return declared
+  throw new Error('IMAGE_FORMAT_UNKNOWN: missing supported image signature or MIME')
+}
+
+export function splitMediaResult (value) {
+  const images = []
+  const clean = (input, location) => {
+    if (Buffer.isBuffer(input)) return undefined
+    if (Array.isArray(input)) return input.map((child, i) => clean(child, `${location}[${i}]`))
+    if (!input || typeof input !== 'object') return input
+    const output = {}
+    for (const [key, child] of Object.entries(input)) {
+      if (!Buffer.isBuffer(child)) output[key] = clean(child, `${location}.${key}`)
+    }
+    const bytes = Buffer.isBuffer(input.annotated) ? input.annotated
+      : Buffer.isBuffer(input.image) ? input.image : Buffer.isBuffer(input.raw) ? input.raw : null
+    if (bytes) {
+      const mime = imageMime(bytes, input.mime)
+      output.mime = mime
+      const coordinates = captureCoordinates(input.region, input.imageSize)
+      if (coordinates) output.coordinates = coordinates
+      images.push({ bytes, mime, location, metadata: output })
+    }
+    return output
+  }
+  const text = clean(value, 'result')
+  const latest = images.at(-1)
+  for (const entry of images) entry.metadata.imageStatus = entry === latest ? 'attached' : 'omitted-earlier-batch-image'
+  return { value: text, ...(latest ? { image: latest.bytes, mime: latest.mime, imageSource: latest.location } : {}) }
+}