dsh-codex-style-computer-use 1.3.1 → 2.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,83 @@
1
+ // One reviewed, non-interactive behavior gate for source and packed artifacts.
2
+ // Tests requiring real input or unisolated desktop hooks are listed explicitly.
3
+ import fs from 'node:fs'
4
+ import os from 'node:os'
5
+ import path from 'node:path'
6
+ import { spawn } from 'node:child_process'
7
+ import { createHash } from 'node:crypto'
8
+
9
+ const sourceRoot = path.resolve(import.meta.dirname, '..')
10
+ const packageIndex = process.argv.indexOf('--package')
11
+ const packageRoot = packageIndex >= 0 ? path.resolve(process.argv[packageIndex + 1] || '') : null
12
+ if (packageIndex >= 0 && !process.argv[packageIndex + 1]) throw Error('--package needs an unpacked package path')
13
+ const run = fs.mkdtempSync(path.join(os.tmpdir(), 'dsh-behavior-'))
14
+ const logs = path.join(run, 'logs'); fs.mkdirSync(logs)
15
+ let root = sourceRoot
16
+ if (packageRoot) {
17
+ if (!fs.existsSync(path.join(packageRoot, 'package.json'))) throw Error('not an unpacked package')
18
+ root = path.join(run, 'package')
19
+ fs.cpSync(packageRoot, root, { recursive: true, filter: p => path.basename(p) !== 'node_modules' })
20
+ fs.mkdirSync(path.join(root, 'build'), { recursive: true })
21
+ // Only test code comes from the checkout; every lib/skill/script under test
22
+ // comes from the actual archive. Never replace packed product code with source.
23
+ for (const item of fs.readdirSync(path.join(sourceRoot, 'build'), { withFileTypes: true })) {
24
+ if (item.isFile() && (/\.(mjs|cs)$/.test(item.name) || item.name === 'key-command-cases.json'))
25
+ fs.copyFileSync(path.join(sourceRoot, 'build', item.name), path.join(root, 'build', item.name))
26
+ }
27
+ fs.cpSync(path.join(sourceRoot, 'build/fixtures'), path.join(root, 'build/fixtures'), { recursive: true })
28
+ }
29
+ const excluded = {
30
+ 'test-evidence-native.mjs': 'requires actual foreground SendInput and keyboard delivery; a never-activated private desktop cannot prove this physical-input property',
31
+ 'test-approval-attention.mjs': 'alias imports test-attention.mjs, which is already run directly',
32
+ }
33
+ const reviewed = new Set([
34
+ 'action-feedback', 'agent-pointer', 'approval-attribution', 'ask-invariants', 'ask-merge', 'ask-orphan', 'ask-release',
35
+ 'attention-native', 'attention', 'audit-completion', 'batch-coverage', 'buildusermessage', 'bundled-skill',
36
+ 'coactive', 'compact-feedback', 'control-targets', 'core-system-recovery', 'cursor', 'cycle-completion',
37
+ 'cycle-no-timeout', 'cycle-open', 'desktop-acceptance-io', 'desktop-ownership', 'drag-options', 'evidence-protocol',
38
+ 'exit-coherence', 'feedback-cache-integration', 'feedback-policy', 'glow-pump', 'glow-visual', 'handoff-continuation',
39
+ 'handoff-lock', 'human-handoff', 'indicator-hold', 'input-stamp', 'key-command',
40
+ 'kickoff-focus', 'mcp-contracts', 'mcp-execution', 'modifier-vk', 'monitor-lifetime', 'native-hardening', 'native-respawn',
41
+ 'native-runtime', 'native-fault-lifecycle', 'observation-archive', 'observations', 'op-classification', 'pointer-lease', 'policy', 'preparation',
42
+ 'release-channel', 'renders', 'root-integration', 'session-identity', 'skill-gate', 'steer-message-shape',
43
+ 'task-rules', 'terminal-cleanup', 'timeout-branch', 'turnend-agent', 'turnend-rootagent',
44
+ 'ui-geometry', 'uia-budget', 'uia-read-timeout', 'uia-native', 'uia-targets-native', 'verification-region', 'window-management-key',
45
+ ].map(name => `test-${name}.mjs`))
46
+ const files = fs.readdirSync(path.join(root, 'build')).filter(name => /^test-.+\.mjs$/.test(name)).sort()
47
+ const unreviewed = files.filter(name => !reviewed.has(name) && !excluded[name])
48
+ if (unreviewed.length) throw Error('Review desktop isolation and add new tests to the gate: ' + unreviewed.join(', '))
49
+ const results = []
50
+ for (const name of files) {
51
+ if (excluded[name]) { results.push({ name, status: 'excluded', reason: excluded[name] }); continue }
52
+ const state = path.join(run, 'state', name); fs.mkdirSync(state, { recursive: true })
53
+ const env = { ...process.env, DSH_HOME: state, LOCALAPPDATA: path.join(state, 'local'),
54
+ DSH_COMPUTER_USE_STOP_FILE: path.join(state, 'STOP'), DSH_COMPUTER_USE_EXIT_FILE: path.join(state, 'EXITED'),
55
+ DSH_COMPUTER_USE_OWNER_FILE: path.join(state, 'owner'), DSH_COMPUTER_USE_AUDIT_LOG: path.join(state, 'worker-audit.log') }
56
+ delete env.DSH_CU_NATIVE_RELEASE_LEG; delete env.DSH_CU_DESKTOP_TESTS
57
+ const started = Date.now(), log = path.join(logs, name + '.log')
58
+ const stream = fs.createWriteStream(log)
59
+ const result = await new Promise(resolve => {
60
+ const child = spawn(process.execPath, [path.join(root, 'build', name)], { cwd: root, env, windowsHide: true, stdio: ['ignore', 'pipe', 'pipe'] })
61
+ let tail = '', timedOut = false
62
+ const consume = b => { stream.write(b); tail = (tail + b.toString()).slice(-5000) }
63
+ child.stdout.on('data', consume); child.stderr.on('data', consume)
64
+ const timer = setTimeout(() => { timedOut = true; child.kill() }, 240_000)
65
+ child.once('error', error => { clearTimeout(timer); stream.end(); resolve({ status: 'failed', error: error.message, log }) })
66
+ child.once('close', code => {
67
+ clearTimeout(timer); stream.end()
68
+ resolve({ status: code === 0 && !timedOut ? 'passed' : 'failed', code, timedOut, log,
69
+ ...(code === 0 ? {} : { error: tail }), subcasesSkipped: /\bSKIP\b/.test(tail) })
70
+ })
71
+ })
72
+ results.push({ name, ...result, elapsedMs: Date.now() - started })
73
+ console.log(`${result.status}: ${name}${result.subcasesSkipped ? ' (optional subcases excluded; see log)' : ''}`)
74
+ if (result.status === 'failed') console.log(result.error)
75
+ }
76
+ const pkg = JSON.parse(fs.readFileSync(path.join(root, 'package.json'), 'utf8'))
77
+ const sourceHash = createHash('sha256').update(fs.readFileSync(path.join(root, 'lib/core/worker.cs'))).digest('hex')
78
+ const report = { package: pkg.name, version: pkg.version, packageRoot, testedRoot: root, sourceHash,
79
+ generatedAt: new Date().toISOString(), passed: results.filter(r => r.status === 'passed').length,
80
+ failed: results.filter(r => r.status === 'failed').length, excluded: results.filter(r => r.status === 'excluded').length, results }
81
+ const reportPath = path.join(run, 'report.json'); fs.writeFileSync(reportPath, JSON.stringify(report, null, 2) + '\n')
82
+ console.log(JSON.stringify({ report: reportPath, passed: report.passed, failed: report.failed, excluded: report.excluded }))
83
+ if (report.failed) process.exitCode = 1
@@ -0,0 +1,21 @@
1
+ // Read-only model-facing schema cost. No tool executes and no native worker starts.
2
+ import fs from 'node:fs'
3
+ import path from 'node:path'
4
+ import { pathToFileURL } from 'node:url'
5
+ const args = process.argv.slice(2)
6
+ const source = path.resolve(args.find(a => !a.startsWith('--')) || path.join(import.meta.dirname, '../lib/tools.js'))
7
+ const definitions = []
8
+ const { registerTools } = await import(pathToFileURL(source).href)
9
+ registerTools({ ctx: { tools: { register: t => definitions.push(t) } }, cu: {},
10
+ config: { annotateMarks: true }, dataDir: '.', log: () => {},
11
+ snapshotPath: () => 'not-created.jpg', defineTool: t => t })
12
+ const tools = definitions.map(({ name, description, parameters }) => {
13
+ const characters = JSON.stringify({ name, description, parameters }).length
14
+ return { name, characters, approximateTokens: Math.round(characters / 4) }
15
+ })
16
+ const characters = tools.reduce((sum, t) => sum + t.characters, 0)
17
+ const result = { source, basis: 'JSON name/description/parameters; chars / 4 is an estimate, not a tokenizer',
18
+ count: tools.length, characters, approximateTokens: Math.round(characters / 4), tools }
19
+ console.log(JSON.stringify(result, null, 2))
20
+ const output = args.find(a => a.startsWith('--out='))?.slice(6)
21
+ if (output && !args.includes('--no-baseline')) fs.writeFileSync(path.resolve(output), JSON.stringify(result, null, 2) + '\n')
@@ -21,19 +21,23 @@ Observation and the safety direction (ask, stop, exit) are not gated by that rec
21
21
  2. Use the cheapest sufficient observation: returned readback, a focused UIA query, a reusable mark map, then an image when visual evidence is needed. Do not read the same state twice merely to satisfy a checklist.
22
22
  3. Prefer `computer_uia {hwnd, role, name, id}` and `computer_uia_act` for controls exposing patterns. `name` is a substring; `id` is the exact AutomationId from a query. Scope with hwnd. Prefer a returned `target` handle for the selected control. If it was replaced, `rebind:true` permits recovery only when native identity or complete, unique semantic evidence still identifies it in the same window. Changed context, ambiguity and incomplete searches refuse. Use either target or name/id/role selectors; do not combine them. Query again with a narrower target; do not pick the first same-name button. Tabs use action:"select"; fields can use setValue with a readback.
23
23
  4. Otherwise use a currently observed mark or measured screen coordinate. A mark is a snapshot ID. It does not follow scrolling. Each mark action checks window identity, position and sampled pixels; STALE_MARK means no input was dispatched. Refresh with `computer_marks {refresh:true}`, then select the intended element by its new ID. Old IDs are retired. Refresh after known scrolling/navigation even if a sample misses the change. dx/dy offsets work in direct and batch clicks.
24
- 5. Batch a coherent set of actions to a natural checkpoint, without a fixed step count. Actions run serially; failure, failed activation or cancellation stops subsequent steps. Reuse the returned evidence. `shot:"auto"` adds at most one observation when needed (image or route-specific UIA state); `shot:"never"` is suitable when text/readback already verifies the result. Direct mouse/keyboard actions also return a checkpoint on tested routes. Batch known inputs to avoid paying for an intermediate image after each keystroke.
24
+ 5. Batch a coherent set of actions to a natural checkpoint, without a fixed step count. Actions run serially; failure, failed activation or cancellation stops subsequent steps. Reuse the returned evidence. `shot:"auto"` adds at most one observation when needed (image or UIA state); `shot:"never"` is suitable when text/readback already verifies the result. Direct mouse/keyboard actions also return a checkpoint chosen from current transport capability and dated route evidence. Batch known inputs to avoid paying for an intermediate image after each keystroke.
25
25
  6. Verify the task outcome: field value, saved record/document, selected range, application status or image. A provider accepting input proves neither a save nor the business result. Report what was actually observed and any unresolved uncertainty, then finish.
26
26
  7. As soon as desktop work is finished, call `computer_ctrl {action:"release"}`. Release the computer-use cycle even when you will continue explaining, reasoning, or using other tools in the same chat. Do not retain desktop ownership merely because the conversation is alive. A later genuine desktop need can open a fresh cycle in the same turn. Normal turn completion releases automatically without a question card; explicit user pauses/questions are preserved.
27
27
 
28
- Repeated `computer_uia` or `computer_state` queries can pass `since` with the previous observation ID and the same query. A diff contains added/changed/removed elements and any order change; omitted elements retain their previous state. No base, expired base, changed scope, partial results or an uneconomical diff returns full. `snapshot` retrieves a retained complete historical observation without another capture; it is not fresh verification. Omit since/snapshot for a fresh full view after losing the baseline. See [progressive-control.md](references/progressive-control.md) for examples, retention and recovery boundaries.
28
+ Repeated `computer_uia` or `computer_state` queries can pass `since` with the previous observation ID and the same query. A diff contains added/changed/removed elements and any order change; omitted elements retain their previous state. No base, expired base, changed scope, partial results or an uneconomical diff returns full. `snapshot` retrieves a retained historical observation without another capture; it is not fresh verification. Use `offset` and `limit` to read a page instead of loading the whole snapshot. A compact feedback snapshot is read with `computer_state {snapshot:"O..."}`. `computer_ctrl {action:"clear_cache"}` clears this Agent's observation cache without opening or driving a desktop cycle. Disk caches are bounded and expire; storage failure falls back to bounded memory, and memory pressure falls back to inline results without a fake snapshot ID. Omit since/snapshot for fresh evidence after losing the baseline. See [progressive-control.md](references/progressive-control.md).
29
29
 
30
30
  ## Screenshots and coordinates
31
31
 
32
32
  `computer_shot` returns a native image on routes that declare image input. With `imageStatus:"attached"`, follow `view` and inspect it directly without another read_image call when vision is usable. With `imageStatus:"path-only"`, follow `view`; the path is not visual evidence. For a text-only route use UIA/text or an image-capable model. `settled:false` needs a fresh capture. A batch attaches only its latest image and labels its step; an earlier image does not verify later actions. `computer_marks {shot:true}` returns an annotated file path whose IDs match the text map.
33
33
 
34
- Verified Desktop routes (2026-10-02): `bigmodel-anthropic/glm-5.3-flash` and `deepseek-official/deepseek-flash` support images returned by tools. Reuse `observation` from direct click/type/key/move/drag/scroll/select; do not call shot/read_image again without a reason. `bigmodel-anthropic/glm-5.3` with max reasoning passed text controls but failed fresh direct and tool-image grounding tests despite declaring image input. Its automatic checkpoint is UIA state; explicit screenshots retain their image/path with `visionStatus:"unverified-route"`. Do not invent visual coordinates or claim visual verification on that route. If UIA is absent or truncated, narrow the query or report the evidence gap. These findings apply to the tested routes/build, not every endpoint with the same model name. `actionFeedback:false` restores manual direct-action feedback.
34
+ Reuse `observation` from direct click/type/key/move/drag/scroll/select; do not call shot/read_image again without a reason. Image-capable routes, including provider aliases, receive image feedback when the attachment channel is available. Missing capabilities, text-only routes or unavailable attachments use UIA/text and return the reason in `feedback`. Image transport does not certify visual grounding: inspect `visionEvidence` and verify the actual target/effect. Configurable `feedbackRules` match provider/model/adapter version and expire; a current failed-grounding record selects text feedback. The historical unversioned GLM 5.3 failure record has a limited validity window and does not permanently classify every endpoint with that model name. Explicit screenshots retain their image/path even with a failed-grounding record. If UIA is absent or truncated, narrow the query or report the evidence gap. `actionFeedback:false` restores manual direct-action feedback.
35
35
 
36
- An `observationError` after input does not undo that input. Verify with a fresh readback before replaying any write. Refusal, cancellation and a human brake suppress automatic follow-up observations. Raw input receipts and full observation snapshots are retained; the displayed checkpoint is not a replacement for them.
36
+ Automatic feedback defaults to `feedback:"compact"`: text feedback uses one local point read instead of a desktop UIA inventory, normally with a 6 KiB additional-text budget. The raw point evidence may be cached locally and indexed by `observation.snapshot`. Disk failure with a valid in-memory snapshot stays compact. If no retrievable snapshot can be retained, existing raw evidence is returned as `observation.inlineEvidence`, with `feedback.detail:"full"` and an explanation; this resource fallback is exempt from the compact budget and performs no additional query or input. `feedback:"full"` requests the previous complete checkpoint; `feedback:"none"` skips automatic readback when a separate authoritative verification follows. The same options apply to all model routes. Image feedback remains full-frame by default; it is only cropped when the caller explicitly supplies a click `verificationRegion:"x,y,w,h"`. This region can be far from the click (e.g. a playback bar). `changedPct` is only sampled pixel difference, and `effectVerified:false` does not claim business success. Do not skip necessary visual verification based only on a successful input receipt or changed pixels.
37
+
38
+ An `observationError` after input does not undo that input. Verify with a fresh readback before replaying any write. Refusal, cancellation and a human brake suppress automatic follow-up observations. Snapshot retention is best-effort and bounded; an uncached or expired result must be re-observed, not reconstructed from old coordinates.
39
+
40
+ `WORKER_REOBSERVE` means the worker was replaced and its old observations are invalid. Obtain fresh state and continue the unfinished task; do not replay the previous write blindly. `runtime` identifies the loaded package/build/protocol and worker generation. `cleanupConfirmed` describes input-release evidence only, never whether the application saved or accepted the intended result.
37
41
 
38
42
  Input uses screen pixels. For image point (u,v):
39
43
 
@@ -43,10 +47,14 @@ Input uses screen pixels. For image point (u,v):
43
47
 
44
48
  Use returned coordinates metadata for crops, scaling and negative monitor origins. Measure stable anchors; invalidate geometry when its witness moves. See [precision.md](references/precision.md) for small icons, tables, dropdowns, tiling or uncertain geometry.
45
49
 
50
+ `capturedAt` is the native pixel-capture time; `ts` is capture-response construction time. `imageSize` and `coordinates` remain available for file-only images. Do not multiply image-to-screen coordinates by DPI or browser zoom again. `settled:true` only means sampled pixels were quiet; it does not prove a resize/navigation finished. Capture again after a known geometry change. For a narrow drag handle, use fresh geometry or a tight crop, hover with `computer_move`, and inspect its cursor evidence. Use optional `expectCursor:"resize-ew"` only when that recognized standard shape is expected. Browser CSS `col-resize`/`row-resize` and custom cursors can be `classification:"unrecognized", shape:"unknown"`; this proves neither a hit nor a miss. Independently confirm such targets with current geometry and hit-testing, then omit the optional cursor check while retaining `hwnd`. `cursorCheck.status:"not-requested"` is not a successful shape check. Native/UIA hit-testing does not identify a DOM selector or CSS value. Compare fresh application geometry or images before claiming the drag worked; after two misses, remeasure or change method. See [precision.md](references/precision.md).
51
+
46
52
  ## Focus and recovery
47
53
 
48
54
  Keyboard input goes to the foreground window. Target the intended field, inspect focus/receiver, then type. The worker reasserts the last activated/clicked target; a missing target or failed refocus refuses keyboard input. Verify the resulting value, particularly after clearing a field. A matching focus report alone does not prove the correct business field was changed.
49
55
 
56
+ Window-management keys (`alt+tab`, `alt+shift+tab`, `alt+esc`, `alt+shift+esc`, or `win`/`rwin` plus `d`, `tab`, `left`, `right`, `up`, `down`) deliberately invalidate the previous keyboard target after any accepted key-down. `receiverChanged` reports observed HWND/PID change, not successful switching; `reobserveRequired:true` and `effectVerified:false` require a new check. Use `computer_state {windows:true}` → `computer_window {hwnd,op:"activate"}` with `activated:true`, then observe the intended target before further input. Until activation, key/type returns `INPUT_REOBSERVE_REQUIRED` without sending more input. A refused follow-up screenshot does not mean the earlier chord was not dispatched. This flow never releases a human brake.
57
+
50
58
  `KEY_ALREADY_DOWN` sends no chord; let the human finish before retrying. Paste preserves supported clipboard formats and skips restoration after a newer copy. A clipboard warning requires a value readback before repeating input. See [shared-input.md](references/shared-input.md) for clipboard fallbacks and cleanup failures.
51
59
 
52
60
  UIA `invoke` does not promise keyboard focus. After invoking a button that opens an editor, explicitly `computer_uia_act {action:"focus", ...}` the observed editor before typing; these can share a batch ending in a value readback. `TYPE_FOCUS_NOT_EDITABLE` means no text was sent: focus the intended editor before retrying. Do not repeat typing just because the window itself matched.
@@ -70,13 +78,17 @@ IME rule (generalizes to every editor you cannot read back): when an IME is acti
70
78
 
71
79
  `computer_wait {mode:"change"}` returns `changeBounds` (absolute px) — WHERE the change landed. A diff at the input area means the box cleared (message sent); the same-size diff blinking at one line is a caret. `mode:"stable"` returns `caretSteady:true` when the only motion is a ~1-2 Hz caret blink — treat it as settled and stop waiting. `computer_marks {uiaOnly:true}` pins the map to window+geometry when an animated neighbour keeps flipping the pixel sample (STALE_MARK loops); the click-time occlusion check still applies.
72
80
 
73
- A cold Chromium accessibility tree can initially expose only chrome/an ancestor; retry after readiness before declaring absence. Query depth defaults to 16, maximum 24; action depth defaults to 24. A truncated query is partial evidence. Read [precision.md](references/precision.md) when focused queries, activation or input still fail.
81
+ A cold accessibility tree can initially expose only window chrome/an ancestor; use a point query and one deliberate re-observation after readiness before declaring absence. Filtered tree queries require `hwnd` (`UIA_SCOPE_REQUIRED` otherwise). Defaults are depth 6, 120 nodes and a 1200 ms soft budget; `budgetMs` can be 100–5000. `depth>6` or `maxNodes>300` requires an observed `anchor:"x,y"` in physical screen pixels inside that window, and starts at that element's subtree (`UIA_ANCHOR_REQUIRED` otherwise). An exact `id` does not bypass traversal limits. This rule applies to all applications and model routes. Budget metadata describes partial traversal; a single synchronous provider call may exceed the budget and remains covered by worker recovery. A truncated result proves neither absence nor uniqueness. Prefer an observed UIA target for actions; selector searches default to depth 6 and 300 nodes. Read [precision.md](references/precision.md) when focused queries, activation or input still fail.
74
82
 
75
83
  ## Human control and permission
76
84
 
77
- The user can operate normally whenever the agent is not actively controlling input. Thinking, observation and the pre-operation warning do not claim keyboard/mouse ownership. Before a new input call or coherent batch, a purple 1.2-second notice says “即将开始操作,请暂停键鼠”. The deep-blue cue lasts only while actual actuation is in progress, with no trailing hold.
85
+ Cyan breathing means the user can freely use the keyboard and mouse. Initial observation is free. Deep-blue steady means input is being dispatched; deep-blue breathing means the agent needs the current page and pointer position preserved while checking or preparing the next step. Actual input retains this scene hold afterward. Use `computer_ctrl {action:"hold"}` before a position-sensitive readback; use `computer_ctrl {action:"free"}` as soon as remaining thinking/reading no longer depends on an unchanged desktop. These actions neither create a new cycle nor release a manual brake. Release the cycle when desktop work is complete.
86
+
87
+ During acting/holding, a compact tilted blue kite temporarily replaces the visible pointer; its tip is the exact native hotspot. Holding breathes; acting is steady. The small center symbol follows recognized native text/resize/etc. shapes; unknown stays neutral. Native cursor handles and logical visibility are preserved for cursor checks and screenshot burn-in, so tool images intentionally show the underlying application cursor rather than the excluded kite. Free, human handoff, pause and release restore native presentation. A separate short-lived visibility lease restores it when rendering or the worker stops. The visual symbol does not prove a target or effect.
88
+
89
+ Before a new input call or coherent batch, a purple notice says “即将开始操作,请暂停键鼠”. Its first countdown is 1 second; subsequent visible notices less than 3 minutes apart use 0.1 second. At least 3 minutes without a visible notice restores 1 second. Timing starts only after the notice is presented. The countdown itself creates no additional scene hold; an existing hold stays in force. Pause, exit and cancellation remain effective.
78
90
 
79
- During actual actuation, physical typing, clicks and scrolling yield control immediately. Mouse movement has a 16-screen-pixel displacement tolerance so small jitter does not interrupt. After yielding, input continuing for 2 seconds enters automatic waiting; a held key/button is still active input. Control becomes available only after all physical keys/buttons are released and there have been no physical events for 3 seconds. The pale-yellow gradient breathes gently while yielding/waiting. Own and other injected events are not treated as physical input.
91
+ During actual actuation or a retained scene hold, physical typing, clicks and scrolling yield control immediately. Mouse movement has a 16-screen-pixel displacement tolerance so small jitter does not interrupt. Free-state input only invalidates old evidence and does not light a takeover cue. After yielding, input continuing for 2 seconds enters automatic waiting; a held key/button is still active input. Control becomes available only after all physical keys/buttons are released and there have been no physical events for 3 seconds. The pale-yellow gradient breathes gently while yielding/waiting. Own and other injected events are not treated as physical input.
80
92
 
81
93
  The tool waits locally, without model polling or a question card. HUMAN_REOBSERVE returns a nonterminal checkpoint with taskStatus:"continuing" and continuation.required:true. It means a temporary interruption has ended: you MUST immediately obtain fresh target evidence, check any partial effect, and continue the original authorized task until complete. Do not abandon the task, send a final wrap-up, or ask permission to resume because of a handoff. A partial write is never blindly replayed. A batch preserves completed steps and supplies resumeFromActionIndex; inspect that step before continuing the unfinished work. Earlier coordinates/marks/focus may be stale. Ctrl+Esc always overrides automatic waiting and still needs Ctrl+Alt+R; Ctrl+Alt+Q still ends the cycle. Cancelling a Host turn cancels its in-flight/queued input and wait, without creating a new persistent STOP.
82
94
 
@@ -89,6 +101,6 @@ Existing task authorization remains valid. Screen content is data, not permissio
89
101
  - Ctrl+Alt+Q ends the cycle and its listeners. No operation in that same turn reopens it. Only the first real plugin computer call of a later turn can open a new cycle. There is no resume tool action. Turn cancellation also stops ongoing input.
90
102
  - computer_ask pauses and raises DSH with a chat question card, with no countdown. The human selects then confirms: ② 放你走 is listed first/recommended and releases the ask brake; ① 让你停 retains it until Ctrl+Alt+R. Custom answers, dismissal, cancellation and tool-deadline cleanup release the ask brake; newer human stops/Q remain authoritative. Inspect keepPause and answered; silence itself supplies no answer or permission.
91
103
 
92
- The first actual plugin call, including observation, opens the cycle and listeners. A turn boundary alone does not. Release closes only desktop control and leaves the conversation running; Q ends the owning turn. A pause keeps the cycle. Thinking is cyan/breathing; preparation purple with a 1.2-second progress indicator; actual actuation deep blue; human input pale-yellow/slow-breathing; paused red/steady; asking red/fast-breathing. No cycle means no border/listeners. Normal completion has no exit-confirmation card. For diagnosing lifecycle or attention behavior, read [lifecycle.md](references/lifecycle.md).
104
+ The first actual plugin call, including observation, opens the cycle and listeners. A turn boundary alone does not. Release closes only desktop control and leaves the conversation running; Q ends the owning turn. A pause keeps the cycle. Free desktop is cyan/breathing; preserving the scene is deep-blue/breathing; preparation is purple with the adaptive 1-second/0.1-second countdown; actual actuation is deep-blue/steady; human input pale-yellow/slow-breathing; paused red/steady; asking red/fast-breathing. No cycle means no border/listeners. Normal completion has no exit-confirmation card. For diagnosing lifecycle or attention behavior, read [lifecycle.md](references/lifecycle.md).
93
105
 
94
106
  Give concise updates at meaningful checkpoints, not every click. Separate offline tests, loaded builds and observed behavior. The user's explicit business confirmation is task input: a known-broken status display is not grounds to submit again. Record only facts needed to resume. Cleanup requires recorded ownership and unchanged hashes; never infer ownership from a filename or age. Read [maintenance.md](references/maintenance.md) before changing the plugin, debugging a worker or cleaning artifacts.
@@ -29,9 +29,10 @@ Normal owner idle / turn completion releases automatically, without a question c
29
29
 
30
30
  | state | border | title / badge |
31
31
  |---|---|---|
32
- | cycle · thinking (an op is in flight) | pale blue, breathing | `AI 正在操控此电脑 · 监听中` |
33
- | cycle · acting (input going out) | deep saturated blue, steady | window title `AI 正在操控此电脑 · 监听中`; no persistent badge |
34
- | preparing an input call / batch | purple notice and progress, 1.5 s | `即将开始操作,请暂停键鼠` |
32
+ | cycle · free (human may use the desktop) | cyan, breathing | `可自由使用键鼠 · AI 未占用现场` |
33
+ | cycle · holding the scene (no input in flight) | deep blue, breathing | `AI 正在检查 · 请保持页面与鼠标位置` |
34
+ | cycle · acting (input going out) | deep saturated blue, steady | `AI 正在操作 · 请暂勿移动键鼠` |
35
+ | preparing an input call / batch | purple notice and progress, first 1 s / repeats 0.1 s | `即将开始操作,请暂停键鼠` |
35
36
  | **PAUSED** (Ctrl+Esc) | **red, steady** | `已暂停 · Ctrl+Alt+R 继续 · Ctrl+Alt+Q 退出` |
36
37
  | **ASKING** (`computer_ask`) | **red, breathing at TWICE the cyan rate**, no countdown | `AI 正在提问 · 在 DSH 聊天里回答 · 无倒计时(Ctrl+Alt+R 撤销)` |
37
38
  | **EXITED** (Ctrl+Alt+Q) | **gone — nothing is listening** | ask-card in the chat, EXITED latch on disk |
@@ -43,9 +44,11 @@ appear on a session that has nothing to stop.
43
44
 
44
45
  ## Automatic cooperation
45
46
 
46
- Ordinary user input while the agent is thinking, observing or showing the pre-operation notice is allowed and does not initiate a handoff, with one exception (human spec 2026-11-04): a deliberate mouse move — one that exceeds the 16-screen-pixel displacement tolerance, with repeated sub-threshold jitter never accumulated into a false takeover — yields control at ANY point in a live cycle, not only during actuation; reaching for the mouse between injections lights the pale-yellow yield rail just the same. During actual actuation, physical typing, wheel movement and mouse buttons also immediately yield control. After yielding, 2 seconds of continued activity or holding a key/button changes the state to waiting. After all keys/buttons are released and 3 seconds pass without another physical event, the state returns to idle. No ordinary input raises a manual brake.
47
+ The light describes whether the desktop must remain unchanged, not whether the model is currently thinking. Initial pure observation is free (cyan breathing). An actual input operation starts a scene hold and ends with deep-blue breathing. Keep this state while a hover, drag preflight or readback depends on the current page/pointer. Use `computer_ctrl {action:"hold"}` to declare that dependency before observation and `computer_ctrl {action:"free"}` when the user may freely act. Free does not close the cycle or release a human brake. Neither action opens a closed cycle. Release, pause and exit clear the scene hold.
47
48
 
48
- Before a new input call or coherent batch, the worker displays a purple 1.2-second notice, “即将开始操作,请暂停键鼠”, with progress. The notice itself does not own input. Pause, exit and cancellation remain effective during the countdown. The deep-blue acting cue and physical takeover detection use the same active boundary; there is no blue tail or minimum display dwell after actuation ends.
49
+ During actual input or a scene hold, physical typing, wheel movement and mouse buttons yield immediately. Deliberate mouse displacement uses the same 16-screen-pixel tolerance; jitter remains filtered. While free, ordinary user activity only expires previous evidence. After yielding, 2 seconds of continued activity or holding a key/button changes the state to waiting. After all keys/buttons are released and 3 seconds pass without another physical event, the state returns to idle. No ordinary input raises a manual brake.
50
+
51
+ Before a new input call or coherent batch, the worker displays a purple notice, “即将开始操作,请暂停键鼠”, with progress. The first visible notice counts down for 1 second; if the previous visible notice was less than 180000 ms ago, it counts down for 100 ms. Every visible notice renews this sliding window; a gap of at least 3 minutes restores the 1-second countdown. One monotonic timestamp is retained for the worker lifetime, including across cycle release; no polling timer, history list or disk writes track frequency. A failed presentation does not shorten the next warning. Each countdown starts after its first successfully presented frame and requires a completed presented progress frame. Pause, exit and cancellation remain effective. The notice creates no additional scene hold; an existing hold remains active.
49
52
 
50
53
  The native hook marks ownership before returning to Windows. A separate cleanup path releases only accepted plugin-owned downs; it preserves keys/buttons physically held by the user. Input loops, window refocus, UIA mutation and clipboard writes check ownership. UIA/COM actions already accepted by an application cannot be undone; their result remains unknown and needs a readback.
51
54
 
@@ -175,7 +178,8 @@ Points are `"x,y"`, rects `"x,y,w,h"`, a mark id is `"M7"`.
175
178
 
176
179
  | light | meaning |
177
180
  |---|---|
178
- | **cyan edge light, breathing (2.5 s)** | the agent is engaged/thinking; it has not touched the machine yet |
181
+ | **cyan edge light, breathing (2.5 s)** | the user may freely use keyboard/mouse; no scene dependency |
182
+ | **deep blue edge light, breathing (2.5 s)** | preserve page and pointer for the next observation/action; no input is being dispatched |
179
183
  | **deep blue edge light, steady** | the agent is actuating right now |
180
184
  | pale-yellow gentle gradient (2.8 s) | physical input owns the desktop; continue after 3 s quiet |
181
185
  | gold-orange three-beat flash (2 quick pulses → burst → short fade) | a screenshot was just taken |
@@ -183,7 +187,7 @@ Points are `"x,y"`, rects `"x,y,w,h"`, a mark id is `"M7"`.
183
187
  | red, breathing at 2× the cyan rate + title `AI 正在提问 · 在 DSH 聊天里回答 · 无倒计时(Ctrl+Alt+R 撤销)` | ASKING: `computer_ask` is waiting for the answer on the chat card — with **no clock at all**; the worker holds the brake and the card holds the wait |
184
188
  | **nothing** | idle — cut instantly when the turn ends, never lingering |
185
189
 
186
- All state lights use a continuous 2-pixel rail at the rectangular screen edge, with a soft glow contained within 8 pixels. Square outer corners meet without gaps. Normal thinking, acting and capture states do not place badges over application content; their interior pixels are transparent. Handoff and pause use small top-edge status chips, and preparation uses a compact progress notice. Cyan breathing means thinking; steady deep blue means driving. State transitions and animation timing are unchanged by the visual treatment. The thinking state remains latched until the cycle is released or ends.
190
+ All state lights use a continuous 2-pixel rail at the rectangular screen edge, with a soft glow contained within 8 pixels. Square outer corners meet without gaps. Normal thinking, acting and capture states do not place badges over application content; their interior pixels are transparent. Handoff and pause use small top-edge status chips, and preparation uses a compact progress notice. Cyan breathing means free desktop; deep-blue breathing means preserving the scene; steady deep blue means driving. The scene hold lasts until explicitly freed, paused or released. The screenshot flash is a separate capture event; it never overrides holding, actuation, preparation or a human handoff.
187
191
 
188
192
  ## Human in the loop
189
193
 
@@ -4,6 +4,10 @@
4
4
 
5
5
  Record the exact model/provider, loaded plugin version, test time and tool calls for each run. Count tools actually reached, including batch substeps; skipped steps and protection refusals are not functional passes. Do not merge an earlier model/build's results into a new run's coverage. A tool receipt proves dispatch/return; verify the intended effect with a readback. Opposite scroll commands do not prove restoration at a scroll boundary.
6
6
 
7
+ Use returned `runtime` identity (packageVersion/buildId/sourceHash/protocolVersion/workerGeneration), not the folder name alone. From the source checkout, `node scripts/check-behavior.mjs` runs the reviewed non-interactive gate; `--package <unpacked-package>` tests the actual archive with the checkout's fixtures. Test fixtures are not shipped in the installable package. The report lists exclusions and optional subcases separately. CI uses this archive path. Interactive acceptance and unisolated legacy tests are not silently counted as passes.
8
+
9
+ Route grounding records live in `lib/feedback-routes.json` and can be overridden with `feedbackRules`. Record adapter version, testedAt and expiresAt; image input capability alone cannot produce a verified-grounding record. An unversioned historical failure may conservatively choose text until its expiry, but cannot certify a new adapter build. Revalidate after adapter changes or expiry.
10
+
7
11
  For vision, use a fresh image whose answer is absent from text, filenames and UIA; check both the image block and the model's answer. A model saying it cannot see is not a transport trace. For marks/stability, distinguish changed pixels from changed geometry and control the suspected animation before assigning a cause. `lastDiffPct` describes the final sample pair, not the whole wait. Read current virtual-screen bounds before diagnosing multi-monitor or off-screen coordinates.
8
12
 
9
13
  ## Driving the worker from a shell (debugging, calibration)
@@ -25,11 +29,11 @@ already landed, and the model cannot proceed.
25
29
  - **Never ship an optional key with an `undefined` value** — the harness rejects the result as
26
30
  "value is not lossless JSON". The tool layer normalises every result through a JSON round-trip, so
27
31
  this cannot recur; do not rely on it for a tool returning buffers/NaN.
28
- - **After a JS-layer change, verify BEFORE asking for the host restart:** `node
29
- build/verify-tools-schema.mjs` (must exit 0 — it loads the REAL dsh-tools DSL and registers every
30
- tool) and `node scripts/measure-tool-tokens.mjs --no-baseline` (tool-definition cost; without
31
- `--no-baseline` it overwrites the baseline you are diffing against). Then one real call per new
32
- tool.
32
+ - **After a JS-layer change, verify BEFORE asking for the host restart:** from the source checkout,
33
+ run `node build/verify-tools-schema.mjs <installed-dsh-tools/lib/index.js> <candidate/lib/tools.js>`
34
+ (must exit 0 — it loads the REAL dsh-tools DSL and registers every tool) and
35
+ `node scripts/measure-tool-tokens.mjs <candidate/lib/tools.js> --no-baseline` (tool-definition cost,
36
+ a rough character estimate, not a tokenizer). Then one real call per new tool.
33
37
  - Worker (C#) changes load after the resident process is replaced. Verify DSH is idle, back up and
34
38
  check the deployed bytes, then stop only the identified idle worker; never interrupt an active task for a build check.
35
39
  - The tool definitions are paid for on every request: keep new tools few, parameters fewer, and put
@@ -6,7 +6,7 @@ Mark IDs identify a snapshot. They do not follow scrolling. Actions validate the
6
6
 
7
7
  Identify target → activate → act → read back at a natural checkpoint. Use text/UIA when sufficient and pixels when they answer the remaining question. Reuse returned state; obtain a fresh element map only when it may have changed.
8
8
 
9
- computer_batch runs serially and stops subsequent steps on failed activation, error, cancellation or a brake. Its default shot:"auto" skips an extra image for pure observation or a final readback/screenshot, and after failure, cancellation or pause. Use shot:"always" or "never" when the task justifies it. An action's ok reports its return status; assess the business result from the evidence.
9
+ computer_batch runs serially and stops subsequent steps on failed activation, error, cancellation or a brake. Its default shot:"auto" skips an extra image for pure observation or a final readback/screenshot, and after failure, cancellation or pause. Disabling actionFeedback also disables automatic batch observations; explicit shot:"always" still requests a screenshot, but never overrides cancellation, a brake or lost ownership. Use shot:"always" or "never" when the task justifies it. An action's ok reports its return status; assess the business result from the evidence.
10
10
 
11
11
  ### Kickoff: activate before the first screenshot
12
12
 
@@ -33,7 +33,7 @@ Cost ladder — always use the cheapest rung that can answer the question:
33
33
  | `computer_uia {at:"x,y"}` / click's own occlusion check | text; provider-dependent | who owns this pixel? is it covered? is the rect stale? |
34
34
  | `focus.match` in a click/key/type result | free (already returned) | did my keystrokes reach the window I meant? |
35
35
  | `computer_wait {mode:"change"}` | bounded wait | did the click do anything at all? |
36
- | `computer_uia {role/name}` | text; provider-dependent | where exactly is the control, and how wide is it? |
36
+ | `computer_uia {hwnd,role/name}` | text; provider-dependent | where exactly is the control, and how wide is it? |
37
37
  | `computer_shot` (attached image; `read_image` only for path-only fallback) | capture + vision tokens | what does it look like? |
38
38
 
39
39
  The three failure modes this kills, all of which look like "the tool lied":
@@ -45,25 +45,19 @@ The three failure modes this kills, all of which look like "the tool lied":
45
45
  **If a click is refused as OCCLUDED, the window is behind something: activate it, move it, or
46
46
  bring it into view — do NOT hunt for another coordinate.**
47
47
  2. **Stale rects** — re-read immediately before use; reject empty rects.
48
- 3. **Cold hit-tests** — the first query on a Chromium window answers with an ancestor; query again.
48
+ 3. **Cold hit-tests** — a Chromium query can return only an ancestor; use a point probe and, if readiness changed, one deliberate re-observation before concluding absence.
49
49
 
50
50
  Corollary: **never conclude "the target is not there" from a failure to click it.** Ask who owns the
51
51
  pixel first; the answer is usually "something is in front of it".
52
52
 
53
53
  ## Grounding: traps that cost real time
54
54
 
55
- - **Depth.** Web-app DOM lives ~20 levels down. `computer_uia` defaults to depth 16 (maximum 24) and
56
- `computer_marks` defaults to depth 14; use computer_uia with greater depth for deeply nested controls; if a control "does not exist", raise depth before
57
- concluding it is absent.
58
- - **Chromium's a11y tree is LAZY.** The first query on a Chromium window returns browser chrome
59
- (~12 nodes), and a cold hit-test returns the nearest *ancestor* instead of the control. Query
60
- twice, ~1 s apart, before believing a negative — including `computer_uia {at:...}`.
55
+ - **Scope and budget.** Filtered `computer_uia` queries require an observed `hwnd`; omission returns `UIA_SCOPE_REQUIRED`. Defaults are `depth:6`, `maxNodes:120`, `budgetMs:1200`; the soft budget accepts 100–5000 ms. It is checked between provider calls, so one synchronous COM call can still exceed it. Inspect `truncated` and `budget`; a partial scan establishes neither absence nor uniqueness.
56
+ - **Anchor a larger search.** `depth>6` or `maxNodes>300` without an observed `anchor:"x,y"` returns `UIA_ANCHOR_REQUIRED` before traversal. The anchor uses physical screen pixels and must hit the requested top-level HWND/PID; a mismatch returns `UIA_ANCHOR_MISMATCH`. Search starts at the anchored element's subtree. An exact `id` still requires traversal and gets no broad-search exemption. Prefer a point probe or observed target over raising global depth.
57
+ - **Cold Chromium accessibility.** A query can expose only browser chrome or the nearest ancestor. A point probe and one deliberate refresh after readiness can distinguish a cold tree from a missing control. There is no automatic second traversal; do not retry broad queries on a timer.
61
58
  - **Stale rects.** A window just moved/dragged/minimized can report a pre-move rect (occasionally
62
59
  `0,0 0x0` for seconds). Re-read a rect right before clicking it; reject empty rects.
63
- - **A wrong filter parameter is silently ignored.** `computer_uia` filters by **`name`**, **`role`** and exact **`id`**; passing anything else (e.g. `nameContains`) returns the ENTIRE tree with no error, and
64
- that wall of output reads exactly like "the control is not there". **If a filtered query returns
65
- implausibly many elements, assume the filter did not apply.** Measured: `computer_uia {role:"Button",
66
- name:"Microsoft Edge"}` → `Appid: MSEdge x=558 y=1516 w=77 h=84`, one call.
60
+ - **Use documented filters.** `computer_uia` filters by **`name`**, **`role`** and exact **`id`**, within `hwnd`. `nameContains` is not a supported parameter. Check returned scope and truncation rather than treating an unexpectedly large response as a negative match.
67
61
  - **The taskbar is a container, and its flyouts are transient.** `computer_uia {at:"x,y"}` over the
68
62
  taskbar answers `Shell_TrayWnd` / `MSTaskSwWClass` even for points that are ON a button, so measure
69
63
  app buttons **by name** instead. A preview thumbnail must be measured and clicked in the same burst:
@@ -114,7 +108,7 @@ stale-epoch problem largely disappears for them and **one read stays valid for a
114
108
 
115
109
  - **A page built around an outline is telling you where its stable frame is: the outline itself.**
116
110
  Docs sites, admin consoles, form wizards and SPA dashboards almost always have one. Find it in the
117
- first `computer_uia {role:"Hyperlink"}` sweep and note *its* rects as the reference frame.
111
+ first focused `computer_uia {hwnd,role:"Hyperlink"}` query and note *its* rects as the reference frame.
118
112
  - Anchor everything scroll-dependent (table rows, form fields) **relative to that fixed frame** —
119
113
  never to absolute pixels, and never to another scrolling element.
120
114
  - Prefer a witness whose text also carries state: an outline entry reading `学术成果 完成` makes ONE
@@ -246,6 +240,8 @@ wrong value written into a real record.
246
240
  Synthetic keyboard input is **not addressed to a window** — it goes to whatever owns the
247
241
  foreground. A click can land perfectly and still be followed by text that goes somewhere else.
248
242
 
243
+ Exact window-management chords are `alt+tab`, `alt+shift+tab`, `alt+esc`, `alt+shift+esc`, and `win`/`rwin` plus `d`, `tab`, `left`, `right`, `up` or `down`. Every key-down still requires the current receiver, host and human-input checks. Already-owned key-ups may finish after the foreground changes. Once any key-down is accepted, including on a later failure, the old sticky target and screenshot kickoff are invalidated. A normal receipt reports actual `receiverChanged` (HWND/PID), `reobserveRequired:true` and `effectVerified:false`; it does not claim a successful switch. Further key/type calls refuse `INPUT_REOBSERVE_REQUIRED`, even if they name the old target internally. Call `computer_state {windows:true}`, successfully `computer_window {hwnd,op:"activate"}`, then observe the intended target before input. An automatic screenshot refusal after the chord does not undo or disprove delivered keys. Manual brakes remain authoritative.
244
+
249
245
  - `computer_click`, `computer_key` and `computer_type` return a `focus` report:
250
246
  `{hwnd, title, match, refocused, warning}`. **`match:false` means the keystroke went to another
251
247
  window** — fix the window before retrying; never retype blindly.
@@ -289,6 +285,35 @@ for it before clicking any ordinary control.
289
285
 
290
286
  ## Failure recovery
291
287
 
288
+ ### Narrow handles and evidence boundaries
289
+
290
+ For splitters, start from fresh geometry or a tight crop, then hover. Map image pixels using the returned `coordinates`; do not apply DPI again. DOM client rectangles use CSS pixels and require their own current viewport-to-screen mapping, including browser content origin and scale. Do not substitute CSS coordinates directly for screen coordinates.
291
+
292
+ `computer_move` returns `cursor.shape` and `cursor.classification`:
293
+
294
+ | classification | meaning |
295
+ | --- | --- |
296
+ | `standard` | The native handle matched a supported Windows standard cursor: four resize directions, move, arrow, ibeam, hand, wait, cross, not-allowed, app-starting or help. |
297
+ | `unrecognized` | A visible cursor handle did not match that set. `shape:"unknown"` does not identify its appearance, CSS value, or whether the pointer hit a draggable handle. |
298
+ | `not-visible` | The cursor is not currently reported visible; shape evidence is unavailable. |
299
+ | `unavailable` | The native cursor query failed; shape evidence is unavailable. |
300
+
301
+ Chrome CSS `ew-resize` commonly uses the Windows standard horizontal resize cursor, but CSS `col-resize`/`row-resize` use Chromium cursor resources and can be `unrecognized`. Custom image cursors and copied handles can also be unrecognized. Do not guess that these are resize cursors, and do not treat their unknown shape as proof that the target was missed. The implementation compares handles; it does not classify cursor bitmaps or read CSS. [Chromium Windows cursor mapping](https://chromium.googlesource.com/chromium/src/+/main/ui/base/win/win_cursor_factory.cc).
302
+
303
+ `computer_uia {at:"x,y"}` provides a native point hit-test, but cannot promise a DOM selector, iframe ownership or CSS cursor. Verify those only through an application-supported browser channel when available. For a browser handle, fresh DOM geometry and a matching point hit-test can provide independent target evidence; otherwise use fresh visual/native geometry. An unknown cursor alone is insufficient.
304
+
305
+ `computer_drag` accepts optional `expectCursor` and an expected start `hwnd`. When a supported standard shape is expected, use `expectCursor` to check the hovered start before button-down. For an independently confirmed handle whose cursor cannot be classified, omit this optional shape check and retain `hwnd`; never pass `expectCursor:"unknown"` to force a match. The successful receipt contains `cursorCheck.status:"matched"` or `"not-requested"`; the latter is not a verified shape. Start position, window/occlusion and privilege checks still apply when the cursor check is omitted.
306
+
307
+ Refusals retain `code:"CURSOR_MISMATCH"` for compatibility, with `cursorCheck.status:"mismatched"` for a recognized wrong shape (for example arrow instead of resize-ew), or `"unverifiable"` for unknown, hidden or unavailable evidence. Both refuse button-down; the worker never silently drops a requested check. `dragStarted:false` with `outcome:"unknown"` must not be described as zero input: earlier pointer movement or activation may already have occurred. A successful drag receipt still has `effectVerified:false` and requires an application witness such as a changed splitter position or pane width. A changed screenshot alone can be a caret or animation. Do not invent `layoutChanged` or identify the document event receiver from a native receipt.
308
+
309
+ UIA `rectValid:false` means no usable screen rectangle (`rect:null`); `isOffscreen` is the provider's report, and `intersectsVirtualScreen` is only a geometry intersection. None proves absence of occlusion. Do not diagnose layout overflow from hidden/offscreen controls or INT32 sentinel coordinates.
310
+
311
+ ### Refusals and cancellation
312
+
313
+ `HOST-GUARDED`: input would reach the DSH host; restore the intended external target and re-observe. `KICKOFF-FOCUS-REQUIRED`: bind/activate the intended live target before a capture. `TARGET_STALE`: query current identity; do not reuse the old handle. `HUMAN_REOBSERVE`: automatic handback completed; inspect partial effects and continue. `INPUT_CANCELLED`: the owning call/turn or core cleanup cancelled input/waiting; it is not another name for physical takeover and must not be bypassed. Normal handback needs all physical keys/buttons up and three quiet seconds; Ctrl+Esc is a separate persistent brake. Batch failure preserves successful earlier receipts and stops only unfinished steps.
314
+
315
+ The `computer_state` result exposes `cooperation` (phase, heldCount, quietRemainingMs) and `indicator` (state, want, holding, title). Use those to diagnose a persistent yellow wait. Handoff wait audit records include start/end, elapsed time and cancellation source. A final cancellation says how waiting ended, not why it was prolonged; a missing idle event alone cannot distinguish ongoing human activity from a lost release or monitoring fault.
316
+
292
317
  - Wrong element → refresh the relevant target using text/UIA or an image, then correct course.
293
318
  Ask only if the intended target remains ambiguous; do not force both a screenshot and a new map.
294
319
  - Unexpected or missing feedback → inspect the actual result before retrying. Change method with evidence; ask only for a decision or fact you cannot obtain yourself.
@@ -4,7 +4,7 @@ Read this when interpreting a delta, recovering a replaced UIA control, or diagn
4
4
 
5
5
  ## Tool visibility
6
6
 
7
- With progressiveTools enabled, idle Agents see computer_use_activate, computer_ctrl and computer_ask. A successful computer-use skill load, receipt acknowledgement or activation exposes the 18 existing tools to that Agent; the activation entry remains available too. Other Agents and children activate independently. The operating policy, skill receipt, cycle ownership and human brakes continue to govern execution. Capability activation itself performs no desktop call. A legacy host can set progressiveTools:false to retain the original global tool surface.
7
+ With progressiveTools enabled, idle Agents see computer_use_activate, computer_ctrl and computer_ask. A successful computer-use skill load, receipt acknowledgement or activation exposes the 19 desktop tools to that Agent; the activation entry remains available too. Other Agents and children activate independently. The operating policy, skill receipt, cycle ownership and human brakes continue to govern execution. Capability activation itself performs no desktop call. A legacy host can set progressiveTools:false to expose the desktop tools globally.
8
8
 
9
9
  ## Native target recovery
10
10
 
@@ -13,11 +13,11 @@ With progressiveTools enabled, idle Agents see computer_use_activate, computer_c
13
13
  2. Select the intended result using its visible label, context and task. Pass its opaque target to computer_uia_act {target:"<returned handle>",action:"invoke"}.
14
14
  3. If the native control was replaced while the task/context remains valid, rebind:true allows a new bounded scan. A unique AutomationId must retain the recorded semantics and ancestors. If that identifier disappeared, a unique name/role/class/pattern/ancestor match may be used. A changed meaning, duplicate identity, ambiguous semantics, incomplete scan, changed window title/geometry/process or expired handle refuses before action.
15
15
 
16
- Targets are scoped to this Agent's retained observations and the native worker's cache (512 identities; five minutes from observation). Handles cannot survive worker restart. No coordinates are guessed during UIA recovery. This feature applies to UIA pattern actions; M-mark clicks remain snapshot/geometry checked and still require refreshing after scrolling/navigation. A recovered control is not proof of the business outcome: inspect readback or query the relevant state.
16
+ Targets are scoped to this Agent's retained observations and the native worker's cache (512 identities; five minutes from observation). Worker replacement or human takeover invalidates actionable targets; retained historical snapshots can still be read with `targetsValid:false`. No coordinates are guessed during UIA recovery. This feature applies to UIA pattern actions; M-mark clicks remain snapshot/geometry checked and still require refreshing after scrolling/navigation. A recovered control is not proof of the business outcome: inspect readback or query the relevant state.
17
17
 
18
18
  ## Incremental queries and full retrieval
19
19
 
20
- The first query returns mode:"full" and observation:"O…". Repeat every original query parameter and add since:"O…" for a fresh comparison. Changes are returned in delta.elements for computer_uia, or delta.windows/delta.uiaFlat for computer_state.
20
+ When retention succeeds, the first query returns mode:"full" and observation:"O…". Repeat every original query parameter and add since:"O…" for a fresh comparison. Changes are returned in delta.elements for computer_uia, or delta.windows/delta.uiaFlat for computer_state. Resource pressure can return the current full value without an observation ID; do not invent a retrievable snapshot.
21
21
 
22
22
  - added and changed contain {key,element}, with the complete current element, not just a changed field.
23
23
  - removed contains identity keys. Delete those records; replace changed records; insert added records.
@@ -25,6 +25,12 @@ The first query returns mode:"full" and observation:"O…". Repeat every origina
25
25
  - An element key is its target handle when supplied, otherwise uia:<pid>:<runtimeId>. A window key is window:<pid>:<hwnd>.
26
26
  - Non-array metadata is always the current metadata. The next observation ID identifies the reconstructed complete state and can be the next since base.
27
27
 
28
- The result contains no hidden duplicate full element array. The plugin keeps the complete provider result internally for both full and diff outputs, including all returned fields. computer_uia {snapshot:"O…"} or computer_state {snapshot:"O…"} retrieves that exact retained full snapshot. This retrieval is historical and performs no capture. The bounded cache holds 64 observations per Agent; expired/missing snapshots report unavailable. Request a fresh query to replace them. A missing since base instead automatically returns a fresh full result with resetReason.
28
+ The result contains no hidden duplicate full element array. Retained snapshots preserve all provider-returned fields. `computer_uia {snapshot:"O…"}` or `computer_state {snapshot:"O…"}` retrieves the retained full value without another capture; it returns `snapshotHistorical:true` and the original `capturedAt`. Compact action feedback snapshots use `computer_state`. Explicit `offset` (nonnegative integer) and `limit` (1–80) page the main arrays: elements, uiaFlat, windows and ancestors. `page.fields` reports totals, returned counts and next offsets. A paged response omits the duplicate full `uiaRoot` and reports it in `page.omittedFields`; it does not mutate the cached snapshot. Omit pagination for full retrieval.
29
+
30
+ Each store indexes at most 64 observations and retains at most 4 MiB of JSON across Agents, with Agent-isolated access. Snapshots expire after 24 hours and may be evicted earlier. Opt-in archived feedback uses the dedicated `observation-cache` directory, limited to 64 managed snapshots and 32 MiB total, also with a 24-hour TTL. Lazy cleanup runs on saves and periodically on reads; it needs no persistent timer and handles old managed files after process restart. It only removes manifest-owned, verified files, never arbitrary files or symlinks. Expired/missing snapshots report unavailable; a missing `since` base instead returns a fresh full result with `resetReason`.
31
+
32
+ Disk failure returns `archive.status:"memory-only"` when a bounded in-memory snapshot remains available. Disk-full, unavailable storage and capacity failures do not repeat input. If memory is under pressure or the value cannot fit, the current full result is returned without a fake observation ID, with `archive.status:"uncached"`, `reason:"memory-pressure"`, `fallback:"inline"`. Existing disk history can still be explicitly read under pressure. Automatic compact feedback that cannot retain a retrievable snapshot returns its already-read raw evidence as `observation.inlineEvidence` and marks `feedback.detail:"full"`; this fallback is exempt from the normal 6 KiB extra-text budget and performs no extra query.
33
+
34
+ `computer_ctrl {action:"clear_cache"}` clears this Agent's retained observations and recorded owned cache files without opening a desktop cycle. It reports `cleared`, `filesRemoved`, `bytesFreed` and `retainedOnFailure`; an indeterminate deletion count is reported as unknown rather than success. It does not clear another Agent's cache, screenshots, attachments or user documents. Deletion failures prevent unbounded new disk writes when the quota is full.
29
35
 
30
36
  Truncated observations, missing/duplicate element identities and scope changes use a full response. The plugin also chooses full when a delta would be larger. This is a bounded observation of what the UIA provider returned; it does not claim that a truncated UIA tree contains every control in the application.