dsh-codex-style-computer-use 1.2.0-rc.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (40) hide show
  1. package/LICENSE +21 -0
  2. package/README.md +144 -0
  3. package/README.zh.md +132 -0
  4. package/cordis.patch.yml +17 -0
  5. package/lib/artifacts.js +35 -0
  6. package/lib/attention-native.js +93 -0
  7. package/lib/attention-worker.cs +194 -0
  8. package/lib/attention.js +140 -0
  9. package/lib/audit.js +30 -0
  10. package/lib/batch.js +52 -0
  11. package/lib/bin/attention-worker.exe +0 -0
  12. package/lib/bin/attention-worker.exe.json +7 -0
  13. package/lib/core/bin/dsh-computer-use-worker.exe +0 -0
  14. package/lib/core/bin/dsh-computer-use-worker.exe.json +7 -0
  15. package/lib/core/compile-worker.ps1 +58 -0
  16. package/lib/core/index.js +844 -0
  17. package/lib/core/native-runtime.js +97 -0
  18. package/lib/core/worker.cs +6375 -0
  19. package/lib/cycle.js +314 -0
  20. package/lib/exposure.js +91 -0
  21. package/lib/feedback.js +35 -0
  22. package/lib/image-output.js +92 -0
  23. package/lib/index.js +1715 -0
  24. package/lib/observations.js +78 -0
  25. package/lib/policy.js +109 -0
  26. package/lib/recovery.js +8 -0
  27. package/lib/skill-gate.js +68 -0
  28. package/lib/tools.js +717 -0
  29. package/mcp/package.json +12 -0
  30. package/mcp/src/index.js +335 -0
  31. package/package.json +88 -0
  32. package/scripts/build-native.mjs +7 -0
  33. package/scripts/smoke-native.mjs +38 -0
  34. package/scripts/verify-package.mjs +28 -0
  35. package/skills/computer-use/SKILL.md +74 -0
  36. package/skills/computer-use/references/lifecycle.md +180 -0
  37. package/skills/computer-use/references/maintenance.md +56 -0
  38. package/skills/computer-use/references/precision.md +297 -0
  39. package/skills/computer-use/references/progressive-control.md +30 -0
  40. package/skills/computer-use/references/shared-input.md +27 -0
package/LICENSE ADDED
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2026 ltmroberthk915
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
package/README.md ADDED
@@ -0,0 +1,144 @@
1
+ # Computer Use · Codex-style desktop control
2
+
3
+ Windows desktop computer use for **DeepSeek Harness**: 18 desktop tools plus an Agent activation entry that observe and drive
4
+ native Windows applications through prebuilt C# workers included in the package. No PowerShell 7,
5
+ SDK, terminal commands or install-time builds are needed.
6
+
7
+ English | [中文](README.zh.md)
8
+
9
+ Windows only (`os: win32`). Requires Node.js ≥ 22 and the .NET Framework 4.x that Windows ships with.
10
+
11
+ ## What it does
12
+
13
+ - **Observe** — window list, screenshots (region / JPEG / PNG / downscaled), a Set-of-Marks map of
14
+ clickable elements, and the UI Automation tree with exact `AutomationId`s.
15
+ - **Act** — click, move, drag, scroll, select a text range, type, press keys, and drive controls
16
+ through their own UI Automation patterns instead of blind coordinates.
17
+ - **Batch** — one `computer_batch` call runs a list of actions serially and stops at the first
18
+ failure, so a known sequence costs one model round trip instead of ten.
19
+ - **Ask** — `computer_ask` raises a question on the same chat card DSH uses for `ask_user_question`,
20
+ with no countdown.
21
+ - **A human brake that reaches the model** — a stop is pushed into the running session, so the agent
22
+ learns it was interrupted instead of discovering it on the next turn.
23
+
24
+ The tools read the desktop; they do not read your files. The only file the plugin writes on its own
25
+ is the Worker binary in `%LOCALAPPDATA%\dsh-computer-use\worker\`, plus the audit log and screenshots
26
+ under `$DSH_HOME/data/computer-use/`.
27
+
28
+ ## Install and update in dsh-market
29
+
30
+ Search **computer-use** and select **dsh-codex-style-computer-use**, by **ltmroberthk915** (npm maintainer: **ltmroberthk**). Click **Install**. Later, click **Update** or **Update all** in the market. The package includes both native helpers; no PowerShell 7 or build permission is needed.
31
+
32
+ If the catalog has not synced yet, use **Settings → Plugins → Add plugin**, enter `dsh-codex-style-computer-use`, and install. This also creates an ordinary registry installation that can be updated from the market. If DSH reports **restart required**, fully exit DSH from its tray menu and reopen it.
33
+
34
+ **Migrating an older Git/tarball installation from this repository:** remove the old **dsh-computer-use** entry in the market, then install **dsh-codex-style-computer-use**. The unscoped npm name `dsh-computer-use` belongs to a different repository; do not install it as an upgrade of this plugin. A Git installation cannot switch its dependency identity just by fetching a new commit. After this one-time UI migration, use normal market updates. Existing `computer_*` tool names and the `computer-use` settings namespace are retained.
35
+
36
+ The GitHub Release also includes `dsh-computer-use.tgz` for offline/manual installation; registry installation is the default for market updates.
37
+
38
+ CLI users (web profile):
39
+
40
+ ```sh
41
+ dsh plugin --profile web add dsh-codex-style-computer-use
42
+ ```
43
+
44
+ ## The bundled skill
45
+
46
+ The driving tools stay locked until the session produces a receipt phrase that exists only inside
47
+ `skills/computer-use/SKILL.md`. That is deliberate — it is the strongest honest version of "read the
48
+ manual before driving", because no tool layer can see whether a model read a document.
49
+
50
+ The plugin registers the bundled skill through `ctx.skills.register()` when available. If that service is absent, `computer_use_activate` returns the same manual, its absolute source path and the scoped tool list. The receipt and all existing approval/brake checks still apply.
51
+
52
+ ## Shared input in 1.2.0-rc.2
53
+
54
+ This pre-release introduces immediate human takeover, automatic waiting after 2 seconds of continued input, and continuation after 3 quiet seconds. Waiting happens locally without model polling. Interrupted writes require a fresh readback and are never blindly replayed. Host cancellation releases accepted automation-owned input without creating a new persistent pause. Existing DSH Pet/main-window and original-topmost handling is retained.
55
+
56
+ The configured GLM 5.3 Max, GLM 5.3 Flash and DeepSeek Flash routes passed a controlled continuation-protocol test. Native input and physical hotkeys were verified separately; this is not a general desktop or vision benchmark. See [release verification](docs/RELEASE-1.2.0-rc.2.md) for scope.
57
+
58
+ ## On-demand controls in 1.1
59
+
60
+ - **Per-Agent tools:** idle Agents see `computer_use_activate`, `computer_ctrl` and `computer_ask`. A successful computer-use skill load, receipt or activation exposes the 18 existing tools plus the activation entry to that Agent. Other Agents and children activate independently. Native calls and Node `run_code` are supported. Activation does not start control or release a human brake.
61
+ - **Observed UIA targets:** a scoped name/id query returns an opaque target handle. Use it for pattern actions; opt-in `rebind:true` allows unique identity or semantic recovery after replacement in the same window. Changed context, ambiguity and incomplete scans refuse. Broad queries skip recovery handles unless `targets:true`; creating witnesses for a large list costs extra native work.
62
+ - **Incremental observations:** pass the previous observation ID as `since` with the same query. Deltas preserve additions, changes, removals and ordering. Each Agent retains 64 complete snapshots; `snapshot` retrieves an exact historical result. Missing bases, changed scope, incomplete evidence or a larger delta fall back to full output.
63
+
64
+ See [the detailed protocol](skills/computer-use/references/progressive-control.md) for cache limits and recovery boundaries. UIA snapshots preserve provider-returned fields within existing provider limits; historical retrieval is not fresh outcome verification.
65
+
66
+ ## Tools
67
+
68
+ | Group | Tools |
69
+ |---|---|
70
+ | Activate | `computer_use_activate` |
71
+ | Observe | `computer_state` `computer_shot` `computer_marks` `computer_uia` |
72
+ | Act | `computer_click` `computer_move` `computer_drag` `computer_scroll` `computer_select` `computer_key` `computer_type` `computer_uia_act` `computer_window` `computer_clip` |
73
+ | Flow | `computer_wait` `computer_batch` |
74
+ | Meta | `computer_ask` `computer_ctrl` |
75
+
76
+ `computer_marks` returns numbered marks for the elements it found on screen; `computer_marks {shot:true}` also saves an annotated image. `computer_shot` returns a plain screenshot and, on image-capable routes, a native image attachment. A mark is a snapshot ID: it is validated against window
77
+ identity and sampled pixels before any input is dispatched, so a stale mark is refused rather than
78
+ clicked at the wrong place.
79
+
80
+ ## Approval modes
81
+
82
+ `automationMode` is set in the plugin's settings and defaults to `standard`:
83
+
84
+ | Mode | Observation | Actuation |
85
+ |---|---|---|
86
+ | `read-only` | allowed | denied |
87
+ | `standard` | allowed | every action asks through the host's approval card |
88
+ | `autonomous` | allowed | allowed, high-risk operations still ask |
89
+ | `unrestricted` | allowed | allowed (worker failsafe and rate limit still apply) |
90
+
91
+ High-risk operations (closing a window, `alt+f4`, `win+r`, and the like) ask in every mode. Password
92
+ fields are detected through UI Automation's `IsPassword` and refused by default.
93
+
94
+ ## Safety model
95
+
96
+ - **Shared input** — physical typing, scrolling, mouse movement and mouse buttons immediately yield control. Continued input or a held key/button for 2 seconds enters automatic waiting. After all keys/buttons are released and 3 seconds pass without physical input, the agent takes a fresh observation and continues the authorized task. A brief touch also waits for the 3-second quiet interval. A gentle pale-yellow edge gradient shows human ownership.
97
+ - **Manual pause** — only physical `Ctrl+Esc` raises a persistent input brake; `Ctrl+Alt+R` resumes. Ordinary Esc, typing, scrolling, pointer distance/frequency and the screen corner do not create this pause. Explicit question/diagnostic holds remain separate.
98
+ - **Exit** — `Ctrl+Alt+Q` ends the session: the overlay goes away, its listeners are torn down, and
99
+ further actuation is refused until a later turn opens a new one.
100
+ - **The brake is a fact about the machine, not one process** — the engaged state is persisted, and
101
+ every worker adopts it, including one started later from a shell.
102
+ - **Audit** — every actuation is appended to `$DSH_HOME/data/computer-use/audit.jsonl` with argument
103
+ sanitisation.
104
+ - **Kill switch for the plugin itself** — `DSH_COMPUTER_USE_KILL=1` makes the plugin refuse to start;
105
+ `dryRun: true` simulates actuations and only logs them.
106
+
107
+ ## Configuration
108
+
109
+ | Key | Default | Meaning |
110
+ |---|---:|---|
111
+ | `automationMode` | `standard` | approval mode (above) |
112
+ | `progressiveTools` | `true` | per-Agent activation; `false` keeps the original 18 global tools |
113
+ | `dryRun` | `false` | simulate actuations, log only |
114
+ | `maxActionsPerMinute` | `60` | core-level actuation rate limit |
115
+ | `annotateMarks` | `true` | draw Set-of-Marks boxes on screenshots |
116
+ | `workerExe` | `""` | explicit worker path; empty = autodiscover or compile |
117
+ | `snapshotDir` | `""` | screenshot directory; empty = `$DSH_HOME/data/computer-use/shots` |
118
+
119
+ ## MCP server
120
+
121
+ `mcp/` is a zero-dependency MCP stdio server over the same core, for clients that speak MCP rather
122
+ than Cordis:
123
+
124
+ ```sh
125
+ node mcp/src/index.js
126
+ ```
127
+
128
+ ## Development
129
+
130
+ The repository ships the guard suite that protects the invariants above — the cycle lifetime, the
131
+ ask/brake protocol, the operation classification, the skill gate, and the bundled-skill wiring:
132
+
133
+ ```sh
134
+ pwsh -NoProfile -ExecutionPolicy Bypass -File build/run-guards.ps1 # all guards
135
+ node build/verify-tools-schema.mjs # tool-schema load guard
136
+ ```
137
+
138
+ The tool definitions are compiled through the real `@deepseek-ai/dsh-tools` DSL, so a schema mistake
139
+ fails here instead of taking the host's plugin tree down on restart. Some guards compile the native
140
+ worker and are Windows-only.
141
+
142
+ ## License
143
+
144
+ MIT — see [LICENSE](LICENSE).
package/README.zh.md ADDED
@@ -0,0 +1,132 @@
1
+ # Computer Use · Codex 风格桌面操控
2
+
3
+ 给 **DeepSeek Harness** 用的 Windows 桌面操控插件:18 个桌面工具加一个 Agent 激活入口,通过一个 C# worker
4
+ 观察并驱动原生 Windows 应用。发布包已包含编译好的原生组件,无需安装 PowerShell 7、SDK 或执行编译命令。
5
+
6
+ [English](README.md) | 中文
7
+
8
+ 仅支持 Windows(`os: win32`)。需要 Node.js ≥ 22 与 Windows 自带的 .NET Framework 4.x。
9
+
10
+ ## 它能做什么
11
+
12
+ - **观察** —— 窗口列表、截图(区域 / JPEG / PNG / 降采样)、可点元素的 Set-of-Marks 编号图,
13
+ 以及带精确 `AutomationId` 的 UI Automation 树。
14
+ - **操作** —— 点击、移动、拖拽、滚动、选中文本区间、键入、按键,并可走控件**自身的 UI
15
+ Automation 模式**,而不是盲猜坐标。
16
+ - **批量** —— 一次 `computer_batch` 串行执行一组动作,遇错即停,因此一段已知流程只花一轮模型往返。
17
+ - **提问** —— `computer_ask` 把问题发到 DSH 与 `ask_user_question` 相同的那张聊天卡片上,且没有倒计时。
18
+ - **能传到模型的人为刹车** —— 停止事件被推入正在运行的会话,agent 当场知道自己被打断,
19
+ 而不是等到下一轮才发现。
20
+
21
+ 工具读的是桌面,不读你的文件。插件自己只写三处:worker 二进制
22
+ (`%LOCALAPPDATA%\dsh-computer-use\worker\`)、审计日志与截图(`$DSH_HOME/data/computer-use/`)。
23
+
24
+ ## 在 dsh-market 安装和更新
25
+
26
+ 搜索 **computer-use**,选择 **dsh-codex-style-computer-use**,作者 **ltmroberthk915**(npm 维护者 **ltmroberthk**),点击**安装**。以后直接点市场里的**更新**或**全部更新**。两份原生组件都已随包提供,无需终端、PowerShell 7、SDK 或手改构建白名单。
27
+
28
+ 目录尚未同步时,可在 **设置 → 插件 → 添加插件** 输入 `dsh-codex-style-computer-use` 安装;这同样是可由市场更新的 npm 安装。如果界面提示**需要重启**,从托盘彻底退出 DSH 后重新打开。
29
+
30
+ **已有本仓库旧 Git/tgz 版本:**在市场卸载旧的 **dsh-computer-use**,再安装 **dsh-codex-style-computer-use**。npm 上的无 scope 包 `dsh-computer-use` 属于另一仓库,不能用作本插件的更新。旧 Git 依赖不会因仓库改名自动迁移;完成这一次界面内迁移后,后续使用普通市场更新即可。原有 `computer_*` 工具名与 `computer-use` 设置命名空间保留。
31
+
32
+ GitHub Release 仍提供 `dsh-computer-use.tgz`,供离线或手动安装。市场自动更新优先使用 npm 包。
33
+
34
+ 网页版的命令行安装方式:
35
+
36
+ ```sh
37
+ dsh plugin --profile web add dsh-codex-style-computer-use
38
+ ```
39
+
40
+ ## 随包 skill
41
+
42
+ 驱动类工具会被锁住,直到会话给出那句只存在于 `skills/computer-use/SKILL.md` 里的回执短语。
43
+ 这是刻意的——没有任何工具层能"看见"模型是否读过文档,这是"先读规程再动手"最诚实的一种实现。
44
+
45
+ 插件在宿主提供 skills 服务时通过 `ctx.skills.register()` 注册随包规程。服务缺失时,`computer_use_activate` 也能返回同一份规程、源文件绝对路径及当前 Agent 的工具列表。回执门禁和既有审批、刹车检查仍然有效。
46
+
47
+ ## 1.2.0-rc.2 的人机共用输入
48
+
49
+ 本预发布版采用“立即让权 → 持续输入 2 秒进入等待 → 静默 3 秒后继续”。等待由本地事件完成,不消耗模型轮询。中途被打断的写入先读回实际结果,避免重复执行。宿主取消任务时释放自动化实际按下的输入,不新增持续暂停。此前 DSH Pet 主窗口识别和原有置顶位恢复修复均保留。
50
+
51
+ 已配置的 GLM 5.3 Max、GLM 5.3 Flash 和 DeepSeek Flash 通过受控续作协议测试;原生操作与实体热键另行验证。这些结果不代表任意真实桌面或视觉任务的成功率,详见[发布验证说明](docs/RELEASE-1.2.0-rc.2.md)。
52
+
53
+ ## 1.1 的按需控制
54
+
55
+ - **按 Agent 加载:**未激活时暴露 `computer_use_activate`、`computer_ctrl`、`computer_ask`。成功读取 computer-use skill、确认规程或调用激活入口后,该 Agent 获得原有 18 个工具,加入口共 19 个。其他 Agent 和子 Agent 独立启用,支持原生调用和 Node `run_code`。激活本身不启动控制,也不释放人为刹车。
56
+ - **已观察的 UIA 目标:**指定窗口和 name/id 的聚焦查询默认生成 target 句柄。控件重建时,显式 `rebind:true` 才允许在同一窗口内进行唯一身份或语义匹配。上下文变化、歧义、扫描不完整时拒绝动作。宽查询需 `targets:true` 才生成恢复句柄,避免普通观察承担额外开销。
57
+ - **增量观察:**重复相同查询并传入上一 observation 的 `since`,返回新增、变化、删除及顺序信息。每 Agent 保留最近 64 份完整快照,`snapshot` 可取回确切的历史结果。缺少基线、范围变化、证据不完整或增量更大时自动回退全量。
58
+
59
+ 保留的是提供器实际返回的完整字段,沿用原有提供器限制;历史快照不代替动作后的新验证。缓存上限和恢复边界见[详细协议](skills/computer-use/references/progressive-control.md)。
60
+
61
+ ## 工具
62
+
63
+ | 分组 | 工具 |
64
+ |---|---|
65
+ | 激活 | `computer_use_activate` |
66
+ | 观察 | `computer_state` `computer_shot` `computer_marks` `computer_uia` |
67
+ | 操作 | `computer_click` `computer_move` `computer_drag` `computer_scroll` `computer_select` `computer_key` `computer_type` `computer_uia_act` `computer_window` `computer_clip` |
68
+ | 流程 | `computer_wait` `computer_batch` |
69
+ | 元 | `computer_ask` `computer_ctrl` |
70
+
71
+ `computer_marks` 返回屏幕上元素的编号列表;`computer_marks {shot:true}` 额外保存对应的标注图。`computer_shot` 返回普通截图,在支持图片输入的模型路由上直接附带图片。mark 是一次**快照
72
+ ID**:派发输入前会核对窗口身份与采样像素,因此过期的 mark 会被拒绝,而不是点到错误的位置。
73
+
74
+ ## 审批模式
75
+
76
+ `automationMode` 在插件设置里配置,默认 `standard`:
77
+
78
+ | 模式 | 观察 | 操作 |
79
+ |---|---|---|
80
+ | `read-only` | 允许 | 拒绝 |
81
+ | `standard` | 允许 | 每个动作都经宿主审批卡确认 |
82
+ | `autonomous` | 允许 | 允许,高危动作仍需确认 |
83
+ | `unrestricted` | 允许 | 允许(worker 自身的失效保护与限速仍在) |
84
+
85
+ 高危动作(关闭窗口、`alt+f4`、`win+r` 等)在任何模式下都要确认。
86
+ 密码框通过 UI Automation 的 `IsPassword` 识别,默认拒绝输入。
87
+
88
+ ## 安全模型
89
+
90
+ - **人机共用输入** —— 实体键盘、滚轮、鼠标移动或按钮操作立即让出控制。持续输入或按住键/按钮 2 秒进入自动等待;全部松开并静默 3 秒后,重新观察目标并继续已授权任务。短暂触碰也等待 3 秒静默。人操作时显示温和的淡黄色边缘渐变。
91
+ - **人工暂停** —— 只有实体 `Ctrl+Esc` 触发持续输入刹车,`Ctrl+Alt+R` 恢复。普通 Esc、打字、滚动、鼠标距离/频率和屏幕左上角不再触发这种暂停;显式提问、诊断暂停保持独立。
92
+ - **退出** —— `Ctrl+Alt+Q` 结束会话:覆盖层消失、监听全部拆除,后续注入被拒,直到之后某一轮重新开启。
93
+ - **刹车是关于机器的事实,不属于某一个进程** —— 闩锁状态落盘,每个 worker 都会遵守,包括之后从命令行启动的。
94
+ - **审计** —— 每次操作连同脱敏后的参数追加到 `$DSH_HOME/data/computer-use/audit.jsonl`。
95
+ - **插件级开关** —— `DSH_COMPUTER_USE_KILL=1` 让插件拒绝启动;`dryRun: true` 只模拟并记录。
96
+
97
+ ## 配置
98
+
99
+ | 键 | 默认 | 含义 |
100
+ |---|---:|---|
101
+ | `automationMode` | `standard` | 审批模式(见上) |
102
+ | `progressiveTools` | `true` | 按 Agent 激活;设为 `false` 可保留原有 18 个全局工具 |
103
+ | `dryRun` | `false` | 只模拟操作并记录 |
104
+ | `maxActionsPerMinute` | `60` | 核心层操作限速 |
105
+ | `annotateMarks` | `true` | 在截图上绘制 Set-of-Marks 方框 |
106
+ | `workerExe` | `""` | 显式指定 worker 路径;留空则自动发现或编译 |
107
+ | `snapshotDir` | `""` | 截图目录;留空为 `$DSH_HOME/data/computer-use/shots` |
108
+
109
+ ## MCP 服务
110
+
111
+ `mcp/` 是建立在同一核心之上的零依赖 MCP stdio 服务,供只讲 MCP、不讲 Cordis 的客户端使用:
112
+
113
+ ```sh
114
+ node mcp/src/index.js
115
+ ```
116
+
117
+ ## 开发
118
+
119
+ 仓库自带守卫套件,保护上面这些不变量——循环生命周期、提问/刹车协议、操作分类、skill 门禁,
120
+ 以及随包 skill 的注册接线:
121
+
122
+ ```sh
123
+ pwsh -NoProfile -ExecutionPolicy Bypass -File build/run-guards.ps1 # 全部守卫
124
+ node build/verify-tools-schema.mjs # 工具 schema 加载守卫
125
+ ```
126
+
127
+ 工具定义走真实的 `@deepseek-ai/dsh-tools` DSL 编译,因此 schema 写错会在这里失败,
128
+ 而不是在宿主重启时把整棵插件树带崩。部分守卫会编译原生 worker,仅 Windows 可跑。
129
+
130
+ ## 许可
131
+
132
+ MIT —— 见 [LICENSE](LICENSE)。
@@ -0,0 +1,17 @@
1
+ # cordis.patch.yml — dsh-computer-use bundle layer.
2
+ # `dsh.bundle.patch` in package.json points here; installing this package into a profile
3
+ # reconciles this insert into the profile's plugin stack.
4
+ #
5
+ # automationMode defaults to `standard` on purpose: observation is free, and every actuation
6
+ # (click / type / key / drag / scroll / clipboard write) goes through the host's approval card.
7
+ # Raise it to `autonomous` (high-risk operations still ask) or `unrestricted` deliberately, in
8
+ # Settings → computer-use, once you have decided the machine may be driven without a prompt.
9
+ - insert:
10
+ - id: computer-use
11
+ name: dsh-codex-style-computer-use
12
+ config:
13
+ enabled: true
14
+ automationMode: standard # read-only | standard | autonomous | unrestricted
15
+ dryRun: false
16
+ maxActionsPerMinute: 60
17
+ annotateMarks: true
@@ -0,0 +1,35 @@
1
+ import { appendFileSync, readFileSync, lstatSync, unlinkSync, writeFileSync } from 'node:fs'
2
+ import { basename, dirname, join, resolve } from 'node:path'
3
+ import { createHash } from 'node:crypto'
4
+
5
+ const digest = bytes => createHash('sha256').update(bytes).digest('hex')
6
+ const manifest = dir => join(dir.root, 'owned-shots.jsonl')
7
+
8
+ export function saveOwnedShot (dir, path, bytes) {
9
+ if (resolve(dirname(path)) !== resolve(dir.shots)) throw new Error('shot path outside its owned directory')
10
+ writeFileSync(path, bytes, { flag: 'wx' })
11
+ // A failed manifest write merely makes this image ineligible for cleanup.
12
+ try { appendFileSync(manifest(dir), JSON.stringify({ file: basename(path), sha256: digest(bytes), size: bytes.length, createdAt: Date.now() }) + '\n') } catch {}
13
+ return path
14
+ }
15
+
16
+ export function cleanupOwnedShots (dir, { before, dryRun = true } = {}) {
17
+ if (!Number.isFinite(before)) throw new Error('an explicit created-before timestamp is required')
18
+ let lines
19
+ try { lines = readFileSync(manifest(dir), 'utf8').split('\n') } catch { return { dryRun, files: [], bytes: 0 } }
20
+ const result = { dryRun, files: [], bytes: 0 }
21
+ const seen = new Set()
22
+ for (const line of lines) {
23
+ try {
24
+ const record = JSON.parse(line)
25
+ if (!record.file || record.file !== basename(record.file) || seen.has(record.file) ||
26
+ !/^shot-[\w-]+\.(jpg|png)$/.test(record.file) || !(record.createdAt < before)) continue
27
+ seen.add(record.file)
28
+ const path = join(dir.shots, record.file), stat = lstatSync(path)
29
+ if (!stat.isFile() || stat.isSymbolicLink() || stat.size !== record.size || digest(readFileSync(path)) !== record.sha256) continue
30
+ if (!dryRun) unlinkSync(path)
31
+ result.files.push(path); result.bytes += stat.size
32
+ } catch { /* missing, locked, modified or malformed entries are retained */ }
33
+ }
34
+ return result
35
+ }
@@ -0,0 +1,93 @@
1
+ import fs from 'node:fs'
2
+ import path from 'node:path'
3
+ import os from 'node:os'
4
+ import { fileURLToPath } from 'node:url'
5
+ import { spawn } from 'node:child_process'
6
+ import { prepareNativeBinary } from './core/native-runtime.js'
7
+
8
+ export class AttentionNative {
9
+ constructor ({ exe = process.execPath, hostPid = process.pid, ownerPid = 0, dataDir, timeoutMs = 1800 } = {}) {
10
+ this.target = { exe, hostPid, ownerPid }
11
+ this.timeoutMs = timeoutMs
12
+ this.dataDir = dataDir || path.join(process.env.DSH_HOME || path.join(os.homedir(), '.dsh'), 'data', 'computer-use')
13
+ this.pending = new Map()
14
+ this.starting = null
15
+ this.child = null
16
+ this.closed = false
17
+ this.journal = path.join(this.dataDir, 'attention-native.jsonl')
18
+ fs.mkdirSync(this.dataDir, { recursive: true })
19
+ }
20
+ log (record) {
21
+ try { fs.appendFileSync(path.join(this.dataDir, 'attention.jsonl'), JSON.stringify({ utc: new Date().toISOString(), version: 'attention-v1', ...record }) + '\n') } catch (error) { console.error('[dsh-attention] log write failed:', error.message) }
22
+ }
23
+ async compile () {
24
+ const source = fileURLToPath(new URL('./attention-worker.cs', import.meta.url))
25
+ return prepareNativeBinary({ source, name: 'attention-worker.exe', cache: 'attention',
26
+ references: ['System.Web.Extensions.dll'] }, { log: message => this.log({ stage: 'helper-prepare', message }) })
27
+ }
28
+ async start () {
29
+ if (this.closed) throw new Error('attention helper disposed')
30
+ if (this.child) return
31
+ if (this.starting) return this.starting
32
+ this.starting = (async () => {
33
+ const binary = await this.compile()
34
+ if (this.closed) throw new Error('attention helper disposed')
35
+ await new Promise((resolve, reject) => {
36
+ const child = spawn(binary, [this.journal], { windowsHide: true, stdio: ['pipe', 'pipe', 'pipe'] })
37
+ this.child = child
38
+ let buffer = '', ready = false
39
+ const timer = setTimeout(() => { reject(new Error('attention helper startup timeout')); child.stdin.end() }, 5000)
40
+ const fail = (error) => {
41
+ clearTimeout(timer)
42
+ if (this.child === child) this.child = null
43
+ for (const p of this.pending.values()) { clearTimeout(p.timer); p.reject(error) }
44
+ this.pending.clear()
45
+ if (!ready) reject(error)
46
+ this.log({ stage: 'helper-exit', error: error.message })
47
+ }
48
+ child.once('error', fail)
49
+ child.once('exit', (code, signal) => fail(new Error(`attention helper exited: ${code}/${signal}`)))
50
+ child.stderr.on('data', b => this.log({ stage: 'helper-stderr', text: String(b).slice(0, 4000) }))
51
+ child.stdin.on('error', e => this.log({ stage: 'helper-stdin-error', error: e.message }))
52
+ child.stdout.on('data', chunk => {
53
+ buffer += chunk.toString('utf8')
54
+ for (let eol; (eol = buffer.indexOf('\n')) >= 0;) {
55
+ const line = buffer.slice(0, eol).trim(); buffer = buffer.slice(eol + 1)
56
+ if (!line) continue
57
+ let event
58
+ try { event = JSON.parse(line) } catch { this.log({ stage: 'helper-protocol-error', line: line.slice(0, 500) }); continue }
59
+ if (event.stage === 'ready') { ready = true; clearTimeout(timer); this.log({ stage: 'helper-ready', ...event.data, binary }); resolve() }
60
+ if ('result' in event || event.error) {
61
+ const pending = this.pending.get(event.id)
62
+ if (!pending) continue
63
+ clearTimeout(pending.timer); this.pending.delete(event.id)
64
+ if (event.error) pending.reject(new Error(event.error)); else pending.resolve(event.result)
65
+ }
66
+ }
67
+ })
68
+ })
69
+ })()
70
+ try { await this.starting } finally { this.starting = null }
71
+ }
72
+ async request (op, options = {}) {
73
+ const started = performance.now()
74
+ await this.start()
75
+ const id = randomUUID()
76
+ const result = await new Promise((resolve, reject) => {
77
+ const timer = setTimeout(() => { this.pending.delete(id); reject(new Error(`attention ${op} timed out (${this.timeoutMs} ms)`)) }, this.timeoutMs)
78
+ this.pending.set(id, { resolve, reject, timer })
79
+ this.child.stdin.write(JSON.stringify({ ...this.target, ...options, op, id }) + '\n', error => {
80
+ if (error) { clearTimeout(timer); this.pending.delete(id); reject(error) }
81
+ })
82
+ })
83
+ return { ...result, roundTripMs: Math.round(performance.now() - started), requestId: id }
84
+ }
85
+ async close () {
86
+ this.closed = true
87
+ if (this.starting) await this.starting.catch(() => {})
88
+ const child = this.child
89
+ if (!child) return
90
+ // EOF executes the native finally; do not terminate it while it owns a topmost lease.
91
+ await new Promise(resolve => { child.once('exit', resolve); child.stdin.end(); setTimeout(resolve, 2000).unref() })
92
+ }
93
+ }
@@ -0,0 +1,194 @@
1
+ // C# 5, Windows inbox .NET Framework only. No input injection or attached queues.
2
+ using System;
3
+ using System.Collections.Generic;
4
+ using System.Diagnostics;
5
+ using System.IO;
6
+ using System.Runtime.InteropServices;
7
+ using System.Text;
8
+ using System.Threading;
9
+ using System.Web.Script.Serialization;
10
+
11
+ class AttentionWorker
12
+ {
13
+ [StructLayout(LayoutKind.Sequential)] struct RECT { public int L, T, R, B; }
14
+ [StructLayout(LayoutKind.Sequential)] struct POINT { public int X, Y; public POINT(int x, int y) { X=x; Y=y; } }
15
+ [StructLayout(LayoutKind.Sequential)] struct FLASH { public uint size; public IntPtr hwnd; public uint flags, count, timeout; }
16
+ [StructLayout(LayoutKind.Sequential, CharSet=CharSet.Unicode)] struct ENTRY {
17
+ public uint size, usage, pid; public IntPtr heap; public uint module, threads, parent; public int priority; public uint flags;
18
+ [MarshalAs(UnmanagedType.ByValTStr, SizeConst=260)] public string exe;
19
+ }
20
+ delegate bool EnumProc(IntPtr h, IntPtr p);
21
+ [DllImport("user32.dll")] static extern bool EnumWindows(EnumProc cb, IntPtr p);
22
+ [DllImport("user32.dll")] static extern bool IsWindow(IntPtr h);
23
+ [DllImport("user32.dll")] static extern bool IsWindowVisible(IntPtr h);
24
+ [DllImport("user32.dll")] static extern bool IsIconic(IntPtr h);
25
+ [DllImport("user32.dll")] static extern IntPtr GetForegroundWindow();
26
+ [DllImport("user32.dll")] static extern uint GetWindowThreadProcessId(IntPtr h, out uint pid);
27
+ [DllImport("user32.dll")] static extern IntPtr GetWindow(IntPtr h, uint cmd);
28
+ [DllImport("user32.dll")] static extern IntPtr GetAncestor(IntPtr h, uint flags);
29
+ [DllImport("user32.dll", CharSet=CharSet.Unicode)] static extern int GetWindowTextLength(IntPtr h);
30
+ [DllImport("user32.dll", CharSet=CharSet.Unicode)] static extern int GetWindowText(IntPtr h, StringBuilder title, int length);
31
+ [DllImport("user32.dll")] static extern int GetWindowLong(IntPtr h, int n);
32
+ [DllImport("user32.dll")] static extern bool GetClientRect(IntPtr h, out RECT r);
33
+ [DllImport("user32.dll")] static extern bool ClientToScreen(IntPtr h, ref POINT p);
34
+ [DllImport("user32.dll")] static extern IntPtr WindowFromPoint(POINT p);
35
+ [DllImport("user32.dll", SetLastError=true)] static extern bool SetWindowPos(IntPtr h, IntPtr after, int x, int y, int w, int t, uint flags);
36
+ [DllImport("user32.dll")] static extern bool ShowWindowAsync(IntPtr h, int cmd);
37
+ [DllImport("user32.dll")] static extern bool SetForegroundWindow(IntPtr h);
38
+ [DllImport("user32.dll")] static extern bool FlashWindowEx(ref FLASH f);
39
+ [DllImport("user32.dll")] static extern IntPtr SetThreadDpiAwarenessContext(IntPtr v);
40
+ [DllImport("dwmapi.dll")] static extern int DwmGetWindowAttribute(IntPtr h, int a, out int v, int size);
41
+ [DllImport("kernel32.dll", SetLastError=true)] static extern IntPtr OpenProcess(uint access, bool inherit, uint pid);
42
+ [DllImport("kernel32.dll", CharSet=CharSet.Unicode, SetLastError=true)] static extern bool QueryFullProcessImageName(IntPtr p, int flags, StringBuilder b, ref int n);
43
+ [DllImport("kernel32.dll")] static extern bool CloseHandle(IntPtr p);
44
+ [DllImport("kernel32.dll")] static extern IntPtr CreateToolhelp32Snapshot(uint flags, uint pid);
45
+ [DllImport("kernel32.dll", CharSet=CharSet.Unicode)] static extern bool Process32First(IntPtr s, ref ENTRY e);
46
+ [DllImport("kernel32.dll", CharSet=CharSet.Unicode)] static extern bool Process32Next(IntPtr s, ref ENTRY e);
47
+
48
+ const uint POSITION = 0x0001 | 0x0002 | 0x0010 | 0x0200 | 0x4000; // no size/move/activate/owner-z, async
49
+ static readonly object outputLock = new object(), leaseLock = new object();
50
+ static readonly Dictionary<long, Lease> leases = new Dictionary<long, Lease>();
51
+ static string journal;
52
+ class Lease { public IntPtr h; public uint pid; public long until; public string id; }
53
+ static Dictionary<string,object> D(params object[] pairs) {
54
+ var d = new Dictionary<string,object>(); for (int i=0;i<pairs.Length;i+=2) d[(string)pairs[i]]=pairs[i+1]; return d;
55
+ }
56
+ static void Write(object d) {
57
+ lock(outputLock) {
58
+ string s = new JavaScriptSerializer().Serialize(d);
59
+ try { Console.WriteLine(s); Console.Out.Flush(); } catch { }
60
+ if (journal != null) try { File.AppendAllText(journal, s+Environment.NewLine, new UTF8Encoding(false)); } catch { }
61
+ }
62
+ }
63
+ static void Stage(string id, string stage, object data) { Write(D("id",id,"utc",DateTime.UtcNow.ToString("o"),"stage",stage,"data",data)); }
64
+ static string S(Dictionary<string,object> d,string k,string fallback="") { return d.ContainsKey(k) ? Convert.ToString(d[k]) : fallback; }
65
+ static long N(Dictionary<string,object> d,string k,long fallback=0) { return d.ContainsKey(k) ? Convert.ToInt64(d[k]) : fallback; }
66
+ static bool B(Dictionary<string,object> d,string k) { return d.ContainsKey(k) && Convert.ToBoolean(d[k]); }
67
+ static uint Pid(IntPtr h) { uint p; GetWindowThreadProcessId(h,out p); return p; }
68
+ static string PathOf(uint pid) {
69
+ IntPtr p=OpenProcess(0x1000,false,pid); if(p==IntPtr.Zero) return "";
70
+ try { var b=new StringBuilder(32768); int n=b.Capacity; return QueryFullProcessImageName(p,0,b,ref n) ? b.ToString() : ""; }
71
+ finally { CloseHandle(p); }
72
+ }
73
+ static HashSet<uint> Family(uint host) {
74
+ var parents=new Dictionary<uint,uint>(); IntPtr snap=CreateToolhelp32Snapshot(2,0);
75
+ try { ENTRY e=new ENTRY(); e.size=(uint)Marshal.SizeOf(typeof(ENTRY));
76
+ if(Process32First(snap,ref e)) do { parents[e.pid]=e.parent; } while(Process32Next(snap,ref e));
77
+ } finally { if(snap!=new IntPtr(-1)) CloseHandle(snap); }
78
+ var family=new HashSet<uint>(); uint p=host;
79
+ for(int i=0;i<32 && p!=0 && family.Add(p);i++) { if(!parents.TryGetValue(p,out p)) break; }
80
+ return family;
81
+ }
82
+ static IntPtr Resolve(Dictionary<string,object> q) {
83
+ string exe=S(q,"exe"); uint wanted=(uint)N(q,"ownerPid");
84
+ var family=Family((uint)N(q,"hostPid")); var candidates=new List<IntPtr>();
85
+ EnumWindows(delegate(IntPtr h,IntPtr unused) {
86
+ uint p=Pid(h);
87
+ if(GetWindow(h,4)!=IntPtr.Zero || GetWindowTextLength(h)==0 || (GetWindowLong(h,-20)&0x80)!=0) return true;
88
+ if(wanted!=0 && p!=wanted) return true;
89
+ var title=new StringBuilder(512); GetWindowText(h,title,title.Capacity);
90
+ if(title.ToString()=="DSH Pet") return true; // legacy independent pet, never the main window
91
+ if(!String.Equals(PathOf(p),exe,StringComparison.OrdinalIgnoreCase)) return true;
92
+ candidates.Add(h); return true;
93
+ },IntPtr.Zero);
94
+ var related=candidates.FindAll(h=>family.Contains(Pid(h)));
95
+ if(related.Count==1) return related[0];
96
+ if(candidates.Count==1) return candidates[0];
97
+ if(candidates.Count>1) throw new Exception("ambiguous-window: "+candidates.Count+" matching windows; no arbitrary selection");
98
+ return IntPtr.Zero;
99
+ }
100
+ static bool Top(IntPtr h) { return (GetWindowLong(h,-20)&8)!=0; }
101
+ static Dictionary<string,object> Probe(IntPtr h) {
102
+ // Coordinates are derived and consumed on this same PMv2 thread.
103
+ SetThreadDpiAwarenessContext(new IntPtr(-4));
104
+ int cloaked=0; int dwm=DwmGetWindowAttribute(h,14,out cloaked,4);
105
+ RECT r; POINT origin=new POINT(0,0);
106
+ bool bounds=GetClientRect(h,out r) && ClientToScreen(h,ref origin);
107
+ bool shown=IsWindow(h) && IsWindowVisible(h) && !IsIconic(h) && dwm==0 && cloaked==0 && bounds && r.R>r.L && r.B>r.T;
108
+ var owners=new List<long>(); int visible=0;
109
+ if(shown) for(int y=1;y<=3;y++) for(int x=1;x<=3;x++) {
110
+ IntPtr at=GetAncestor(WindowFromPoint(new POINT(origin.X+(r.R-r.L)*x/4,origin.Y+(r.B-r.T)*y/4)),2);
111
+ owners.Add(at.ToInt64()); if(at==h) visible++;
112
+ }
113
+ return D("hwnd",h.ToInt64(),"pid",Pid(h),"foreground",GetForegroundWindow().ToInt64(),
114
+ "focused",GetForegroundWindow()==h,"shown",shown,"minimized",IsIconic(h),"cloaked",cloaked,"dwmResult",dwm,
115
+ "topmost",Top(h),"clientRect",new int[]{origin.X,origin.Y,r.R-r.L,r.B-r.T},
116
+ "sampleOwners",owners,"exposedPoints",visible,"sampledVisible",shown && visible==9);
117
+ }
118
+ static bool Flash(IntPtr h, bool start) {
119
+ FLASH f=new FLASH(); f.size=(uint)Marshal.SizeOf(typeof(FLASH)); f.hwnd=h;
120
+ f.flags=start ? 2u|12u : 0u; f.count=uint.MaxValue;
121
+ return FlashWindowEx(ref f); // PREVIOUS caption state, not success!
122
+ }
123
+ static void Cleanup(bool all) {
124
+ lock(leaseLock) {
125
+ foreach(var item in new List<Lease>(leases.Values)) {
126
+ if(!all && DateTime.UtcNow.Ticks<item.until && GetForegroundWindow()!=item.h) continue;
127
+ bool same=IsWindow(item.h) && Pid(item.h)==item.pid;
128
+ bool requested=false;
129
+ if(same) {
130
+ requested=SetWindowPos(item.h,new IntPtr(-2),0,0,0,0,POSITION);
131
+ for(int i=0;i<20 && Top(item.h);i++) Thread.Sleep(10);
132
+ if(GetForegroundWindow()==item.h) Flash(item.h,false);
133
+ }
134
+ bool restored=!same || !Top(item.h);
135
+ Stage(item.id,"cleanup",D("hwnd",item.h.ToInt64(),"sameOwner",same,"requested",requested,"restored",restored));
136
+ if(restored || all) leases.Remove(item.h.ToInt64());
137
+ }
138
+ }
139
+ }
140
+ static object Run(Dictionary<string,object> q) {
141
+ string id=S(q,"id"), op=S(q,"op","probe");
142
+ if(op=="cleanup") { Cleanup(true); return D("status","cleanup-complete"); }
143
+ var clock=Stopwatch.StartNew(); IntPtr h=Resolve(q);
144
+ if(h==IntPtr.Zero) return D("status","no-window","elapsedMs",clock.ElapsedMilliseconds,"visible",false,"focused",false);
145
+ var before=Probe(h); Stage(id,"resolved",before);
146
+ if(op=="probe") return before;
147
+ if(op!="raise") throw new Exception("unsupported operation: "+op);
148
+ bool foregroundApi=false, topApi=false, flashPrevious=false;
149
+ bool already=(bool)before["focused"] && (bool)before["sampledVisible"];
150
+ if(!already) {
151
+ flashPrevious=Flash(h,true); Stage(id,"flash",D("state","requested","previousActive",flashPrevious,"observed",false));
152
+ if(IsIconic(h)) ShowWindowAsync(h,9); else if(!IsWindowVisible(h)) ShowWindowAsync(h,4);
153
+ if(!B(q,"skipFocus")) foregroundApi=SetForegroundWindow(h);
154
+ // SetForegroundWindow has an asynchronous cross-queue phase; do not judge immediately.
155
+ for(int i=0;i<5 && GetForegroundWindow()!=h;i++) Thread.Sleep(10);
156
+ Stage(id,"focus",D("apiReturn",foregroundApi,"observed",GetForegroundWindow()==h,"skipped",B(q,"skipFocus")));
157
+ var afterFocus=Probe(h);
158
+ if(!(bool)afterFocus["sampledVisible"] || GetForegroundWindow()!=h || B(q,"skipFocus")) {
159
+ lock(leaseLock) {
160
+ // Remember original state BEFORE promotion. A repeat extends the same lease.
161
+ Lease lease;
162
+ if(leases.TryGetValue(h.ToInt64(),out lease)) lease.until=DateTime.UtcNow.AddMilliseconds(4000).Ticks;
163
+ else if(!Top(h)) leases[h.ToInt64()]=new Lease {h=h,pid=Pid(h),id=id,until=DateTime.UtcNow.AddMilliseconds(4000).Ticks};
164
+ topApi=SetWindowPos(h,new IntPtr(-1),0,0,0,0,POSITION|0x0040);
165
+ }
166
+ Stage(id,"topmost-request",D("apiReturn",topApi,"lastError",topApi ? 0 : Marshal.GetLastWin32Error()));
167
+ }
168
+ }
169
+ Dictionary<string,object> after=Probe(h);
170
+ for(int i=0;i<25 && !(bool)after["sampledVisible"];i++) { Thread.Sleep(10); after=Probe(h); }
171
+ bool visible=(bool)after["sampledVisible"], focused=(bool)after["focused"];
172
+ return D("status",visible ? (focused ? "foreground" : "visible-without-focus") : "not-visible",
173
+ "visible",visible,"focused",focused,"already",already,"elapsedMs",clock.ElapsedMilliseconds,
174
+ "foregroundApi",foregroundApi,"topmostApi",topApi,"flash",already ? "not-needed" : "requested-unverified",
175
+ "before",before,"after",after,"cleanupLeaseMs",4000);
176
+ }
177
+ static int Main(string[] args) {
178
+ Console.InputEncoding=new UTF8Encoding(false); Console.OutputEncoding=new UTF8Encoding(false);
179
+ if(args.Length>0) { journal=args[0]; Directory.CreateDirectory(System.IO.Path.GetDirectoryName(journal)); }
180
+ SetThreadDpiAwarenessContext(new IntPtr(-4));
181
+ var timer=new Timer(_=>{try { Cleanup(false); } catch(Exception e) { Stage("","cleanup-error",e.ToString()); }},null,100,100);
182
+ Stage("","ready",D("pid",Process.GetCurrentProcess().Id,"version","attention-v1","inputInjection",false,"attachedQueues",false));
183
+ try {
184
+ string line;
185
+ while((line=Console.ReadLine())!=null) {
186
+ string id="";
187
+ try { var q=new JavaScriptSerializer().Deserialize<Dictionary<string,object>>(line); id=S(q,"id");
188
+ object r=Run(q); Write(D("id",id,"utc",DateTime.UtcNow.ToString("o"),"result",r));
189
+ } catch(Exception e) { Write(D("id",id,"utc",DateTime.UtcNow.ToString("o"),"error",e.ToString())); }
190
+ }
191
+ } finally { timer.Dispose(); Cleanup(true); }
192
+ return 0;
193
+ }
194
+ }