nexus-windows-codex 0.0.0-stage → 3.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +304 -0
- package/LICENSE +21 -0
- package/README.md +438 -3
- package/SECURITY.md +13 -0
- package/bin/nexus-windows-codex.mjs +45 -0
- package/garde-fous.json +176 -0
- package/lib/garde-fous.mjs +340 -0
- package/lib/journal.mjs +56 -0
- package/lib/mcp.mjs +281 -0
- package/lib/memory.mjs +109 -0
- package/lib/native-provider.mjs +37 -0
- package/lib/native.mjs +215 -0
- package/lib/presence.mjs +77 -0
- package/lib/session.mjs +1490 -0
- package/lib/stop.mjs +184 -0
- package/lib/tree.mjs +152 -0
- package/native/bin/NexusCodex.UIA.exe +4 -0
- package/native/bin/logo.png +0 -0
- package/package.json +44 -6
package/README.md
CHANGED
|
@@ -1,3 +1,438 @@
|
|
|
1
|
-
#
|
|
2
|
-
|
|
3
|
-
|
|
1
|
+
# NexusWindows for Codex
|
|
2
|
+
|
|
3
|
+
<!-- mcp-name: io.github.pironjulien/nexus-windows-codex -->
|
|
4
|
+
|
|
5
|
+
[](https://github.com/pironjulien/NexusWindows-Codex/releases)
|
|
6
|
+
[](https://github.com/pironjulien/NexusWindows-Codex/actions/workflows/tests.yml)
|
|
7
|
+
[](https://scorecard.dev/viewer/?uri=github.com/pironjulien/NexusWindows-Codex)
|
|
8
|
+
[](https://github.com/pironjulien/NexusWindows-Codex/blob/main/LICENSE)
|
|
9
|
+
[](https://www.npmjs.com/package/nexus-windows-codex)
|
|
10
|
+
[](https://github.com/pironjulien/NexusWindows-Codex/blob/main/package.json)
|
|
11
|
+
[](https://github.com/pironjulien/NexusWindows-Codex/tree/main/native)
|
|
12
|
+
[](#installation)
|
|
13
|
+
|
|
14
|
+
[](https://julienpiron.fr/nexuswindows/)
|
|
15
|
+
|
|
16
|
+
**Windows computer use for Codex, in the background.** An MCP server (`nexus-windows-codex`, one `npx` command) that
|
|
17
|
+
lets OpenAI Codex see and operate Windows desktop applications while the user keeps their mouse, keyboard and
|
|
18
|
+
foreground window. Codex's own Computer Use came to macOS first; on Windows, OpenAI's documentation says to keep the
|
|
19
|
+
target app visible on the active desktop while it runs ([Codex docs](https://developers.openai.com/codex/app/computer-use)).
|
|
20
|
+
NexusWindows for Codex has its own local .NET helper for everything: window inventory, UI Automation tree, PrintWindow
|
|
21
|
+
capture of covered windows, Win32 messages and UI Automation patterns without focus, and its own real mouse and
|
|
22
|
+
keyboard as the last resort, under a shared foreground token. Every action is judged from the application's real
|
|
23
|
+
state, and a batch needs an explicit postcondition per step. It also ships as a Codex skill that layers the same session over Codex's Computer Use API (`@oai/sky`).
|
|
24
|
+
|
|
25
|
+
Version 3.0.0 · Windows 10/11 · Node 22+ · .NET 8 · MIT licence
|
|
26
|
+
|
|
27
|
+
Part of the Nexus family: [NexusWindows](https://julienpiron.fr/nexuswindows/)
|
|
28
|
+
([Claude](https://github.com/pironjulien/NexusWindows-Claude) ·
|
|
29
|
+
[Codex](https://github.com/pironjulien/NexusWindows-Codex) ·
|
|
30
|
+
[Antigravity](https://github.com/pironjulien/NexusWindows-Antigravity)),
|
|
31
|
+
[NexusBrowser](https://julienpiron.fr/nexusbrowser/) ([repo](https://github.com/pironjulien/NexusBrowser)),
|
|
32
|
+
[NexusAndroid](https://julienpiron.fr/nexusandroid/) ([repo](https://github.com/pironjulien/NexusAndroid)).
|
|
33
|
+
Project page: [julienpiron.fr/nexuswindows](https://julienpiron.fr/nexuswindows/).
|
|
34
|
+
|
|
35
|
+
## Why
|
|
36
|
+
|
|
37
|
+
Codex's built-in Computer Use acts with the physical mouse and keyboard: it brings the target window to the front and
|
|
38
|
+
the user has to stop working while it acts. Many Windows applications can be driven without that. NexusWindows for
|
|
39
|
+
Codex tries the background techniques first, judges the result from the application's real state, and only uses its
|
|
40
|
+
helper's real mouse and keyboard as a fallback, after a pause in the user's own typing and under a token shared with
|
|
41
|
+
the other agents on the PC.
|
|
42
|
+
|
|
43
|
+
It is an MCP server (Node, no npm dependency) with a self-contained .NET helper: no listening network port, no token
|
|
44
|
+
and no private Electron API. The optional Codex skill runs the same session in Codex's Node REPL instead, over Codex's
|
|
45
|
+
Computer Use API.
|
|
46
|
+
|
|
47
|
+
## NexusWindows family
|
|
48
|
+
|
|
49
|
+
Three MCP servers, one per agent, built separately because each model needs its tools shaped for it, with the same
|
|
50
|
+
promise: the user keeps their mouse, keyboard and foreground window, and every action is read back.
|
|
51
|
+
|
|
52
|
+
| | Claude | Codex | Antigravity |
|
|
53
|
+
|---|---|---|---|
|
|
54
|
+
| Install | `uvx --python 3.14 nexus-windows` | `npx -y nexus-windows-codex` | `uvx --python 3.14 nexus-windows-antigravity` |
|
|
55
|
+
| Package | [PyPI](https://pypi.org/project/nexus-windows/) | [npm](https://www.npmjs.com/package/nexus-windows-codex) | [PyPI](https://pypi.org/project/nexus-windows-antigravity/) |
|
|
56
|
+
| Tools | `windows` `look` `act` `launch` `window`, plus `notifications` and `logs` | `windows` `look` `act` `launch` `window` | `windows` `look` `act` `launch` `window` |
|
|
57
|
+
| Repository | [NexusWindows-Claude](https://github.com/pironjulien/NexusWindows-Claude) | [NexusWindows-Codex](https://github.com/pironjulien/NexusWindows-Codex) | [NexusWindows-Antigravity](https://github.com/pironjulien/NexusWindows-Antigravity) |
|
|
58
|
+
|
|
59
|
+
Every step reports ✓ verified, = already done, ~ unknown (never replayed) or ✗ failed or refused. The three share the
|
|
60
|
+
guardrail policy, the foreground token, the emergency stop (Ctrl+Alt+Shift+Esc), a common journal format and a common
|
|
61
|
+
bench.
|
|
62
|
+
|
|
63
|
+
## Installation
|
|
64
|
+
|
|
65
|
+
Requirements: Windows 10 or 11 and Node 22+. The npm package carries a self-contained helper: no .NET install.
|
|
66
|
+
|
|
67
|
+
From [npm](https://www.npmjs.com/package/nexus-windows-codex), in Codex's `~/.codex/config.toml`:
|
|
68
|
+
|
|
69
|
+
```toml
|
|
70
|
+
[mcp_servers.nexus-windows]
|
|
71
|
+
command = "cmd"
|
|
72
|
+
args = ["/c", "npx", "-y", "nexus-windows-codex"]
|
|
73
|
+
```
|
|
74
|
+
|
|
75
|
+
On Windows, `npx` is a `.cmd` script, hence `cmd /c`. The same command works in any MCP client
|
|
76
|
+
(`"command": "cmd", "args": ["/c", "npx", "-y", "nexus-windows-codex"]`).
|
|
77
|
+
|
|
78
|
+
### Codex skill (alternative, from a clone)
|
|
79
|
+
|
|
80
|
+
The skill runs the same session inside Codex's Node REPL and can fall back on Codex's Computer Use API. It needs the
|
|
81
|
+
Codex desktop app with its Computer Use plugin and the .NET 8 SDK:
|
|
82
|
+
|
|
83
|
+
```powershell
|
|
84
|
+
git clone https://github.com/pironjulien/NexusWindows-Codex.git
|
|
85
|
+
cd NexusWindows-Codex
|
|
86
|
+
pwsh -NoProfile -File install.ps1 # build the helper, install the skill into %USERPROFILE%\.codex\skills\nexus-windows
|
|
87
|
+
pwsh -NoProfile -File install.ps1 -CheckOnly # verify the installation matches the sources
|
|
88
|
+
```
|
|
89
|
+
|
|
90
|
+
`install.ps1` keeps versioned helper builds under `%LOCALAPPDATA%\NexusCodex\ComputerUse` (or `-StateRoot`), writes a
|
|
91
|
+
source-linked `entry.mjs` next to the skill (so keep the clone where it is), checks the four .NET runtime files by
|
|
92
|
+
hash, and serialises concurrent installations. `CODEX_HOME` or `-CodexHome` changes the Codex folder. Re-run it after
|
|
93
|
+
updating the clone. Note: the build path must stay under the Windows 260-character limit. The MCP server also finds
|
|
94
|
+
this helper when it runs from a clone.
|
|
95
|
+
|
|
96
|
+
The skill is picked up by new Codex sessions. A REPL that already imported an older version must be reset, because
|
|
97
|
+
the JavaScript module cache does not reload.
|
|
98
|
+
|
|
99
|
+
## Tools
|
|
100
|
+
|
|
101
|
+
| Tool | Purpose |
|
|
102
|
+
|---|---|
|
|
103
|
+
| `windows(filter)` | Application windows (id, app, title, pid), like the taskbar; password managers never listed |
|
|
104
|
+
| `look(window, mode)` | One window by id or a unique part of its title or app: UI Automation tree, plus a capture that works on covered windows with `mode="image"` or `"both"`; selects the window for `act` |
|
|
105
|
+
| `act(steps)` | One step, or up to 32 each with its own `expect`, on the window of the last `look`; stops at the first step not verified |
|
|
106
|
+
| `launch(app)` | Start an application (name, path, Store app id), a folder, a document or a Settings page (`ms-settings:`) and give the foreground back |
|
|
107
|
+
| `window(action, window)` | `action="close"`: one close message, read back; a "save?" prompt is reported, never answered |
|
|
108
|
+
|
|
109
|
+
Step kinds: `click` (selector, or point and native capture id; `button`, `count`), `setValue`, `typeText`, `key`,
|
|
110
|
+
`select`, `menu`, `secondary`, `scroll`, `drag`. Each step reports ✓ verified, = already done, ~ unknown (look again,
|
|
111
|
+
never replayed) or ✗ failed or refused.
|
|
112
|
+
|
|
113
|
+
## Usage
|
|
114
|
+
|
|
115
|
+
As an MCP server: `windows`, then `look` at one window, then `act` with selectors from that look and an `expect`
|
|
116
|
+
that proves the effect. An observation serves one `act` and expires after 60 s.
|
|
117
|
+
|
|
118
|
+
As a skill, the agent follows [SKILL.md](https://github.com/pironjulien/NexusWindows-Codex/blob/main/skill/nexus-windows/SKILL.md) and the [API reference](https://github.com/pironjulien/NexusWindows-Codex/blob/main/skill/nexus-windows/references/api.md).
|
|
119
|
+
In Codex's Node REPL:
|
|
120
|
+
|
|
121
|
+
```js
|
|
122
|
+
if (!globalThis.sky) { const { sky } = await import('@oai/sky'); globalThis.sky = sky; }
|
|
123
|
+
const home = nodeRepl.homeDir ?? (await import('node:os')).homedir();
|
|
124
|
+
globalThis.nexusWindows = await import('file:///' + home.replaceAll('\\', '/') + '/.codex/skills/nexus-windows/entry.mjs');
|
|
125
|
+
globalThis.win = nexusWindows.createWindowsSession(sky);
|
|
126
|
+
nodeRepl.write(await win.windows()); // pick exactly one window from this list
|
|
127
|
+
await win.bind({ id: chosenWindowId, app: chosenAppId });
|
|
128
|
+
globalThis.obs = await win.observe();
|
|
129
|
+
globalThis.result = await win.act(obs, {
|
|
130
|
+
kind: 'click', selector: { automationId: 'okButton', role: 'button' },
|
|
131
|
+
expect: { selector: { automationId: 'status' }, field: 'name', equals: 'Saved' }
|
|
132
|
+
});
|
|
133
|
+
```
|
|
134
|
+
|
|
135
|
+
Read `result.status`, `result.technique` and `result.foregroundTaken`, then continue from `result.observation`.
|
|
136
|
+
Call `await win.close()` at the end.
|
|
137
|
+
|
|
138
|
+
Local files:
|
|
139
|
+
|
|
140
|
+
| Location | Content |
|
|
141
|
+
|---|---|
|
|
142
|
+
| `%USERPROFILE%\.codex\skills\nexus-windows` | installed skill and `entry.mjs` |
|
|
143
|
+
| `%LOCALAPPDATA%\NexusCodex\ComputerUse\native\<fingerprint>` | versioned helper builds |
|
|
144
|
+
| `%LOCALAPPDATA%\NexusCodex\ComputerUse\recipes` | verified recipes (static UI labels only) |
|
|
145
|
+
| `%LOCALAPPDATA%\NexusCodex\ComputerUse\reports` | explicit measurements, no window content or typed text |
|
|
146
|
+
| `%LOCALAPPDATA%\NexusCodex\ComputerUse\captures` | native captures, 10 minutes and 32 files at most |
|
|
147
|
+
|
|
148
|
+
## What it does
|
|
149
|
+
|
|
150
|
+
### Background first, physical input last
|
|
151
|
+
|
|
152
|
+
Each semantic `click`, `setValue`, `typeText`, `secondary`, `select` and `menu` goes first to the helper:
|
|
153
|
+
|
|
154
|
+
| UI family | Background technique | Never used |
|
|
155
|
+
|---|---|---|
|
|
156
|
+
| Win32 | `BN_CLICKED` to the parent (`BM_SETCHECK` for check boxes and radio groups), `EM_SETSEL` + `EM_REPLACESEL`, combo box list opened and closed by `CB_SHOWDROPDOWN` (read back by `CB_GETDROPPEDSTATE`), option chosen by `CB_SETCURSEL` + `CBN_SELCHANGE` + `CBN_SELENDOK` (an open list is closed first: that close is a cancel which restores the previous choice), `LB_SETCURSEL` + `LBN_SELCHANGE`, `TCM_SETCURFOCUS`, menu `WM_COMMAND`; UIA Toggle | UIA Invoke (the Win32 proxy calls `SetForegroundWindow`), UIA Select on a list (silent) |
|
|
157
|
+
| WinForms | UIA Invoke/Toggle; messages for Edit, ComboBox (same as Win32, open list included) and ListBox | `BN_CLICKED` on a check box, UIA Select on a list |
|
|
158
|
+
| WPF, UWP/WinUI, Chromium/Electron | COM UI Automation with `AutoSetFocus = FALSE`: Invoke, Toggle, Value, ExpandCollapse (state read back), SelectionItem; a combo box is closed again after any choice | |
|
|
159
|
+
|
|
160
|
+
A measured detail drives the design: once the managed `System.Windows.Automation` client has been used in a process,
|
|
161
|
+
a COM Invoke or Toggle from that same process activates the target window despite `AutoSetFocus = FALSE` (WPF and
|
|
162
|
+
WinForms on .NET 8). UI Automation patterns are therefore sent from a child process (`--com-worker`) that never loads
|
|
163
|
+
the managed client. Since 1.4.0 the managed client is not loaded anywhere: the tree is read with the COM UI
|
|
164
|
+
Automation client in **one round trip** (`CacheRequest`, `Subtree` scope, control view; about 50 ms for the bench
|
|
165
|
+
forms instead of 150 to 300 ms). For an action, only the target and its ancestors are read again, and any difference
|
|
166
|
+
from the observation is refused.
|
|
167
|
+
|
|
168
|
+
The options of a Win32/WinForms combo box are read by messages (`CB_GETCOUNT`, `CB_GETLBTEXT`, `CB_GETCURSEL`), open
|
|
169
|
+
or not, as `list`/`listitem` nodes under it; `expanded` comes from `CB_GETDROPPEDSTATE`. Measured: the UIA subtree of
|
|
170
|
+
an open WinForms combo box answers "Malformed Cacheresponse" after about 7 s and then stops answering, so it is never
|
|
171
|
+
walked.
|
|
172
|
+
|
|
173
|
+
Each technique reads its effect back: `applied`, `none` (nothing changed) or `sent` (no read-back possible).
|
|
174
|
+
Physical fallback happens only when no technique applies, or when the `expect` is not met **and** nothing observable
|
|
175
|
+
changed. A partial effect is never repeated physically; `fallback:false` forbids the physical path. Every result says
|
|
176
|
+
which technique acted (`background:messages`, `background:uia`, `foreground:uia`, `physical:sky`, `sky:semantic`),
|
|
177
|
+
whether the foreground was taken, and every attempt.
|
|
178
|
+
|
|
179
|
+
### Explicit postconditions
|
|
180
|
+
|
|
181
|
+
`expect` distinguishes `verified`, `unmet`, `already-satisfied` and unknown results. An observation is consumed once;
|
|
182
|
+
a stale observation, another window, a missing or ambiguous control, or a changed control identity aborts the action
|
|
183
|
+
before any input. A truncated UIA value cannot satisfy an exact equality, and an empty tree never proves that a
|
|
184
|
+
control disappeared. Native password fields are refused.
|
|
185
|
+
|
|
186
|
+
A postcondition already true **before** the action skips it (`already-satisfied`, nothing sent) only when it is a
|
|
187
|
+
precise state (`equals`) that the action changes: `value`, `checked`, `selected`, `expanded` or `focused` on the
|
|
188
|
+
target, or a result on another element. A postcondition that only restates the target ("the button exists") is
|
|
189
|
+
refused, and a non-precise one already true (`exists`, `includes`, the target's label) is refused before any input:
|
|
190
|
+
it could not tell the effect from the previous state. A combo box choice is proven by the combo box `value`, never by
|
|
191
|
+
its option element.
|
|
192
|
+
|
|
193
|
+
### Clean capture with PrintWindow
|
|
194
|
+
|
|
195
|
+
`observe({mode:'image', capture:'native'})` captures the bound window with `PrintWindow(PW_RENDERFULLCONTENT)`
|
|
196
|
+
without bringing it forward, even when covered: per-monitor DPI aware, cropped to the DWM frame, DPI-unaware windows
|
|
197
|
+
drawn at their logical size and scaled. In the Codex skill, a blank, minimized or failed capture falls back to Sky's
|
|
198
|
+
screenshot; the MCP server has no such fallback and reports the failure. Native images are local PNG files (32 files
|
|
199
|
+
at most, removed after 10 minutes at the next capture, at server start and at a clean stop): evidence for the model,
|
|
200
|
+
and the coordinate frame of a point action that names the image (`screenshotId`).
|
|
201
|
+
|
|
202
|
+
### Verified batches (`actBatch`)
|
|
203
|
+
|
|
204
|
+
`actBatch(obs, steps)` runs up to 32 actions in one call. Every step needs an `expect`; all steps are validated before
|
|
205
|
+
any input; each step acts on the observation refreshed by the previous one; the batch stops at the first step that is
|
|
206
|
+
not `verified` or `already-satisfied`.
|
|
207
|
+
|
|
208
|
+
### Multiple windows
|
|
209
|
+
|
|
210
|
+
The session follows dialogs and popups without a new `bind`: windows owned by the followed windows, windows that
|
|
211
|
+
disable the window in front (modal), and windows of the same process that appear during an action. When a dialog
|
|
212
|
+
closes, the session returns to its nearest open owner. Each observation reports `windowStack` and `windowChange`.
|
|
213
|
+
Transient popups (caption-less `WS_POPUP` windows with `WS_EX_NOACTIVATE`: an open drop-down list, a tooltip, a WPF
|
|
214
|
+
popup; or a Win32 `ComboLBox`) are never followed: they are read through the window's own tree and listed in
|
|
215
|
+
`obs.popups`.
|
|
216
|
+
|
|
217
|
+
### Guardrails: one shared policy
|
|
218
|
+
|
|
219
|
+
Since 1.5.0 the guardrails come from the policy shared by the three NexusWindows editions (Codex, Claude,
|
|
220
|
+
Antigravity), `garde-fous.json`, with a `codex` entry. A copy is embedded at the root of this repository; inside the
|
|
221
|
+
author's private workspace the live shared file is read instead (a test checks both are byte-identical).
|
|
222
|
+
`NEXUS_GARDE_FOUS` can point to another file; an imposed path is never silently replaced. A missing or invalid policy
|
|
223
|
+
refuses every action (fail closed), and the .NET helper refuses every window request without a readable policy.
|
|
224
|
+
|
|
225
|
+
**Hard block** (refused before any read or input): the secure desktop (UAC, lock screen, sign-in), Windows credential
|
|
226
|
+
prompts, password managers (KeePass, Bitwarden, 1Password, Dashlane, NordPass, RoboForm, LastPass, by process
|
|
227
|
+
prefix), and Codex's own application (Codex, ChatGPT), because Codex Computer Use asks for app access inside its
|
|
228
|
+
main window. Consoles and terminals, File Explorer and administration tools are no longer refused (they were until
|
|
229
|
+
1.4.1): the layer is meant for a user who already lets Codex run commands. Password fields stay refused everywhere,
|
|
230
|
+
and their value is never read.
|
|
231
|
+
|
|
232
|
+
**Confirmation** (`confirm:true` on the action, set only on the user's explicit order or after their yes): paying,
|
|
233
|
+
calling, sending, publishing, deleting permanently, formatting, uninstalling, resetting, cancelling a subscription,
|
|
234
|
+
making a repository public, plus a generic verb (Validate, OK, Start) in a window whose title is about payment or
|
|
235
|
+
formatting. The check is on what the gesture triggers, not on its form, before any input and whatever technique
|
|
236
|
+
follows: a click by selector or by screenshot coordinates (single or double; a text or icon inside a button counts as
|
|
237
|
+
the button), Enter, Ctrl+Enter, Space, Alt+letter (access key or underlined letter), typed text containing a line
|
|
238
|
+
break, a menu command, a secondary action other than expand, collapse, toggle, select or scroll. The refusal
|
|
239
|
+
(`error.confirmationRequired`) sends nothing; a batch stops at that step. A learned recipe never carries
|
|
240
|
+
`confirm`. Direct calls to Sky bypass this layer: always go through the session.
|
|
241
|
+
|
|
242
|
+
### The user's presence (Raw Input)
|
|
243
|
+
|
|
244
|
+
A long-lived `--stop-guard` process registers Raw Input (`RIDEV_INPUTSINK`, message-only window, no hook). It reports
|
|
245
|
+
only the instant of each physical keyboard or mouse input, never its content. Injected input, including Sky's own
|
|
246
|
+
`SendInput`, has no source device and is ignored. Before any physical input the session waits for 1.618 s without
|
|
247
|
+
user input (at most 8 s, otherwise the action is deferred and nothing is sent), then types in chunks of at most
|
|
248
|
+
48 graphemes and stops (`interrupted`, never resent) if the user takes the keyboard or mouse back. Without Raw
|
|
249
|
+
Input it falls back to `GetLastInputInfo`, excluding its own injections.
|
|
250
|
+
|
|
251
|
+
### Application memory
|
|
252
|
+
|
|
253
|
+
A verified semantic click or secondary action can be saved as a recipe with its expected effect, a fingerprint of
|
|
254
|
+
the executable's metadata and one of the control structure. Any change forces a new inspection. No control number or
|
|
255
|
+
pixel is ever replayed, and typed content is never stored.
|
|
256
|
+
|
|
257
|
+
### Agent at work
|
|
258
|
+
|
|
259
|
+
While an action runs (`act`, `actBatch`, `activate`; never an observation), a small logo spins on itself in the
|
|
260
|
+
bottom-right corner of the bound window's screen, like a game loading screen, with a breathing blue halo. It goes away
|
|
261
|
+
1.6 s after the last action. The emergency-stop guard, already alive for the whole REPL, draws it on the commands
|
|
262
|
+
`busy <hwnd>` and `idle` (`native/BusyIndicator.cs`): a layered window that is always on top, borderless,
|
|
263
|
+
click-through, never activated, absent from the taskbar and excluded from screen captures. Rendering is analytic
|
|
264
|
+
(halo, shadow, coin) from `native/logo.png` decoded by GDI+, with no added dependency. `NEXUS_WINDOWS_INDICATOR=0`
|
|
265
|
+
turns it off.
|
|
266
|
+
|
|
267
|
+
## Working alongside other agents
|
|
268
|
+
|
|
269
|
+
This layer runs next to two other desktop agents on the same PC (a Claude MCP server,
|
|
270
|
+
[NexusWindows for Claude](https://github.com/pironjulien/NexusWindows-Claude), and an Antigravity layer).
|
|
271
|
+
They coordinate through two file-based protocols that any agent can implement:
|
|
272
|
+
|
|
273
|
+
**Foreground token** (`%USERPROFILE%\.nexus\foreground.lock`): held only while really using the mouse or the
|
|
274
|
+
foreground (Sky physical input, activation, the foreground UIA fallback), never for reads or background techniques.
|
|
275
|
+
Holding it means an exclusive `LockFileEx` on 1 byte at offset 2^32 (in C#: `FileStream` with
|
|
276
|
+
`FileShare.ReadWrite | FileShare.Delete`, then `Lock(1L << 32, 1)`). Windows releases it when the handle closes,
|
|
277
|
+
including when the process dies. The holder writes `{"owner", "pid", "since", "purpose"}` for diagnostics only. A
|
|
278
|
+
waiter waits at most 8 s, then reports a deferred action and sends nothing.
|
|
279
|
+
|
|
280
|
+
**Emergency stop** (Ctrl+Alt+Shift+Esc, files in `%USERPROFILE%\.nexus\`): one process per PC holds
|
|
281
|
+
`RegisterHotKey(MOD_NOREPEAT)` (others retry every 3 s) and beats in `stop.owner` every 3 s; a press rewrites `stop`
|
|
282
|
+
with `{"by", "pid", "at", "hold_s": 120, "nonce"}`. Here the `--stop-guard` helper takes the hotkey when nobody else
|
|
283
|
+
holds it, for as long as the Codex REPL lives. The JavaScript session watches `stop` every 0.2 s and before each
|
|
284
|
+
action, and the helper checks it again before each UIA action and token acquisition. A new version younger than 120 s
|
|
285
|
+
blocks every desktop action for the remaining time; observations stay allowed. The stop is latched in memory on a
|
|
286
|
+
monotonic clock: deleting the file or changing the clock does not lift it.
|
|
287
|
+
|
|
288
|
+
## Architecture
|
|
289
|
+
|
|
290
|
+
```mermaid
|
|
291
|
+
flowchart LR
|
|
292
|
+
Codex["Codex"] --> MCP["MCP server bin and lib mcp.mjs"]
|
|
293
|
+
MCP --> Provider["lib native-provider.mjs"]
|
|
294
|
+
Provider --> Session["lib session.mjs"]
|
|
295
|
+
Codex --> Skill["skill entry.mjs in the Node REPL"]
|
|
296
|
+
Skill --> Session
|
|
297
|
+
Session -. skill only .-> Sky["Codex Computer Use API"]
|
|
298
|
+
Session --> Native["lib native.mjs"]
|
|
299
|
+
Native --> Helper["NexusCodex.UIA server"]
|
|
300
|
+
Helper --> Messages["Win32 messages"]
|
|
301
|
+
Helper --> ComWorker["COM only worker for UIA patterns"]
|
|
302
|
+
Helper --> Capture["PrintWindow capture"]
|
|
303
|
+
Session --> Guard["stop guard with Raw Input"]
|
|
304
|
+
Guard --> Shared["shared files in USERPROFILE nexus"]
|
|
305
|
+
Helper --> Shared
|
|
306
|
+
```
|
|
307
|
+
|
|
308
|
+
```
|
|
309
|
+
bin/nexus-windows-codex.mjs the MCP server command (stdio)
|
|
310
|
+
lib/
|
|
311
|
+
mcp.mjs MCP protocol and tools (no npm dependency)
|
|
312
|
+
native-provider.mjs window inventory by the helper, every window driven by the helper alone
|
|
313
|
+
session.mjs WindowsSession: bind, observe, act, actBatch, recipes, window following
|
|
314
|
+
native.mjs client for the .NET helper (one reused process, correlated requests, timeouts)
|
|
315
|
+
tree.mjs accessibility tree normalisation and digests
|
|
316
|
+
memory.mjs recipes and metrics (atomic, locked writes)
|
|
317
|
+
presence.mjs user presence from the stop guard
|
|
318
|
+
stop.mjs emergency stop watcher and guard process
|
|
319
|
+
native/ .NET 8 helper: Program.cs (modes --server, --com-worker, --stop-guard, --foreground-token),
|
|
320
|
+
Background.cs, ComUia.cs, ComWorker.cs, Capture.cs, Presence.cs, EmergencyStop.cs,
|
|
321
|
+
ForegroundToken.cs, Win32.cs
|
|
322
|
+
skill/nexus-windows/ SKILL.md and references (API, per-application notes)
|
|
323
|
+
tests/ node:test suites; tests/live: real validation fixture (WinForms, Win32, WPF)
|
|
324
|
+
install.ps1 builds the helper and installs the skill
|
|
325
|
+
test.ps1 offline check; -Full adds the real validation
|
|
326
|
+
```
|
|
327
|
+
|
|
328
|
+
## Measurements
|
|
329
|
+
|
|
330
|
+
From [VALIDATION.md](VALIDATION.md) (October 15, 2026, real validation on the repository's own fixture; the file is in
|
|
331
|
+
French). Windows shown without activation at the bottom of the z-order, the Win32 one covered by the WinForms one.
|
|
332
|
+
Each action: postcondition verified, event counted by the application, foreground unchanged, cursor still.
|
|
333
|
+
|
|
334
|
+
| Action | Technique | Duration (input and read-backs) |
|
|
335
|
+
|---|---|---|
|
|
336
|
+
| Win32 button, check box, typing, append at caret, combo box, list, menu | messages | 105 to 159 ms |
|
|
337
|
+
| WinForms typing, combo box, list | messages | 292 to 338 ms |
|
|
338
|
+
| WinForms check box, button, dialog opened then closed (followed automatically) | COM UIA Toggle / Invoke | 205 to 441 ms |
|
|
339
|
+
| WPF button, check box, value | COM UIA Invoke / Toggle / Value | 193 to 241 ms |
|
|
340
|
+
| Batch of 3 steps, stopped at step 2 (step 3 never sent) | messages | 658 ms |
|
|
341
|
+
| Capture of the covered Win32 window | PrintWindow, not blank | 19 ms |
|
|
342
|
+
|
|
343
|
+
Offline suite: 108 JavaScript tests, 103 passed and 5 real-environment tests skipped outside `-Full`
|
|
344
|
+
(`test.ps1`, October 16, 2026). Durations are dominated by full UIA tree read-backs. The physical fallback that takes
|
|
345
|
+
the foreground was skipped in that run because the user was active; it is not counted as passed.
|
|
346
|
+
|
|
347
|
+
### Cross-agent bench (October 16, 2026)
|
|
348
|
+
|
|
349
|
+
The same task was run by the three Windows desktop agents used on this PC (this layer, the Claude server and the
|
|
350
|
+
Antigravity layer) on WinForms and WPF forms hosted by `powershell.exe`, created without activation at the bottom of
|
|
351
|
+
the z-order while the user kept working: fill three fields, tick a box, open the "Formule" drop-down list, choose
|
|
352
|
+
"Bravo" in the open list, validate. Proof is the result label read back in the application. Here the real session
|
|
353
|
+
and .NET helper ran with a fake Sky that refuses and counts any physical input.
|
|
354
|
+
|
|
355
|
+
| Check | WinForms | WPF |
|
|
356
|
+
|---|---|---|
|
|
357
|
+
| Three fields | ✓ background:messages, 103 to 124 ms | ✓ background:uia, 170 to 189 ms |
|
|
358
|
+
| Open the list (`expanded = true`) | ✓ verified (1.3.0: wrongly `already-satisfied`) | ✓ verified (1.3.0: `unmet`, then the form's controls were lost) |
|
|
359
|
+
| Choose "Bravo" in the open list, validate | ✓ verified | ✓ verified (1.3.0: validate skipped) |
|
|
360
|
+
| Trivial postcondition ("the button exists") | ✓ refused before any input | ✓ |
|
|
361
|
+
| Application result | `Formule=Bravo;Envois=1` | `Formule=Bravo;Envois=1` |
|
|
362
|
+
| Physical input / foreground taken | 0 / 0 of 123 samples | 0 / 0 |
|
|
363
|
+
| Time (observe + batch) | 0.12 + 1.16 s | 0.11 + 1.41 s (1.3.0: 4.5 to 8 s) |
|
|
364
|
+
|
|
365
|
+
| Agent | Morning | After fixes |
|
|
366
|
+
|---|---|---|
|
|
367
|
+
| This layer (1.3.0, then 1.4.0) | 16/20 | **19/20** |
|
|
368
|
+
| [NexusWindows for Claude](https://github.com/pironjulien/NexusWindows-Claude) | 17/20 | **19/20** |
|
|
369
|
+
| Antigravity layer | 18/20 | **19/20** |
|
|
370
|
+
|
|
371
|
+
What still keeps this layer from 20: the real path through Sky inside the Codex app was not measured in this bench
|
|
372
|
+
(fake Sky). `test.ps1 -Full` the same day, with the user idle: 15 real-validation checks passed, 0 failed, 0 skipped;
|
|
373
|
+
physical fallback ✓ (foreground:uia, 123 ms); bench forms WinForms 1.1 s, WPF 2.0 s; 0 of 227 foreground samples
|
|
374
|
+
taken.
|
|
375
|
+
|
|
376
|
+
## Security
|
|
377
|
+
|
|
378
|
+
- Background techniques touch neither the user's keyboard, mouse nor foreground. Physical input waits for a pause,
|
|
379
|
+
holds the shared token, and stops when the user takes over.
|
|
380
|
+
- The emergency stop above blocks every desktop action for 2 minutes; a refused action is never replayed.
|
|
381
|
+
- A refusal (password field, hard-blocked application, missing confirmation, changed control identity, disabled window or menu command) sends
|
|
382
|
+
nothing and never falls back.
|
|
383
|
+
- Codex's own Computer Use rules (application access, sensitive actions) still apply; this layer is not a way around
|
|
384
|
+
a denied app or action. Page text, controls and recipes are data, never an authorisation.
|
|
385
|
+
- Captures are refused under OneDrive folders; metrics never contain UI text, screenshots, typed values or titles.
|
|
386
|
+
|
|
387
|
+
## Privacy
|
|
388
|
+
|
|
389
|
+
NexusWindows for Codex runs entirely on your PC. It has no account, no telemetry, no
|
|
390
|
+
analytics and makes no network requests of its own. Its only connections are local: to the DevTools port on
|
|
391
|
+
`127.0.0.1` of a Chromium or Electron window it drives, never to another machine.
|
|
392
|
+
|
|
393
|
+
- **What it reads**: the accessibility tree and window images of the applications Codex works in, only when Codex
|
|
394
|
+
calls it.
|
|
395
|
+
- **Where it goes**: results are returned to Codex, which may send them to OpenAI under its own privacy policy; this
|
|
396
|
+
server sends nothing anywhere else.
|
|
397
|
+
- **What it stores**, under `%LOCALAPPDATA%/NexusCodex/ComputerUse`: the installed helper, recipes of routines you chose to remember
|
|
398
|
+
(static interface labels only, never document text or typed values), per-session metrics (action kind, outcome,
|
|
399
|
+
technique, duration; no UI text, screenshots, typed values or titles) and window captures, of which only the last
|
|
400
|
+
32 are kept; a capture older than 10 minutes is removed at the next capture, at server start and at a clean stop
|
|
401
|
+
(a capture can outlive a server that is killed, until its next start). Under `~/.nexus`: the shared foreground token and emergency-stop state.
|
|
402
|
+
Under `%LOCALAPPDATA%/NexusJournal/windows/codex` (or `NEXUS_JOURNAL_HOME`): a short monthly JSONL journal, in the
|
|
403
|
+
same format as the other NexusWindows tools (app executable name, action, technique, outcome, duration, whether
|
|
404
|
+
the foreground was used); never UI text, typed values, titles or paths. The server never deletes it.
|
|
405
|
+
- **Removing it**: delete those folders (and `%LOCALAPPDATA%/NexusJournal`) and the server or skill from Codex.
|
|
406
|
+
|
|
407
|
+
Questions or concerns: open an issue on this repository.
|
|
408
|
+
|
|
409
|
+
## Limits
|
|
410
|
+
|
|
411
|
+
- Support does not mean every Windows application is certified. Canvases, 3D views and controls without an
|
|
412
|
+
accessibility provider still need screenshots and physical gestures.
|
|
413
|
+
- Message techniques emulate the notification a real click would send; an app that only reacts to real mouse input
|
|
414
|
+
will not see them, and the postcondition decides.
|
|
415
|
+
- Raw clicks, wheels, keyboard shortcuts and rich editors without an Edit control still need physical input. Tk has
|
|
416
|
+
no UI Automation and ignores posted input.
|
|
417
|
+
- An effect with no trace in the accessibility tree cannot be told from "no effect": give a postcondition that
|
|
418
|
+
observes it, or `fallback:false`.
|
|
419
|
+
- `launch` attributes a window to the launch by its process, and says how (`proof`): `process`, the process the
|
|
420
|
+
shell returned; or `application`, for an application named by its executable or Store id (whose launcher may hand
|
|
421
|
+
over to another process), a process of that application born after the request. With `application`, another
|
|
422
|
+
instance of the same application started by something else at the same moment could be taken for it. Files and
|
|
423
|
+
folders need an absolute path; their windows come back as candidates unless the shell returned the process that
|
|
424
|
+
shows them.
|
|
425
|
+
- In the skill, Sky's physical input cannot be tested outside the Codex REPL; its orchestration (pause, token,
|
|
426
|
+
chunks, interruption) is covered by the offline tests only. The MCP server never uses Sky.
|
|
427
|
+
- The skill, comments and some messages are written for one user (they name the user) and partly in French.
|
|
428
|
+
|
|
429
|
+
## Development
|
|
430
|
+
|
|
431
|
+
```powershell
|
|
432
|
+
pwsh -NoProfile -File test.ps1 # node tests + helper build in a temp folder; never touches %USERPROFILE%\.nexus
|
|
433
|
+
pwsh -NoProfile -File test.ps1 -Full # also real token/stop tests and the background validation on the fixture
|
|
434
|
+
```
|
|
435
|
+
|
|
436
|
+
## Licence
|
|
437
|
+
|
|
438
|
+
MIT, see [LICENSE](https://github.com/pironjulien/NexusWindows-Codex/blob/main/LICENSE). Copyright (c) 2026 Julien Piron.
|
package/SECURITY.md
ADDED
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
# Security policy
|
|
2
|
+
|
|
3
|
+
NexusWindows for Codex drives real Windows applications, so safety issues matter more than usual: a way to make the
|
|
4
|
+
agent pay, send, publish, delete permanently or confirm without `confirm: true`, to read or drive a password manager,
|
|
5
|
+
a credential prompt or the secure desktop, to type into a password field, to take the user's mouse, keyboard or
|
|
6
|
+
foreground outside the shared token and the user's pause, or to act during the emergency stop.
|
|
7
|
+
|
|
8
|
+
Please report such issues privately through
|
|
9
|
+
[GitHub private vulnerability reporting](https://github.com/pironjulien/NexusWindows-Codex/security/advisories/new),
|
|
10
|
+
not in a public issue. Include the tool call, the application involved and what happened. You will get an answer
|
|
11
|
+
within a few days.
|
|
12
|
+
|
|
13
|
+
Only the latest release is supported.
|
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
// Commande `nexus-windows-codex` : serveur MCP autonome (stdio) de NexusWindows for Codex, sans Sky.
|
|
3
|
+
// Helper natif cherché dans l'ordre : NEXUS_WINDOWS_CODEX_HELPER, celui du paquet npm (native/bin, construit par la
|
|
4
|
+
// CI), puis celui de install.ps1 (installation.json de l'état local).
|
|
5
|
+
import { existsSync, readFileSync } from 'node:fs';
|
|
6
|
+
import path from 'node:path';
|
|
7
|
+
import { fileURLToPath } from 'node:url';
|
|
8
|
+
import { NativeUIA } from '../lib/native.mjs';
|
|
9
|
+
import { nativeProvider } from '../lib/native-provider.mjs';
|
|
10
|
+
import { WindowsSession } from '../lib/session.mjs';
|
|
11
|
+
import { emergencyStop } from '../lib/stop.mjs';
|
|
12
|
+
import { WindowsTools, serve } from '../lib/mcp.mjs';
|
|
13
|
+
|
|
14
|
+
const root = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..');
|
|
15
|
+
const stateRoot = path.join(process.env.LOCALAPPDATA ?? path.join(process.env.USERPROFILE ?? '.', 'AppData', 'Local'), 'NexusCodex', 'ComputerUse');
|
|
16
|
+
|
|
17
|
+
function helperPath() {
|
|
18
|
+
// Chemin imposé : lui seul (absent = aucun helper), jamais un autre trouvé à sa place.
|
|
19
|
+
const forced = process.env.NEXUS_WINDOWS_CODEX_HELPER;
|
|
20
|
+
if (forced) return existsSync(forced) ? forced : null;
|
|
21
|
+
const candidates = [path.join(root, 'native', 'bin', 'NexusCodex.UIA.exe')];
|
|
22
|
+
try { candidates.push(JSON.parse(readFileSync(path.join(stateRoot, 'installation.json'), 'utf8')).NativeExecutable); } catch { /* pas d'install.ps1 */ }
|
|
23
|
+
return candidates.find(file => file && existsSync(file)) ?? null;
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
const version = JSON.parse(readFileSync(path.join(root, 'package.json'), 'utf8')).version;
|
|
27
|
+
// Hors Windows ou sans helper, le serveur démarre et liste ses outils (annuaires MCP qui le lancent dans un conteneur
|
|
28
|
+
// Linux) ; chaque appel répond la cause, rien n'est tenté.
|
|
29
|
+
const unavailable = reason => ({ async call() { throw new Error(reason); } });
|
|
30
|
+
const helper = process.platform === 'win32' ? helperPath() : null;
|
|
31
|
+
if (process.platform !== 'win32') {
|
|
32
|
+
await serve({ tools: unavailable('NexusWindows for Codex drives the Windows desktop: Windows 10 or 11 is required.'), version });
|
|
33
|
+
} else if (!helper) {
|
|
34
|
+
await serve({ tools: unavailable('Native helper not found: install the npm package, run install.ps1, or set NEXUS_WINDOWS_CODEX_HELPER.'), version });
|
|
35
|
+
} else {
|
|
36
|
+
const stop = emergencyStop();
|
|
37
|
+
stop.startGuard(helper);
|
|
38
|
+
const native = new NativeUIA(helper, { emergencyStop: stop });
|
|
39
|
+
const session = new WindowsSession(nativeProvider(native), { stateRoot, native, emergencyStop: stop, maxAgeMs: 60000 });
|
|
40
|
+
await session.pruneCaptures(); // captures d'une session précédente : purgées dès le démarrage
|
|
41
|
+
await serve({ tools: new WindowsTools(session), version });
|
|
42
|
+
await session.close().catch(() => {});
|
|
43
|
+
native.close();
|
|
44
|
+
}
|
|
45
|
+
process.exit(0);
|