nexus-windows-codex 3.1.0 → 3.1.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +39 -0
- package/README.md +6 -4
- package/lib/garde-fous.mjs +2 -2
- package/lib/journal.mjs +7 -3
- package/lib/mcp.mjs +19 -3
- package/lib/native.mjs +2 -1
- package/lib/session.mjs +30 -10
- package/lib/tree.mjs +25 -0
- package/native/bin/NexusCodex.UIA.exe +2 -2
- package/package.json +1 -1
package/CHANGELOG.md
CHANGED
|
@@ -7,6 +7,45 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
|
|
|
7
7
|
|
|
8
8
|
## [Unreleased]
|
|
9
9
|
|
|
10
|
+
## [3.1.2] - 2026-10-25 — Docs that match the code
|
|
11
|
+
|
|
12
|
+
### Fixed
|
|
13
|
+
- The shared action journal recorded version `3.0.0` for every release: the version is now read from `package.json`.
|
|
14
|
+
- The shared policy is looked up in the author's current layout; the embedded `garde-fous.json` stays the copy used by
|
|
15
|
+
the npm package (no behaviour change for npm installs).
|
|
16
|
+
- SKILL.md and the API reference describe the code as it is: physical fallback by the helper's own real input
|
|
17
|
+
(`physical:native`) before Sky's, native WebP captures usable as `screenshotId` for a background click or wheel,
|
|
18
|
+
background keys and Ctrl/Shift shortcuts in Win32 and WinForms, wheel at a point in Win32, WinForms, WPF, Tk and
|
|
19
|
+
Chromium started with a DevTools port; product name NexusWindows for Codex; installer path neutral.
|
|
20
|
+
- README: the measurements section no longer points to the French-only validation archive.
|
|
21
|
+
|
|
22
|
+
### Removed
|
|
23
|
+
- `VALIDATION.md` (French archive of the 1.x validations).
|
|
24
|
+
|
|
25
|
+
## [3.1.1] - 2026-10-24 — Numbers on the image
|
|
26
|
+
|
|
27
|
+
### Added
|
|
28
|
+
- `look(marks="auto"|"all")` (mode both, native tree): the tree is read just before the capture, once, under the same
|
|
29
|
+
lock; each kept element gets its exact tree line number drawn on the image (digits only, built-in 5×7 font ×2, thin
|
|
30
|
+
outline), after scaling and before encoding, so at a constant size in the final image. `auto`: unnamed controls with
|
|
31
|
+
a rectangle; `all`: every element with a rectangle. Default `none`.
|
|
32
|
+
- Guards: the window frame is read before and after the tree and after PrintWindow; if it moved or was resized, no
|
|
33
|
+
number and the reason is given. Elements of another window (popup menu absent from PrintWindow) are never labelled;
|
|
34
|
+
a blank capture is judged before any drawing; a label that does not fit whole is omitted, never cut; labels without
|
|
35
|
+
room, past 120, or past 400 eligible elements are counted in the report.
|
|
36
|
+
|
|
37
|
+
### Fixed
|
|
38
|
+
- A zoom region computed on one window's image, after a dialog has been followed, is refused before any capture
|
|
39
|
+
(before, the parent's area was read in the dialog).
|
|
40
|
+
- A refused or failed `look` no longer leaves the previous observation usable by `act`.
|
|
41
|
+
- Live test of the foreground fallback: it had never run (always skipped while the user was active) and failed as
|
|
42
|
+
soon as it ran, 3.0.1 included, because it required `focused` after the foreground was given back. It is now judged
|
|
43
|
+
by the fixture's focus counter, the technique and the give-back, with no postcondition.
|
|
44
|
+
|
|
45
|
+
### Security
|
|
46
|
+
- Reviewed by the Codex agent over three rounds (cut label reported as drawn, window resized during PrintWindow, zoom
|
|
47
|
+
region applied to a followed dialog, stale observation after a refused look, silent 400 cap).
|
|
48
|
+
|
|
10
49
|
## [3.1.0] - 2026-10-24 — Tk in the background, common image rule, zoom
|
|
11
50
|
|
|
12
51
|
### Added
|
package/README.md
CHANGED
|
@@ -101,7 +101,7 @@ the JavaScript module cache does not reload.
|
|
|
101
101
|
| Tool | Purpose |
|
|
102
102
|
|---|---|
|
|
103
103
|
| `windows(filter)` | Application windows (id, app, title, pid), like the taskbar; password managers never listed |
|
|
104
|
-
| `look(window, mode, region)` | One window by id or a unique part of its title or app: UI Automation tree, plus a capture that works on covered windows with `mode="image"` or `"both"`; `region` zooms on an area of the last image; selects the window for `act` |
|
|
104
|
+
| `look(window, mode, region, marks)` | One window by id or a unique part of its title or app: UI Automation tree, plus a capture that works on covered windows with `mode="image"` or `"both"`; `region` zooms on an area of the last image; `marks` draws tree line numbers on the image; selects the window for `act` |
|
|
105
105
|
| `act(steps)` | One step, or up to 32 each with its own `expect`, on the window of the last `look`; stops at the first step not verified |
|
|
106
106
|
| `launch(app)` | Start an application (name, path, Store app id), a folder, a document or a Settings page (`ms-settings:`) and give the foreground back |
|
|
107
107
|
| `window(action, window)` | `action="close"`: one close message, read back; a "save?" prompt is reported, never answered |
|
|
@@ -202,7 +202,10 @@ evidence for the model, and the coordinate frame of a point action that names th
|
|
|
202
202
|
to the screen pixel. `look(region=[x1, y1, x2, y2])` reads an area of the last full image at full resolution, enlarged
|
|
203
203
|
up to 2x for small text; its own `screenshotId` takes points. An image identical to the last one sent for that window
|
|
204
204
|
(same pixels and frame) is not attached again; the reply names the id that takes points, and `mode="image"` always
|
|
205
|
-
attaches it.
|
|
205
|
+
attaches it. `look(mode="both", marks="auto"|"all")` reads the tree just before the capture and draws on the image the
|
|
206
|
+
tree line number of each kept element (`auto`: unnamed controls; `all`: every element with a rectangle), at a constant
|
|
207
|
+
size after scaling; no number when the window moved between the two reads, on another window's popup, or when the
|
|
208
|
+
label does not fit whole. Default `none`. Numbers link the image to the tree; `act` still targets a selector or a point.
|
|
206
209
|
|
|
207
210
|
### Verified batches (`actBatch`)
|
|
208
211
|
|
|
@@ -332,8 +335,7 @@ test.ps1 offline check; -Full adds the real validation
|
|
|
332
335
|
|
|
333
336
|
## Measurements
|
|
334
337
|
|
|
335
|
-
|
|
336
|
-
French). Windows shown without activation at the bottom of the z-order, the Win32 one covered by the WinForms one.
|
|
338
|
+
Real validation of October 15, 2026 on the repository's own fixture (`tests/live`, run by `test.ps1 -Full`). Windows shown without activation at the bottom of the z-order, the Win32 one covered by the WinForms one.
|
|
337
339
|
Each action: postcondition verified, event counted by the application, foreground unchanged, cursor still.
|
|
338
340
|
|
|
339
341
|
| Action | Technique | Duration (input and read-backs) |
|
package/lib/garde-fous.mjs
CHANGED
|
@@ -5,7 +5,7 @@
|
|
|
5
5
|
// résilier, passer un dépôt en public) exige `confirm:true` sur l'action, posé seulement sur l'ordre explicite de Julien
|
|
6
6
|
// ou après son oui. Le contrôle porte sur CE QUE LE GESTE DÉCLENCHE, pas sur sa forme : clic par sélecteur ou en
|
|
7
7
|
// coordonnées, Entrée, Ctrl+Entrée, Espace, Alt+lettre, saut de ligne tapé, menu, action secondaire.
|
|
8
|
-
// Fichier lu : NEXUS_GARDE_FOUS s'il est donné, sinon la politique commune de
|
|
8
|
+
// Fichier lu : NEXUS_GARDE_FOUS s'il est donné, sinon la politique commune de NexusWindows (`../../commun`), sinon la
|
|
9
9
|
// copie embarquée (version publique ; un test vérifie qu'elle est identique). Absent ou invalide : refus (échec fermé).
|
|
10
10
|
import { createHash, randomBytes } from 'node:crypto';
|
|
11
11
|
import { readFileSync, statSync, existsSync } from 'node:fs';
|
|
@@ -13,7 +13,7 @@ import path from 'node:path';
|
|
|
13
13
|
import { fileURLToPath } from 'node:url';
|
|
14
14
|
|
|
15
15
|
const here = path.dirname(fileURLToPath(import.meta.url));
|
|
16
|
-
export const SHARED_POLICY = path.resolve(here, '..', '..', '
|
|
16
|
+
export const SHARED_POLICY = path.resolve(here, '..', '..', 'commun', 'garde-fous.json');
|
|
17
17
|
export const EMBEDDED_POLICY = path.resolve(here, '..', 'garde-fous.json');
|
|
18
18
|
export const AGENT = 'codex';
|
|
19
19
|
const TRIGGER_DEPTH = 8; // ancêtres remontés depuis un texte ou une icône jusqu'à son bouton
|
package/lib/journal.mjs
CHANGED
|
@@ -1,11 +1,15 @@
|
|
|
1
|
-
// Journal commun des trois Nexus Windows (NexusHarness/
|
|
1
|
+
// Journal commun des trois Nexus Windows (NexusHarness/NexusShared/journal/FORMAT.md) : une ligne par action, sans texte
|
|
2
2
|
// d'écran (ni texte tapé, ni libellé, ni titre). Écriture synchrone et silencieuse : un journal ne bloque jamais une
|
|
3
3
|
// action, y compris dans un runtime REPL sans objet global `process` (relecture de Codex du 23/10).
|
|
4
|
-
import { appendFileSync, mkdirSync } from 'node:fs';
|
|
4
|
+
import { appendFileSync, mkdirSync, readFileSync } from 'node:fs';
|
|
5
5
|
import os from 'node:os';
|
|
6
6
|
import path from 'node:path';
|
|
7
7
|
|
|
8
|
-
|
|
8
|
+
// Version de l'outil : celle de package.json (jamais écrite en dur, comme stop.mjs).
|
|
9
|
+
const VERSION = (() => {
|
|
10
|
+
try { return JSON.parse(readFileSync(new URL('../package.json', import.meta.url), 'utf8')).version ?? 'unknown'; }
|
|
11
|
+
catch { return 'unknown'; }
|
|
12
|
+
})();
|
|
9
13
|
const ISSUES = {
|
|
10
14
|
verified: '✓', already: '=', 'already-satisfied': '=', observed: '~', 'sent-unconfirmed': '~', unmet: '~',
|
|
11
15
|
unknown: '~', interrupted: '~', refused: 'refus', 'confirmation-required': 'refus', 'julien-active': 'différé',
|
package/lib/mcp.mjs
CHANGED
|
@@ -45,6 +45,8 @@ const ACTION = {
|
|
|
45
45
|
const OBSERVE_MODE = { type: 'string', enum: ['text', 'image', 'both'], description: 'text (UI Automation tree, default), image (native capture), both' };
|
|
46
46
|
const REGION = { type: 'array', items: { type: 'number' }, minItems: 4, maxItems: 4,
|
|
47
47
|
description: '[x1, y1, x2, y2] in pixels of the last native image of this window: that area enlarged up to 2x (small text); its own screenshotId takes points' };
|
|
48
|
+
const NUMBERS = { type: 'string', enum: ['none', 'auto', 'all'],
|
|
49
|
+
description: 'Numbers drawn on the image of mode both, each the index of its tree line: none (default), auto (unnamed controls), all' };
|
|
48
50
|
const WINDOW = { type: ['integer', 'string'], description: 'Window id from windows, or a unique part of its title or app; omitted: the window of the last look' };
|
|
49
51
|
const READ = { readOnlyHint: true, destructiveHint: false, idempotentHint: true, openWorldHint: false };
|
|
50
52
|
const WRITE = { readOnlyHint: false, destructiveHint: true, idempotentHint: false, openWorldHint: true };
|
|
@@ -60,7 +62,7 @@ export const TOOLS = [
|
|
|
60
62
|
'with mode image or both, a native capture that works even when the window is covered (WebP, at most 1568 px; ' +
|
|
61
63
|
'unchanged since the last image sent: not attached again, mode image always attaches it). Selects that window for act; ' +
|
|
62
64
|
'an observation serves one act and expires after 60 s.',
|
|
63
|
-
inputSchema: { type: 'object', additionalProperties: false, properties: { window: WINDOW, mode: OBSERVE_MODE, region: REGION } } },
|
|
65
|
+
inputSchema: { type: 'object', additionalProperties: false, properties: { window: WINDOW, mode: OBSERVE_MODE, region: REGION, marks: NUMBERS } } },
|
|
64
66
|
{ name: 'act', title: 'Act in the window', annotations: { ...WRITE, title: 'Act in the window' },
|
|
65
67
|
description: 'Steps on the window of the last look, in the background first (Win32 messages, UI Automation, DevTools for Chromium), ' +
|
|
66
68
|
'the real mouse/keyboard only as a fallback under the shared foreground token. One step, or up to 32 each with its own expect; ' +
|
|
@@ -95,7 +97,7 @@ function errorStatus(error) {
|
|
|
95
97
|
}
|
|
96
98
|
|
|
97
99
|
// Observation lisible par le modèle : arbre et état, plus l'image native jointe (fichier local, jamais copiée ailleurs).
|
|
98
|
-
// Pas d'image en double (règle commune des produits Nexus,
|
|
100
|
+
// Pas d'image en double (règle commune des produits Nexus, NexusShared/images.md) : mêmes octets (encodeur WebP déterministe :
|
|
99
101
|
// mêmes pixels), mêmes dimensions et même cadre écran que la dernière image envoyée pour cette fenêtre (ou ce zoom) →
|
|
100
102
|
// non jointe, la réponse dit quel identifiant prend les points. `always` (mode image) la joint toujours. `received` :
|
|
101
103
|
// appelé pour chaque image que le modèle a réellement (jointe, ou identique à la dernière envoyée), jamais pour une
|
|
@@ -119,11 +121,24 @@ async function observationContent(observation, sent = null, always = false, rece
|
|
|
119
121
|
}
|
|
120
122
|
sent?.set(key, { print, id: image.id });
|
|
121
123
|
content.push({ type: 'image', data: data.toString('base64'), mimeType });
|
|
124
|
+
if (image.marks) content.push(text(numbersLine(image)));
|
|
122
125
|
received?.(image, w.id);
|
|
123
126
|
}
|
|
124
127
|
return content;
|
|
125
128
|
}
|
|
126
129
|
|
|
130
|
+
// Numéros dessinés sur une image jointe : lesquels, et ceux qui manquent (place, autre fenêtre, hors zone).
|
|
131
|
+
function numbersLine(image) {
|
|
132
|
+
const m = image.marks;
|
|
133
|
+
if (m.reason) return `Image ${image.id}: ${m.reason}.`;
|
|
134
|
+
const parts = [`Image ${image.id}: numbers = tree line indexes, drawn for ${m.drawn?.length ?? 0} element(s)`];
|
|
135
|
+
if (m.omitted?.length) parts.push(`not drawn for lack of room: ${m.omitted.join(', ')}`);
|
|
136
|
+
if (m.foreign?.length) parts.push(`other windows (not in this capture): ${m.foreign.join(', ')}`);
|
|
137
|
+
if (m.outside) parts.push(`${m.outside} outside the captured area`);
|
|
138
|
+
if (m.capped) parts.push(`${m.capped} more element(s) not considered (limit 400)`);
|
|
139
|
+
return parts.join('; ') + '.';
|
|
140
|
+
}
|
|
141
|
+
|
|
127
142
|
// Zone [x1, y1, x2, y2] (pixels inclus) d'une image native → zone écran [left, top, width, height] : bords des pixels de
|
|
128
143
|
// l'image ramenés au cadre (même échelle que les points), bornés au cadre.
|
|
129
144
|
export function regionToScreen(image, region) {
|
|
@@ -194,12 +209,13 @@ export class WindowsTools {
|
|
|
194
209
|
return [text(wanted ? listed.filter(w => w.title.toLowerCase().includes(wanted) || w.app.toLowerCase().includes(wanted)) : listed)];
|
|
195
210
|
}
|
|
196
211
|
case 'look': {
|
|
212
|
+
this.#observation = null; // refus ou échec : plus d'observation ancienne pour act (relecture de Codex du 24/10)
|
|
197
213
|
await this.#bind(args.window);
|
|
198
214
|
// Repère : dernière image complète envoyée de la fenêtre que cette liaison observe (dialogue suivi compris).
|
|
199
215
|
const shown = this.#observed.get(this.#bound?.id) ?? this.#bound?.id;
|
|
200
216
|
const zoom = args.region === undefined ? null : regionToScreen(this.#images.get(shown), args.region);
|
|
201
217
|
const mode = zoom && (args.mode ?? 'text') === 'text' ? 'both' : args.mode ?? 'text';
|
|
202
|
-
this.#observation = await s.observe({ mode, backend: 'native', capture: 'native', zoom });
|
|
218
|
+
this.#observation = await s.observe({ mode, backend: 'native', capture: 'native', zoom, ...(zoom ? { zoomWindow: shown } : {}), ...(args.marks && args.marks !== 'none' ? { marks: args.marks } : {}) });
|
|
203
219
|
this.#remember(this.#observation);
|
|
204
220
|
return observationContent(this.#observation, this.#sent, mode === 'image', this.#received);
|
|
205
221
|
}
|
package/lib/native.mjs
CHANGED
|
@@ -138,7 +138,8 @@ export class NativeUIA {
|
|
|
138
138
|
}
|
|
139
139
|
inspect(window) { return this.call(window, 'inspect'); }
|
|
140
140
|
// Lectures sans entrée (permises pendant l'arrêt d'urgence).
|
|
141
|
-
|
|
141
|
+
// marks : numéros à dessiner ({ marks: [{index, bounds, scope}], treeFrame }), lus avec l'arbre juste avant (session.mjs).
|
|
142
|
+
capture(window, directory, zoom, marks = null) { return this.call(window, 'capture', { directory, ...(zoom ? { zoom } : {}), ...(marks ?? {}) }); }
|
|
142
143
|
related(window) { return this.call(window, 'related'); }
|
|
143
144
|
alive(window) { return this.call(window, 'alive'); }
|
|
144
145
|
// Page des Paramètres (ms-settings: seul), ouverte sous le jeton et premier plan rendu ; rend la fenêtre notée.
|
package/lib/session.mjs
CHANGED
|
@@ -2,7 +2,7 @@ import { randomUUID } from 'node:crypto';
|
|
|
2
2
|
import { mkdir, writeFile, rename, stat, unlink, readdir } from 'node:fs/promises';
|
|
3
3
|
import path from 'node:path';
|
|
4
4
|
import { ApplicationMemory } from './memory.mjs';
|
|
5
|
-
import { parseTree, uniqueElement, findElements, structureHash, stateDigest, checkExpectation, validateExpectation, digest } from './tree.mjs';
|
|
5
|
+
import { parseTree, markCandidates, uniqueElement, findElements, structureHash, stateDigest, checkExpectation, validateExpectation, digest } from './tree.mjs';
|
|
6
6
|
import { emergencyStop } from './stop.mjs';
|
|
7
7
|
import { JulienActiveError, PresenceUnknownError, QUIET_MS, MAX_WAIT_MS } from './presence.mjs';
|
|
8
8
|
import * as gardeFous from './garde-fous.mjs';
|
|
@@ -539,12 +539,13 @@ export class WindowsSession {
|
|
|
539
539
|
if (i >= CAPTURE_KEEP || now - entry.at > CAPTURE_RETENTION_MS) await unlink(entry.file).catch(() => {});
|
|
540
540
|
}
|
|
541
541
|
}
|
|
542
|
-
async #capture(mode, backend = 'auto', { capture = 'sky', windowChange = null, zoom = null } = {}) {
|
|
542
|
+
async #capture(mode, backend = 'auto', { capture = 'sky', windowChange = null, zoom = null, marks = 'none' } = {}) {
|
|
543
543
|
this.#invalidate();
|
|
544
544
|
if (!this.#window) throw new Error('Bind a returned window before observing');
|
|
545
545
|
if (!['text', 'image', 'both'].includes(mode)) throw new Error('mode must be text, image or both');
|
|
546
546
|
if (!['auto', 'sky', 'native'].includes(backend)) throw new Error('backend must be auto, sky or native');
|
|
547
547
|
if (!['sky', 'native'].includes(capture)) throw new Error('capture must be sky or native');
|
|
548
|
+
if (!['none', 'auto', 'all'].includes(marks)) throw new Error('marks must be none, auto or all');
|
|
548
549
|
if (backend === 'native' && !this.#native) throw new Error('Native UIA is not configured');
|
|
549
550
|
const nativeOnly = this.#window.nativeOnly === true;
|
|
550
551
|
if (nativeOnly) {
|
|
@@ -556,22 +557,33 @@ export class WindowsSession {
|
|
|
556
557
|
if (!sameWindow(current, this.#window)) throw new Error('Window identity changed; bind it again');
|
|
557
558
|
const useNative = nativeOnly || backend === 'native' || (backend === 'auto' && this.#nativePreferred);
|
|
558
559
|
// Capture propre (PrintWindow, fenêtre recouverte, sans premier plan) ; image vide ou échec : capture Sky.
|
|
560
|
+
// Numéros [N] (NexusShared/images.md point 5, avis de Codex du 24/10) : arbre natif lu AVANT la capture, une seule fois et
|
|
561
|
+
// sous le même verrou ; ses index et rectangles vont au helper, qui ne les dessine que si le cadre de la fenêtre est
|
|
562
|
+
// resté celui de la lecture. L'âge de l'observation part du début de cette lecture (FRESH_MS).
|
|
563
|
+
let marksNote = null, preread = null, treeStartedAt = null, candidates = null;
|
|
564
|
+
if (marks !== 'none') {
|
|
565
|
+
if (mode !== 'both') marksNote = 'numbers need mode both (tree and image together): none drawn';
|
|
566
|
+
else if (capture !== 'native' || !useNative || !this.#native?.inspect || !this.#native?.capture || !this.#root) marksNote = 'numbers need the native tree and capture: none drawn';
|
|
567
|
+
else { treeStartedAt = Date.now(); preread = await this.#native.inspect(current); }
|
|
568
|
+
}
|
|
559
569
|
let images = [], imageFallback = null;
|
|
560
570
|
if (capture === 'native' && mode !== 'text') {
|
|
561
571
|
if (!this.#native?.capture || !this.#root) imageFallback = 'Native capture is not configured';
|
|
562
572
|
else {
|
|
563
573
|
const directory = path.join(this.#root, 'captures');
|
|
564
574
|
try {
|
|
565
|
-
const shot = await this.#native.capture(current, directory, zoom
|
|
575
|
+
const shot = await this.#native.capture(current, directory, zoom,
|
|
576
|
+
preread ? { marks: (candidates = markCandidates(preread.elements, marks)), treeFrame: preread.frame ?? null } : null);
|
|
566
577
|
if (shot.blank) imageFallback = 'PrintWindow returned a uniform (blank) image';
|
|
567
578
|
// id : repère de coordonnées natif (banc commun du 21/10, plan de Codex étape 5), distinct des captures Sky ;
|
|
568
579
|
// un point de cette image vise l'écran à frame.left + x * frame.width / width (image réduite au plus à
|
|
569
580
|
// 1568 px et 1,15 Mpx, règle commune des produits Nexus : nativeScreen).
|
|
570
|
-
else images = [{ id: `native-${++this.#nativeShots}`, path: shot.path, width: shot.width, height: shot.height, method: shot.method, scale: shot.scale, ms: shot.ms, frame: shot.frame, windowFrame: shot.windowFrame ?? shot.frame, ...(zoom ? { zoom: true } : {}) }];
|
|
581
|
+
else images = [{ id: `native-${++this.#nativeShots}`, path: shot.path, width: shot.width, height: shot.height, method: shot.method, scale: shot.scale, ms: shot.ms, frame: shot.frame, windowFrame: shot.windowFrame ?? shot.frame, ...(zoom ? { zoom: true } : {}), ...(preread ? { marks: { ...(shot.marks ?? { reason: 'the helper drew no numbers' }), ...(candidates?.capped ? { capped: candidates.capped } : {}) } } : {}) }];
|
|
571
582
|
} catch (error) { imageFallback = error.message; }
|
|
572
583
|
await this.#pruneCaptures(directory);
|
|
573
584
|
}
|
|
574
585
|
}
|
|
586
|
+
if (preread && !images.length) marksNote = 'no native image: none drawn';
|
|
575
587
|
const skyText = mode !== 'image' && !useNative;
|
|
576
588
|
const skyImage = mode !== 'text' && !images.length && !nativeOnly;
|
|
577
589
|
const state = !skyText && !skyImage
|
|
@@ -588,14 +600,14 @@ export class WindowsSession {
|
|
|
588
600
|
const nativeRead = mode !== 'image' && this.#native?.inspect && (useNative || (!elements.length && backend === 'auto'));
|
|
589
601
|
let revision = nativeRead ? null : await this.#getRevision(state.window.app); // une lecture native donne la sienne
|
|
590
602
|
if (nativeRead) {
|
|
591
|
-
const native = await this.#native.inspect(state.window);
|
|
603
|
+
const native = preread ?? await this.#native.inspect(state.window);
|
|
592
604
|
accessibility = native.accessibility; elements = native.elements; source = 'native-uia';
|
|
593
605
|
revision = native.revision;
|
|
594
606
|
this.#nativePreferred = true;
|
|
595
607
|
}
|
|
596
608
|
else if (mode !== 'image') this.#nativePreferred = false;
|
|
597
609
|
const observation = freeze({
|
|
598
|
-
token: randomUUID(), epoch: this.#coordinator.epoch, capturedAt: Date.now(), mode, source, revision,
|
|
610
|
+
token: randomUUID(), epoch: this.#coordinator.epoch, capturedAt: treeStartedAt ?? Date.now(), mode, source, revision,
|
|
599
611
|
window: clone(state.window), windowStack: this.#stack.map(w => w.id), windowChange: clone(windowChange), popups: clone(this.#popups),
|
|
600
612
|
elements, structureHash: structureHash(elements),
|
|
601
613
|
accessibility: clone(accessibility),
|
|
@@ -603,16 +615,23 @@ export class WindowsSession {
|
|
|
603
615
|
screenshots: (state.screenshots ?? []).map(({ id, width, height, originX, originY, zIndex }) =>
|
|
604
616
|
({ id, width, height, originX, originY, zIndex })),
|
|
605
617
|
// Captures natives : fichiers locaux (LocalAppData) à regarder avec view_image ; jamais des cibles de coordonnées.
|
|
606
|
-
images, imageFallback,
|
|
618
|
+
images, imageFallback, ...(marksNote ? { marksNote } : {}),
|
|
607
619
|
});
|
|
608
620
|
this.#observation = observation;
|
|
609
621
|
return observation;
|
|
610
622
|
}
|
|
611
623
|
// zoom : zone écran [left, top, width, height] d'une capture native (lookZoom), agrandie jusqu'à ×2.
|
|
612
|
-
|
|
624
|
+
// marks : none (défaut), auto (contrôles sans nom), all ; mode both et lecture native seulement (sinon marksNote).
|
|
625
|
+
// zoomWindow : fenêtre de l'image d'où vient la zone ; un dialogue suivi entre-temps la rend caduque (refus, rien lu
|
|
626
|
+
// dans un autre repère ; relecture de Codex du 24/10).
|
|
627
|
+
async observe({ mode = 'text', backend = 'auto', capture = 'sky', zoom = null, marks = 'none', zoomWindow = null } = {}) {
|
|
613
628
|
return this.#exclusive(async () => {
|
|
629
|
+
this.#invalidate(); // un refus ci-dessous ne laisse aucune observation utilisable (relecture de Codex du 24/10)
|
|
614
630
|
const windowChange = await this.#follow(false);
|
|
615
|
-
|
|
631
|
+
if (zoom && zoomWindow != null && this.#window?.id !== zoomWindow) {
|
|
632
|
+
throw new Error(`region comes from an image of window ${zoomWindow}, but the observed window is now ${this.#window?.id}: look again without region`);
|
|
633
|
+
}
|
|
634
|
+
return this.#capture(mode, backend, { capture, windowChange, zoom, marks });
|
|
616
635
|
});
|
|
617
636
|
}
|
|
618
637
|
#take(observation) {
|
|
@@ -1465,7 +1484,7 @@ export class WindowsSession {
|
|
|
1465
1484
|
}
|
|
1466
1485
|
#record(kind, status, durationMs, technique = null, foreground = null) {
|
|
1467
1486
|
this.#metrics.push({ kind, status, technique, durationMs, at: new Date().toISOString() });
|
|
1468
|
-
// Journal commun des trois Nexus Windows (
|
|
1487
|
+
// Journal commun des trois Nexus Windows (NexusShared/journal/FORMAT.md) : sans texte d'écran, jamais bloquant.
|
|
1469
1488
|
try { writeJournal(journalLine({ session: this.#sessionId, app: this.#window?.app, kind, status, technique, durationMs, foreground })); }
|
|
1470
1489
|
catch { /* journal : jamais bloquant */ }
|
|
1471
1490
|
}
|
|
@@ -1492,5 +1511,6 @@ export function summarize(observation) {
|
|
|
1492
1511
|
windowStack: observation.windowStack, windowChange: observation.windowChange, popups: observation.popups,
|
|
1493
1512
|
tree: observation.accessibility?.tree ?? '', focused: observation.accessibility?.focused_element ?? '',
|
|
1494
1513
|
screenshots: observation.screenshots, images: observation.images, imageFallback: observation.imageFallback,
|
|
1514
|
+
...(observation.marksNote ? { marksNote: observation.marksNote } : {}),
|
|
1495
1515
|
};
|
|
1496
1516
|
}
|
package/lib/tree.mjs
CHANGED
|
@@ -20,6 +20,31 @@ const stateFlags = new Set(['disabled', 'selected', 'unselected', 'selectable',
|
|
|
20
20
|
'settable', 'string', 'float', 'integer', 'boolean']);
|
|
21
21
|
export const digest = value => createHash('sha256').update(value).digest('hex');
|
|
22
22
|
|
|
23
|
+
// Numéros [N] sur l'image (NexusShared/images.md point 5) : éléments natifs avec un rectangle, hors racine et hors éléments
|
|
24
|
+
// dits hors écran. auto : contrôles sans nom (espaces compris) qu'on peut viser ; all : tous. scope : HWND porteur le plus
|
|
25
|
+
// proche (l'élément ou un ancêtre), que le helper compare à la fenêtre capturée (menu surgissant d'une autre fenêtre :
|
|
26
|
+
// jamais d'étiquette fantôme). Index EXACT de la ligne de l'arbre, jamais renuméroté.
|
|
27
|
+
const MARKS_LIMIT = 400;
|
|
28
|
+
const MARKABLE = new Set(['button', 'splitbutton', 'edit', 'checkbox', 'radio', 'combobox', 'listitem', 'menuitem', 'tab',
|
|
29
|
+
'treeitem', 'link', 'slider', 'spinner', 'dataitem', 'headeritem', 'image']);
|
|
30
|
+
export function markCandidates(elements = [], marks = 'auto') {
|
|
31
|
+
const byIndex = new Map(elements.map(e => [e.index, e]));
|
|
32
|
+
const out = [];
|
|
33
|
+
let capped = 0; // éléments admissibles au-delà du plafond : dits dans le rapport (all n'est alors pas exhaustif)
|
|
34
|
+
for (const e of elements) {
|
|
35
|
+
const b = e.bounds;
|
|
36
|
+
if (!Array.isArray(b) || b.length !== 4 || !b.every(Number.isFinite) || !(b[2] > 0) || !(b[3] > 0)) continue;
|
|
37
|
+
if (!e.ancestors?.length || e.flags?.includes('offscreen')) continue;
|
|
38
|
+
if (marks === 'auto' && (String(e.name ?? '').trim() || !(MARKABLE.has(e.role) || e.invoke === true))) continue;
|
|
39
|
+
const chain = [e, ...[...e.ancestors].reverse().map(i => byIndex.get(i))];
|
|
40
|
+
const scope = chain.find(x => x?.nativeWindow)?.nativeWindow ?? 0;
|
|
41
|
+
if (out.length >= MARKS_LIMIT) { capped++; continue; }
|
|
42
|
+
out.push({ index: e.index, bounds: b.map(Math.round), scope });
|
|
43
|
+
}
|
|
44
|
+
if (capped) Object.defineProperty(out, 'capped', { value: capped }); // hors JSON : seul le compte part au rapport
|
|
45
|
+
return out;
|
|
46
|
+
}
|
|
47
|
+
|
|
23
48
|
export function parseTree(tree = '') {
|
|
24
49
|
const elements = [], stack = [];
|
|
25
50
|
for (const line of tree.split(/\r?\n/)) {
|
|
@@ -1,4 +1,4 @@
|
|
|
1
1
|
[diffend] Oversized file quarantined before diffing.
|
|
2
2
|
name: package/native/bin/NexusCodex.UIA.exe
|
|
3
|
-
size:
|
|
4
|
-
sha256:
|
|
3
|
+
size: 35244346 bytes
|
|
4
|
+
sha256: 15e9d75e1756c8b0bb63c0bdaad3ee5e18aa1c5a09ab6047686b7326716c44b2
|
package/package.json
CHANGED