nexus-windows-codex 3.1.0 → 3.1.1

package/CHANGELOG.md

@@ -7,6 +7,30 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+## [3.1.1] - 2026-10-24 — Numbers on the image
+
+### Added
+- `look(marks="auto"|"all")` (mode both, native tree): the tree is read just before the capture, once, under the same
+  lock; each kept element gets its exact tree line number drawn on the image (digits only, built-in 5×7 font ×2, thin
+  outline), after scaling and before encoding, so at a constant size in the final image. `auto`: unnamed controls with
+  a rectangle; `all`: every element with a rectangle. Default `none`.
+- Guards: the window frame is read before and after the tree and after PrintWindow; if it moved or was resized, no
+  number and the reason is given. Elements of another window (popup menu absent from PrintWindow) are never labelled;
+  a blank capture is judged before any drawing; a label that does not fit whole is omitted, never cut; labels without
+  room, past 120, or past 400 eligible elements are counted in the report.
+
+### Fixed
+- A zoom region computed on one window's image, after a dialog has been followed, is refused before any capture
+  (before, the parent's area was read in the dialog).
+- A refused or failed `look` no longer leaves the previous observation usable by `act`.
+- Live test of the foreground fallback: it had never run (always skipped while the user was active) and failed as
+  soon as it ran, 3.0.1 included, because it required `focused` after the foreground was given back. It is now judged
+  by the fixture's focus counter, the technique and the give-back, with no postcondition.
+
+### Security
+- Reviewed by the Codex agent over three rounds (cut label reported as drawn, window resized during PrintWindow, zoom
+  region applied to a followed dialog, stale observation after a refused look, silent 400 cap).
+
 ## [3.1.0] - 2026-10-24 — Tk in the background, common image rule, zoom
 
 ### Added

package/README.md

@@ -101,7 +101,7 @@ the JavaScript module cache does not reload.
 | Tool | Purpose |
 |---|---|
 | `windows(filter)` | Application windows (id, app, title, pid), like the taskbar; password managers never listed |
-| `look(window, mode, region)` | One window by id or a unique part of its title or app: UI Automation tree, plus a capture that works on covered windows with `mode="image"` or `"both"`; `region` zooms on an area of the last image; selects the window for `act` |
+| `look(window, mode, region, marks)` | One window by id or a unique part of its title or app: UI Automation tree, plus a capture that works on covered windows with `mode="image"` or `"both"`; `region` zooms on an area of the last image; `marks` draws tree line numbers on the image; selects the window for `act` |
 | `act(steps)` | One step, or up to 32 each with its own `expect`, on the window of the last `look`; stops at the first step not verified |
 | `launch(app)` | Start an application (name, path, Store app id), a folder, a document or a Settings page (`ms-settings:`) and give the foreground back |
 | `window(action, window)` | `action="close"`: one close message, read back; a "save?" prompt is reported, never answered |
@@ -202,7 +202,10 @@ evidence for the model, and the coordinate frame of a point action that names th
 to the screen pixel. `look(region=[x1, y1, x2, y2])` reads an area of the last full image at full resolution, enlarged
 up to 2x for small text; its own `screenshotId` takes points. An image identical to the last one sent for that window
 (same pixels and frame) is not attached again; the reply names the id that takes points, and `mode="image"` always
-attaches it.
+attaches it. `look(mode="both", marks="auto"|"all")` reads the tree just before the capture and draws on the image the
+tree line number of each kept element (`auto`: unnamed controls; `all`: every element with a rectangle), at a constant
+size after scaling; no number when the window moved between the two reads, on another window's popup, or when the
+label does not fit whole. Default `none`. Numbers link the image to the tree; `act` still targets a selector or a point.
 
 ### Verified batches (`actBatch`)
 

package/lib/mcp.mjs

@@ -45,6 +45,8 @@ const ACTION = {
 const OBSERVE_MODE = { type: 'string', enum: ['text', 'image', 'both'], description: 'text (UI Automation tree, default), image (native capture), both' };
 const REGION = { type: 'array', items: { type: 'number' }, minItems: 4, maxItems: 4,
   description: '[x1, y1, x2, y2] in pixels of the last native image of this window: that area enlarged up to 2x (small text); its own screenshotId takes points' };
+const NUMBERS = { type: 'string', enum: ['none', 'auto', 'all'],
+  description: 'Numbers drawn on the image of mode both, each the index of its tree line: none (default), auto (unnamed controls), all' };
 const WINDOW = { type: ['integer', 'string'], description: 'Window id from windows, or a unique part of its title or app; omitted: the window of the last look' };
 const READ = { readOnlyHint: true, destructiveHint: false, idempotentHint: true, openWorldHint: false };
 const WRITE = { readOnlyHint: false, destructiveHint: true, idempotentHint: false, openWorldHint: true };
@@ -60,7 +62,7 @@ export const TOOLS = [
       'with mode image or both, a native capture that works even when the window is covered (WebP, at most 1568 px; ' +
       'unchanged since the last image sent: not attached again, mode image always attaches it). Selects that window for act; ' +
       'an observation serves one act and expires after 60 s.',
-    inputSchema: { type: 'object', additionalProperties: false, properties: { window: WINDOW, mode: OBSERVE_MODE, region: REGION } } },
+    inputSchema: { type: 'object', additionalProperties: false, properties: { window: WINDOW, mode: OBSERVE_MODE, region: REGION, marks: NUMBERS } } },
   { name: 'act', title: 'Act in the window', annotations: { ...WRITE, title: 'Act in the window' },
     description: 'Steps on the window of the last look, in the background first (Win32 messages, UI Automation, DevTools for Chromium), ' +
       'the real mouse/keyboard only as a fallback under the shared foreground token. One step, or up to 32 each with its own expect; ' +
@@ -119,11 +121,24 @@ async function observationContent(observation, sent = null, always = false, rece
     }
     sent?.set(key, { print, id: image.id });
     content.push({ type: 'image', data: data.toString('base64'), mimeType });
+    if (image.marks) content.push(text(numbersLine(image)));
     received?.(image, w.id);
   }
   return content;
 }
 
+// Numéros dessinés sur une image jointe : lesquels, et ceux qui manquent (place, autre fenêtre, hors zone).
+function numbersLine(image) {
+  const m = image.marks;
+  if (m.reason) return `Image ${image.id}: ${m.reason}.`;
+  const parts = [`Image ${image.id}: numbers = tree line indexes, drawn for ${m.drawn?.length ?? 0} element(s)`];
+  if (m.omitted?.length) parts.push(`not drawn for lack of room: ${m.omitted.join(', ')}`);
+  if (m.foreign?.length) parts.push(`other windows (not in this capture): ${m.foreign.join(', ')}`);
+  if (m.outside) parts.push(`${m.outside} outside the captured area`);
+  if (m.capped) parts.push(`${m.capped} more element(s) not considered (limit 400)`);
+  return parts.join('; ') + '.';
+}
+
 // Zone [x1, y1, x2, y2] (pixels inclus) d'une image native → zone écran [left, top, width, height] : bords des pixels de
 // l'image ramenés au cadre (même échelle que les points), bornés au cadre.
 export function regionToScreen(image, region) {
@@ -194,12 +209,13 @@ export class WindowsTools {
         return [text(wanted ? listed.filter(w => w.title.toLowerCase().includes(wanted) || w.app.toLowerCase().includes(wanted)) : listed)];
       }
       case 'look': {
+        this.#observation = null; // refus ou échec : plus d'observation ancienne pour act (relecture de Codex du 24/10)
         await this.#bind(args.window);
         // Repère : dernière image complète envoyée de la fenêtre que cette liaison observe (dialogue suivi compris).
         const shown = this.#observed.get(this.#bound?.id) ?? this.#bound?.id;
         const zoom = args.region === undefined ? null : regionToScreen(this.#images.get(shown), args.region);
         const mode = zoom && (args.mode ?? 'text') === 'text' ? 'both' : args.mode ?? 'text';
-        this.#observation = await s.observe({ mode, backend: 'native', capture: 'native', zoom });
+        this.#observation = await s.observe({ mode, backend: 'native', capture: 'native', zoom, ...(zoom ? { zoomWindow: shown } : {}), ...(args.marks && args.marks !== 'none' ? { marks: args.marks } : {}) });
         this.#remember(this.#observation);
         return observationContent(this.#observation, this.#sent, mode === 'image', this.#received);
       }

package/lib/native.mjs

@@ -138,7 +138,8 @@ export class NativeUIA {
   }
   inspect(window) { return this.call(window, 'inspect'); }
   // Lectures sans entrée (permises pendant l'arrêt d'urgence).
-  capture(window, directory, zoom) { return this.call(window, 'capture', { directory, ...(zoom ? { zoom } : {}) }); }
+  // marks : numéros à dessiner ({ marks: [{index, bounds, scope}], treeFrame }), lus avec l'arbre juste avant (session.mjs).
+  capture(window, directory, zoom, marks = null) { return this.call(window, 'capture', { directory, ...(zoom ? { zoom } : {}), ...(marks ?? {}) }); }
   related(window) { return this.call(window, 'related'); }
   alive(window) { return this.call(window, 'alive'); }
   // Page des Paramètres (ms-settings: seul), ouverte sous le jeton et premier plan rendu ; rend la fenêtre notée.

package/lib/session.mjs

@@ -2,7 +2,7 @@ import { randomUUID } from 'node:crypto';
 import { mkdir, writeFile, rename, stat, unlink, readdir } from 'node:fs/promises';
 import path from 'node:path';
 import { ApplicationMemory } from './memory.mjs';
-import { parseTree, uniqueElement, findElements, structureHash, stateDigest, checkExpectation, validateExpectation, digest } from './tree.mjs';
+import { parseTree, markCandidates, uniqueElement, findElements, structureHash, stateDigest, checkExpectation, validateExpectation, digest } from './tree.mjs';
 import { emergencyStop } from './stop.mjs';
 import { JulienActiveError, PresenceUnknownError, QUIET_MS, MAX_WAIT_MS } from './presence.mjs';
 import * as gardeFous from './garde-fous.mjs';
@@ -539,12 +539,13 @@ export class WindowsSession {
       if (i >= CAPTURE_KEEP || now - entry.at > CAPTURE_RETENTION_MS) await unlink(entry.file).catch(() => {});
     }
   }
-  async #capture(mode, backend = 'auto', { capture = 'sky', windowChange = null, zoom = null } = {}) {
+  async #capture(mode, backend = 'auto', { capture = 'sky', windowChange = null, zoom = null, marks = 'none' } = {}) {
     this.#invalidate();
     if (!this.#window) throw new Error('Bind a returned window before observing');
     if (!['text', 'image', 'both'].includes(mode)) throw new Error('mode must be text, image or both');
     if (!['auto', 'sky', 'native'].includes(backend)) throw new Error('backend must be auto, sky or native');
     if (!['sky', 'native'].includes(capture)) throw new Error('capture must be sky or native');
+    if (!['none', 'auto', 'all'].includes(marks)) throw new Error('marks must be none, auto or all');
     if (backend === 'native' && !this.#native) throw new Error('Native UIA is not configured');
     const nativeOnly = this.#window.nativeOnly === true;
     if (nativeOnly) {
@@ -556,22 +557,33 @@ export class WindowsSession {
     if (!sameWindow(current, this.#window)) throw new Error('Window identity changed; bind it again');
     const useNative = nativeOnly || backend === 'native' || (backend === 'auto' && this.#nativePreferred);
     // Capture propre (PrintWindow, fenêtre recouverte, sans premier plan) ; image vide ou échec : capture Sky.
+    // Numéros [N] (shared/images.md point 5, avis de Codex du 24/10) : arbre natif lu AVANT la capture, une seule fois et
+    // sous le même verrou ; ses index et rectangles vont au helper, qui ne les dessine que si le cadre de la fenêtre est
+    // resté celui de la lecture. L'âge de l'observation part du début de cette lecture (FRESH_MS).
+    let marksNote = null, preread = null, treeStartedAt = null, candidates = null;
+    if (marks !== 'none') {
+      if (mode !== 'both') marksNote = 'numbers need mode both (tree and image together): none drawn';
+      else if (capture !== 'native' || !useNative || !this.#native?.inspect || !this.#native?.capture || !this.#root) marksNote = 'numbers need the native tree and capture: none drawn';
+      else { treeStartedAt = Date.now(); preread = await this.#native.inspect(current); }
+    }
     let images = [], imageFallback = null;
     if (capture === 'native' && mode !== 'text') {
       if (!this.#native?.capture || !this.#root) imageFallback = 'Native capture is not configured';
       else {
         const directory = path.join(this.#root, 'captures');
         try {
-          const shot = await this.#native.capture(current, directory, zoom);
+          const shot = await this.#native.capture(current, directory, zoom,
+            preread ? { marks: (candidates = markCandidates(preread.elements, marks)), treeFrame: preread.frame ?? null } : null);
           if (shot.blank) imageFallback = 'PrintWindow returned a uniform (blank) image';
           // id : repère de coordonnées natif (banc commun du 21/10, plan de Codex étape 5), distinct des captures Sky ;
           // un point de cette image vise l'écran à frame.left + x * frame.width / width (image réduite au plus à
           // 1568 px et 1,15 Mpx, règle commune des produits Nexus : nativeScreen).
-          else images = [{ id: `native-${++this.#nativeShots}`, path: shot.path, width: shot.width, height: shot.height, method: shot.method, scale: shot.scale, ms: shot.ms, frame: shot.frame, windowFrame: shot.windowFrame ?? shot.frame, ...(zoom ? { zoom: true } : {}) }];
+          else images = [{ id: `native-${++this.#nativeShots}`, path: shot.path, width: shot.width, height: shot.height, method: shot.method, scale: shot.scale, ms: shot.ms, frame: shot.frame, windowFrame: shot.windowFrame ?? shot.frame, ...(zoom ? { zoom: true } : {}), ...(preread ? { marks: { ...(shot.marks ?? { reason: 'the helper drew no numbers' }), ...(candidates?.capped ? { capped: candidates.capped } : {}) } } : {}) }];
         } catch (error) { imageFallback = error.message; }
         await this.#pruneCaptures(directory);
       }
     }
+    if (preread && !images.length) marksNote = 'no native image: none drawn';
     const skyText = mode !== 'image' && !useNative;
     const skyImage = mode !== 'text' && !images.length && !nativeOnly;
     const state = !skyText && !skyImage
@@ -588,14 +600,14 @@ export class WindowsSession {
     const nativeRead = mode !== 'image' && this.#native?.inspect && (useNative || (!elements.length && backend === 'auto'));
     let revision = nativeRead ? null : await this.#getRevision(state.window.app); // une lecture native donne la sienne
     if (nativeRead) {
-      const native = await this.#native.inspect(state.window);
+      const native = preread ?? await this.#native.inspect(state.window);
       accessibility = native.accessibility; elements = native.elements; source = 'native-uia';
       revision = native.revision;
       this.#nativePreferred = true;
     }
     else if (mode !== 'image') this.#nativePreferred = false;
     const observation = freeze({
-      token: randomUUID(), epoch: this.#coordinator.epoch, capturedAt: Date.now(), mode, source, revision,
+      token: randomUUID(), epoch: this.#coordinator.epoch, capturedAt: treeStartedAt ?? Date.now(), mode, source, revision,
       window: clone(state.window), windowStack: this.#stack.map(w => w.id), windowChange: clone(windowChange), popups: clone(this.#popups),
       elements, structureHash: structureHash(elements),
       accessibility: clone(accessibility),
@@ -603,16 +615,23 @@ export class WindowsSession {
       screenshots: (state.screenshots ?? []).map(({ id, width, height, originX, originY, zIndex }) =>
         ({ id, width, height, originX, originY, zIndex })),
       // Captures natives : fichiers locaux (LocalAppData) à regarder avec view_image ; jamais des cibles de coordonnées.
-      images, imageFallback,
+      images, imageFallback, ...(marksNote ? { marksNote } : {}),
     });
     this.#observation = observation;
     return observation;
   }
   // zoom : zone écran [left, top, width, height] d'une capture native (lookZoom), agrandie jusqu'à ×2.
-  async observe({ mode = 'text', backend = 'auto', capture = 'sky', zoom = null } = {}) {
+  // marks : none (défaut), auto (contrôles sans nom), all ; mode both et lecture native seulement (sinon marksNote).
+  // zoomWindow : fenêtre de l'image d'où vient la zone ; un dialogue suivi entre-temps la rend caduque (refus, rien lu
+  // dans un autre repère ; relecture de Codex du 24/10).
+  async observe({ mode = 'text', backend = 'auto', capture = 'sky', zoom = null, marks = 'none', zoomWindow = null } = {}) {
     return this.#exclusive(async () => {
+      this.#invalidate(); // un refus ci-dessous ne laisse aucune observation utilisable (relecture de Codex du 24/10)
       const windowChange = await this.#follow(false);
-      return this.#capture(mode, backend, { capture, windowChange, zoom });
+      if (zoom && zoomWindow != null && this.#window?.id !== zoomWindow) {
+        throw new Error(`region comes from an image of window ${zoomWindow}, but the observed window is now ${this.#window?.id}: look again without region`);
+      }
+      return this.#capture(mode, backend, { capture, windowChange, zoom, marks });
     });
   }
   #take(observation) {
@@ -1492,5 +1511,6 @@ export function summarize(observation) {
     windowStack: observation.windowStack, windowChange: observation.windowChange, popups: observation.popups,
     tree: observation.accessibility?.tree ?? '', focused: observation.accessibility?.focused_element ?? '',
     screenshots: observation.screenshots, images: observation.images, imageFallback: observation.imageFallback,
+    ...(observation.marksNote ? { marksNote: observation.marksNote } : {}),
   };
 }

package/lib/tree.mjs

@@ -20,6 +20,31 @@ const stateFlags = new Set(['disabled', 'selected', 'unselected', 'selectable',
   'settable', 'string', 'float', 'integer', 'boolean']);
 export const digest = value => createHash('sha256').update(value).digest('hex');
 
+// Numéros [N] sur l'image (shared/images.md point 5) : éléments natifs avec un rectangle, hors racine et hors éléments
+// dits hors écran. auto : contrôles sans nom (espaces compris) qu'on peut viser ; all : tous. scope : HWND porteur le plus
+// proche (l'élément ou un ancêtre), que le helper compare à la fenêtre capturée (menu surgissant d'une autre fenêtre :
+// jamais d'étiquette fantôme). Index EXACT de la ligne de l'arbre, jamais renuméroté.
+const MARKS_LIMIT = 400;
+const MARKABLE = new Set(['button', 'splitbutton', 'edit', 'checkbox', 'radio', 'combobox', 'listitem', 'menuitem', 'tab',
+  'treeitem', 'link', 'slider', 'spinner', 'dataitem', 'headeritem', 'image']);
+export function markCandidates(elements = [], marks = 'auto') {
+  const byIndex = new Map(elements.map(e => [e.index, e]));
+  const out = [];
+  let capped = 0; // éléments admissibles au-delà du plafond : dits dans le rapport (all n'est alors pas exhaustif)
+  for (const e of elements) {
+    const b = e.bounds;
+    if (!Array.isArray(b) || b.length !== 4 || !b.every(Number.isFinite) || !(b[2] > 0) || !(b[3] > 0)) continue;
+    if (!e.ancestors?.length || e.flags?.includes('offscreen')) continue;
+    if (marks === 'auto' && (String(e.name ?? '').trim() || !(MARKABLE.has(e.role) || e.invoke === true))) continue;
+    const chain = [e, ...[...e.ancestors].reverse().map(i => byIndex.get(i))];
+    const scope = chain.find(x => x?.nativeWindow)?.nativeWindow ?? 0;
+    if (out.length >= MARKS_LIMIT) { capped++; continue; }
+    out.push({ index: e.index, bounds: b.map(Math.round), scope });
+  }
+  if (capped) Object.defineProperty(out, 'capped', { value: capped }); // hors JSON : seul le compte part au rapport
+  return out;
+}
+
 export function parseTree(tree = '') {
   const elements = [], stack = [];
   for (const line of tree.split(/\r?\n/)) {

package/native/bin/NexusCodex.UIA.exe

@@ -1,4 +1,4 @@
 [diffend] Oversized file quarantined before diffing.
 name: package/native/bin/NexusCodex.UIA.exe
-size: 35239104 bytes
-sha256: 5173778c736d462f737f3fc1142d09d1b160ae7cb01976203da67ab46c21954f
+size: 35244346 bytes
+sha256: e348309a77c415cd4e4102bb6f20045b413589c79a48d815560ba99d67273a68

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "nexus-windows-codex",
-  "version": "3.1.0",
+  "version": "3.1.1",
   "description": "NexusWindows for Codex: Windows computer use in the background for OpenAI Codex (MCP server and Codex skill).",
   "type": "module",
   "license": "MIT",