jevway 0.3.0 → 0.4.0

package/dist/browser/code-boxes.js

@@ -0,0 +1,121 @@
+const MIN_BOXES = 4;
+const MAX_BOXES = 8;
+// One character fits in a box this wide at any common font size; a field for a word is wider.
+const ONE_CHARACTER_PX = 64;
+const CODE_MODES = ['numeric', 'decimal', 'text', 'tel'];
+const CODE_TYPES = ['tel', 'number'];
+// Words a box's own label shares with its neighbours that name no field ("Digit 1 of 6").
+const GENERIC_WORDS = /^(?:digit|character|char|box|number|input|field|code|otp|pin|textbox|spinbutton)s?$/i;
+const FALLBACK_NAME = 'Verification code';
+const holdsOneCharacter = (action) => {
+    const { box, rect } = action;
+    if (action.kind !== 'fill' || !box || !rect || action.hover)
+        return false;
+    if (box.max === 1)
+        return true;
+    return box.max < 0 && rect.w <= ONE_CHARACTER_PX && (CODE_MODES.includes(box.mode) || CODE_TYPES.includes(box.type) || box.otp);
+};
+const centreY = (box) => box.rect.y + box.rect.h / 2;
+// One form, one container (every box in the same parent, or each wrapped once in the same grandparent), one row, read
+// left to right, all about the same width.
+const joins = (run, next) => {
+    const [first] = run;
+    const last = run.at(-1);
+    const widths = [...run, next].map((box) => box.rect.w);
+    return (next.box.form === first.box.form &&
+        [0, 1].some((level) => run.every((box) => box.box.within[level] === next.box.within[level])) &&
+        Math.abs(centreY(next) - centreY(first)) <= first.rect.h / 2 &&
+        next.rect.x > last.rect.x &&
+        Math.max(...widths) <= Math.min(...widths) * 1.5);
+};
+// Runs of one-character boxes that follow one another among the listed fields; another field between them ends a run.
+const runsOf = (actions) => {
+    const runs = [];
+    let run = [];
+    for (const action of actions) {
+        if (action.kind !== 'fill')
+            continue;
+        if (holdsOneCharacter(action) && run.length && joins(run, action)) {
+            run.push(action);
+            continue;
+        }
+        if (run.length)
+            runs.push(run);
+        run = holdsOneCharacter(action) ? [action] : [];
+    }
+    if (run.length)
+        runs.push(run);
+    return runs.filter((boxes) => boxes.length >= MIN_BOXES && boxes.length <= MAX_BOXES);
+};
+// "Security code character 3" → "Security code character": a box's label without its place in the group.
+const stemOf = (label) => label
+    .replace(/\d+/g, ' ')
+    .replace(/\bof\b/gi, ' ')
+    .replace(/^[^\p{L}]+|[^\p{L}]+$/gu, '')
+    .replace(/\s+/g, ' ');
+const nameOf = (run) => {
+    if (run[0].box.name)
+        return run[0].box.name;
+    const stems = new Set(run.map((box) => stemOf(box.label)));
+    const [stem] = stems;
+    return stems.size === 1 && stem && !GENERIC_WORDS.test(stem) ? stem : FALLBACK_NAME;
+};
+const unionOf = (run) => {
+    const left = Math.min(...run.map((box) => box.rect.x));
+    const top = Math.min(...run.map((box) => box.rect.y));
+    const right = Math.max(...run.map((box) => box.rect.x + box.rect.w));
+    const bottom = Math.max(...run.map((box) => box.rect.y + box.rect.h));
+    return { x: left, y: top, w: right - left, h: bottom - top };
+};
+const withoutFacts = ({ box: _box, ...action }) => action;
+/**
+ * A code split into one-character boxes (4–8 `maxlength=1` boxes, or narrow numeric/one-time-code ones with no
+ * `maxlength`, following one another in one form and container, left to right on one row) becomes one `fill` target on
+ * its first box: `boxes` lists every box, `value` joins their values, `rect` covers them all, and the label is the
+ * group's name with the count. The other boxes and their `Open …` clicks are not listed. The reader's facts are removed
+ * from every action, and element ids are renumbered when anything was grouped.
+ *
+ * @example
+ * groupCodeBoxes(snapshot.actions) // [{ kind: 'fill', node: 11, label: 'Verification code (6 boxes)', boxes: [11, …, 16], … }, …]
+ */
+export const groupCodeBoxes = (actions) => {
+    const runs = runsOf(actions);
+    if (!runs.length)
+        return actions.some((action) => action.box) ? actions.map(withoutFacts) : actions;
+    const groupOf = new Map();
+    for (const run of runs) {
+        const group = {
+            first: run[0].node,
+            boxes: run.map((box) => box.node),
+            label: `${nameOf(run)} (${run.length} boxes)`,
+            value: run.map((box) => box.value ?? '').join(''),
+            rect: unionOf(run),
+        };
+        for (const node of group.boxes)
+            groupOf.set(node, group);
+    }
+    let next = 0;
+    return actions.flatMap((action) => {
+        const group = action.node === undefined ? undefined : groupOf.get(action.node);
+        if (group && action.node !== group.first)
+            return [];
+        const { label, value, boxes, rect } = group ?? {};
+        const listed = group ? { ...withoutFacts(action), label: action.kind === 'click' ? `Open ${label}` : label, value, boxes, rect } : withoutFacts(action);
+        return [action.node === undefined ? listed : { ...listed, id: `e${++next}` }];
+    });
+};
+/**
+ * The characters a code puts into `count` boxes, one each: the text as given, or without its spaces and hyphens when
+ * only that fits ("739 214" into 6 boxes). Undefined when neither fits.
+ *
+ * @example
+ * codeCharacters('739-214', 6) // ['7', '3', '9', '2', '1', '4']
+ */
+export const codeCharacters = (text, count) => {
+    for (const form of [text, text.replace(/[\s-]/g, '')]) {
+        const characters = [...form];
+        if (characters.length === count)
+            return characters;
+    }
+    return undefined;
+};

package/dist/browser/frames.js

@@ -1,5 +1,6 @@
 import { centerOf, contains } from '../jev/box.js';
 import { CdpError } from './cdp.js';
+import { groupCodeBoxes } from './code-boxes.js';
 // jevway: every frame is read with the same reader (snapshot.js), each in an isolated world of its own, so ids, guards and
 // the input counter live where the frame's elements live. The main frame keeps the main world, exactly as before frames.
 /** Each frame owns a range of element ids this wide; the main frame is range 0, so its ids are what they always were. */
@@ -247,7 +248,7 @@ export class Frames {
         const state = response.exceptionDetails ? null : response.result?.value;
         if (!state)
             return null;
-        const placed = placeFrameActions(state.actions, { w: state.w, h: state.h }, placement, frame.id);
+        const placed = placeFrameActions(groupCodeBoxes(state.actions), { w: state.w, h: state.h }, placement, frame.id);
         // On-screen controls must be on top in every document around the frame: a banner or sheet over the frame hides them.
         const shown = placed.filter((action) => contains(placement.region, centerOf(action.rect)));
         const onTop = await this.#hits(levels, shown.map((action) => centerOf(action.rect)));

package/dist/browser/snapshot.js

@@ -332,6 +332,34 @@
       if (['auto', 'scroll'].includes(getComputedStyle(p).overflowX) && p.scrollWidth > p.clientWidth) return row;
     return null;
   };
+  // jevway: a short text input may be one box of a code split into one-character boxes. The tab groups them from these
+  // facts (code-boxes.ts); only the group's name is read here, where the page is: an aria name or legend around the
+  // boxes, else a short line right before them. Container and form keys are unique within this read.
+  const containerKeys = new Map();
+  const keyOf = (n) => (n ? containerKeys.get(n) ?? containerKeys.set(n, containerKeys.size + 1).get(n) : 0);
+  const codeGroupName = (e) => {
+    for (let p = e.parentElement, up = 0; p && up < 3 && p !== document.body && p.tagName !== 'FORM'; p = p.parentElement, up++) {
+      const legend = p.tagName === 'FIELDSET' ? p.querySelector('legend') : null;
+      const own = p.getAttribute('aria-label') || (p.hasAttribute('aria-labelledby') ? name(p) : '') || legend?.innerText || '';
+      if (own.trim()) return cap(own.replace(/\s+/g, ' ').trim(), 80);
+    }
+    for (let p = e, up = 0; p && up < 3 && p !== document.body; p = p.parentElement, up++) {
+      const before = p.previousElementSibling;
+      if (!before || before.matches(interactive) || before.querySelector(interactive) || !visible(before)) continue;
+      const line = before.innerText.replace(/\s+/g, ' ').trim();
+      if (line) return line.length <= 80 ? line : '';
+    }
+    return '';
+  };
+  const codeBoxFacts = (e) => {
+    if (e.tagName !== 'INPUT') return null;
+    const max = e.maxLength;
+    if (!(max >= 1 && max <= 4) && !(max < 0 && e.getBoundingClientRect().width <= 64)) return null;
+    const parent = e.parentElement;
+    const within = [keyOf(parent), keyOf(parent?.parentElement)];
+    const facts = { max, mode: e.inputMode || '', otp: e.autocomplete === 'one-time-code', type: e.type, form: keyOf(e.form), within };
+    return { box: { ...facts, name: codeGroupName(e) } };
+  };
   for (const e of candidates) {
     const shown = eligible(e);
     const hover = !shown && revealedByHover(e);
@@ -402,7 +430,7 @@
           (rname === 'combobox' && (['INPUT', 'TEXTAREA'].includes(e.tagName) || e.isContentEditable)));
       const value =
         'value' in e ? String(e.value) : e.isContentEditable || rname === 'combobox' ? e.innerText.trim() : '';
-      actions.push({ ...base, kind: editable ? 'fill' : 'click', value });
+      actions.push({ ...base, kind: editable ? 'fill' : 'click', value, ...(editable && !hover && codeBoxFacts(e)) });
       if (editable) actions.push({ ...base, kind: 'click', value, label: 'Open ' + base.label });
     }
   }

package/dist/browser/tab.js

@@ -3,6 +3,7 @@ import { createHash } from 'node:crypto';
 import { readFileSync } from 'node:fs';
 import { isDeepStrictEqual } from 'node:util';
 import { CdpError } from './cdp.js';
+import { codeCharacters, groupCodeBoxes } from './code-boxes.js';
 import { DEVICES } from './devices.js';
 import { TabDownloads } from './downloads.js';
 import { Frames, idBaseOf, mergeFrames, whenAborted } from './frames.js';
@@ -453,7 +454,7 @@ export class Tab {
             try {
                 const main = await this.#evaluate(`window.__jevWanted=${JSON.stringify(this.#wanted)};window.__jevFiles=${this.#files};${READ_STATE}`);
                 if (main) {
-                    const state = await this.#withFrames(main);
+                    const state = await this.#withFrames({ ...main, actions: groupCodeBoxes(main.actions) });
                     const named = placeholders ? { ...state, text: `Loading: ${placeholders} placeholder blocks on screen\n${state.text}` } : state;
                     return { ...named, fingerprint: fingerprint(named) };
                 }
@@ -522,8 +523,13 @@ export class Tab {
         ]);
         return isDeepStrictEqual([main, current], [page.page_key, [observed.page_key, page.guards[String(action.node)]]]);
     }
-    /** Runs one observed action. A scroll resolves to how far it moved the page (see `#scroll`). */
+    /**
+     * Runs one observed action. A scroll resolves to how far it moved the page (see `#scroll`); typing into a split code's
+     * boxes, to whether they ended up holding the code (`kept`, absent when they can't be read).
+     */
     async act(action, page, text, { evenIfChanged = false } = {}) {
+        if (action.kind === 'fill' && action.boxes)
+            return this.#typeBoxes(action, page, text ?? '');
         // Only a page-level action may skip the check, and only when the loop asks: Enter still needs its field focused (#pressEnter).
         const skipCheck = evenIfChanged && PAGE_LEVEL_KINDS.includes(action.kind);
         if (!skipCheck && !(await this.fresh(page, action)))
@@ -542,6 +548,39 @@ export class Tab {
         this.#afterInput = action;
         return outcome;
     }
+    // jevway: a code split into one-character boxes is typed box by box through `act`, so every box gets the freshness
+    // check, the hit test, the delivered-press check and real key events. Typing changes the page, so each box after the
+    // first is checked against a fresh read, which also lets the page's own move to the next box settle. A box that
+    // already holds its character (the page moved or spread it) is left alone, so nothing is typed twice.
+    async #typeBoxes(group, page, text) {
+        const boxes = group.boxes;
+        const characters = codeCharacters(text, boxes.length);
+        if (!characters)
+            throw new Error(`The code has ${[...text].length} characters but "${group.label}" has ${boxes.length} boxes`);
+        if (!(await this.fresh(page, group)))
+            throw new StalePage('Page changed since this decision');
+        let current = page;
+        for (const [i, node] of boxes.entries()) {
+            if (i)
+                current = await this.observe();
+            const listed = current.actions.find((action) => action.kind === 'fill' && action.node === group.node && isDeepStrictEqual(action.boxes, boxes));
+            if (!listed)
+                throw new StalePage('The code boxes changed while typing');
+            const [held] = await this.#boxValues(group, [node]);
+            if (held === characters[i])
+                continue;
+            await this.act({ ...listed, node, boxes: undefined, label: `${listed.label}, box ${i + 1}` }, current, characters[i]);
+        }
+        const values = await this.#boxValues(group, boxes).catch(() => undefined);
+        return values ? { kept: values.join('') === characters.join('') } : {};
+    }
+    // What each box holds now, read in the group's frame; null for a box that is gone.
+    async #boxValues(group, nodes) {
+        const read = `(function jevBoxValues(ids) {
+      return ids.map((id) => { const e = window.__jevFast?.nodes.get(id); return e?.isConnected ? String(e.value ?? '') : null; });
+    })(${JSON.stringify(nodes)})`;
+        return (await this.#evaluateFor(this.#frameOf(group.node), read)) ?? [];
+    }
     // jevway: the wheel goes where the reader found the pane that scrolls (or the fixed point on a page whose document
     // scrolls), and the offsets before and after say whether it moved anything. The wheel lands asynchronously, so the
     // offsets are polled until they stop moving: 500 ms with no movement means nothing moved, 1.5 s at most.

package/dist/task/run-task.js

@@ -1,6 +1,7 @@
 import { tmpdir } from 'node:os';
 import { join } from 'node:path';
 import { findBotWall } from '../browser/bot-wall.js';
+import { codeCharacters } from '../browser/code-boxes.js';
 import { PAGE_LEVEL_KINDS, StalePage, TargetUnreachable } from '../browser/tab.js';
 import { showsTarget } from '../browser/words.js';
 import { buildActionSpace } from '../jev/action-space.js';
@@ -8,7 +9,7 @@ import { centerOf, contains } from '../jev/box.js';
 import { decide, evidenceRequest, foldedFavourite, overflowFolds, readEvidence } from '../jev/decide.js';
 import { estimateCostUsd } from '../jev/model.js';
 import { ANY_HOST, narrowHosts, outsideHosts } from './hosts.js';
-import { scrubberFor } from '../jev/secrets.js';
+import { scrubberFor, secretMarker } from '../jev/secrets.js';
 import { describeFailure, runChecks } from './checks.js';
 import { deepLink, quoteSpans } from './evidence.js';
 import { writeCall } from '../output/run-folder.js';
@@ -199,6 +200,13 @@ export class BrowserTask {
     #toggled = new Map();
     /** Nodes of fields the model chose with no supplied value to type: each is refused once before the caller is asked. */
     #refusedForValue = new Set();
+    /** First-box nodes of split-code groups a value of the wrong length was chosen for: each is fed back once before the caller is asked. */
+    #codeLengthNoted = new Set();
+    /**
+     * Boxes of a split code a secret was typed into, with the secret's key and their document (ids restart in a new one):
+     * their values read as its marker from then on.
+     */
+    #secretBoxes = new Map();
     /** The caller's authenticator secrets by key: only `decide` (as a marker) and the typing see them. */
     #totp = new Map();
     /** Secrets that are not supplied values, by key: each authenticator secret as given and as base32, every code typed. Kept for the session. */
@@ -274,12 +282,20 @@ export class BrowserTask {
             title: scrub(page.title),
             text: scrub(page.text),
             ...(page.passwordFields && { passwordFields: page.passwordFields.map((field) => ({ ...field, name: scrub(field.name) })) }),
-            actions: page.actions.map((action) => ({
-                ...action,
-                label: scrub(action.label),
-                ...(action.value !== undefined && action.kind !== 'select' && { value: scrub(action.value) }),
-                ...(action.current_value !== undefined && { current_value: scrub(action.current_value) }),
-            })),
+            actions: page.actions.map((action) => {
+                // A box of a split secret code may hold part of it, which no value match finds: it shows the key's marker.
+                const document = documentOf(page, action);
+                const boxKey = [action.node, ...(action.boxes ?? [])]
+                    .map((node) => this.#secretBoxes.get(node))
+                    .find((noted) => noted?.document === document)?.key;
+                return {
+                    ...action,
+                    label: scrub(action.label),
+                    ...(action.value !== undefined && action.kind !== 'select' && { value: scrub(action.value) }),
+                    ...(action.current_value !== undefined && { current_value: scrub(action.current_value) }),
+                    ...(boxKey && action.value && { value: secretMarker(boxKey) }),
+                };
+            }),
         };
     }
     /** Reads the session's tab with no model call and no input, its secret values already masked: what `jevway_look` shows. With `pdf`, the page as a PDF too. */
@@ -913,7 +929,41 @@ export class BrowserTask {
                 }
                 const action = decision.action;
                 const text = decision.valueKey ? await this.#textFor(decision.valueKey, values) : undefined;
+                // A CLICK_AT carries no action.
+                if (action?.boxes && decision.valueKey && text !== undefined) {
+                    const characters = codeCharacters(text, action.boxes.length);
+                    if (!characters) {
+                        // A value that doesn't fit the boxes is a wrong turn first, and only the second time a question for the caller.
+                        record(false);
+                        const [count, boxes] = [[...text].length, action.boxes.length];
+                        if (this.#codeLengthNoted.has(action.node)) {
+                            // A hand-back decided on a page that has since changed is stale: read it again instead.
+                            if (!(await this.#tab.fresh(page))) {
+                                page = await this.#observe();
+                                continue;
+                            }
+                            return stop('needs_input', `"${action.label}" has ${boxes} boxes but the "${decision.valueKey}" value has ${count} characters; supply it as ${boxes} characters`, decision);
+                        }
+                        this.#codeLengthNoted.add(action.node);
+                        this.#history.push({
+                            operation: decision.operation,
+                            element: action.label,
+                            not_executed: `the "${decision.valueKey}" value has ${count} characters but this field has ${boxes} boxes; type nothing here — choose another value or step, or BLOCKED if the goal needs a value you were not given`,
+                            confidence: round(decision.confidence),
+                        });
+                        page = await this.#observe();
+                        continue;
+                    }
+                    // Masked before it is typed, so the page read right after typing already shows the marker.
+                    if (this.#options.secretKeys.includes(decision.valueKey)) {
+                        for (const node of action.boxes)
+                            this.#secretBoxes.set(node, { key: decision.valueKey, document: documentOf(page, action) });
+                        if (characters.join('') !== text)
+                            this.#moreSecrets.push([decision.valueKey, characters.join('')]);
+                    }
+                }
                 let element;
+                let kept;
                 // jevway: the dialog a confirmed click opens is the same action asking again; any other dialog still gets the answer that changes nothing.
                 if (isRiskyClick(decision) && this.#confirmedCommit)
                     this.#tab.armConfirm([this.#confirmTarget ?? '', bareLabel(decision.action?.label ?? '')]);
@@ -945,6 +995,7 @@ export class BrowserTask {
                         const outcome = action.kind === 'upload'
                             ? await this.#tab.upload(action, page, this.#options.files[decision.fileKey])
                             : await this.#tab.act(action, page, text, forced ? { evenIfChanged: true } : undefined);
+                        kept = outcome?.kept;
                         // A scroll that moved nothing says so, or Jev keeps asking for it.
                         element = outcome?.scrolled === 0 ? `${action.label} (nothing moved: there is no more to scroll this way)` : action.label;
                     }
@@ -1069,7 +1120,8 @@ export class BrowserTask {
                 this.#history.push({
                     ...ran,
                     // A field that is gone (navigated, submitted) can't say; one still on the page must show what was typed.
-                    ...(field && { value_kept: field.value === this.#scrub(text) }),
+                    // Split code boxes report what they hold right after typing; any other field, on the page read after it.
+                    ...(kept !== undefined ? { value_kept: kept } : field && { value_kept: field.value === this.#scrub(text) }),
                     page_changed: page.fingerprint !== before,
                     confidence,
                 });
@@ -1775,6 +1827,8 @@ const decoded = (url) => {
         return url;
     }
 };
+// The document an action's node belongs to (its frame's, or the page's): its start time, as the reader's keys hold it.
+const documentOf = (page, action) => JSON.stringify(action.frame ? page.frames?.find((frame) => frame.id === action.frame)?.identity : page.page_key?.[0]);
 const bareLabel = (label) => label.replace(/ \(appears on hover\)$/, '');
 /** Up to 3 labels, in list order and cut at 40 characters, of the listed controls inside the layer covering a target; none when nothing usable names it. */
 const coveringLabels = (coverNodes, page) => page.actions

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "jevway",
-  "version": "0.3.0",
+  "version": "0.4.0",
   "description": "Hand a browser goal to TypeSafe Jev; get a short verified summary back. MCP server + CLI.",
   "keywords": [
     "mcp",