@e2e-dev/decision 0.2.0-nightly-20261009094406 → 0.2.0

package/dist/elements.js

@@ -1,3 +1,4 @@
+import { fnv1a } from './hash.js';
 /** TypeSafe's per-question choice limit; the AI SDK itself has none. */
 const MAX_CHOICES = 255;
 const tappable = new Set([
@@ -13,11 +14,40 @@ const tappable = new Set([
 const typable = new Set(['textbox', 'searchbox', 'spinbutton', 'combobox']);
 const secretTypable = new Set(['textbox', 'searchbox', 'combobox']);
 const checkable = new Set(['checkbox', 'radio', 'switch']);
+/** Live regions whose text reports what the app just did; a log can run long, so each line and their count are bounded. */
+const liveRegions = new Set(['status', 'alert', 'log']);
+const MAX_STATUSES = 20;
+const MAX_STATUS_LENGTH = 300;
+/** Roles that can receive a dropped node. */
+const droppable = new Set(['region', 'list', 'listitem', 'group', 'cell', 'gridcell', 'row', 'article', 'section', 'tabpanel']);
+/**
+ * Roles of nodes that are no control yet take a pointer: a card that opens
+ * a menu on hover, a file row that takes a right-click, a paragraph to
+ * scroll to, a column to drop on. The empty role is a plain text node.
+ */
+const passiveRoles = new Set([
+    '',
+    'listitem',
+    'article',
+    'region',
+    'group',
+    'cell',
+    'gridcell',
+    'row',
+    'img',
+    'figure',
+    'heading',
+    'paragraph',
+    'list',
+    'section',
+    'tabpanel',
+]);
 /** Builds the element table and bound targets from the newest observation. */
 export function actionSpace(ctx, observation, canType) {
     const verbs = ctx.target.verbs;
     const rows = [];
     const pageText = [];
+    const statuses = [];
     /** Walks the tree, collecting interactive rows and page text. */
     const visit = (node, underNativeSelect) => {
         if (node.states?.disabled !== true && node.states?.hidden !== true) {
@@ -27,9 +57,11 @@ export function actionSpace(ctx, observation, canType) {
             const nativeSelect = role === 'combobox' && (node.children ?? []).some((child) => child.role === 'option');
             const childSelect = underNativeSelect || nativeSelect;
             const checkedRadio = role === 'radio' && node.states?.checked === true;
-            if (verbs.has('tap') && tappable.has(role) && !checkable.has(role) && !(role === 'option' && childSelect)) {
+            const fileInput = node.attributes?.['type'] === 'file';
+            const inViewport = intersects(node, observation.viewport);
+            const pressable = tappable.has(role) && !checkable.has(role) && !(role === 'option' && childSelect) && !fileInput;
+            if (verbs.has('tap') && pressable)
                 operations.push('tap');
-            }
             if (typable.has(role) && !nativeSelect) {
                 if (!password && verbs.has('type') && canType)
                     operations.push('type');
@@ -44,34 +76,78 @@ export function actionSpace(ctx, observation, canType) {
             // `check` toggles, so a checked radio would be unchecked: offer radios only when unchecked.
             if (verbs.has('check') && checkable.has(role) && !checkedRadio)
                 operations.push('check');
+            // Paths come from the text model, as typed values do.
+            if (verbs.has('upload') && fileInput && canType)
+                operations.push('upload');
+            const interactive = operations.length > 0;
+            if (pressable) {
+                if (verbs.has('doubleTap'))
+                    operations.push('double_tap');
+                if (verbs.has('longPress'))
+                    operations.push('long_press');
+            }
+            // Pointer and scroll operations reach any labeled node, not only
+            // controls: a card menu that opens on hover, a file row that takes a
+            // right-click, a footnote to bring into view.
+            const pointable = (interactive || (passiveRoles.has(role) && !childSelect)) && nodeLabel(node) !== '' && node !== observation.tree;
+            if (pointable) {
+                if (verbs.has('hover'))
+                    operations.push('hover');
+                if (verbs.has('secondaryTap'))
+                    operations.push('secondary_tap');
+                if (verbs.has('drag'))
+                    operations.push('drag');
+                if (verbs.has('scrollTo') && node.rect !== undefined && !inViewport)
+                    operations.push('scroll_to');
+            }
             if (operations.length > 0) {
-                rows.push({ node, operations, inViewport: intersects(node, observation.viewport) });
+                rows.push({ node, operations, inViewport, passive: !interactive });
             }
-            else {
+            if (!interactive) {
                 const line = pageLine(node);
                 // A checked radio leaves the table, so its state rides on the page text.
                 if (line !== '')
                     pageText.push(checkedRadio ? `${line} (checked)` : line);
+                if (liveRegions.has(role) && statuses.length < MAX_STATUSES) {
+                    const text = (node.text ?? '').replace(/\s+/g, ' ').trim();
+                    if (text !== '')
+                        statuses.push(clip(`${(node.name ?? '').trim() || role}: ${text}`, MAX_STATUS_LENGTH));
+                }
             }
         }
         for (const child of node.children ?? [])
             visit(child, underNativeSelect || node.role === 'combobox');
     };
     visit(observation.tree, false);
-    // Viewport nodes first, then tree order; drop the rest past the cap so
-    // the model can scroll to them, and never block on a large screen.
+    // Controls before passive rows, viewport nodes first within each, then
+    // tree order; drop the rest past the cap so the model can scroll to them,
+    // and never block on a large screen. A control below the fold still
+    // outranks any amount of visible prose.
     const ordered = [...rows]
         .map((row, order) => ({ row, order }))
-        .toSorted((a, b) => Number(b.row.inViewport) - Number(a.row.inViewport) || a.order - b.order)
+        .toSorted((a, b) => Number(a.row.passive) - Number(b.row.passive) || Number(b.row.inViewport) - Number(a.row.inViewport) || a.order - b.order)
         .map(({ row }) => row);
     const omitted = Math.max(0, ordered.length - MAX_CHOICES);
     const kept = ordered.slice(0, MAX_CHOICES);
+    // Drop targets come from the kept rows: a drag with nowhere to drop is no operation.
+    const destinations = new Map();
+    if (verbs.has('drag')) {
+        for (const [position, row] of kept.entries()) {
+            const role = row.node.role ?? '';
+            const label = clip(nodeLabel(row.node), 120);
+            if (droppable.has(role) && label !== '')
+                destinations.set(String(position + 1), { id: row.node.id, label, role });
+        }
+    }
     const targets = new Map();
+    const byIndex = new Map();
     let selectCount = 0;
     const elements = kept.map((row, position) => {
         const index = String(position + 1);
         const label = clip(nodeLabel(row.node), 120);
-        for (const operation of row.operations) {
+        const role = row.node.role ?? '';
+        const operations = destinations.size === 0 ? row.operations.filter((operation) => operation !== 'drag') : row.operations;
+        for (const operation of operations) {
             if (operation === 'select') {
                 const options = new Map();
                 for (const [optionIndex, child] of (row.node.children ?? []).entries()) {
@@ -88,11 +164,7 @@ export function actionSpace(ctx, observation, canType) {
                         continue;
                     const key = `${index}:${optionIndex}`;
                     const optionLabel = child.name ?? child.text ?? '';
-                    options.set(key, {
-                        description: `select option ${JSON.stringify(optionLabel)} in ${label} [${row.node.id}]`,
-                        optionLabel,
-                        run: () => ctx.actions.select({ id: row.node.id }, optionLabel),
-                    });
+                    options.set(key, { id: row.node.id, description: `select option ${JSON.stringify(optionLabel)} in ${named(role, label)}`, optionLabel });
                     selectCount += 1;
                 }
                 if (options.size > 0) {
@@ -103,57 +175,81 @@ export function actionSpace(ctx, observation, canType) {
                 }
             }
             else {
-                const key = index;
-                const target = bind(row.node, operation, label, ctx);
                 const group = targets.get(operation) ?? new Map();
-                group.set(key, target);
+                group.set(index, {
+                    id: row.node.id,
+                    description: `${VERBS[operation]} ${named(role, label)}`,
+                    ...(row.node.states?.checked === undefined ? {} : { checked: row.node.states.checked }),
+                });
                 targets.set(operation, group);
             }
         }
-        return {
+        const element = {
             index,
-            role: row.node.role ?? '',
+            role,
             label,
             ...(row.node.value === undefined ? {} : { value: row.node.value }),
             ...(row.node.states?.checked === undefined ? {} : { checked: row.node.states.checked }),
             ...(row.node.states?.expanded === undefined ? {} : { expanded: row.node.states.expanded }),
-            operations: row.operations,
+            operations,
         };
+        byIndex.set(index, element);
+        return element;
     });
-    const controls = new Map();
+    const tapAt = observation.pixels !== undefined && observation.locates === true && verbs.has('tapAt');
+    const controls = new Set();
     if (verbs.has('scroll')) {
-        controls.set('scroll_up', { description: 'scroll viewport up', run: () => ctx.actions.scroll('up') });
-        controls.set('scroll_down', { description: 'scroll viewport down', run: () => ctx.actions.scroll('down') });
-    }
-    if (verbs.has('back')) {
-        controls.set('back', { description: 'back one step in history', run: () => ctx.actions.back() });
+        controls.add('scroll_up');
+        controls.add('scroll_down');
     }
+    if (verbs.has('back'))
+        controls.add('back');
     return {
         elements,
+        element: (key) => byIndex.get(targetKeyIndex(key)),
         targets,
         controls,
+        operation: (choice) => (targets.has(choice) || (tapAt && choice === 'tap_at') ? choice : undefined),
+        control: (choice) => (controls.has(choice) ? choice : undefined),
+        destinations,
+        tapAt,
         omitted,
         pageText: clip(pageText.join('\n'), 6000),
-        fingerprint: fingerprint(observation.path ?? '', observation.tree, observation.viewport),
+        statuses,
+        fingerprint: fingerprint(observation.path ?? '', observation.tree, observation.viewport, observation.pixels),
     };
 }
-/** Binds one element operation to the runner's actions. */
-function bind(node, operation, label, ctx) {
-    const target = { id: node.id };
-    const where = `${label} [${node.id}]`;
-    switch (operation) {
-        case 'tap':
-            return { description: `tap ${where}`, run: () => ctx.actions.tap(target) };
-        case 'type':
-            return { description: `type into ${where}`, run: (argument = '') => ctx.actions.type(target, argument) };
-        case 'typeSecret':
-            return { description: `typeSecret into ${where}`, run: (argument = '') => ctx.actions.typeSecret(target, argument) };
-        case 'submit':
-            return { description: `submit ${where}`, run: () => ctx.actions.press(target, 'Enter') };
-        case 'check':
-            return { description: `check ${where}`, run: () => ctx.actions.check(target, !(node.states?.checked ?? false)) };
-        case 'select': throw new Error('select binds per option, not per element');
-    }
+/**
+ * A node as an action line names it: `checkbox "Agree to terms"`. Measured
+ * on gpt-6-luna over captured requests, the role and the quoted label lift
+ * a completion verdict the bare label with a node id left inconclusive
+ * (holds 0.17 to 0.83 on a checked checkbox), with no loss on the
+ * operation questions.
+ */
+function named(role, label) {
+    return role === '' ? JSON.stringify(label) : `${role} ${JSON.stringify(label)}`;
+}
+/** How each operation reads in a target description and in history. */
+const VERBS = {
+    select: 'select',
+    tap_at: 'tap at',
+    tap: 'tap',
+    type: 'type into',
+    typeSecret: 'typeSecret into',
+    submit: 'submit',
+    check: 'check',
+    hover: 'hover',
+    secondary_tap: 'right-click',
+    double_tap: 'double-tap',
+    long_press: 'long-press',
+    scroll_to: 'scroll to',
+    upload: 'upload to',
+    drag: 'drag',
+};
+/** Element index behind a target key (`7` for both `7` and `7:2`). */
+export function targetKeyIndex(key) {
+    const at = key.indexOf(':');
+    return at === -1 ? key : key.slice(0, at);
 }
 /** Label the model reads: name, placeholder, or text. */
 function nodeLabel(node) {
@@ -183,41 +279,24 @@ function clip(text, limit) {
     return text.length > limit ? text.slice(0, limit) : text;
 }
 /**
- * Stable hash of the path, the tree content with node ids removed, and
- * whether each node meets the viewport. Ids change on every capture, so they
- * stay out. In-view membership lets a scroll that brings other nodes into
- * view count as progress, while one that moves nothing (the page bottom)
- * still reads as unchanged; raw coordinates stay out so small layout shifts
- * do not count.
+ * Stable hash of the path, the tree content with node ids removed, whether
+ * each node meets the viewport, and the pixels when the step sees them. Ids
+ * change on every capture, so they stay out. In-view membership lets a
+ * scroll that brings other nodes into view count as progress, while one
+ * that moves nothing (the page bottom) still reads as unchanged; raw
+ * coordinates stay out so small layout shifts do not count. Pixels are in
+ * because a drawn control changes nothing in the tree: a keypad digit
+ * entered on a canvas is progress only the screenshot shows.
  */
-function fingerprint(path, tree, viewport) {
-    let hash = 2166136261;
-    const feed = (text) => {
-        for (let index = 0; index < text.length; index += 1) {
-            hash ^= text.charCodeAt(index);
-            hash = Math.imul(hash, 16777619);
-        }
-    };
-    feed(path);
-    feed('\0');
+function fingerprint(path, tree, viewport, pixels) {
+    const parts = [path];
+    if (pixels !== undefined)
+        parts.push(pixels.data);
     const visit = (node) => {
-        feed(node.role ?? '');
-        feed('\0');
-        feed(node.name ?? '');
-        feed('\0');
-        feed(node.text ?? '');
-        feed('\0');
-        feed(node.value ?? '');
-        feed('\0');
-        feed(JSON.stringify(node.states ?? null));
-        feed('\0');
-        feed(JSON.stringify(node.attributes ?? null));
-        feed('\0');
-        feed(intersects(node, viewport) ? 'v' : '-');
-        feed('\0');
+        parts.push(node.role ?? '', node.name ?? '', node.text ?? '', node.value ?? '', JSON.stringify(node.states ?? null), JSON.stringify(node.attributes ?? null), intersects(node, viewport) ? 'v' : '-');
         for (const child of node.children ?? [])
             visit(child);
     };
     visit(tree);
-    return (hash >>> 0).toString(36);
+    return fnv1a(parts);
 }