@ddwang/magnitude-core 0.3.1-ddwang.13 → 0.3.1-ddwang.15

package/dist/web/baseSelect.d.ts

@@ -0,0 +1,3 @@
+import type { Frame } from 'playwright';
+/** Applies to one document; the transformer calls it for each frame load and navigation. */
+export declare function renderSelectPickersInFrame(frame: Frame): Promise<void>;

package/dist/web/baseSelect.js

@@ -0,0 +1,20 @@
+// Native select popups render outside the page, so screenshots never show their options.
+// Chrome 135+ draws a base-select picker in the page's top layer, where screenshots include it.
+// Multi-selects and list boxes already render in the page.
+// Dropdowns are single-choice selects without a list-box size. !important outranks author and inline
+// styles such as appearance: none, which would otherwise keep the native, unscreenshotted popup.
+const DROPDOWNS = ['select:not([multiple]):not([size])', 'select:not([multiple])[size="0"]', 'select:not([multiple])[size="1"]'];
+const CSS = `${DROPDOWNS.join(', ')}, ${DROPDOWNS.map(selector => `${selector}::picker(select)`).join(', ')} { appearance: base-select !important; }`;
+// A constructed stylesheet adds no DOM node, so the page's own markup is unchanged.
+function install(css) {
+    const sheets = document.adoptedStyleSheets;
+    if (sheets.some(sheet => sheet.cssRules[0]?.cssText.includes('base-select')))
+        return;
+    const sheet = new CSSStyleSheet();
+    sheet.replaceSync(css);
+    document.adoptedStyleSheets = [...sheets, sheet];
+}
+/** Applies to one document; the transformer calls it for each frame load and navigation. */
+export async function renderSelectPickersInFrame(frame) {
+    await frame.evaluate(install, CSS).catch(() => { }); // A detached or navigating frame has no document to style.
+}

package/dist/web/groundedControls.d.ts

@@ -5,10 +5,38 @@ export declare const GROUNDED_CLICK_REJECTED: Readonly<{
     reason: "target_unavailable";
     instruction: "No click was submitted. Replan from the new observation; do not reuse the rejected reference.";
 }>;
-/** One disposable snapshot, owned by the operation that observed it. No DOM attributes are injected. */
+export declare const GROUNDED_INPUT_REJECTED: Readonly<{
+    changed: false;
+    reason: "target_unavailable";
+    instruction: "No value was set. Replan from the new observation; do not reuse the rejected reference.";
+}>;
+/** Results that submitted nothing; the executor stops the remaining batch after any of them. */
+export declare const GROUNDED_REJECTIONS: ReadonlySet<unknown>;
+declare const REJECTIONS: {
+    target_unavailable: Readonly<{
+        changed: false;
+        reason: "target_unavailable";
+        instruction: "No value was set. Replan from the new observation; do not reuse the rejected reference.";
+    }>;
+    option_unavailable: Readonly<{
+        changed: false;
+        reason: "option_unavailable";
+        instruction: "No value was set. Choose exactly one enabled option label from the observation.";
+    }>;
+    invalid_value: Readonly<{
+        changed: false;
+        reason: "invalid_value";
+        instruction: "No value was set. Use the input's ISO format, such as yyyy-mm-dd for date, HH:MM for time, or yyyy-mm for month.";
+    }>;
+};
+/** One disposable snapshot per frame, owned by the operation that observed it. No DOM attributes are injected. */
 export declare class GroundedControls {
-    private snapshot?;
+    private snapshots;
+    private fieldChanged;
     clear(): Promise<void>;
+    /** Called before planning: the planner sees only the latest snapshot. */
+    keepLatest(): Promise<void>;
+    private release;
     observe(page: Page): Promise<{
         scope: string;
         truncated: boolean;
@@ -17,9 +45,18 @@ export declare class GroundedControls {
             label: string;
             context: string;
             enabled: boolean;
+            value?: string;
+            options?: string[];
             ambiguous: boolean;
             ref: string;
         }[];
     }>;
+    private entry;
     click(harness: WebHarness, ref: string): Promise<boolean>;
+    /** Sets a native select or date-like input through the DOM, where its picker can't be seen. */
+    setValue(harness: WebHarness, ref: string, value: string, kind: 'select' | 'fill'): Promise<{
+        changed: true;
+        value: string;
+    } | (typeof REJECTIONS)[keyof typeof REJECTIONS]>;
 }
+export {};

package/dist/web/groundedControls.js

@@ -2,8 +2,19 @@ import { randomUUID } from 'node:crypto';
 import { checkOperation, currentOperation } from '@/common/operation';
 export const GROUNDED_CLICK_REJECTED = Object.freeze({ clicked: false, reason: 'target_unavailable',
     instruction: 'No click was submitted. Replan from the new observation; do not reuse the rejected reference.' });
-/** A deliberately small, current-viewport subset; this is not an accessibility tree. */
-function captureControls() {
+export const GROUNDED_INPUT_REJECTED = Object.freeze({ changed: false, reason: 'target_unavailable',
+    instruction: 'No value was set. Replan from the new observation; do not reuse the rejected reference.' });
+const OPTION_UNAVAILABLE = Object.freeze({ changed: false, reason: 'option_unavailable',
+    instruction: 'No value was set. Choose exactly one enabled option label from the observation.' });
+const INVALID_VALUE = Object.freeze({ changed: false, reason: 'invalid_value',
+    instruction: 'No value was set. Use the input\'s ISO format, such as yyyy-mm-dd for date, HH:MM for time, or yyyy-mm for month.' });
+/** Results that submitted nothing; the executor stops the remaining batch after any of them. */
+export const GROUNDED_REJECTIONS = new Set([GROUNDED_CLICK_REJECTED, GROUNDED_INPUT_REJECTED, OPTION_UNAVAILABLE, INVALID_VALUE]);
+const REJECTIONS = { target_unavailable: GROUNDED_INPUT_REJECTED, option_unavailable: OPTION_UNAVAILABLE, invalid_value: INVALID_VALUE };
+// Single-choice native form controls whose pickers render outside the page screenshot.
+const FIELDS = 'select:not([multiple]),input[type="date"],input[type="time"],input[type="datetime-local"],input[type="month"],input[type="week"]';
+/** A deliberately small, current-viewport subset of one frame; this is not an accessibility tree. */
+function captureControls(fields) {
     const documentAtCapture = document;
     const url = location.href;
     const visibility = { checkOpacity: true, checkVisibilityCSS: true };
@@ -13,11 +24,27 @@ function captureControls() {
         + '[role~="menuitem"],[role~="menuitemcheckbox"],[role~="menuitemradio"],[role~="option"],'
         + '[role~="combobox"],[role~="listbox"],[role~="textbox"],[role~="searchbox"],'
         + '[role~="slider"],[role~="spinbutton"],[role~="scrollbar"],[role~="tab"],[role~="treeitem"]';
+    const text = (value) => (value ?? '').replace(/\s+/g, ' ').trim();
+    // Read form attributes through the prototype: a control named "action" would shadow form.action.
+    const formProperty = (form, key) => Object.getOwnPropertyDescriptor(HTMLFormElement.prototype, key).get.call(form);
+    // Options a person could choose: neither the option nor its group is disabled or hidden.
+    // Computed styles work while the picker is closed; layout-based visibility checks do not.
+    const concealed = (element) => {
+        const style = getComputedStyle(element);
+        return element.hidden || style.display === 'none' || style.visibility === 'hidden' || style.visibility === 'collapse';
+    };
+    const eligible = (select) => Array.from(select.options).filter(option => {
+        const group = option.parentElement instanceof HTMLOptGroupElement ? option.parentElement : null;
+        return !option.disabled && !concealed(option) && !(group && (group.disabled || concealed(group)));
+    });
     function describe(node) {
-        if (!(node instanceof HTMLAnchorElement || node instanceof HTMLButtonElement)
+        const field = node.matches(fields) ? node : null;
+        const submit = node instanceof HTMLInputElement && node.type === 'submit' ? node : null;
+        if (!(field || submit || node instanceof HTMLAnchorElement || node instanceof HTMLButtonElement)
             || !node.isConnected || !node.checkVisibility(visibility) || node.closest('[hidden],[inert],[aria-hidden="true"]'))
             return null;
-        if (node instanceof HTMLButtonElement && node.form && node.type !== 'button')
+        // Submit buttons are listed; reset buttons only discard input.
+        if (node instanceof HTMLButtonElement && node.type === 'reset')
             return null;
         if (node instanceof HTMLAnchorElement && !/^https?:$/.test(node.protocol))
             return null;
@@ -26,70 +53,82 @@ function captureControls() {
             || box.right > innerWidth || box.bottom > innerHeight)
             return null;
         const labelledBy = node.getAttribute('aria-labelledby');
-        const label = (labelledBy
+        const label = text(labelledBy
             ? labelledBy.split(/\s+/).map(id => document.getElementById(id)?.textContent ?? '').join(' ')
-            : node.getAttribute('aria-label') || node.innerText || node.getAttribute('title') || '').trim();
+            : node.getAttribute('aria-label') || (field
+                // A wrapping label also contains the field's own text, such as its option labels.
+                ? Array.from(field.labels?.[0]?.childNodes ?? []).filter(child => !(child instanceof Element && child.matches('input,select,textarea,button')))
+                    .map(child => child.textContent).join(' ').trim() || node.getAttribute('title') || node.getAttribute('name')
+                : (submit ? submit.value : node.innerText) || node.getAttribute('title')));
         if (!label || label.length > 256)
             return null;
         const container = node.closest('tr,[role="row"],li,fieldset,form,dialog,[role="dialog"],section,article');
-        const context = (container?.matches('tr,[role="row"],li')
+        const context = text(container?.matches('tr,[role="row"],li')
             ? container.innerText
-            : container?.getAttribute('aria-label') || container?.querySelector(':scope > legend,:scope > h1,:scope > h2,:scope > h3')?.textContent || '').trim();
+            : container?.getAttribute('aria-label') || container?.querySelector(':scope > legend,:scope > h1,:scope > h2,:scope > h3')?.textContent);
         if (context.length > 256)
             return null;
-        const enabled = !node.matches(':disabled') && !node.closest('[aria-disabled="true"]');
-        const role = node.getAttribute('role') || (node instanceof HTMLAnchorElement ? 'link' : 'button');
+        const enabled = !node.matches(':disabled') && !node.closest('[aria-disabled="true"]') && !field?.readOnly;
+        const role = field instanceof HTMLSelectElement ? 'select' : field ? field.type
+            : node.getAttribute('role') || (node instanceof HTMLAnchorElement ? 'link' : 'button');
+        const value = field instanceof HTMLSelectElement ? text(field.selectedOptions[0]?.label) : field ? field.value : undefined;
+        const form = node instanceof HTMLButtonElement || field || submit ? node.form : null;
+        // Where a submit control sends its form: its own overrides, else the form's attributes.
+        const submitter = submit ?? (node instanceof HTMLButtonElement && node.type === 'submit' ? node : null);
+        const submission = submitter && form ? [
+            submitter.hasAttribute('formaction') ? submitter.formAction : formProperty(form, 'action'),
+            submitter.hasAttribute('formmethod') ? submitter.formMethod : formProperty(form, 'method'),
+            submitter.hasAttribute('formtarget') ? submitter.formTarget : formProperty(form, 'target'),
+            submitter.hasAttribute('formenctype') ? submitter.formEnctype : formProperty(form, 'enctype'),
+            submitter.formNoValidate || formProperty(form, 'noValidate'),
+        ] : null;
+        const options = field instanceof HTMLSelectElement ? eligible(field).slice(0, 40).map(option => text(option.label)) : undefined;
         // Keep activation attributes and context identity local, not in diagnostics.
-        const identity = JSON.stringify([label, context, role, enabled,
+        const identity = JSON.stringify([label, context, role, enabled, node.getAttribute('name'),
             node.getAttribute('href'), node instanceof HTMLAnchorElement ? node.href : null,
             node.getAttribute('target'), node.getAttribute('download'), node.getAttribute('type'),
             node.getAttribute('popovertarget'), node.getAttribute('popovertargetaction'),
             node.getAttribute('commandfor'), node.getAttribute('command'),
-            node.getAttribute('aria-expanded'), node.getAttribute('aria-selected'), node.getAttribute('aria-pressed')]);
-        const form = node instanceof HTMLButtonElement ? node.form : null;
-        return { label, context, role, enabled, identity, container, form, x: box.x + box.width / 2, y: box.y + box.height / 2 };
+            node.getAttribute('aria-expanded'), node.getAttribute('aria-selected'), node.getAttribute('aria-pressed'), submission]);
+        return { label, context, role, enabled, value, options, identity, container, form,
+            box: { x: box.x, y: box.y, width: box.width, height: box.height } };
     }
     const nodes = [];
     const walker = document.createTreeWalker(document, NodeFilter.SHOW_ELEMENT, {
-        acceptNode: node => node.matches('a[href],button') ? NodeFilter.FILTER_ACCEPT : NodeFilter.FILTER_SKIP,
+        acceptNode: node => node.matches(`a[href],button,input[type="submit"],${fields}`) ? NodeFilter.FILTER_ACCEPT : NodeFilter.FILTER_SKIP,
     }); // Does not cross frames or shadow roots; stop collecting at the first match beyond the cap.
     while (nodes.length <= 512 && walker.nextNode())
         nodes.push(walker.currentNode);
     const entries = [];
-    let truncated = nodes.length > 512;
+    const truncated = nodes.length > 512;
     if (!truncated)
         for (const node of nodes) {
             const state = describe(node);
             if (state)
                 entries.push({ node, state });
         }
-    const counts = new Map();
-    const key = (state) => JSON.stringify([state.role, state.label, state.context]);
-    for (const { state } of entries)
-        counts.set(key(state), (counts.get(key(state)) ?? 0) + 1);
-    let bytes = 0;
-    const controls = [];
-    for (const { state } of entries) {
-        const item = { role: state.role, label: state.label, context: state.context, enabled: state.enabled, ambiguous: counts.get(key(state)) > 1 };
-        bytes += new TextEncoder().encode(JSON.stringify(item)).length + 100; // Reserve reference/JSON overhead.
-        if (controls.length === 64 || bytes > 16_000) {
-            truncated = true;
-            break;
-        }
-        controls.push(item);
+    // Re-check a reference against the node it was captured from, without dispatching anything.
+    function current(index) {
+        const entry = entries[index];
+        if (document !== documentAtCapture || location.href !== url || !entry)
+            return null;
+        const state = describe(entry.node);
+        if (!state || !state.enabled || state.container !== entry.state.container
+            || state.form !== entry.state.form || state.identity !== entry.state.identity)
+            return null;
+        return { entry, state };
     }
     return {
-        controls, truncated,
-        resolve(index) {
-            const entry = entries[index];
-            if (document !== documentAtCapture || location.href !== url || !entry || !controls[index]
-                || controls[index].ambiguous)
-                return null;
-            const state = describe(entry.node);
-            if (!state || !state.enabled || state.container !== entry.state.container
-                || state.form !== entry.state.form || state.identity !== entry.state.identity)
+        truncated,
+        controls: entries.map(({ state }) => ({ role: state.role, label: state.label, context: state.context, enabled: state.enabled,
+            ...(state.value !== undefined ? { value: state.value } : {}), ...(state.options ? { options: state.options } : {}), box: state.box })),
+        point(index) {
+            const resolved = current(index);
+            if (!resolved)
                 return null;
-            const hit = document.elementFromPoint(state.x, state.y);
+            const { entry, state } = resolved;
+            const x = state.box.x + state.box.width / 2, y = state.box.y + state.box.height / 2;
+            const hit = document.elementFromPoint(x, y);
             // Text/icon descendants belong to this control; nested controls do not.
             if (!hit || hit.closest(interactive) !== entry.node)
                 return null;
@@ -99,44 +138,200 @@ function captureControls() {
                 if (descendant.shadowRoot || descendant.localName.includes('-') || descendant.hasAttribute('is'))
                     return null;
             }
-            return { x: state.x, y: state.y };
+            return { x, y };
+        },
+        // Checks a field change without making it: the reference is current, the field receives
+        // input at its center, and the value is one the field accepts.
+        check(index, kind, value) {
+            const at = this.point(index);
+            const node = entries[index]?.node;
+            if (!at || !node?.matches(fields) || (kind === 'select') !== node instanceof HTMLSelectElement)
+                return { ok: false, reason: 'target_unavailable' };
+            if (node instanceof HTMLSelectElement) {
+                const matches = eligible(node).filter(option => text(option.label) === value);
+                return matches.length === 1 ? { ok: true, point: at, option: matches[0].index } : { ok: false, reason: 'option_unavailable' };
+            }
+            // A detached input of the same type sanitizes an invalid value to empty, without touching the page.
+            const probe = document.createElement('input');
+            probe.type = node.type;
+            probe.value = value;
+            return probe.value === value ? { ok: true, point: at, option: -1 } : { ok: false, reason: 'invalid_value' };
+        },
+        // Rechecks and changes the field in one step, through the native setter, then reports its value.
+        apply(index, kind, value) {
+            const checked = this.check(index, kind, value);
+            if (!checked.ok)
+                return checked;
+            const node = entries[index].node;
+            if (node instanceof HTMLSelectElement) {
+                Object.getOwnPropertyDescriptor(HTMLSelectElement.prototype, 'selectedIndex').set.call(node, checked.option);
+            }
+            else {
+                Object.getOwnPropertyDescriptor(HTMLInputElement.prototype, 'value').set.call(node, value);
+            }
+            node.dispatchEvent(new Event('input', { bubbles: true }));
+            node.dispatchEvent(new Event('change', { bubbles: true }));
+            return { ok: true, value: node instanceof HTMLSelectElement ? text(node.selectedOptions[0]?.label) : node.value };
         },
     };
 }
-/** One disposable snapshot, owned by the operation that observed it. No DOM attributes are injected. */
+// Large enough to hold any frame, so offsets are computed without clipping.
+const UNCLIPPED = { x: -1e9, y: -1e9, width: 2e9, height: 2e9 };
+// Where a frame's content box sits in main-viewport coordinates, clipped to what is visible.
+async function frameArea(frame, viewport) {
+    const parent = frame.parentFrame();
+    if (!parent)
+        return { ...viewport, dx: 0, dy: 0 };
+    const outer = await frameArea(parent, viewport);
+    const owner = outer && await frame.frameElement().catch(() => null);
+    if (!outer || !owner)
+        return null;
+    try {
+        const box = await owner.evaluate(node => {
+            const element = node;
+            // An iframe that is invisible, hidden, or inert hides its controls, as it would a control of its own.
+            if (!element.checkVisibility({ checkOpacity: true, checkVisibilityCSS: true })
+                || element.closest('[hidden],[inert],[aria-hidden="true"]'))
+                return null;
+            // A transformed or zoomed iframe doesn't map its coordinates by offset; don't ground its controls.
+            for (let ancestor = element; ancestor; ancestor = ancestor.parentElement) {
+                const style = getComputedStyle(ancestor);
+                if (style.transform !== 'none' || style.scale !== 'none' || style.rotate !== 'none'
+                    || style.translate !== 'none' || (style.zoom && style.zoom !== '1'))
+                    return null;
+            }
+            const rect = element.getBoundingClientRect(), style = getComputedStyle(element);
+            const left = rect.left + element.clientLeft + parseFloat(style.paddingLeft);
+            const top = rect.top + element.clientTop + parseFloat(style.paddingTop);
+            return { left, top, width: element.clientWidth - parseFloat(style.paddingLeft) - parseFloat(style.paddingRight),
+                height: element.clientHeight - parseFloat(style.paddingTop) - parseFloat(style.paddingBottom) };
+        });
+        if (!box)
+            return null;
+        const dx = outer.dx + box.left, dy = outer.dy + box.top;
+        const x = Math.max(outer.x, dx), y = Math.max(outer.y, dy);
+        const width = Math.min(outer.x + outer.width, dx + box.width) - x, height = Math.min(outer.y + outer.height, dy + box.height) - y;
+        return width > 0 && height > 0 ? { x, y, width, height, dx, dy } : null;
+    }
+    finally {
+        await owner.dispose();
+    }
+}
+// Each ancestor frame must show its iframe at the point; another element covering it would take the click.
+async function frameVisibleAt(frame, x, y) {
+    const parent = frame.parentFrame();
+    if (!parent)
+        return true;
+    const owner = await frame.frameElement().catch(() => null);
+    if (!owner)
+        return false;
+    try {
+        const local = await frameArea(parent, UNCLIPPED);
+        if (!local)
+            return false;
+        const hit = await owner.evaluate((element, point) => document.elementFromPoint(point.x, point.y) === element, { x: x - local.dx, y: y - local.dy });
+        return hit && await frameVisibleAt(parent, x, y);
+    }
+    finally {
+        await owner.dispose();
+    }
+}
+/** One disposable snapshot per frame, owned by the operation that observed it. No DOM attributes are injected. */
 export class GroundedControls {
-    snapshot;
+    // The latest snapshot, preceded by the ones observed since the plan began, when only field changes happened in between.
+    snapshots = [];
+    fieldChanged = false;
     async clear() {
-        const snapshot = this.snapshot;
-        this.snapshot = undefined;
-        await snapshot?.handle.dispose();
+        await this.release(this.snapshots.splice(0));
+    }
+    /** Called before planning: the planner sees only the latest snapshot. */
+    async keepLatest() {
+        await this.release(this.snapshots.splice(0, this.snapshots.length - 1));
+    }
+    async release(snapshots) {
+        await Promise.all(snapshots.flatMap(snapshot => snapshot.handles.map(handle => handle.dispose().catch(() => { }))));
     }
     async observe(page) {
-        await this.clear();
+        // A successful select or fill keeps earlier refs usable for the rest of the batch; any other action expires them.
+        if (!this.fieldChanged)
+            await this.clear();
+        this.fieldChanged = false;
         checkOperation();
+        const scope = 'viewport-links-buttons-and-native-fields';
         const operation = currentOperation();
         if (!operation)
-            return { controls: [], truncated: false, scope: 'viewport-native-links-and-buttons' };
-        const handle = await page.evaluateHandle(captureControls);
+            return { controls: [], truncated: false, scope };
+        const size = page.viewportSize() ?? await page.evaluate(() => ({ width: innerWidth, height: innerHeight }));
+        const viewport = { x: 0, y: 0, ...size };
+        const handles = [], entries = [];
+        const candidates = [];
+        let truncated = false;
         const prefix = randomUUID();
-        this.snapshot = { handle, page, operation, prefix };
+        this.snapshots.push({ handles, entries, page, operation, prefix });
+        for (const frame of page.frames()) {
+            const area = await frameArea(frame, viewport).catch(() => null);
+            if (!area)
+                continue;
+            const handle = await frame.evaluateHandle(captureControls, FIELDS).catch(() => null);
+            checkOperation();
+            if (!handle)
+                continue;
+            handles.push(handle);
+            const data = await handle.evaluate(({ controls, truncated }) => ({ controls, truncated })).catch(() => null);
+            if (!data)
+                continue;
+            truncated ||= data.truncated;
+            data.controls.forEach(({ box, ...control }, index) => {
+                const x = area.dx + box.x, y = area.dy + box.y;
+                // Only controls fully inside the frame's visible area of the main viewport.
+                if (x < area.x || y < area.y || x + box.width > area.x + area.width || y + box.height > area.y + area.height)
+                    return;
+                candidates.push({ frame, handle, index, control });
+            });
+        }
         checkOperation();
-        const data = await handle.evaluate(({ controls, truncated }) => ({ controls, truncated }));
-        return { scope: 'viewport-native-links-and-buttons', truncated: data.truncated,
-            controls: data.controls.map((control, index) => ({ ref: `${prefix}:${index}`, ...control })) };
+        // Count descriptions before limiting the list, so truncation can't hide a duplicate.
+        const key = (control) => JSON.stringify([control.role, control.label, control.context]);
+        const counts = new Map();
+        for (const { control } of candidates)
+            counts.set(key(control), (counts.get(key(control)) ?? 0) + 1);
+        const listed = [];
+        let bytes = 0;
+        for (const { frame, handle, index, control } of candidates) {
+            bytes += new TextEncoder().encode(JSON.stringify(control)).length + 100; // Reserve reference/JSON overhead.
+            if (listed.length === 64 || bytes > 16_000) {
+                truncated = true;
+                break;
+            }
+            const ambiguous = counts.get(key(control)) > 1;
+            listed.push({ ...control, ambiguous });
+            entries.push({ frame, handle, index, role: control.role, ambiguous });
+        }
+        return { scope, truncated, controls: listed.map((control, index) => ({ ref: `${prefix}:${index}`, ...control })) };
     }
-    async click(harness, ref) {
-        const snapshot = this.snapshot;
+    entry(ref) {
         const index = Number(ref.slice(ref.lastIndexOf(':') + 1));
-        if (!snapshot || snapshot.operation !== currentOperation() || !Number.isInteger(index)
-            || ref !== `${snapshot.prefix}:${index}`)
+        const snapshot = this.snapshots.find(snapshot => ref === `${snapshot.prefix}:${index}`);
+        if (!snapshot || snapshot.operation !== currentOperation() || !snapshot.entries[index] || snapshot.entries[index].ambiguous)
+            return null;
+        return { snapshot, entry: snapshot.entries[index] };
+    }
+    async click(harness, ref) {
+        const found = this.entry(ref);
+        if (!found)
             return false;
+        const { snapshot, entry } = found;
         return harness.clickGrounded(async () => {
             checkOperation();
-            if (this.snapshot !== snapshot || harness.page !== snapshot.page)
+            if (!this.snapshots.includes(snapshot) || harness.page !== snapshot.page)
                 return null;
             try {
-                return await snapshot.handle.evaluate((state, index) => state.resolve(index), index);
+                const area = await frameArea(entry.frame, UNCLIPPED);
+                const point = area && await entry.handle.evaluate((state, index) => state.point(index), entry.index);
+                if (!area || !point)
+                    return null;
+                const x = area.dx + point.x, y = area.dy + point.y;
+                return await frameVisibleAt(entry.frame, x, y) ? { x, y } : null;
             }
             catch {
                 checkOperation();
@@ -144,4 +339,40 @@ export class GroundedControls {
             } // A destroyed document rejects the read; never replay a click.
         });
     }
+    /** Sets a native select or date-like input through the DOM, where its picker can't be seen. */
+    async setValue(harness, ref, value, kind) {
+        const found = this.entry(ref);
+        if (!found || (kind === 'select') !== (found.entry.role === 'select'))
+            return GROUNDED_INPUT_REJECTED;
+        const { snapshot, entry } = found;
+        const args = { index: entry.index, kind, value };
+        try {
+            // Nothing has changed yet, so any failure here is a rejection.
+            checkOperation();
+            if (!this.snapshots.includes(snapshot) || harness.page !== snapshot.page)
+                return GROUNDED_INPUT_REJECTED;
+            const area = await frameArea(entry.frame, UNCLIPPED);
+            const checked = area && await entry.handle.evaluate((state, { index, kind, value }) => state.check(index, kind, value), args);
+            if (!area || !checked)
+                return GROUNDED_INPUT_REJECTED;
+            if (!checked.ok)
+                return REJECTIONS[checked.reason];
+            if (!await frameVisibleAt(entry.frame, area.dx + checked.point.x, area.dy + checked.point.y))
+                return GROUNDED_INPUT_REJECTED;
+        }
+        catch {
+            checkOperation();
+            return GROUNDED_INPUT_REJECTED;
+        }
+        checkOperation();
+        if (!this.snapshots.includes(snapshot))
+            return GROUNDED_INPUT_REJECTED;
+        // The field may change from here, so failures propagate as unknown outcomes rather than rejections.
+        const applied = await entry.handle.evaluate((state, { index, kind, value }) => state.apply(index, kind, value), args);
+        if (!applied.ok)
+            return REJECTIONS[applied.reason];
+        this.fieldChanged = true;
+        await harness.waitForStability();
+        return { changed: true, value: applied.value };
+    }
 }

package/dist/web/tabs.js

@@ -210,13 +210,6 @@ export class TabManager {
         logger.debug(`Active tab changed to: ${page.url()}`);
         this.activePage = page;
         this.events.emit('tabChanged', page);
-        page.removeAllListeners('framenavigated');
-        page.on('framenavigated', async (frame) => {
-            if (frame === page.mainFrame()) {
-                // const url = frame.url();
-                // await logNavigation(url);
-            }
-        });
     }
     async switchTab(index) {
         const pages = this.context.pages();

package/dist/web/transformer.js

@@ -1,4 +1,5 @@
 import getShadowDOMInputAdapterScript from './scripts/shadowDOMInputAdapter';
+import { renderSelectPickersInFrame } from './baseSelect';
 import logger from '@/logger';
 export class DOMTransformer {
     initializedPages = new WeakSet(); // Track pages for which 'load' listener is set
@@ -10,12 +11,20 @@ export class DOMTransformer {
                 // Pass 'newPage' (the page that triggered the 'load' event) to setupScriptForPage.
                 await this.setupScriptForPage(newPage);
             });
+            // An iframe can navigate without a page 'load' event; its selects still need the picker style.
+            newPage.on('framenavigated', async (frame) => {
+                if (!frame.parentFrame())
+                    return;
+                await frame.waitForLoadState('domcontentloaded').catch(() => { });
+                await renderSelectPickersInFrame(frame);
+            });
             this.initializedPages.add(newPage); // Mark this Page object as having its 'load' listener set up.
             // A page that loaded before it was tracked never fires 'load' again.
             void this.setupScriptForPage(newPage);
         }
     }
     async setupScriptForPage(targetPage) {
+        await Promise.all(targetPage.frames().map(renderSelectPickersInFrame));
         try {
             // Check if a marker for the script already exists on the page for this load cycle.
             const scriptAlreadyInjected = await targetPage.evaluate(() => {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@ddwang/magnitude-core",
-  "version": "0.3.1-ddwang.13",
+  "version": "0.3.1-ddwang.15",
   "description": "Magnitude e2e testing agent",
   "publishConfig": {
     "access": "public"