@evan_zhong/browser-attach 0.17.2 → 0.19.0

package/extension/controller.js

@@ -18,6 +18,7 @@ import { waitFor } from './wait.js';
 import { boundedRead, installOutcomeObserver } from './outcomes.js';
 import { createDrag, dragOptions } from './drag.js';
 import { createFetches, capturePage } from './fetch.js';
+import { errorData } from './errors.js';
 
 export function createController(api, notify = () => {}) {
   const claims = new Map();
@@ -390,7 +391,7 @@ export function createController(api, notify = () => {}) {
           const stepBudget = budget.child(step.timeout_ms ?? 30000, command === 'wait' ? 'wait' : 'input');
           results.push(await perform(tabId, command, { ...inputArgs, dialog_policy: step.dialog_policy ?? args.dialog_policy ?? 'ask', budget: stepBudget }));
         } catch (error) {
-          return { completed: false, results, started_at: args.start_at ?? 0, failed_step: index, next_unattempted_step: index + 1 < args.steps.length ? index + 1 : null, retry_failed_step_safe: error.input_dispatched === false, error: error.message, ...(error.post_action ? { post_action: error.post_action } : {}),
+          return { completed: false, results, started_at: args.start_at ?? 0, failed_step: index, next_unattempted_step: index + 1 < args.steps.length ? index + 1 : null, retry_failed_step_safe: error.input_dispatched === false, ...errorData(error),
             application_result: 'not_checked', observe_again: true };
         }
       }
@@ -401,7 +402,7 @@ export function createController(api, notify = () => {}) {
       if (args.result_type != null && !['json', 'element'].includes(args.result_type)) throw new Error('result_type must be json or element');
       if (args.cursor == null) expireCursor(tabId);
       if (args.cursor != null) return await evaluations.read(tabId, args);
-      const budget = deadline(args.timeout_ms ?? 10000, 'evaluation'), version = documents.get(tabId), claim = claims.get(tabId);
+      const budget = args.budget || deadline(args.timeout_ms ?? 10000, 'evaluation'), version = documents.get(tabId), claim = claims.get(tabId);
       const frameId = await resolveFrame(cdp, value, sessions, tabId, args, budget), target = sessions.target(tabId, frameId);
       const contextId = args.frame || args.frame_id ? pageContexts.get(JSON.stringify([tabId, target.sessionId, frameId])) : undefined;
       if ((args.frame || args.frame_id) && contextId == null) throw new Error('Page frame context is unavailable; observe again');
@@ -431,6 +432,11 @@ export function createController(api, notify = () => {}) {
       });
     }
     if (command === 'read_text') {
+      if (args.selector_type === 'role') {
+        const budget = deadline(5000), node = await resolveSelector(cdp, value, sessions, tabId, args, budget);
+        try { return { ...await budget.run(() => readText(cdp, value, node.target, { ...args, node })), scope: args.frame || args.frame_id ? 'frame' : 'main_frame', frame_id: node.frameId }; }
+        finally { await boundedRead(() => cdp(node.target, 'Runtime.releaseObjectGroup', { objectGroup: 'browser-attach' }), 100).catch(() => {}); }
+      }
       if (!args.frame && !args.frame_id) return await readText(cdp, value, tabId, args);
       const budget = deadline(5000), frameId = await resolveFrame(cdp, value, sessions, tabId, args, budget), target = sessions.target(tabId, frameId);
       const contextId = pageContexts.get(JSON.stringify([tabId, target.sessionId, frameId]));
@@ -440,7 +446,7 @@ export function createController(api, notify = () => {}) {
     if (command === 'screenshot') {
       const view = await viewport(tabId);
       const hasTarget = args.ref != null || args.selector != null;
-      if (!hasTarget && (args.frame || args.frame_id || args.nth != null || args.selector_type)) throw new Error('Element screenshot frame/nth/type requires an element selector');
+      if (!hasTarget && (args.frame || args.frame_id || args.nth != null || args.selector_type || args.name != null)) throw new Error('Element screenshot frame/nth/type/name requires an element selector');
       if (hasTarget && args.full_page) throw new Error('Element target and full_page are mutually exclusive');
       const clip = hasTarget ? await elementClip(tabId, args, view) : args.full_page ? await fullPageClip(tabId) : null;
       const includeIndicators = args.include_indicators === true;
@@ -498,8 +504,8 @@ export function createController(api, notify = () => {}) {
     if (command === 'select_option') selectOptions(args);
     if (command === 'type') typeOptions(args);
     if (command === 'scroll') scrollOptions(args);
-    if (['scroll', 'upload_file'].includes(command) && ['ref', 'selector', 'frame', 'frame_id', 'selector_type', 'nth'].some(key => args[key] != null)) targetOptions(args);
-    if (['click', 'fill', 'type', 'hover', 'select_option'].includes(command)) targetOptions(args, { coordinates: command === 'hover' || command === 'click' && inputMode(args) !== 'dom' });
+    if (['scroll', 'upload_file'].includes(command) && ['ref', 'selector', 'frame', 'frame_id', 'selector_type', 'name', 'nth'].some(key => args[key] != null)) targetOptions(args);
+    if (['click', 'fill', 'type', 'hover', 'select_option'].includes(command)) targetOptions(args, { coordinates: command === 'hover' || command === 'click' && inputMode(args) !== 'dom', focused: command === 'type' });
     if (command === 'click') clickOptions(args);
     if (['click', 'fill', 'type', 'press_key', 'scroll', 'hover', 'select_option', 'upload_file'].includes(command)) args = { ...args, budget: args.budget || deadline(args.timeout_ms) };
     // Check before any renderer preparation or ref invalidation. A rejected
@@ -636,6 +642,9 @@ export function createController(api, notify = () => {}) {
 
   async function perform(tabId, command, args) {
     if (args.dialog_policy != null && !['ask', 'accept', 'dismiss'].includes(args.dialog_policy)) throw new Error('dialog_policy must be ask, accept or dismiss');
+    if (!args.budget && (['click', 'fill', 'type', 'press_key', 'hover', 'scroll', 'select_option', 'upload_file'].includes(command) || command === 'evaluate' && args.cursor == null)) {
+      args = { ...args, budget: deadline(args.timeout_ms ?? (command === 'evaluate' ? 10000 : 5000), command === 'evaluate' ? 'evaluation' : 'input') };
+    }
     const parent = dialogPolicies.get(tabId), state = { policy: args.dialog_policy ?? parent?.policy ?? 'ask', messages: [], parent };
     dialogPolicies.set(tabId, state);
     const version = documents.get(tabId), startTab = await api.tabs.get(tabId);
@@ -644,7 +653,7 @@ export function createController(api, notify = () => {}) {
     const startObservation = async () => { if (!watch || observer || dialogs.has(tabId)) return; try {
       observer = await boundedRead(async () => {
         if (args.ref) observationFrame = snapshots.get(tabId)?.refs.get(args.ref)?.frameId || observationFrame;
-        else if (args.frame || args.frame_id) observationFrame = await resolveFrame(cdp, value, sessions, tabId, args, deadline(Math.max(100, Math.min(200, args.budget?.remaining() ?? 200))));
+        else if (args.frame || args.frame_id) observationFrame = await resolveFrame(cdp, value, sessions, tabId, args, deadline(Math.max(100, Math.min(200, args.budget?.remaining() ?? 200))), { wait: false });
         observationTarget = sessions.target(tabId, observationFrame);
         observedVersion = frameStates.get(tabId)?.versions.get(observationFrame);
         const { executionContextId } = await cdp(observationTarget, 'Page.createIsolatedWorld', { frameId: observationFrame, worldName: 'browser-attach-outcome' });
@@ -672,7 +681,6 @@ export function createController(api, notify = () => {}) {
           current_url: current?.url ?? null, dom_changed: dom?.dom_changed ?? null,
           assessment: dialogs.has(tabId) || state.messages.length || documents.get(tabId) !== version || frameChanged || dom?.dom_changed ? 'observable_effects' : 'no_observable_effects',
           note: 'These observations do not prove business success or that an unchanged page received no input. Do not replay automatically.' };
-        error.message += ` Post-action observation: ${JSON.stringify(error.post_action)}`;
       } else if (state.messages.length) error.message += ` Dialogs: ${JSON.stringify(state.messages)}`;
       throw error;
     } finally {

package/extension/errors.js

@@ -0,0 +1,13 @@
+const fields = ['input_dispatched', 'post_action', 'diagnostics'];
+
+export function errorData(error) {
+  const data = { error: error.message || String(error) };
+  for (const key of fields) if (error[key] !== undefined) data[key] = error[key];
+  return data;
+}
+
+export function errorFromData(data) {
+  const error = new Error(String(data.error));
+  for (const key of fields) if (data[key] !== undefined) error[key] = data[key];
+  return error;
+}

package/extension/evaluate.js

@@ -109,8 +109,8 @@ export function createEvaluations(cdp) {
     }
     const { kind, items, offset, result } = state;
     const length = kind === 'scalar' ? 1 : items.length;
-    const make = end => kind === 'string' ? items.slice(offset, end) : kind === 'array' ? items.slice(offset, end) :
-      kind === 'object' ? Object.fromEntries(items.slice(offset, end)) : items;
+    const make = (end, start = offset) => kind === 'string' || kind === 'array' ? items.slice(start, end) :
+      kind === 'object' ? Object.fromEntries(items.slice(start, end)) : items;
     let low = offset, high = length;
     while (low < high) {
       const mid = Math.ceil((low + high) / 2);
@@ -124,7 +124,9 @@ export function createEvaluations(cdp) {
     }
     state.offset = low;
     state.cursor = low < length ? `${state.token}@${low}` : null;
+    const nextRequired = low < length ? JSON.stringify(make(low + 1, low)).length : null;
     return { ...result, value: make(low), result_kind: kind, has_more: low < length,
+      ...(nextRequired > max ? { next_item: { ...(kind === 'object' ? { key: items[low][0] } : { index: low }), required_chars: nextRequired, exceeds_max_chars: nextRequired > 100000 } } : {}),
       next_cursor: state.cursor, capture_truncated: result.truncated, truncated: result.truncated || low < length };
   }
   return { read, forgetFrame(tabId, frameId) { if (results.get(tabId)?.frameId === frameId) results.delete(tabId); }, forget: tabId => results.delete(tabId) };

package/extension/input.js

@@ -32,6 +32,30 @@ export function createInput(cdp, value, sessions) {
   }
   function cancel(tabId) { pending.get(tabId)?.reject(new Error('Tab detached during keyboard input; outcome is unknown. Observe before retrying.')); }
 
+  async function focusedNode(tabId, budget, worlds, frames) {
+    frames ||= await budget.run(() => sessions.frames(tabId));
+    let frameId = frames[0].id;
+    const visited = new Set();
+    while (true) {
+      if (visited.has(frameId)) throw new Error('Focus frame cycle; no input was dispatched.');
+      visited.add(frameId);
+      let world = worlds?.get(frameId);
+      if (!world) {
+        if (worlds || !frames.some(frame => frame.id === frameId)) throw new Error('Focused frame is unavailable; no input was dispatched.');
+        const target = sessions.target(tabId, frameId);
+        const { executionContextId } = await budget.run(() => cdp(target, 'Page.createIsolatedWorld', { frameId, worldName: 'browser-attach-input' }));
+        world = { target, contextId: executionContextId };
+      }
+      const active = await budget.run(() => cdp(world.target, 'Runtime.evaluate', { contextId: world.contextId,
+        expression: `(${activeElement.toString()})()`, objectGroup: 'browser-attach' }));
+      value(active);
+      if (!active.result?.objectId) throw new Error('No focused page element is available; no input was dispatched.');
+      const { node } = await budget.run(() => cdp(world.target, 'DOM.describeNode', { objectId: active.result.objectId }));
+      if (node.frameId) frameId = node.frameId;
+      else return { target: world.target, frameId, objectId: active.result.objectId, backendNodeId: node.backendNodeId };
+    }
+  }
+
   async function pressKey(tabId, chord, args = {}) {
     const budget = args.budget || deadline(args.timeout_ms);
     const send = (...params) => budget.run(() => cdp(...params));
@@ -66,34 +90,20 @@ export function createInput(cdp, value, sessions) {
           omittedFrames.push({ frame_id: frame.id, reason: error.message });
         }
       }
-      let focusTarget = worlds.get(frames[0].id), focusedNode;
-      const visited = new Set();
-      while (focusTarget) {
-        const { target, contextId } = focusTarget;
-        if (visited.has(contextId + ':' + target.sessionId)) throw new Error('Focus frame cycle; no input was dispatched.');
-        visited.add(contextId + ':' + target.sessionId);
-        const active = await send(target, 'Runtime.evaluate', { contextId,
-          expression: `(${activeElement.toString()})()`, objectGroup: 'browser-attach' });
-        if (!active.result?.objectId) throw new Error('No focused page element is available; no input was dispatched.');
-        const { node } = await send(target, 'DOM.describeNode', { objectId: active.result.objectId });
-        if (node.frameId) {
-          focusTarget = worlds.get(node.frameId);
-          if (!focusTarget) throw new Error('Focused frame is unavailable; no input was dispatched.');
-        } else { focusedNode = { objectId: active.result.objectId }; break; }
-      }
-      await waitForTarget(cdp, value, focusTarget.target, focusedNode, budget);
+      const focused = await focusedNode(tabId, budget, worlds, frames);
+      await waitForTarget(cdp, value, focused.target, focused, budget);
       if (args.expectedNode) {
         const expected = args.expectedNode;
-        const focused = focusTarget.frameId === expected.frameId && value(await send(expected.target, 'Runtime.callFunctionOn', { objectId: expected.objectId,
+        const matches = focused.frameId === expected.frameId && value(await send(expected.target, 'Runtime.callFunctionOn', { objectId: expected.objectId,
           functionDeclaration: 'function() { const a=this.getRootNode().activeElement; return this.isConnected && (a===this || this.contains(a)); }', returnByValue: true }));
-        if (!focused) throw new Error('Typing target lost focus during key preparation; no further key was dispatched');
+        if (!matches) throw new Error('Typing target lost focus during key preparation; no further key was dispatched');
       }
       budget.dispatch();
       // Queue the complete chord before waiting for acknowledgements. Always
       // enqueue keyUp, even if keyDown fails or opens a modal.
-      const commands = [send(focusTarget.target, 'Input.dispatchKeyEvent', { type: args.character != null ? 'keyDown' : 'rawKeyDown', ...(args.character != null ? { text, unmodifiedText: text } : {}), ...params })];
-      if (text != null && args.character == null) commands.push(send(focusTarget.target, 'Input.dispatchKeyEvent', { type: 'char', text, unmodifiedText: text, ...params }));
-      commands.push(send(focusTarget.target, 'Input.dispatchKeyEvent', { type: 'keyUp', ...params }));
+      const commands = [send(focused.target, 'Input.dispatchKeyEvent', { type: args.character != null ? 'keyDown' : 'rawKeyDown', ...(args.character != null ? { text, unmodifiedText: text } : {}), ...params })];
+      if (text != null && args.character == null) commands.push(send(focused.target, 'Input.dispatchKeyEvent', { type: 'char', text, unmodifiedText: text, ...params }));
+      commands.push(send(focused.target, 'Input.dispatchKeyEvent', { type: 'keyUp', ...params }));
       await budget.run(() => delivery(Promise.all([Promise.all(commands), receipt])));
       return { sent: chord, method: 'native', delivery_verified: true, application_result: 'not_checked',
         ...(omittedFrames.length ? { verification_omitted_frames: omittedFrames } : {}), observe_again: true };
@@ -135,21 +145,28 @@ export function createInput(cdp, value, sessions) {
   async function selectOption(tabId, node, args) {
     selectOptions(args);
     const budget = args.budget || deadline(args.timeout_ms);
-    await waitForTarget(cdp, value, node.target, node, budget);
+    const choice = { value: args.value, label: args.label, index: args.index };
+    value(await budget.run(() => cdp(node.target, 'Runtime.callFunctionOn', {
+      objectId: node.objectId, functionDeclaration: selectElement.toString(),
+      arguments: [{ value: choice }, { value: false }], returnByValue: true,
+    })));
     budget.dispatch();
     const result = value(await budget.run(() => cdp(node.target, 'Runtime.callFunctionOn', {
       objectId: node.objectId, functionDeclaration: selectElement.toString(),
-      arguments: [{ value: { value: args.value, label: args.label, index: args.index } }], returnByValue: true,
+      arguments: [{ value: choice }], returnByValue: true,
     })));
     return { ...result, method: 'dom', events_trusted: false, application_result: 'not_checked', observe_again: true };
   }
   async function typeText(tabId, node, args) {
     typeOptions(args);
     const budget = args.budget || deadline(args.timeout_ms);
+    if (args.focused) node = await focusedNode(tabId, budget);
     await waitForTarget(cdp, value, node.target, node, budget, { editable: true });
-    budget.dispatch();
-    value(await budget.run(() => cdp(node.target, 'Runtime.callFunctionOn', { objectId: node.objectId,
-      functionDeclaration: 'function() { this.focus(); if (this.getRootNode().activeElement !== this && !this.contains(this.getRootNode().activeElement)) throw new Error("Field did not receive focus"); }', returnByValue: true })));
+    if (!args.focused) {
+      budget.dispatch();
+      value(await budget.run(() => cdp(node.target, 'Runtime.callFunctionOn', { objectId: node.objectId,
+        functionDeclaration: 'function() { this.focus(); if (this.getRootNode().activeElement !== this && !this.contains(this.getRootNode().activeElement)) throw new Error("Field did not receive focus"); }', returnByValue: true })));
+    }
     let typed = 0; const length = Array.from(args.text).length;
     for (const character of args.text) {
       const focused = value(await budget.run(() => cdp(node.target, 'Runtime.callFunctionOn', { objectId: node.objectId,
@@ -286,14 +303,17 @@ export function selectOptions(args) {
   for (const key of ['value', 'label']) if (args[key] != null && typeof args[key] !== 'string') throw new Error(`${key} must be a string`);
 }
 
-export function selectElement(choice) {
+export function selectElement(choice, apply = true) {
   if (this.tagName !== 'SELECT' || !this.isConnected) throw new Error('Target must be a connected HTML select');
-  if (this.matches(':disabled') || this.closest('[inert]')) throw new Error('Select is disabled or inert');
+  for (let parent = this; parent; parent = parent.assignedSlot || parent.parentElement || parent.getRootNode().host) {
+    if (parent.matches(':disabled') || parent.hasAttribute('inert')) throw new Error('Select is disabled or inert');
+  }
   const options = Array.from(this.options);
   const matches = options.filter((option, index) => choice.index != null ? index === choice.index : choice.value != null ? option.value === choice.value : option.label === choice.label);
   if (matches.length !== 1) throw new Error(`Option must match exactly once; matched ${matches.length}. Use an explicit index for duplicate labels/values. No value was set.`);
   const option = matches[0];
   if (option.disabled || option.parentElement?.tagName === 'OPTGROUP' && option.parentElement.disabled) throw new Error('Option is disabled; no value was set');
+  if (!apply) return true;
   const win = this.ownerDocument.defaultView;
   const index = options.indexOf(option);
   Object.getOwnPropertyDescriptor(win.HTMLSelectElement.prototype, 'selectedIndex').set.call(this, index);

package/extension/locator.js

@@ -1,16 +1,18 @@
 // Selectors are reusable queries, never identities inferred from old AX text.
-export function targetOptions(args, { coordinates = false } = {}) {
+export function targetOptions(args, { coordinates = false, focused = false } = {}) {
   const hasRef = args.ref != null, hasSelector = args.selector != null;
   const hasPoint = args.x != null || args.y != null;
-  if (Number(hasRef) + Number(hasSelector) + Number(hasPoint) !== 1 ||
+  if (args.focused != null && (args.focused !== true || !focused)) throw new Error('focused=true is only supported by type');
+  if (Number(hasRef) + Number(hasSelector) + Number(hasPoint) + Number(args.focused === true) !== 1 ||
       (hasPoint && (!coordinates || !Number.isFinite(args.x) || !Number.isFinite(args.y)))) {
-    throw new Error(coordinates ? 'Provide exactly one of ref, selector, or both finite x and y' : 'Provide exactly one of ref or selector');
+    throw new Error(focused ? 'Provide exactly one of ref, selector or focused=true' : coordinates ? 'Provide exactly one of ref, selector, or both finite x and y' : 'Provide exactly one of ref or selector');
   }
   if (hasRef && (typeof args.ref !== 'string' || !args.ref)) throw new Error('ref must be nonempty');
   if (hasSelector && (typeof args.selector !== 'string' || !args.selector.trim() || args.selector.length > 10000)) throw new Error('selector must contain 1..10000 characters');
-  if (args.selector_type != null && !['css', 'xpath'].includes(args.selector_type)) throw new Error('selector_type must be css or xpath');
+  if (args.selector_type != null && !['css', 'xpath', 'label', 'placeholder', 'role'].includes(args.selector_type)) throw new Error('selector_type must be css, xpath, label, placeholder or role');
+  if (args.name != null && (args.selector_type !== 'role' || typeof args.name !== 'string' || args.name.length > 10000)) throw new Error('name requires selector_type=role and must be a string of at most 10000 characters');
   frameOptions(args);
-  if (!hasSelector && (args.frame_id != null || args.frame != null || args.selector_type != null)) throw new Error('frame_id and selector_type require selector');
+  if (!hasSelector && (args.frame_id != null || args.frame != null || args.selector_type != null)) throw new Error('frame/frame_id and selector_type require selector');
   if (args.nth != null && (!hasSelector || !Number.isSafeInteger(args.nth) || args.nth < 0)) throw new Error('nth requires selector and must be a nonnegative safe integer (zero-based)');
   if (args.coordinate_space != null && (!hasPoint || !['viewport', 'document'].includes(args.coordinate_space))) throw new Error('coordinate_space must be viewport or document and applies to coordinate clicks only');
 }
@@ -22,7 +24,13 @@ export function queryElement(selector, type = 'css', nth, waitForMatch = false)
     const result = document.evaluate(selector, document, null, XPathResult.ORDERED_NODE_SNAPSHOT_TYPE, null);
     count = result.snapshotLength; at = index => result.snapshotItem(index);
   } else {
-    const elements = document.querySelectorAll(selector);
+    const normalize = text => text.replace(/\s+/g, ' ').trim();
+    const elements = type === 'label'
+      ? Array.from(document.querySelectorAll('input,textarea,select,button,meter,output,progress')).filter(element =>
+        Array.from(element.labels || []).some(label => normalize(label.textContent || '') === normalize(selector)))
+      : type === 'placeholder'
+        ? Array.from(document.querySelectorAll('[placeholder]')).filter(element => element.getAttribute('placeholder') === selector)
+        : document.querySelectorAll(selector);
     count = elements.length; at = index => elements[index];
   }
   if (waitForMatch && (count === 0 || (nth != null && nth >= count))) return null;
@@ -43,27 +51,55 @@ export function queryElement(selector, type = 'css', nth, waitForMatch = false)
   return element;
 }
 
-export async function resolveSelector(cdp, value, sessions, tabId, args, budget) {
+export async function resolveSelector(cdp, value, sessions, tabId, args, budget, { wait = true } = {}) {
+  targetOptions(args);
   const frameId = await resolveFrame(cdp, value, sessions, tabId, args, budget);
   const target = sessions.target(tabId, frameId);
   const send = (method, params) => budget.run(() => cdp(target, method, params));
-  const { executionContextId } = await send('Page.createIsolatedWorld', { frameId, worldName: 'browser-attach-locator' });
-  let result;
+  const role = args.selector_type === 'role';
+  const context = role ? null : await send('Page.createIsolatedWorld', { frameId, worldName: 'browser-attach-locator' });
+  let backendNodeId;
   while (true) {
-    result = await send('Runtime.evaluate', { contextId: executionContextId, objectGroup: 'browser-attach',
-      expression: `(${queryElement.toString()})(${JSON.stringify(args.selector)},${JSON.stringify(args.selector_type ?? 'css')},${args.nth ?? 'undefined'},true)` });
-    value(result);
-    if (result.result?.subtype !== 'null') break;
-    budget.describe?.(`selector ${JSON.stringify(args.selector)}${args.nth == null ? '' : ` nth=${args.nth}`} to appear; current=no matching element`);
+    if (role) {
+      const { nodes } = await send('Accessibility.getFullAXTree', { frameId });
+      backendNodeId = matchRole(nodes, args)?.backendDOMNodeId;
+    } else {
+      const result = await send('Runtime.evaluate', { contextId: context.executionContextId, objectGroup: 'browser-attach',
+        expression: `(${queryElement.toString()})(${JSON.stringify(args.selector)},${JSON.stringify(args.selector_type ?? 'css')},${args.nth ?? 'undefined'},true)` });
+      value(result);
+      if (result.result?.subtype !== 'null') {
+        if (!result.result?.objectId) throw new Error('Selector did not resolve to an element');
+        const { node } = await send('DOM.describeNode', { objectId: result.result.objectId });
+        backendNodeId = node.backendNodeId;
+      }
+    }
+    if (backendNodeId != null) break;
+    if (!wait) return null;
+    budget.describe?.(`selector ${JSON.stringify(args.selector)}${args.name == null ? '' : ` name=${JSON.stringify(args.name)}`}${args.nth == null ? '' : ` nth=${args.nth}`} to appear; current=no matching element`);
     await budget.run(() => new Promise(resolve => setTimeout(resolve, 20)));
   }
-  if (!result.result?.objectId) throw new Error('Selector did not resolve to an element');
-  const { node } = await send('DOM.describeNode', { objectId: result.result.objectId });
   // Hit-test refs resolve in the default world too. CDP forbids passing object
   // arguments between worlds, even when both wrappers refer to the same node.
-  const resolved = await send('DOM.resolveNode', { backendNodeId: node.backendNodeId, objectGroup: 'browser-attach' });
+  const resolved = await send('DOM.resolveNode', { backendNodeId, objectGroup: 'browser-attach' });
   if (!resolved.object?.objectId) throw new Error('Selector target detached before resolution; no input was dispatched');
-  return { target, frameId, objectId: resolved.object.objectId, backendNodeId: node.backendNodeId };
+  return { target, frameId, objectId: resolved.object.objectId, backendNodeId };
+}
+
+export function matchRole(nodes, args) {
+  const byId = new Map(nodes.map(node => [node.nodeId, node]));
+  const children = new Set(nodes.flatMap(node => node.childIds || []));
+  const pending = nodes.filter(node => !children.has(node.nodeId)).reverse(), seen = new Set(), matches = new Map();
+  while (pending.length) {
+    const node = pending.pop();
+    if (!node || seen.has(node.nodeId)) continue;
+    seen.add(node.nodeId);
+    if (!node.ignored && node.backendDOMNodeId && node.role?.value === args.selector &&
+        (args.name == null || String(node.name?.value || '') === args.name)) matches.set(node.backendDOMNodeId, node);
+    for (let i = (node.childIds?.length || 0) - 1; i >= 0; i--) pending.push(byId.get(node.childIds[i]));
+  }
+  const candidates = [...matches.values()];
+  if (args.nth == null && candidates.length > 1) throw new Error(`Role selector must match exactly one element; matched ${candidates.length}. Provide name or an explicit zero-based nth. Candidates: ${JSON.stringify(candidates.slice(0, 10).map((node, nth) => ({ nth, role: node.role?.value, name: String(node.name?.value || '').slice(0, 120) })))}`);
+  return candidates[args.nth ?? 0] || null;
 }
 
 export function frameOptions(args) {
@@ -78,25 +114,37 @@ export function frameOptions(args) {
 async function resolveFrameOnce(cdp, value, sessions, tabId, args, budget) {
   frameOptions(args);
   const frames = await budget.run(() => sessions.frames(tabId));
+  const failure = (error, scope = {}) => {
+    const candidates = frames.slice(0, 10).map(frame => ({ frame_id: frame.id, parent_frame_id: frame.parentId ?? null, url: frame.url }));
+    error.diagnostics = { reason: 'frame_locator', ...scope, candidates, omitted_candidates: Math.max(0, frames.length - candidates.length) };
+    error.message += `; frame scope=${JSON.stringify(scope)}; current frames=${JSON.stringify(candidates)}${frames.length > candidates.length ? ` (${frames.length - candidates.length} more omitted)` : ''}`;
+    return error;
+  };
   if (args.frame_id != null) {
-    if (!frames.some(f => f.id === args.frame_id)) throw new Error('frame_id is unknown; take a new snapshot');
+    if (!frames.some(f => f.id === args.frame_id)) throw failure(new Error('frame_id is unknown; take a new snapshot'), { frame_id: args.frame_id });
     return args.frame_id;
   }
   if (!args.frame) return frames[0].id;
   const scope = args.frame;
   let frameId = frames[0].id;
-  for (const selector of scope.path || []) {
-    const target = sessions.target(tabId, frameId);
-    const send = (method, params) => budget.run(() => cdp(target, method, params));
-    const { executionContextId } = await send('Page.createIsolatedWorld', { frameId, worldName: 'browser-attach-locator' });
-    const result = await send('Runtime.evaluate', { contextId: executionContextId, expression: `(${queryElement.toString()})(${JSON.stringify(selector)},"css")`, objectGroup: 'browser-attach-frame' });
-    value(result);
+  for (const [pathIndex, selector] of (scope.path || []).entries()) {
+    const parentFrameId = frameId;
     try {
-      const { node } = await send('DOM.describeNode', { objectId: result.result.objectId });
-      if (!['IFRAME', 'FRAME'].includes(node.nodeName)) throw new Error('frame.path must resolve to iframe/frame elements');
-      frameId = node.frameId || node.contentDocument?.frameId;
-      if (!frameId) throw new Error('Frame document is not available yet');
-    } finally { await send('Runtime.releaseObjectGroup', { objectGroup: 'browser-attach-frame' }); }
+      const target = sessions.target(tabId, frameId);
+      const send = (method, params) => budget.run(() => cdp(target, method, params));
+      const { executionContextId } = await send('Page.createIsolatedWorld', { frameId, worldName: 'browser-attach-locator' });
+      const result = await send('Runtime.evaluate', { contextId: executionContextId, expression: `(${queryElement.toString()})(${JSON.stringify(selector)},"css")`, objectGroup: 'browser-attach-frame' });
+      value(result);
+      try {
+        const { node } = await send('DOM.describeNode', { objectId: result.result.objectId });
+        if (!['IFRAME', 'FRAME'].includes(node.nodeName)) throw new Error('frame.path must resolve to iframe/frame elements');
+        frameId = node.frameId || node.contentDocument?.frameId;
+        if (!frameId) throw new Error('Frame document is not available yet');
+      } finally { await send('Runtime.releaseObjectGroup', { objectGroup: 'browser-attach-frame' }); }
+    } catch (error) {
+      error.message += '; frame.path entries are CSS selectors, e.g. "#content_iframe"';
+      throw failure(error, { parent_frame_id: parentFrameId, path_index: pathIndex, selector });
+    }
   }
   const candidates = scope.path ? frames.filter(f => f.id === frameId) : frames.filter(f => f.id !== frames[0].id);
   const matches = [];
@@ -110,16 +158,17 @@ async function resolveFrameOnce(cdp, value, sessions, tabId, args, budget) {
     }
     matches.push(frame.id);
   }
-  if (matches.length !== 1) throw new Error(`Frame locator must match exactly once; matched ${matches.length}`);
+  if (matches.length !== 1) throw failure(new Error(`Frame locator must match exactly once; matched ${matches.length}`), { locator: scope, matched_frame_ids: matches.slice(0, 10) });
   return matches[0];
 }
 
-export async function resolveFrame(cdp, value, sessions, tabId, args, budget) {
+export async function resolveFrame(cdp, value, sessions, tabId, args, budget, { wait = true } = {}) {
   while (true) {
     try { return await resolveFrameOnce(cdp, value, sessions, tabId, args, budget); }
     catch (error) {
+      if (!wait) throw error;
       if (!/Frame locator.*matched 0|Frame document.*not available|Selector.*matched 0|context.*destroyed|cannot find context/i.test(error.message)) throw error;
-      budget.describe?.(`frame ${JSON.stringify(args.frame)}; current=${error.message}`);
+      budget.describe?.(`frame ${JSON.stringify(args.frame)}; current=${error.message}`, error.diagnostics);
       await budget.run(() => new Promise(resolve => setTimeout(resolve, 20)));
     }
   }

package/extension/manifest.json

@@ -1,7 +1,7 @@
 {
   "manifest_version": 3,
   "name": "BrowserAttach",
-  "version": "0.17.2",
+  "version": "0.19.0",
   "description": "Connect existing Chrome and Edge tabs to a locally paired MCP client.",
   "minimum_chrome_version": "125",
   "permissions": [

package/extension/scroll.js

@@ -27,10 +27,16 @@ export function scrollState() {
 
 export function findScrollText(text, selector) {
   if (!this.isConnected) throw new Error('Scroll container detached');
-  const elements = Array.from(this.querySelectorAll(selector || '*'));
+  const elements = this.querySelectorAll(selector || '*');
   if (elements.length > 10000) throw new Error('Scroll search exceeds 10000 candidates; scope until.selector');
-  const matches = elements.filter(e => (e.textContent || '').includes(text) && e.getBoundingClientRect().width > 0 && e.getBoundingClientRect().height > 0 && e.ownerDocument.defaultView.getComputedStyle(e).visibility === 'visible');
-  const leaves = matches.filter(e => !matches.some(other => other !== e && e.contains(other)));
+  const matches = [];
+  for (const element of elements) {
+    if (!(element.textContent || '').includes(text)) continue;
+    const rect = element.getBoundingClientRect();
+    if (rect.width > 0 && rect.height > 0 && element.ownerDocument.defaultView.getComputedStyle(element).visibility === 'visible') matches.push(element);
+  }
+  // Document order keeps each subtree contiguous, even after filtering.
+  const leaves = matches.filter((element, index) => index === matches.length - 1 || !element.contains(matches[index + 1]));
   if (leaves.length > 1) throw new Error(`Scroll text must match one element; matched ${leaves.length}. Scope until.selector`);
   return leaves[0] || null;
 }

package/extension/snapshot.js

@@ -15,11 +15,61 @@ export function renderAX(nodes, token, refs, frameId, sessionId, options = snaps
   const textRoles = new Set(['StaticText', 'InlineTextBox', 'ListMarker']);
   const controls = new Set(['button', 'checkbox', 'combobox', 'DisclosureTriangle', 'link', 'listbox', 'menuitem',
     'menuitemcheckbox', 'menuitemradio', 'option', 'radio', 'scrollbar', 'searchbox', 'slider', 'spinbutton', 'switch', 'tab', 'textbox', 'treeitem']);
+  const fields = new Set(['checkbox', 'combobox', 'radio', 'searchbox', 'slider', 'spinbutton', 'switch', 'textbox']);
+  const parents = new Map();
+  for (const node of nodes) for (const child of node.childIds || []) parents.set(child, node.nodeId);
+  function contextText(node) {
+    const pending = [node], parts = [];
+    let visited = 0, length = 0;
+    while (pending.length && ++visited <= 32) {
+      const current = pending.pop();
+      if (!current) return null;
+      const role = current.role?.value;
+      if (current.ignored && !['none', 'generic'].includes(role)) continue;
+      if (role === 'StaticText') {
+        const text = String(current.name?.value || '');
+        const separator = parts.length ? 1 : 0;
+        if (length + text.length + separator > 160) return null;
+        parts.push(text); length += text.length + separator;
+      }
+      else {
+        if (!['none', 'generic', 'LabelText', 'paragraph'].includes(role)) return null;
+        if (visited + pending.length + (current.childIds?.length || 0) > 32) return null;
+        for (let i = (current.childIds?.length || 0) - 1; i >= 0; i--) pending.push(byId.get(current.childIds[i]));
+      }
+    }
+    const text = parts.join(' ').replace(/\s+/g, ' ').trim();
+    return !pending.length && text && text.length <= 160 ? text : null;
+  }
+  // Nearby text is observation context, never a replacement for AX identity or
+  // an inferred label locator. Bound both the ancestor search and subtree work.
+  function fieldContext(node) {
+    let child = node;
+    for (let depth = 0; depth < 4; depth++) {
+      const parent = byId.get(parents.get(child.nodeId));
+      if (!parent || !['none', 'generic', 'group'].includes(parent.role?.value)) return null;
+      const pending = [parent];
+      let count = 0, visited = 0;
+      while (pending.length && ++visited <= 64 && count < 2) {
+        const current = pending.pop();
+        if (!current) return null;
+        if (!current.ignored && controls.has(current.role?.value)) count++;
+        if (visited + pending.length + (current.childIds?.length || 0) > 64) return null;
+        for (const id of current.childIds || []) pending.push(byId.get(id));
+      }
+      if (pending.length || count !== 1) return null;
+      const index = parent.childIds.indexOf(child.nodeId);
+      if (index > 0) {
+        const text = contextText(byId.get(parent.childIds[index - 1]));
+        if (text) return text;
+      }
+      child = parent;
+    }
+    return null;
+  }
   let kept;
   if (options.profile === 'interactive') {
     kept = new Set();
-    const parents = new Map();
-    for (const node of nodes) for (const child of node.childIds || []) parents.set(child, node.nodeId);
     function keepText(node) {
       const pending = [node];
       while (pending.length) {
@@ -78,6 +128,8 @@ export function renderAX(nodes, token, refs, frameId, sessionId, options = snaps
       ['disabled', 'focused', 'expanded', 'checked', 'selected', 'required', 'invalid', 'level'].includes(prop.name)
     ).map(prop => `${prop.name}=${JSON.stringify(prop.value?.value)}`);
     if (node.value?.value != null) states.push(`value=${JSON.stringify(node.value.value)}`);
+    const context = fields.has(role) && !node.ignored ? fieldContext(node) : null;
+    if (context && context !== name) states.push(`context=${JSON.stringify(context)}`);
     const attributes = states.join(' ');
     const skip = node.ignored || (!options.text && textRoles.has(role)) || (['none', 'generic'].includes(role) && !name && !attributes) ||
       (role === 'StaticText' && name === parentName);
@@ -88,7 +140,7 @@ export function renderAX(nodes, token, refs, frameId, sessionId, options = snaps
     }
     let nextDepth = depth;
     if (!skip) {
-      if (query == null || identityName.toLowerCase().includes(query) || String(node.value?.value ?? '').toLowerCase().includes(query)) {
+      if (query == null || identityName.toLowerCase().includes(query) || String(node.value?.value ?? '').toLowerCase().includes(query) || context?.toLowerCase().includes(query)) {
         const ref = node.backendDOMNodeId ? `${token}:${refs.size + 1}` : null;
         if (ref) refs.set(ref, { backendNodeId: node.backendDOMNodeId, frameId, sessionId, role: node.role?.value, name: String(node.name?.value || '') });
         entries.push({ node: true, frameId, line: `${'  '.repeat(Math.min(depth, 24))}${ref ? `[${ref}] ` : ''}${role} ${JSON.stringify(name)}${attributes ? ' ' + attributes : ''}` });

package/extension/text.js

@@ -1,21 +1,23 @@
-import { queryElement } from './locator.js';
+import { queryElement, targetOptions } from './locator.js';
 
 export async function readText(cdp, value, tabId, args) {
   const max = args.max_chars ?? 20000, offset = args.offset ?? 0, mode = args.mode ?? 'visible';
   if (!Number.isInteger(max) || max < 1 || max > 100000) throw new Error('max_chars must be 1..100000');
   if (!Number.isSafeInteger(offset) || offset < 0) throw new Error('offset must be 0..9007199254740991');
   if (!['visible', 'text_content', 'value', 'editor'].includes(mode)) throw new Error('read_text mode must be visible, text_content, value or editor');
-  if (args.selector != null && (typeof args.selector !== 'string' || !args.selector.trim() || args.selector.length > 10000)) throw new Error('selector must contain 1..10000 characters');
-  if (args.selector_type != null && !['css', 'xpath'].includes(args.selector_type)) throw new Error('selector_type must be css or xpath');
+  if (args.selector != null || args.selector_type != null || args.name != null || args.nth != null) targetOptions(args);
   if (mode === 'value' || mode === 'editor') {
     if (!args.selector) throw new Error(`${mode} requires selector to identify exactly one field/editor`);
   }
+  const options = { selector: args.selector, selector_type: args.selector_type, nth: args.nth, mode, offset, max_chars: max };
+  if (args.node) return value(await cdp(tabId, 'Runtime.callFunctionOn', { objectId: args.node.objectId, returnByValue: true,
+    functionDeclaration: `function(args) { return (${extractText.toString()})(args, () => this); }`, arguments: [{ value: options }] }));
   return value(await cdp(tabId, 'Runtime.evaluate', { returnByValue: true, ...(args.context_id != null ? { contextId: args.context_id } : {}),
-    expression: `(${extractText.toString()})(${JSON.stringify({ selector: args.selector, selector_type: args.selector_type, mode, offset, max_chars: max })},${queryElement.toString()})` }));
+    expression: `(${extractText.toString()})(${JSON.stringify(options)},${queryElement.toString()})` }));
 }
 
 export function extractText(args, query) {
-  const element = args.selector ? query(args.selector, args.selector_type ?? 'css') : document.body;
+  const element = args.selector ? query(args.selector, args.selector_type ?? 'css', args.nth) : document.body;
   let text, source = args.mode, complete = true;
   if (args.mode === 'visible') { text = element?.innerText ?? ''; complete = false; }
   else if (args.mode === 'text_content') { text = element?.textContent ?? ''; complete = false; }