@mobileaidev/ai-app-bridge-web 0.4.5 → 0.4.7

package/README.md

@@ -23,6 +23,14 @@ unchecked. Use the current SDK with the current Host; missing `checked` is an
 invalid snapshot. A semantic click must still be followed by a fresh observation
 and an independent business check where persistence matters.
 
+Visible controls without text, an ARIA label or a placeholder may also expose
+`context.ancestors`: up to two distinct text samples from their actual DOM
+ancestors, within six parent levels and 160 characters per sample. Collection
+is bounded to 64 nodes per ancestor, excludes form values and hidden branches,
+and stops before BODY/HTML. Truncation flags describe the limits. This fresh
+observation helps an agent distinguish otherwise unlabelled record menus; it
+does not infer a business row or change selectors and action identity.
+
 ## Capture windows and network bodies
 
 Network body capture is explicitly enabled with `captureRequestBodies: true`

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@mobileaidev/ai-app-bridge-web",
-  "version": "0.4.5",
+  "version": "0.4.7",
   "description": "Browser SDK for AI App Bridge Web sessions.",
   "main": "src/index.js",
   "browser": "src/index.js",

package/src/index.js

@@ -187,8 +187,11 @@
           return;
         }
         if (message.payload?.name !== 'domSnapshot') { reply(message, { ok: false, error: 'unknown_web_read' }); return; }
-        const dom = observeDom(message.payload.args ?? {});
-        reply(message, dom.ok ? { ok: true, dom } : dom); return;
+        try {
+          const dom = observeDom(message.payload.args ?? {});
+          reply(message, dom.ok ? { ok: true, dom } : dom);
+        } catch (error) { reply(message, { ok: false, error: 'web_dom_query_failed', message: error.message }); }
+        return;
       }
       if (message.type === 'cancel') {
         const identity = message.payload;
@@ -326,7 +329,8 @@
   }
 
   function documentTarget(doc) {
-    if (!documentTargets.has(doc)) documentTargets.set(doc, { navigationId: newIdentity(), available: true, elements: new WeakMap(), nextElement: 0 });
+    if (!documentTargets.has(doc)) documentTargets.set(doc, { navigationId: newIdentity(), available: true,
+      elements: new WeakMap(), elementsById: new Map(), nextElement: 0 });
     return documentTargets.get(doc);
   }
   function documentPageRef(binding) {
@@ -358,6 +362,7 @@
   function elementIdentity(element, doc = element.ownerDocument || globalValue('document')) {
     const state = documentTarget(doc);
     if (!state.elements.has(element)) state.elements.set(element, `e${++state.nextElement}`);
+    state.elementsById.set(state.elements.get(element), element);
     const sensitive = isSensitiveInput(element);
     return { elementId: state.elements.get(element), tag: text(element.tagName).toLowerCase(), id: text(element.id),
       name: attr(element, 'name'), type: attr(element, 'type'), role: attr(element, 'role'), ariaLabel: attr(element, 'aria-label'),
@@ -408,26 +413,86 @@
       role: attr(hit, 'role'), className: typeof hit.className === 'string' ? trimText(hit.className, 300) : '' } : null };
   }
 
+  function ancestorContext(element, identity, cache) {
+    if ([identity.text, identity.ariaLabel, identity.placeholder].some(value => value.trim())) return null;
+    const ancestors = [], seen = new Set();
+    let parent = element.parentElement, depth = 1;
+    while (parent && !['BODY', 'HTML'].includes(parent.tagName) && depth <= 6 && ancestors.length < 2) {
+      const sample = ancestorText(parent, cache);
+      if (sample.text && !seen.has(sample.text)) {
+        seen.add(sample.text);
+        const id = text(parent.id), role = attr(parent, 'role');
+        ancestors.push({ depth, tag: text(parent.tagName).toLowerCase(), id: id.slice(0, 80), role: role.slice(0, 40),
+          text: sample.text, textTruncated: sample.truncated, identityTruncated: id.length > 80 || role.length > 40 });
+      }
+      parent = parent.parentElement; depth++;
+    }
+    // DOM context describes actual ancestors, not a guessed business row or a visibility guarantee.
+    return ancestors.length ? { source: 'ancestor-dom-text', ancestors,
+      truncated: ancestors.some(item => item.textTruncated || item.identityTruncated)
+        || Boolean(parent && !['BODY', 'HTML'].includes(parent.tagName)) } : null;
+  }
+
+  function ancestorText(root, cache) {
+    if (cache.has(root)) return cache.get(root);
+    let node = root, visited = 0, value = '', truncated = false;
+    while (node && visited < 64) {
+      visited++;
+      let descend = true;
+      if (node.nodeType === 1) {
+        const style = node.ownerDocument?.defaultView?.getComputedStyle(node);
+        descend = !['SCRIPT', 'STYLE', 'TEMPLATE', 'NOSCRIPT', 'INPUT', 'TEXTAREA', 'SELECT'].includes(node.tagName)
+          && !isSensitiveInput(node) && !node.hidden && attr(node, 'aria-hidden') !== 'true'
+          && style?.display !== 'none' && !['hidden', 'collapse'].includes(style?.visibility) && style?.opacity !== '0';
+      } else if (node.nodeType === 3) {
+        const raw = text(node.nodeValue);
+        // Normalize a bounded raw sample so indentation does not consume the readable-text budget.
+        const part = raw.slice(0, 1024).replace(/\s+/g, ' ').trim();
+        const next = value + (value && part ? ' ' : '') + part;
+        value = next.slice(0, 160);
+        if (raw.length > 1024 || next.length > 160) { truncated = true; break; }
+      }
+      if (descend && node.firstChild) node = node.firstChild;
+      else {
+        while (node !== root && !node.nextSibling) node = node.parentNode;
+        node = node === root ? null : node.nextSibling;
+      }
+    }
+    const result = { text: value, truncated: truncated || Boolean(node) };
+    cache.set(root, result);
+    return result;
+  }
+
   function snapshotDom(options = {}) {
     const doc = options.document || globalValue('document');
     if (!doc || !doc.querySelectorAll) {
       return { ok: false, error: 'document_unavailable' };
     }
     const selector = options.selector || defaultControlSelector;
+    if (options.query && options.selector) return { ok: false, error: 'invalid_web_dom_query' };
+    const state = documentTarget(doc);
+    for (const [id, element] of state.elementsById) {
+      if (!element.isConnected || element.ownerDocument !== doc) state.elementsById.delete(id);
+    }
     const maxControls = Number(options.maxControls || 200);
+    const offset = options.offset ?? 0;
     const controls = [];
-    const matched = doc.querySelectorAll(selector);
-    const elements = Array.prototype.slice.call(matched, 0, maxControls);
+    const contextCache = new WeakMap();
+    const matched = options.query ? matchingElements(options.query, doc) : doc.querySelectorAll(selector);
+    const elements = Array.prototype.slice.call(matched, offset, offset + maxControls);
     elements.forEach((element, index) => {
       const sensitive = isSensitiveInput(element);
       const elementValue = text(element.value);
+      const identity = elementIdentity(element, doc), visible = elementVisible(element);
+      const context = visible ? ancestorContext(element, identity, contextCache) : null;
       controls.push({
-        ...elementIdentity(element, doc),
-        index,
+        ...identity,
+        ...(context ? { context } : {}),
+        index: offset + index,
         sensitive,
         ...(sensitive ? { valueLength: elementValue.length } : {}),
         disabled: elementDisabled(element),
-        visible: elementVisible(element),
+        visible,
         editable: elementEditable(element),
         checked: elementChecked(element),
         ariaChecked: element.getAttribute('aria-checked'),
@@ -445,7 +510,10 @@
       bodyTextTruncated: text(doc.body && doc.body.innerText).length > 20000,
       controls,
       controlCount: controls.length,
-      truncated: matched.length > maxControls,
+      totalControls: matched.length, offset,
+      nextOffset: offset + controls.length < matched.length ? offset + controls.length : null,
+      ...(options.query ? { query: options.query } : {}),
+      truncated: offset > 0 || matched.length > controls.length,
       updatedAtMs: Date.now(),
     };
   }
@@ -453,7 +521,7 @@
   function clickElement(args, task) {
     const selected = actionTarget(args, task);
     ensureTarget(selected, task, { hit: true }); task.permit(); selected.element.click();
-    return { ok: true, resolved: selected.ref };
+    return { ok: true, resolved: selected.ref, ...targetWarnings(selected) };
   }
 
   function inputElement(args, task) {
@@ -479,11 +547,31 @@
       element.dispatchEvent(new win.InputEvent('input', { bubbles: true, inputType: 'insertReplacementText', data: value }));
       ensureTarget(selected, task, { focus: true, edited: true }); task.permit(); element.dispatchEvent(new win.Event('change', { bubbles: true }));
     }
-    ensureTarget(selected, task, { edited: true });
+    // The final input/change dispatch may collapse, disable or remove its own
+    // editor. Report what happened to that original object; never reselect it
+    // or treat its resulting layout as a precondition for the completed input.
+    const targetState = { samePage: samePage(task.pageRef(), selected.ref.pageRef),
+      sameDocument: element.ownerDocument === globalValue('document'), connected: element.isConnected,
+      visible: elementVisible(element), disabled: elementDisabled(element), editable: elementEditable(element),
+      focused: doc.activeElement === element, bounds: bounds(element) };
+    if (!targetState.samePage || !targetState.sameDocument || !targetState.connected) {
+      return { ok: false, error: targetState.samePage ? 'web_element_changed' : 'reobserve_required',
+        resolved: selected.ref, inputDispatched: true, valueState: 'unconfirmed', targetState,
+        message: 'Input was dispatched, but its original element is no longer in the current page. Reobserve before continuing.' };
+    }
+    recordTargetChanges(selected, elementIdentity(element), true);
+    const warnings = targetWarnings(selected).warnings ?? [];
+    if (!targetState.visible || targetState.disabled || !targetState.editable || !targetState.focused)
+      warnings.push({ code: 'target_state_changed_after_input',
+        message: 'Input was dispatched; the original element state changed afterwards. Inspect targetState before another action.' });
     const sensitive = isSensitiveInput(element);
     return {
       ok: true,
       resolved: selected.ref,
+      inputDispatched: true,
+      valueState: 'observed',
+      targetState,
+      ...(warnings.length ? { warnings } : {}),
       sensitive,
       valueLength: (richText ? element.innerText : element.value).length,
       ...(sensitive ? {} : { value: richText ? element.innerText : element.value }),
@@ -499,7 +587,7 @@
       key: args.key, code: args.key, bubbles: true, cancelable: true, composed: true,
     });
     task.permit(); element.dispatchEvent(event);
-    return { ok: true, resolved: selected.ref, key: args.key, eventType: 'keydown',
+    return { ok: true, resolved: selected.ref, ...targetWarnings(selected), key: args.key, eventType: 'keydown',
       trusted: event.isTrusted, defaultPrevented: event.defaultPrevented };
   }
 
@@ -539,12 +627,12 @@
     if (args.mode === 'into-view') {
       if (!selected) return { ok: false, error: 'web_selector_required' };
       ensureTarget(selected, task); task.permit(); selected.element.scrollIntoView({ block: 'center', inline: 'center', behavior: 'instant' });
-      return { ok: true, resolved: selected.ref };
+      return { ok: true, resolved: selected.ref, ...targetWarnings(selected) };
     }
     if (args.mode !== 'by') return { ok: false, error: 'invalid_web_scroll_mode' };
     if (selected) {
       ensureTarget(selected, task); task.permit(); selected.element.scrollBy({ left: deltaX, top: deltaY, behavior: 'instant' });
-      return { ok: true, resolved: selected.ref };
+      return { ok: true, resolved: selected.ref, ...targetWarnings(selected) };
     }
     const win = globalValue('window');
     if (win && win.scrollBy) {
@@ -554,20 +642,28 @@
     return { ok: false, error: 'scroll_unavailable' };
   }
 
-  function findElement(selector) {
-    const doc = globalValue('document');
-    if (!doc) return null;
+  function matchingElements(selector, doc) {
     if (!selector || typeof selector !== 'object') throw new Error('invalid_web_selector');
     const identities = ['elementId', 'text', 'ariaLabel', 'css'].filter(key => typeof selector[key] === 'string' && selector[key].length > 0);
-    if (identities.length !== 1) throw new Error('invalid_web_selector');
-    const elements = doc.querySelectorAll(selector.css ?? defaultControlSelector);
-    if (elements.length > 1000) throw new Error('web_selector_scan_limit');
-    const matches = Array.from(elements).filter(element => {
-      if (!elementVisible(element)) return false;
+    if (!identities.length) throw new Error('invalid_web_selector');
+    const known = selector.elementId === undefined ? null : documentTarget(doc).elementsById.get(selector.elementId);
+    const elements = selector.elementId === undefined ? doc.querySelectorAll(selector.css ?? defaultControlSelector) : known ? [known] : [];
+    return Array.from(elements).filter(element => {
+      if (element.ownerDocument !== doc || !elementVisible(element) || selector.elementId !== undefined && selector.css !== undefined && !element.matches(selector.css)) return false;
       const ref = elementIdentity(element);
       return ['elementId', 'text', 'ariaLabel', 'tag', 'role'].every(key => selector[key] === undefined || selector[key] === ref[key]);
     });
-    if (matches.length > 1) throw new Error('web_element_ambiguous');
+  }
+  function findElement(selector) {
+    const doc = globalValue('document');
+    if (!doc) return null;
+    const matches = matchingElements(selector, doc);
+    if (matches.length > 1) throw Object.assign(new Error('web_element_ambiguous'), { details: {
+      selectionRequired: true, matched: matches.length, candidatesTruncated: matches.length > 20,
+      ...(matches.length > 20 ? { candidateQuery: { query: selector, offset: 20, maxControls: 200 } } : {}),
+      candidates: matches.slice(0, 20).map(element => { const ref = elementIdentity(element); return { text: ref.text, tag: ref.tag,
+        role: ref.role, ariaLabel: ref.ariaLabel, disabled: elementDisabled(element), bounds: bounds(element), selector: { elementId: ref.elementId } }; }),
+    } });
     return matches[0] ?? null;
   }
 
@@ -576,8 +672,19 @@
     if (!element) throw new Error('web_element_not_found');
     const ref = { pageRef: task.pageRef(), element: elementIdentity(element) };
     if (args.expectedTarget && (!samePage(ref.pageRef, args.expectedTarget.pageRef)
-      || !elementKeys.every(key => ref.element[key] === args.expectedTarget.element[key]))) throw new Error('reobserve_required');
-    return { element, ref };
+      || ref.element.elementId !== args.expectedTarget.element.elementId)) throw new Error('reobserve_required');
+    const selected = { element, ref, observed: args.expectedTarget?.element ?? ref.element, changedFields: new Set() };
+    recordTargetChanges(selected, ref.element);
+    return selected;
+  }
+  function recordTargetChanges(selected, current, edited = false) {
+    for (const key of elementKeys) {
+      if ((!edited || key !== 'text') && current[key] !== selected.observed[key]) selected.changedFields.add(key);
+    }
+  }
+  function targetWarnings(selected) {
+    return selected.changedFields.size ? { warnings: [{ code: 'target_attributes_changed', fields: [...selected.changedFields],
+      message: 'The observed DOM element is still present; its attributes changed.' }] } : {};
   }
   function samePage(a, b) {
     return ['schemaVersion', 'sessionId', 'runtimeEpoch', 'targetId', 'navigationId', 'url'].every(key => a[key] === b?.[key]);
@@ -590,7 +697,8 @@
     if (!element.isConnected || element.ownerDocument !== doc || elementDisabled(element) || !elementVisible(element)
       || (focus && doc.activeElement !== element)) throw new Error('web_element_changed');
     const current = elementIdentity(element);
-    if (!elementKeys.filter(key => !edited || key !== 'text').every(key => current[key] === ref.element[key])) throw new Error('reobserve_required');
+    if (current.elementId !== ref.element.elementId) throw new Error('reobserve_required');
+    recordTargetChanges(selected, current, edited);
     if (hit) {
       const interaction = interactionState(element);
       if (interaction.status !== 'ready') throw Object.assign(new Error(interaction.status === 'outside-viewport'