@nuanu-ai/agentbrowse 0.2.21 → 0.2.23

package/dist/commands/observe-signals.js

@@ -0,0 +1,461 @@
+import { readFrameHostDescriptor } from './observe-inventory.js';
+const DOM_SIGNAL_COLLECTION_LIMIT = 24;
+function normalizeObservedSignalTextValue(value) {
+    return (value ?? '').replace(/\s+/g, ' ').trim();
+}
+export function inferObservedSignalKindFromFacts(facts) {
+    const role = (facts.role ?? '').trim().toLowerCase();
+    const ariaLive = (facts.ariaLive ?? '').trim().toLowerCase();
+    const classBlob = (facts.classBlob ?? '').toLowerCase();
+    if (role === 'dialog' || facts.ariaModal === true) {
+        return 'dialog';
+    }
+    if (role === 'alert' || ariaLive === 'assertive') {
+        return 'alert';
+    }
+    if (role === 'status' ||
+        ariaLive === 'polite' ||
+        facts.ariaBusy === true ||
+        facts.progressLike === true) {
+        return 'status';
+    }
+    if (/toast|snackbar|banner|notice|warning|error|success/.test(classBlob)) {
+        return 'notice';
+    }
+    return 'notice';
+}
+export function isObservedSignalOutcomeText(text) {
+    const normalized = normalizeObservedSignalTextValue(text);
+    return (normalized.length > 0 &&
+        /(?:thanks|thank you|success(?:ful|fully)?|receipt|order\s+(?:confirmed|complete(?:d)?|successful)|payment\s+(?:successful|complete(?:d)?|confirmed)|purchase\s+(?:complete(?:d)?|successful)|declin(?:e|ed)|fail(?:ed|ure)|error|unable|try again|verification|verify|challenge|captcha|processing|pending|approved|denied)/i.test(normalized));
+}
+export function shouldUseObservedDecoratedNoticeSurface(facts) {
+    if (facts.hasVisibleInteractiveDescendant === true) {
+        return false;
+    }
+    if ((facts.directSemanticTextCount ?? 0) > 0) {
+        return false;
+    }
+    return true;
+}
+export function shouldUseStandaloneOutcomeCandidate(facts) {
+    const normalized = normalizeObservedSignalTextValue(facts.text);
+    const maxLength = Math.max(24, facts.maxLength ?? 160);
+    if (!normalized || normalized.length < 12 || normalized.length > maxLength) {
+        return false;
+    }
+    if (!isObservedSignalOutcomeText(normalized)) {
+        return false;
+    }
+    if (facts.withinKnownSignalSurface === true) {
+        return false;
+    }
+    if (facts.hasVisibleInteractiveDescendant === true) {
+        return false;
+    }
+    if (facts.hasNestedOutcomeCandidate === true) {
+        return false;
+    }
+    return true;
+}
+const OBSERVE_SIGNAL_HELPER_SCRIPT = `
+  const normalizeObservedSignalTextValue = ${normalizeObservedSignalTextValue.toString()};
+  const inferObservedSignalKindFromFacts = ${inferObservedSignalKindFromFacts.toString()};
+  const isObservedSignalOutcomeText = ${isObservedSignalOutcomeText.toString()};
+  const shouldUseObservedDecoratedNoticeSurface = ${shouldUseObservedDecoratedNoticeSurface.toString()};
+  const shouldUseStandaloneOutcomeCandidate = ${shouldUseStandaloneOutcomeCandidate.toString()};
+`;
+function normalizeInheritedFramePath(framePath) {
+    return Array.isArray(framePath) && framePath.length > 0 ? [...framePath] : undefined;
+}
+function normalizeInheritedFrameUrl(frameUrl) {
+    return typeof frameUrl === 'string' && frameUrl.trim().length > 0 ? frameUrl : undefined;
+}
+function applyInheritedSignalMetadata(signal, options) {
+    const normalizedFramePath = normalizeInheritedFramePath(options?.framePath) ??
+        normalizeInheritedFramePath(signal.framePath);
+    const normalizedFrameUrl = normalizeInheritedFrameUrl(options?.frameUrl) ?? signal.frameUrl;
+    return {
+        ...signal,
+        framePath: normalizedFramePath,
+        frameUrl: normalizedFrameUrl,
+    };
+}
+async function collectPageSignalsFromDocument(context, options) {
+    const inheritedFramePath = JSON.stringify(options?.framePath ?? []);
+    const inheritedFrameUrl = JSON.stringify(options?.frameUrl ?? '');
+    const observedSignals = await context.evaluate(String.raw `(() => {
+    const inheritedFramePath = ${inheritedFramePath};
+    const inheritedFrameUrl = ${inheritedFrameUrl};
+    const limit = ${DOM_SIGNAL_COLLECTION_LIMIT};
+    ${OBSERVE_SIGNAL_HELPER_SCRIPT}
+    const interactiveSelector =
+      'button, a[href], input:not([type="hidden"]), textarea, select, [role="button"], [role="link"], [role="textbox"], [role="combobox"], [role="option"], [role="gridcell"], [contenteditable="true"], [tabindex]:not([tabindex="-1"])';
+    const semanticSurfaceSelector = [
+      '[role="alert"]',
+      '[role="status"]',
+      '[aria-live="assertive"]',
+      '[aria-live="polite"]',
+      '[role="dialog"]',
+      '[aria-modal="true"]',
+      '[aria-busy="true"]',
+      '[role="progressbar"]',
+    ].join(', ');
+    const decoratedNoticeSelector = [
+      '[class*="toast"]',
+      '[class*="snackbar"]',
+      '[class*="banner"]',
+      '[class*="notice"]',
+      '[class*="success"]',
+      '[class*="error"]',
+      '[class*="warning"]',
+      '[data-testid*="toast"]',
+      '[data-testid*="banner"]',
+      '[data-testid*="alert"]',
+      '[data-testid*="success"]',
+      '[data-testid*="error"]',
+    ].join(', ');
+    const directSemanticTextSelector =
+      'h1, h2, h3, h4, h5, h6, [role="heading"], legend, p, [role="alert"], [role="status"]';
+    const standaloneOutcomeSelector = 'h1, h2, h3, [role="heading"], p';
+
+    const normalizeText = (value) => (value ?? '').replace(/\s+/g, ' ').trim();
+    const sampleText = (value, maxLength) => {
+      const normalized = normalizeText(value);
+      if (!normalized) {
+        return '';
+      }
+      if (normalized.length <= maxLength) {
+        return normalized;
+      }
+      return normalized.slice(0, maxLength - 1).trimEnd() + '…';
+    };
+    const joinUniqueText = (values, maxLength = 240) => {
+      const seen = new Set();
+      const accepted = [];
+      for (const value of values) {
+        const normalized = sampleText(value, maxLength);
+        if (!normalized) {
+          continue;
+        }
+        const key = normalized.toLowerCase();
+        if (seen.has(key)) {
+          continue;
+        }
+        seen.add(key);
+        accepted.push(normalized);
+      }
+      return sampleText(accepted.join(' '), maxLength);
+    };
+    const visibleTextOf = (element, maxLength = 160) => {
+      if (!(element instanceof HTMLElement)) {
+        return '';
+      }
+      const rawInnerText = typeof element.innerText === 'string' ? element.innerText : undefined;
+      const preferredText = rawInnerText !== undefined ? rawInnerText : element.textContent || '';
+      return sampleText(preferredText, maxLength);
+    };
+    const isVisible = (element) => {
+      if (!(element instanceof HTMLElement)) {
+        return false;
+      }
+      const style = element.ownerDocument?.defaultView?.getComputedStyle(element);
+      if (!style || style.display === 'none' || style.visibility === 'hidden') {
+        return false;
+      }
+      const rect = element.getBoundingClientRect();
+      return rect.width > 0 && rect.height > 0;
+    };
+    const relationTextOf = (element, attribute) => {
+      if (!(element instanceof HTMLElement)) {
+        return '';
+      }
+      const relation = element.getAttribute(attribute)?.trim();
+      if (!relation) {
+        return '';
+      }
+      return sampleText(
+        relation
+          .split(/\s+/)
+          .map((id) => {
+            const relatedElement = document.getElementById(id);
+            return relatedElement instanceof HTMLElement
+              ? relatedElement.innerText || relatedElement.textContent || ''
+              : '';
+          })
+          .join(' '),
+        240
+      );
+    };
+    const hasVisibleInteractiveDescendant = (element) => {
+      if (!(element instanceof HTMLElement)) {
+        return false;
+      }
+      return Array.from(element.querySelectorAll(interactiveSelector)).some(
+        (candidate) => candidate !== element && candidate instanceof HTMLElement && isVisible(candidate)
+      );
+    };
+    const textOf = (element) => visibleTextOf(element, 160);
+    const directSemanticTextsOf = (element) => {
+      if (!(element instanceof HTMLElement)) {
+        return [];
+      }
+
+      if (element.matches(directSemanticTextSelector)) {
+        return [textOf(element)];
+      }
+
+      const directChildren = Array.from(element.children)
+        .filter(
+          (candidate) =>
+            candidate instanceof HTMLElement &&
+            candidate.matches(directSemanticTextSelector) &&
+            isVisible(candidate)
+        )
+        .slice(0, 3);
+
+      return directChildren.map((candidate) => textOf(candidate));
+    };
+    const nonInteractiveSurfaceTextOf = (element) => {
+      if (!(element instanceof HTMLElement)) {
+        return '';
+      }
+
+      const clone = element.cloneNode(true);
+      if (!(clone instanceof HTMLElement)) {
+        return '';
+      }
+
+      for (const nested of Array.from(
+        clone.querySelectorAll(
+          [
+            interactiveSelector,
+            semanticSurfaceSelector,
+            decoratedNoticeSelector,
+            '[hidden]',
+            '[aria-hidden="true"]',
+          ].join(', ')
+        )
+      )) {
+        if (nested instanceof HTMLElement) {
+          nested.remove();
+        }
+      }
+
+      return visibleTextOf(clone, 160);
+    };
+    const signalTextOf = (element) => {
+      if (!(element instanceof HTMLElement)) {
+        return '';
+      }
+
+      return (
+        joinUniqueText(
+          [
+            element.getAttribute('aria-label'),
+            relationTextOf(element, 'aria-labelledby'),
+            element.getAttribute('title'),
+            relationTextOf(element, 'aria-describedby'),
+            ...directSemanticTextsOf(element),
+            nonInteractiveSurfaceTextOf(element),
+          ],
+          240
+        ) || ''
+      );
+    };
+    const isKnownDecoratedNoticeSurface = (element) => {
+      if (!(element instanceof HTMLElement) || !element.matches(decoratedNoticeSelector) || !isVisible(element)) {
+        return false;
+      }
+      return shouldUseObservedDecoratedNoticeSurface({
+        hasVisibleInteractiveDescendant: hasVisibleInteractiveDescendant(element),
+        directSemanticTextCount: directSemanticTextsOf(element).length,
+      });
+    };
+    const isWithinKnownSignalSurface = (element) => {
+      if (!(element instanceof HTMLElement)) {
+        return false;
+      }
+      let current = element.parentElement;
+      while (current instanceof HTMLElement) {
+        if (current.matches(semanticSurfaceSelector) && isVisible(current)) {
+          return true;
+        }
+        if (isKnownDecoratedNoticeSurface(current)) {
+          return true;
+        }
+        current = current.parentElement;
+      }
+      return false;
+    };
+
+    const seen = new Set();
+    const signals = [];
+    const pushSignal = (kind, text) => {
+      const normalized = sampleText(text, 240);
+      if (!normalized) {
+        return;
+      }
+      const key = kind + '|' + normalized.toLowerCase();
+      if (seen.has(key)) {
+        return;
+      }
+      seen.add(key);
+      signals.push({
+        kind,
+        text: normalized,
+        framePath: inheritedFramePath.length > 0 ? inheritedFramePath : undefined,
+        frameUrl: inheritedFrameUrl || undefined,
+        source: 'dom',
+      });
+      if (signals.length > limit) {
+        signals.length = limit;
+      }
+    };
+
+    for (const element of Array.from(document.querySelectorAll(semanticSurfaceSelector))) {
+      if (!(element instanceof HTMLElement) || !isVisible(element)) {
+        continue;
+      }
+      const text = signalTextOf(element);
+      if (!text) {
+        continue;
+      }
+      const classBlob =
+        ((element.getAttribute('class') || '') + ' ' + Object.values(element.dataset || {}).join(' ')).toLowerCase();
+      pushSignal(
+        inferObservedSignalKindFromFacts({
+          role: element.getAttribute('role') || undefined,
+          ariaLive: element.getAttribute('aria-live') || undefined,
+          ariaModal: element.getAttribute('aria-modal') === 'true',
+          ariaBusy: element.hasAttribute('aria-busy'),
+          progressLike: element.matches('[role="progressbar"]'),
+          classBlob,
+        }),
+        text
+      );
+      if (signals.length >= limit) {
+        return signals;
+      }
+    }
+
+    for (const element of Array.from(document.querySelectorAll(decoratedNoticeSelector))) {
+      if (!(element instanceof HTMLElement) || !isVisible(element)) {
+        continue;
+      }
+      if (element.matches(semanticSurfaceSelector)) {
+        continue;
+      }
+      if (
+        !shouldUseObservedDecoratedNoticeSurface({
+          hasVisibleInteractiveDescendant: hasVisibleInteractiveDescendant(element),
+          directSemanticTextCount: directSemanticTextsOf(element).length,
+        })
+      ) {
+        continue;
+      }
+      const text = signalTextOf(element);
+      if (!text) {
+        continue;
+      }
+      pushSignal('notice', text);
+      if (signals.length >= limit) {
+        break;
+      }
+    }
+
+    for (const element of Array.from(document.querySelectorAll(standaloneOutcomeSelector))) {
+      if (!(element instanceof HTMLElement) || !isVisible(element)) {
+        continue;
+      }
+      const text = signalTextOf(element);
+      if (
+        !shouldUseStandaloneOutcomeCandidate({
+          text,
+          withinKnownSignalSurface: isWithinKnownSignalSurface(element),
+          hasVisibleInteractiveDescendant: hasVisibleInteractiveDescendant(element),
+          hasNestedOutcomeCandidate: false,
+          maxLength: 160,
+        })
+      ) {
+        continue;
+      }
+      pushSignal('notice', text);
+      if (signals.length >= limit) {
+        break;
+      }
+    }
+
+    return signals;
+  })()`);
+    if (!Array.isArray(observedSignals)) {
+        return [];
+    }
+    return observedSignals
+        .filter((signal) => Boolean(signal &&
+        typeof signal === 'object' &&
+        typeof signal.kind === 'string' &&
+        typeof signal.text === 'string' &&
+        signal.text.length > 0))
+        .map((signal) => applyInheritedSignalMetadata(signal, {
+        framePath: options?.framePath,
+        frameUrl: options?.frameUrl,
+    }));
+}
+export async function collectPageSignals(page) {
+    if (typeof page.mainFrame !== 'function') {
+        return collectPageSignalsFromDocument(page);
+    }
+    const collected = [];
+    const seen = new Set();
+    const pushSignals = (signals) => {
+        for (const signal of signals) {
+            const key = [
+                signal.kind,
+                signal.text.toLowerCase(),
+                signal.framePath?.join('>') ?? 'top',
+            ].join('|');
+            if (seen.has(key)) {
+                continue;
+            }
+            seen.add(key);
+            collected.push(signal);
+            if (collected.length >= DOM_SIGNAL_COLLECTION_LIMIT) {
+                break;
+            }
+        }
+    };
+    const walk = async (frame, framePath) => {
+        if (collected.length >= DOM_SIGNAL_COLLECTION_LIMIT) {
+            return;
+        }
+        const frameUrl = frame.url().trim() || undefined;
+        const signals = await collectPageSignalsFromDocument(frame, {
+            framePath,
+            frameUrl,
+        }).catch(() => []);
+        pushSignals(signals);
+        if (collected.length >= DOM_SIGNAL_COLLECTION_LIMIT) {
+            return;
+        }
+        for (const childFrame of frame.childFrames().slice(0, 20)) {
+            if (collected.length >= DOM_SIGNAL_COLLECTION_LIMIT) {
+                break;
+            }
+            const frameHost = await readFrameHostDescriptor(childFrame);
+            if (!frameHost?.selector || !frameHost.userVisible) {
+                continue;
+            }
+            await walk(childFrame, [...(framePath ?? []), frameHost.selector]);
+        }
+    };
+    await walk(page.mainFrame());
+    return collected;
+}
+export const __testObserveSignals = {
+    collectPageSignalsFromDocument,
+    inferObservedSignalKindFromFacts,
+    isObservedSignalOutcomeText,
+    shouldUseObservedDecoratedNoticeSurface,
+    shouldUseStandaloneOutcomeCandidate,
+};

package/dist/commands/observe-stagehand.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"observe-stagehand.d.ts","sourceRoot":"","sources":["../../src/commands/observe-stagehand.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,iBAAiB,CAAC;AAQ5C,OAAO,EAAE,0BAA0B,EAAE,4BAA4B,EAAE,MAAM,wBAAwB,CAAC;AAElG,KAAK,sBAAsB,GAAG;IAC5B,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,SAAS,CAAC,EAAE,OAAO,EAAE,CAAC;CACvB,CAAC;AAgBF,wBAAsB,qBAAqB,CACzC,OAAO,EAAE,MAAM,EACf,MAAM,EAAE,sBAAsB,EAC9B,IAAI,EAAE,IAAI,EACV,aAAa,CAAC,EAAE,MAAM;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA8FvB;AAED,eAAO,MAAM,yBAAyB;;;;CAIrC,CAAC"}
+{"version":3,"file":"observe-stagehand.d.ts","sourceRoot":"","sources":["../../src/commands/observe-stagehand.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,iBAAiB,CAAC;AAS5C,OAAO,EAAE,0BAA0B,EAAE,4BAA4B,EAAE,MAAM,wBAAwB,CAAC;AAElG,KAAK,sBAAsB,GAAG;IAC5B,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,SAAS,CAAC,EAAE,OAAO,EAAE,CAAC;CACvB,CAAC;AAEF,wBAAsB,qBAAqB,CACzC,OAAO,EAAE,MAAM,EACf,MAAM,EAAE,sBAAsB,EAC9B,IAAI,EAAE,IAAI,EACV,aAAa,CAAC,EAAE,MAAM;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA2FvB;AAED,eAAO,MAAM,yBAAyB;;;;CAIrC,CAAC"}

package/dist/commands/observe-stagehand.js

@@ -1,18 +1,7 @@
 import { inferAcceptancePolicyFromFacts, inferAllowedActionsFromFacts, inferAvailabilityFromFacts, inferControlFamilyFromFacts, } from '../control-semantics.js';
 import { buildLocator, resolveLocatorRoot } from './action-fallbacks.js';
+import { buildObserveDisplayLabel, normalizeObserveLiveValue } from './observe-display-label.js';
 import { normalizeStagehandSelector, readStagehandLocatorSnapshot } from './observe-inventory.js';
-function normalizeStagehandLiveValue(value) {
-    const normalized = value?.replace(/\s+/g, ' ').trim();
-    if (!normalized || normalized.length > 80) {
-        return undefined;
-    }
-    return normalized;
-}
-function stagehandLabelIncludesValue(label, value) {
-    const normalizedLabel = label.replace(/\s+/g, ' ').trim().toLowerCase();
-    const normalizedValue = value.replace(/\s+/g, ' ').trim().toLowerCase();
-    return Boolean(normalizedValue) && normalizedLabel.includes(normalizedValue);
-}
 export async function toStagehandDescriptor(pageRef, action, page, pageSignature) {
     const normalizedSelector = normalizeStagehandSelector(typeof action.selector === 'string' ? action.selector.trim() : '');
     const selector = normalizedSelector.selector;
@@ -54,13 +43,11 @@ export async function toStagehandDescriptor(pageRef, action, page, pageSignature
     const allowedActions = inferAllowedActionsFromFacts(facts);
     const acceptancePolicy = inferAcceptancePolicyFromFacts(facts, allowedActions);
     const controlFamily = inferControlFamilyFromFacts(facts, allowedActions);
-    const liveValue = normalizeStagehandLiveValue(domFacts?.currentValue) ??
-        normalizeStagehandLiveValue(domFacts?.value) ??
-        normalizeStagehandLiveValue(domFacts?.text);
+    const liveValue = normalizeObserveLiveValue(domFacts?.currentValue) ??
+        normalizeObserveLiveValue(domFacts?.value) ??
+        normalizeObserveLiveValue(domFacts?.text);
     const baseLabel = action.description ?? action.method ?? 'observed target';
-    const displayLabel = liveValue && !stagehandLabelIncludesValue(baseLabel, liveValue)
-        ? `${baseLabel} — ${liveValue}`
-        : undefined;
+    const displayLabel = buildObserveDisplayLabel(baseLabel, liveValue);
     const actionable = !shadowOrUnsupported && locatorCandidates.length > 0 && allowedActions.length > 0;
     const availability = inferAvailabilityFromFacts(domFacts?.states, undefined, {
         readonlyInteractive: controlFamily === 'select' ||

package/dist/commands/observe-surfaces.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"observe-surfaces.d.ts","sourceRoot":"","sources":["../../src/commands/observe-surfaces.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EACV,gBAAgB,EAChB,iBAAiB,EACjB,aAAa,EACb,gBAAgB,EACjB,MAAM,qBAAqB,CAAC;AAE7B,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,wBAAwB,CAAC;AAgBhE,wBAAgB,qBAAqB,CACnC,MAAM,EAAE,aAAa,CAAC,iBAAiB,CAAC,EACxC,OAAO,EAAE,aAAa,CACpB,IAAI,CAAC,gBAAgB,EAAE,YAAY,GAAG,gBAAgB,GAAG,kBAAkB,CAAC,CAC7E,EACD,kBAAkB,CAAC,EAAE,WAAW,CAAC,MAAM,CAAC,GACvC,iBAAiB,EAAE,CA+CrB;AAED,wBAAgB,eAAe,CAAC,MAAM,EAAE,iBAAiB,GAAG,MAAM,GAAG,SAAS,CAa7E;AAyBD,wBAAgB,2BAA2B,CACzC,MAAM,EAAE,IAAI,CAAC,iBAAiB,EAAE,eAAe,GAAG,WAAW,GAAG,UAAU,GAAG,cAAc,GAAG,SAAS,CAAC,GACvG,MAAM,GAAG,SAAS,CAiBpB;AAED,wBAAgB,qBAAqB,CACnC,OAAO,EAAE,IAAI,CAAC,iBAAiB,EAAE,SAAS,GAAG,eAAe,GAAG,WAAW,GAAG,MAAM,GAAG,OAAO,CAAC,GAC7F,MAAM,CAMR;AAED,wBAAgB,gBAAgB,CAAC,OAAO,EAAE,aAAa,GAAG,SAAS,GAAG,MAAM,GAAG,SAAS,CA4CvF;AAED,wBAAgB,0BAA0B,CACxC,GAAG,EAAE,gBAAgB,EAAE,EACvB,SAAS,EAAE,gBAAgB,GAAG,SAAS,GACtC,IAAI,CAON;AAwDD,wBAAgB,yBAAyB,CACvC,OAAO,EAAE,MAAM,EACf,OAAO,EAAE,aAAa,CAAC,iBAAiB,CAAC,oCAoE1C"}
+{"version":3,"file":"observe-surfaces.d.ts","sourceRoot":"","sources":["../../src/commands/observe-surfaces.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EACV,gBAAgB,EAChB,iBAAiB,EACjB,aAAa,EACb,gBAAgB,EACjB,MAAM,qBAAqB,CAAC;AAE7B,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,wBAAwB,CAAC;AAgBhE,wBAAgB,qBAAqB,CACnC,MAAM,EAAE,aAAa,CAAC,iBAAiB,CAAC,EACxC,OAAO,EAAE,aAAa,CACpB,IAAI,CAAC,gBAAgB,EAAE,YAAY,GAAG,gBAAgB,GAAG,kBAAkB,CAAC,CAC7E,EACD,kBAAkB,CAAC,EAAE,WAAW,CAAC,MAAM,CAAC,GACvC,iBAAiB,EAAE,CAiDrB;AAED,wBAAgB,eAAe,CAAC,MAAM,EAAE,iBAAiB,GAAG,MAAM,GAAG,SAAS,CAa7E;AAyBD,wBAAgB,2BAA2B,CACzC,MAAM,EAAE,IAAI,CACV,iBAAiB,EACjB,eAAe,GAAG,WAAW,GAAG,UAAU,GAAG,cAAc,GAAG,SAAS,CACxE,GACA,MAAM,GAAG,SAAS,CAkBpB;AAED,wBAAgB,qBAAqB,CACnC,OAAO,EAAE,IAAI,CAAC,iBAAiB,EAAE,SAAS,GAAG,eAAe,GAAG,WAAW,GAAG,MAAM,GAAG,OAAO,CAAC,GAC7F,MAAM,CAMR;AAED,wBAAgB,gBAAgB,CAAC,OAAO,EAAE,aAAa,GAAG,SAAS,GAAG,MAAM,GAAG,SAAS,CA4CvF;AAED,wBAAgB,0BAA0B,CACxC,GAAG,EAAE,gBAAgB,EAAE,EACvB,SAAS,EAAE,gBAAgB,GAAG,SAAS,GACtC,IAAI,CAON;AAwDD,wBAAgB,yBAAyB,CACvC,OAAO,EAAE,MAAM,EACf,OAAO,EAAE,aAAa,CAAC,iBAAiB,CAAC,oCAoE1C"}

package/dist/commands/observe-surfaces.js

@@ -90,13 +90,14 @@ export function buildSyntheticFormSurfaceId(target) {
     if (!pageKey) {
         return undefined;
     }
-    const formKey = normalizeSurfaceText(target.formSelector) ??
-        semanticFormContextLabel(target)?.toLowerCase();
+    const formKey = normalizeSurfaceText(target.formSelector) ?? semanticFormContextLabel(target)?.toLowerCase();
     if (!formKey) {
         return undefined;
     }
     const frameKey = target.framePath?.join('>') ??
-        (normalizeSurfaceText(target.frameUrl) ? `url:${normalizeSurfaceText(target.frameUrl)}` : 'top');
+        (normalizeSurfaceText(target.frameUrl)
+            ? `url:${normalizeSurfaceText(target.frameUrl)}`
+            : 'top');
     return `${pageKey}|${frameKey}|form|${formKey}`;
 }
 export function surfaceReplacementKey(surface) {
@@ -219,7 +220,7 @@ export function collectSurfaceDescriptors(pageRef, targets) {
             kind: existing?.kind ?? target.surfaceKind ?? 'surface',
             label: existing?.label && existing.label !== 'Active surface'
                 ? existing.label
-                : target.surfaceLabel ?? target.surfaceKind ?? existing?.label ?? 'Active surface',
+                : (target.surfaceLabel ?? target.surfaceKind ?? existing?.label ?? 'Active surface'),
             lifecycle: 'live',
             availability: { state: 'available' },
             locatorCandidates,

package/dist/commands/observe.d.ts

@@ -10,12 +10,6 @@ export declare const __testDomTargetCollection: {
         framePath?: string[];
         frameUrl?: string;
     }) => Promise<import("./observe-inventory.js").DomObservedTarget[]>;
-    collectPageSignalsFromDocument: (context: {
-        evaluate<T>(pageFunction: string): Promise<T>;
-    }, options?: {
-        framePath?: string[];
-        frameUrl?: string;
-    }) => Promise<import("./observe-inventory.js").ObservedPageSignal[]>;
     inferStructuredCellVariantFromEvidence: typeof import("./observe-inventory.js").inferStructuredCellVariantFromEvidence;
     locatorDomSignatureScript: string;
 };

package/dist/commands/observe.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"observe.d.ts","sourceRoot":"","sources":["../../src/commands/observe.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,eAAe,CAAC;AAoDnD,OAAO,EAAE,qBAAqB,EAAE,MAAM,wBAAwB,CAAC;AAU/D,eAAO,MAAM,yBAAyB;;;;iBA+Ri2G,CAAC;gBAAwB,CAAC;;;;;iBAAg1mF,CAAC;gBAAwB,CAAC;;;;CA/RtstF,CAAC;AACtE,eAAO,MAAM,yBAAyB;;;;;;;;;;;;;;;;;;;;;;;;;;;;CAGrC,CAAC;AAEF,wBAAsB,OAAO,CAAC,OAAO,EAAE,aAAa,EAAE,WAAW,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAwRzF"}
+{"version":3,"file":"observe.d.ts","sourceRoot":"","sources":["../../src/commands/observe.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,eAAe,CAAC;AAqDnD,OAAO,EAAE,qBAAqB,EAAE,MAAM,wBAAwB,CAAC;AAU/D,eAAO,MAAM,yBAAyB;;;;iBA4TwQ,CAAC;gBAAwB,CAAC;;;;CA5TnQ,CAAC;AACtE,eAAO,MAAM,yBAAyB;;;;;;;;;;;;;;;;;;;;;;;;;;;;CAGrC,CAAC;AAEF,wBAAsB,OAAO,CAAC,OAAO,EAAE,aAAa,EAAE,WAAW,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAqTzF"}

package/dist/commands/observe.js

@@ -8,7 +8,9 @@ import { outputContractFailure, outputJSON } from '../output.js';
 import { domRuntimeResolution, stagehandRuntimeResolution } from '../runtime-resolution.js';
 import { withStagehand } from '../stagehand-runtime.js';
 import { normalizePageSignature } from './descriptor-validation.js';
-import { collectDomTargets, collectPageSignals, __testDomTargetCollection as inventoryDomTargetCollection, __testStagehandDescriptor as inventoryStagehandDescriptor, } from './observe-inventory.js';
+import { collectDomTargets, __testDomTargetCollection as inventoryDomTargetCollection, __testStagehandDescriptor as inventoryStagehandDescriptor, } from './observe-inventory.js';
+import { enrichDomTargetsWithAccessibility } from './observe-accessibility.js';
+import { collectPageSignals } from './observe-signals.js';
 import { attachObservedTargetOwners, linkObservedSurfaceGraph, reconcileObservedTargetsForPage, persistObservedSurfacesForPage, toDomDescriptor, } from './observe-persistence.js';
 import { clearProtectedFillableFormsForPage, persistProtectedFillableFormsForPage, } from './observe-protected.js';
 import { classifyObservePageState, shouldSuppressFillableFormsForObserve, } from './observe-page-state.js';
@@ -34,7 +36,18 @@ export async function observe(session, instruction) {
             browser = await connectPlaywright(session.cdpUrl);
             const page = await resolvePageByRef(browser, session, pageRef);
             const { url, title } = await syncSessionPage(session, pageRef, page);
-            const domTargets = compressSemanticallyDuplicateTargets(orderBySurfaceCompetition(annotateDomTargets(await collectDomTargets(page))));
+            const collectedTargets = await collectDomTargets(page);
+            let observeAccessibilityStats;
+            const domTargets = compressSemanticallyDuplicateTargets(orderBySurfaceCompetition(annotateDomTargets(await enrichDomTargetsWithAccessibility(page, collectedTargets, {
+                onStats: (stats) => {
+                    observeAccessibilityStats = stats;
+                },
+            }))));
+            if (observeAccessibilityStats) {
+                incrementMetric(session, 'observeAxAttempts', observeAccessibilityStats.axAttempts);
+                incrementMetric(session, 'observeAxHits', observeAccessibilityStats.axHits);
+                incrementMetric(session, 'observeFallbackUses', observeAccessibilityStats.fallbackUses);
+            }
             const signals = compactSignals(await collectPageSignals(page).catch(() => []));
             const pageState = classifyObservePageState(signals);
             const persisted = persistObservedSurfacesForPage(session, pageRef, domTargets);
@@ -85,7 +98,20 @@ export async function observe(session, instruction) {
             }
             const page = await resolvePageByRef(browser, session, pageRef);
             const { url, title } = await syncSessionPage(session, pageRef, page);
-            const domTargets = compressSemanticallyDuplicateTargets(orderBySurfaceCompetition(annotateDomTargets(await collectDomTargets(page, { includeActivationAffordances: true }))));
+            const collectedTargets = await collectDomTargets(page, {
+                includeActivationAffordances: true,
+            });
+            let observeAccessibilityStats;
+            const domTargets = compressSemanticallyDuplicateTargets(orderBySurfaceCompetition(annotateDomTargets(await enrichDomTargetsWithAccessibility(page, collectedTargets, {
+                onStats: (stats) => {
+                    observeAccessibilityStats = stats;
+                },
+            }))));
+            if (observeAccessibilityStats) {
+                incrementMetric(session, 'observeAxAttempts', observeAccessibilityStats.axAttempts);
+                incrementMetric(session, 'observeAxHits', observeAccessibilityStats.axHits);
+                incrementMetric(session, 'observeFallbackUses', observeAccessibilityStats.fallbackUses);
+            }
             const signals = compactSignals(await collectPageSignals(page).catch(() => []));
             const pageState = classifyObservePageState(signals);
             const surfaceInputs = collectSurfaceDescriptors(pageRef, domTargets);
@@ -149,9 +175,7 @@ export async function observe(session, instruction) {
                 });
                 return;
             }
-            else {
-                stagehandFallbackReason = 'deterministic-observe-empty';
-            }
+            stagehandFallbackReason = 'deterministic-observe-empty';
         }
         catch (err) {
             domPassError = err instanceof Error ? err.message : String(err);

package/dist/commands/observe.test-harness.d.ts

@@ -17,12 +17,6 @@ export declare const observe: typeof import("./observe.js").observe, __testDomTa
         framePath?: string[];
         frameUrl?: string;
     }) => Promise<import("./observe-inventory.js").DomObservedTarget[]>;
-    collectPageSignalsFromDocument: (context: {
-        evaluate<T>(pageFunction: string): Promise<T>;
-    }, options?: {
-        framePath?: string[];
-        frameUrl?: string;
-    }) => Promise<import("./observe-inventory.js").ObservedPageSignal[]>;
     inferStructuredCellVariantFromEvidence: typeof import("./observe-inventory.js").inferStructuredCellVariantFromEvidence;
     locatorDomSignatureScript: string;
 }, __testStagehandDescriptor: {

package/dist/commands/observe.test-harness.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"observe.test-harness.d.ts","sourceRoot":"","sources":["../../src/commands/observe.test-harness.ts"],"names":[],"mappings":"AAAA,OAAO,EAAyB,EAAE,EAAE,MAAM,QAAQ,CAAC;AACnD,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,eAAe,CAAC;AAGnD,KAAK,MAAM,GAAG,UAAU,CAAC,OAAO,EAAE,CAAC,EAAE,CAAC,CAAC;AA0EvC,eAAO,MAAM,kBAAkB,EAAE,MAAiC,CAAC;AACnE,eAAO,MAAM,oBAAoB,EAAE,MAAmC,CAAC;AACvE,eAAO,MAAM,yBAAyB,EAAE,MAAwC,CAAC;AACjF,eAAO,MAAM,eAAe,EAAE,MAA8B,CAAC;AAC7D,eAAO,MAAM,iBAAiB,EAAE,MAAgC,CAAC;AACjE,eAAO,MAAM,cAAc,EAAE,MAA6B,CAAC;AAC3D,eAAO,MAAM,2BAA2B,EAAE,MAA0C,CAAC;AACrF,eAAO,MAAM,eAAe,EAAE,MAA8B,CAAC;AAC7D,eAAO,MAAM,qBAAqB,EAAE,OAAO,EAAgC,CAAC;AA+B5E,eAAO,MAAQ,OAAO,yCAAE,yBAAyB;;;;iBAqE0kT,CAAC;gBAAwB,CAAC;;;;;iBAAg1mF,CAAC;gBAAwB,CAAC;;;;GArE585F,yBAAyB;;;;;;;;;;;;;;;;;;;;;;;;;;;;CAE3E,CAAC;AACF,eAAO,MAAQ,0BAA0B;;;;;CAA+C,CAAC;AAMzF,wBAAgB,WAAW,IAAI,aAAa,CAW3C;AAYD,wBAAgB,oBAAoB,CAAC,CAAC,SAAS,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EACpE,IAAI,EAAE,CAAC,EACP,YAAY,SAAkC,GAC7C,CAAC,GAAG;IAAE,OAAO,EAAE,UAAU,CAAC,OAAO,EAAE,CAAC,EAAE,CAAC,CAAA;CAAE,CAM3C;AAED,wBAAgB,uBAAuB,IAAI,IAAI,CAyB9C"}
+{"version":3,"file":"observe.test-harness.d.ts","sourceRoot":"","sources":["../../src/commands/observe.test-harness.ts"],"names":[],"mappings":"AAAA,OAAO,EAAyB,EAAE,EAAE,MAAM,QAAQ,CAAC;AACnD,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,eAAe,CAAC;AAGnD,KAAK,MAAM,GAAG,UAAU,CAAC,OAAO,EAAE,CAAC,EAAE,CAAC,CAAC;AA0EvC,eAAO,MAAM,kBAAkB,EAAE,MAAiC,CAAC;AACnE,eAAO,MAAM,oBAAoB,EAAE,MAAmC,CAAC;AACvE,eAAO,MAAM,yBAAyB,EAAE,MAAwC,CAAC;AACjF,eAAO,MAAM,eAAe,EAAE,MAA8B,CAAC;AAC7D,eAAO,MAAM,iBAAiB,EAAE,MAAgC,CAAC;AACjE,eAAO,MAAM,cAAc,EAAE,MAA6B,CAAC;AAC3D,eAAO,MAAM,2BAA2B,EAAE,MAA0C,CAAC;AACrF,eAAO,MAAM,eAAe,EAAE,MAA8B,CAAC;AAC7D,eAAO,MAAM,qBAAqB,EAAE,OAAO,EAAgC,CAAC;AA+B5E,eAAO,MAAQ,OAAO,yCAAE,yBAAyB;;;;iBAqEw7P,CAAC;gBAAwB,CAAC;;;;GArEh9P,yBAAyB;;;;;;;;;;;;;;;;;;;;;;;;;;;;CAE3E,CAAC;AACF,eAAO,MAAQ,0BAA0B;;;;;CAA+C,CAAC;AAMzF,wBAAgB,WAAW,IAAI,aAAa,CAW3C;AAYD,wBAAgB,oBAAoB,CAAC,CAAC,SAAS,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EACpE,IAAI,EAAE,CAAC,EACP,YAAY,SAAkC,GAC7C,CAAC,GAAG;IAAE,OAAO,EAAE,UAAU,CAAC,OAAO,EAAE,CAAC,EAAE,CAAC,CAAA;CAAE,CAM3C;AAED,wBAAgB,uBAAuB,IAAI,IAAI,CAyB9C"}

package/dist/commands/screenshot.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"screenshot.d.ts","sourceRoot":"","sources":["../../src/commands/screenshot.ts"],"names":[],"mappings":"AAAA;;GAEG;AAGH,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,eAAe,CAAC;AAYnD,wBAAsB,UAAU,CAAC,OAAO,EAAE,aAAa,EAAE,QAAQ,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAqDzF"}
+{"version":3,"file":"screenshot.d.ts","sourceRoot":"","sources":["../../src/commands/screenshot.ts"],"names":[],"mappings":"AAAA;;GAEG;AAGH,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,eAAe,CAAC;AAYnD,wBAAsB,UAAU,CAAC,OAAO,EAAE,aAAa,EAAE,QAAQ,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAsDzF"}

package/dist/commands/select-action-executor.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"select-action-executor.d.ts","sourceRoot":"","sources":["../../src/commands/select-action-executor.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,YAAY,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM,iBAAiB,CAAC;AACnE,OAAO,EAEL,KAAK,qBAAqB,EAC3B,MAAM,8BAA8B,CAAC;AAEtC,KAAK,WAAW,GAAG,IAAI,GAAG,YAAY,GAAG,OAAO,CAAC;AAMjD,KAAK,mBAAmB,GAAG;IACzB,YAAY,EAAE,CAAC,IAAI,EAAE,IAAI,EAAE,OAAO,EAAE,OAAO,EAAE,QAAQ,EAAE,MAAM,EAAE,KAAK,OAAO,CAAC,IAAI,CAAC,CAAC;IAClF,0BAA0B,EAAE,CAAC,OAAO,EAAE,OAAO,EAAE,QAAQ,EAAE,MAAM,EAAE,KAAK,OAAO,CAAC,IAAI,CAAC,CAAC;CACrF,CAAC;AAoFF,wBAAsB,qBAAqB,CACzC,IAAI,EAAE,IAAI,EACV,IAAI,EAAE,WAAW,EACjB,OAAO,EAAE,OAAO,EAChB,KAAK,EAAE,MAAM,EACb,QAAQ,EAAE,MAAM,EAAE,EAClB,OAAO,EAAE,mBAAmB,EAC5B,MAAM,CAAC,EAAE,qBAAqB,GAC7B,OAAO,CAAC,OAAO,CAAC,CA0BlB"}
+{"version":3,"file":"select-action-executor.d.ts","sourceRoot":"","sources":["../../src/commands/select-action-executor.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,YAAY,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM,iBAAiB,CAAC;AACnE,OAAO,EAA2B,KAAK,qBAAqB,EAAE,MAAM,8BAA8B,CAAC;AAEnG,KAAK,WAAW,GAAG,IAAI,GAAG,YAAY,GAAG,OAAO,CAAC;AAMjD,KAAK,mBAAmB,GAAG;IACzB,YAAY,EAAE,CAAC,IAAI,EAAE,IAAI,EAAE,OAAO,EAAE,OAAO,EAAE,QAAQ,EAAE,MAAM,EAAE,KAAK,OAAO,CAAC,IAAI,CAAC,CAAC;IAClF,0BAA0B,EAAE,CAAC,OAAO,EAAE,OAAO,EAAE,QAAQ,EAAE,MAAM,EAAE,KAAK,OAAO,CAAC,IAAI,CAAC,CAAC;CACrF,CAAC;AAoFF,wBAAsB,qBAAqB,CACzC,IAAI,EAAE,IAAI,EACV,IAAI,EAAE,WAAW,EACjB,OAAO,EAAE,OAAO,EAChB,KAAK,EAAE,MAAM,EACb,QAAQ,EAAE,MAAM,EAAE,EAClB,OAAO,EAAE,mBAAmB,EAC5B,MAAM,CAAC,EAAE,qBAAqB,GAC7B,OAAO,CAAC,OAAO,CAAC,CA0BlB"}

package/dist/commands/select-action-executor.js

@@ -1,4 +1,4 @@
-import { runActionExecutionGuard, } from './action-execution-guards.js';
+import { runActionExecutionGuard } from './action-execution-guards.js';
 const OPTION_RESOLVE_TIMEOUT_MS = 250;
 const ASYNC_OPTION_WAIT_MS = 1_000;
 const OPTION_POLL_INTERVAL_MS = 100;