@nuanu-ai/agentbrowse 0.2.21 → 0.2.23

package/dist/commands/observe-accessibility.js

@@ -0,0 +1,497 @@
+import { resolveLocatorRoot } from './action-fallbacks.js';
+import { buildObserveDisplayLabel } from './observe-display-label.js';
+import { fallbackContextNodeLabelOf, fallbackHintTextOf, fallbackSurfaceLabelOf, fallbackTargetLabelOf, } from './observe-fallback-semantics.js';
+const AX_ENRICHMENT_CONCURRENCY = 12;
+const AX_SNAPSHOT_TIMEOUT_MS = 250;
+const GENERIC_FALLBACK_LABELS = new Set([
+    'button',
+    'link',
+    'combobox',
+    'text input',
+    'email input',
+    'password input',
+    'phone input',
+    'search input',
+    'date input',
+    'text area',
+    'option',
+    'menu item',
+    'grid cell',
+]);
+const NON_CONTAINER_AX_ROLES = new Set([
+    'button',
+    'checkbox',
+    'combobox',
+    'gridcell',
+    'img',
+    'link',
+    'menuitem',
+    'option',
+    'radio',
+    'switch',
+    'tab',
+    'textbox',
+]);
+function emptyStats() {
+    return {
+        axAttempts: 0,
+        axHits: 0,
+        fallbackUses: 0,
+    };
+}
+function normalizeText(value) {
+    return (value ?? '').replace(/\s+/g, ' ').trim();
+}
+function isMeaningfulName(value) {
+    const normalized = normalizeText(value);
+    return normalized.length >= 2 && normalized !== '[object Object]';
+}
+function normalizeLabelKey(value) {
+    return normalizeText(value).toLowerCase();
+}
+export function parseObserveAriaSnapshot(snapshot) {
+    const rawSnapshot = typeof snapshot === 'string' ? snapshot : '';
+    if (!normalizeText(rawSnapshot)) {
+        return null;
+    }
+    const lines = rawSnapshot
+        .split(/\r?\n/)
+        .map((line) => line.trim())
+        .filter(Boolean);
+    const headerLine = lines.find((line) => line.startsWith('- '));
+    if (!headerLine) {
+        return null;
+    }
+    const header = headerLine
+        .replace(/^-+\s*/, '')
+        .replace(/:\s*$/, '')
+        .trim();
+    if (!header) {
+        return null;
+    }
+    const roleMatch = header.match(/^([a-z][a-z0-9_-]*)\b/i);
+    const quotedParts = [...header.matchAll(/"([^"]+)"/g)].map((match) => normalizeText(match[1]));
+    const stateParts = [...header.matchAll(/\[([^\]]+)\]/g)].map((match) => normalizeText(match[1]).toLowerCase());
+    const states = {};
+    for (const state of stateParts) {
+        if (!state) {
+            continue;
+        }
+        if (state === 'checked')
+            states.checked = true;
+        else if (state === 'disabled')
+            states.disabled = true;
+        else if (state === 'expanded')
+            states.expanded = true;
+        else if (state === 'pressed')
+            states.pressed = true;
+        else if (state === 'selected')
+            states.selected = true;
+        else if (state.startsWith('level=')) {
+            const level = Number(state.slice('level='.length));
+            if (Number.isFinite(level)) {
+                states.level = level;
+            }
+        }
+    }
+    return {
+        role: roleMatch?.[1]?.toLowerCase(),
+        name: quotedParts.length > 0 ? quotedParts.join(' ') : undefined,
+        states: Object.keys(states).length > 0 ? states : undefined,
+    };
+}
+function shouldPreferAccessibilityName(target, accessibilityName) {
+    const currentLabel = normalizeText(target.label);
+    const currentKey = normalizeLabelKey(currentLabel);
+    const nextLabel = normalizeText(accessibilityName);
+    const visibleText = normalizeText(target.text);
+    if (!isMeaningfulName(nextLabel)) {
+        return false;
+    }
+    if (!isMeaningfulName(currentLabel)) {
+        return true;
+    }
+    if (GENERIC_FALLBACK_LABELS.has(currentKey)) {
+        return true;
+    }
+    if (currentLabel === nextLabel) {
+        return false;
+    }
+    if (isMeaningfulName(visibleText) &&
+        visibleText === nextLabel &&
+        nextLabel.startsWith(currentLabel) &&
+        nextLabel.length > currentLabel.length) {
+        return true;
+    }
+    return currentLabel.startsWith(nextLabel) && currentLabel.length - nextLabel.length >= 12;
+}
+function patchDisplayLabel(displayLabel, previousLabel, nextLabel) {
+    if (!displayLabel || !previousLabel) {
+        return displayLabel;
+    }
+    const normalizedPrevious = normalizeText(previousLabel);
+    if (!normalizedPrevious) {
+        return displayLabel;
+    }
+    return displayLabel.startsWith(previousLabel)
+        ? nextLabel + displayLabel.slice(previousLabel.length)
+        : displayLabel;
+}
+function shouldUseCurrentValueDisplayLabel(target) {
+    return target.controlFamily === 'select' && target.kind !== 'select';
+}
+function chooseDisplayLabel(target, nextLabel) {
+    if (shouldUseCurrentValueDisplayLabel(target)) {
+        const currentValueDisplayLabel = buildObserveDisplayLabel(nextLabel ?? target.label, target.currentValue);
+        if (currentValueDisplayLabel) {
+            return currentValueDisplayLabel;
+        }
+    }
+    return nextLabel && nextLabel !== target.label
+        ? patchDisplayLabel(target.displayLabel, target.label, nextLabel)
+        : target.displayLabel;
+}
+function framePathKey(framePath) {
+    return framePath?.join('>') ?? '__top__';
+}
+function accessibilityCacheKeyOf(target) {
+    return `${framePathKey(target.framePath)}::${target.selector?.trim() ?? ''}`;
+}
+function hasAccessibilitySemantics(semantics) {
+    return Boolean(semantics &&
+        (isMeaningfulName(semantics.name) ||
+            isMeaningfulName(semantics.role) ||
+            (semantics.states && Object.keys(semantics.states).length > 0)));
+}
+async function readAccessibilitySemantics(page, target, rootCache, snapshotCache, stats) {
+    const selector = target.selector?.trim();
+    if (!selector) {
+        return null;
+    }
+    const cacheKey = accessibilityCacheKeyOf(target);
+    const cachedSnapshot = snapshotCache.get(cacheKey);
+    if (cachedSnapshot) {
+        return cachedSnapshot;
+    }
+    stats.axAttempts += 1;
+    const pendingSnapshot = (async () => {
+        try {
+            const rootKey = framePathKey(target.framePath);
+            const root = rootCache.get(rootKey) ??
+                (() => {
+                    const resolvedRoot = resolveLocatorRoot(page, target.framePath);
+                    rootCache.set(rootKey, resolvedRoot);
+                    return resolvedRoot;
+                })();
+            const locator = root.locator(selector).first();
+            const snapshot = await locator.ariaSnapshot({ timeout: AX_SNAPSHOT_TIMEOUT_MS });
+            const semantics = parseObserveAriaSnapshot(snapshot);
+            if (hasAccessibilitySemantics(semantics)) {
+                stats.axHits += 1;
+            }
+            return semantics;
+        }
+        catch {
+            return null;
+        }
+    })();
+    snapshotCache.set(cacheKey, pendingSnapshot);
+    return pendingSnapshot;
+}
+async function readDomFallbackSemantics(page, target, rootCache, fallbackCache) {
+    const selector = target.selector?.trim();
+    if (!selector) {
+        return null;
+    }
+    const cacheKey = accessibilityCacheKeyOf(target);
+    const cachedFallback = fallbackCache.get(cacheKey);
+    if (cachedFallback) {
+        return cachedFallback;
+    }
+    const pendingFallback = (async () => {
+        try {
+            const rootKey = framePathKey(target.framePath);
+            const root = rootCache.get(rootKey) ??
+                (() => {
+                    const resolvedRoot = resolveLocatorRoot(page, target.framePath);
+                    rootCache.set(rootKey, resolvedRoot);
+                    return resolvedRoot;
+                })();
+            const locator = root.locator(selector).first();
+            return await locator.evaluate((element) => {
+                if (!(element instanceof HTMLElement)) {
+                    return null;
+                }
+                const normalizeText = (value) => {
+                    const normalized = (value ?? '').replace(/\s+/g, ' ').trim();
+                    return normalized || undefined;
+                };
+                const relationTextOf = (attribute) => {
+                    const relation = element.getAttribute(attribute)?.trim();
+                    if (!relation) {
+                        return undefined;
+                    }
+                    const text = relation
+                        .split(/\s+/)
+                        .map((id) => {
+                        const related = element.ownerDocument.getElementById(id);
+                        return related instanceof HTMLElement
+                            ? related.innerText || related.textContent || ''
+                            : '';
+                    })
+                        .join(' ');
+                    return normalizeText(text);
+                };
+                const heading = element.querySelector('h1, h2, h3, h4, h5, h6, [role="heading"]');
+                const headingText = heading instanceof HTMLElement && heading !== element && !heading.contains(element)
+                    ? normalizeText(heading.getAttribute('aria-label') ||
+                        heading.getAttribute('title') ||
+                        heading.innerText ||
+                        heading.textContent)
+                    : undefined;
+                return {
+                    label: normalizeText(element.getAttribute('aria-label') ||
+                        relationTextOf('aria-labelledby') ||
+                        element.getAttribute('title') ||
+                        headingText),
+                    hintText: relationTextOf('aria-describedby'),
+                };
+            });
+        }
+        catch {
+            return null;
+        }
+    })();
+    fallbackCache.set(cacheKey, pendingFallback);
+    return pendingFallback;
+}
+function chooseSemanticName(fallbackValue, semantics, stats) {
+    const normalizedFallback = normalizeText(fallbackValue);
+    const normalizedSemanticName = normalizeText(semantics?.name);
+    if (isMeaningfulName(normalizedSemanticName) && normalizedSemanticName === normalizedFallback) {
+        return normalizedSemanticName;
+    }
+    if (isMeaningfulName(fallbackValue)) {
+        stats.fallbackUses += 1;
+        return fallbackValue;
+    }
+    if (isMeaningfulName(normalizedSemanticName)) {
+        return normalizedSemanticName;
+    }
+    return fallbackValue;
+}
+function normalizedContainerKind(value) {
+    return normalizeText(value).toLowerCase();
+}
+function shouldUseContainerAccessibilityName(expectedKind, semantics) {
+    if (!isMeaningfulName(semantics?.name)) {
+        return false;
+    }
+    const role = normalizedContainerKind(semantics?.role);
+    if (!role) {
+        return false;
+    }
+    if (NON_CONTAINER_AX_ROLES.has(role)) {
+        return false;
+    }
+    const expected = normalizedContainerKind(expectedKind);
+    if (!expected) {
+        return true;
+    }
+    if (expected === role) {
+        return true;
+    }
+    if (expected === 'form') {
+        return role === 'form' || role === 'search';
+    }
+    if (expected === 'region' || expected === 'section' || expected === 'aside') {
+        return role === 'region' || role === 'complementary';
+    }
+    if (expected === 'fieldset' || expected === 'group') {
+        return role === 'group';
+    }
+    if (expected === 'dialog') {
+        return role === 'dialog' || role === 'alertdialog';
+    }
+    if (expected === 'datepicker') {
+        return role === 'dialog' || role === 'grid';
+    }
+    if (expected === 'grid') {
+        return role === 'grid' || role === 'treegrid';
+    }
+    return false;
+}
+function chooseContainerSemanticName(fallbackValue, expectedKind, semantics, stats) {
+    if (shouldUseContainerAccessibilityName(expectedKind, semantics)) {
+        return semantics?.name;
+    }
+    if (isMeaningfulName(fallbackValue)) {
+        stats.fallbackUses += 1;
+    }
+    return fallbackValue;
+}
+function enrichContextNode(node, semantics, domFallback, stats) {
+    if (!node) {
+        return undefined;
+    }
+    return {
+        ...node,
+        kind: node.kind ?? semantics?.role,
+        label: chooseContainerSemanticName(fallbackContextNodeLabelOf(node) ?? domFallback?.label, node.kind, semantics, stats),
+    };
+}
+function chooseHintText(context, nextContext, domFallbackHint, stats) {
+    const fallbackHint = fallbackHintTextOf(context) ?? normalizeText(domFallbackHint);
+    const nextHintCandidate = [
+        nextContext?.container?.label,
+        nextContext?.landmark?.label,
+        nextContext?.group?.label,
+    ].find(isMeaningfulName);
+    if (isMeaningfulName(nextHintCandidate)) {
+        return nextHintCandidate;
+    }
+    if (fallbackHint) {
+        stats.fallbackUses += 1;
+        return fallbackHint;
+    }
+    return undefined;
+}
+function chooseStructuredGridZone(target, nextContext, surfaceSemantics, surfaceDomFallback, containerSemantics, containerDomFallback, landmarkSemantics, landmarkDomFallback, stats) {
+    if (target.structure?.family !== 'structured-grid' || target.structure.variant !== 'seat-cell') {
+        return target.structure?.zone;
+    }
+    const blockedZoneKeys = new Set([fallbackSurfaceLabelOf(target), target.surfaceLabel]
+        .map((value) => normalizeLabelKey(value))
+        .filter(Boolean));
+    const chooseCandidate = (candidates) => {
+        for (const candidate of candidates) {
+            const normalized = normalizeText(candidate);
+            if (!isMeaningfulName(normalized)) {
+                continue;
+            }
+            if (blockedZoneKeys.has(normalizeLabelKey(normalized))) {
+                continue;
+            }
+            return normalized;
+        }
+        return undefined;
+    };
+    const semanticZone = chooseCandidate([
+        surfaceSemantics?.name,
+        landmarkSemantics?.name,
+        containerSemantics?.name,
+    ]);
+    if (semanticZone) {
+        return semanticZone;
+    }
+    const fallbackZone = chooseCandidate([
+        surfaceDomFallback?.label,
+        landmarkDomFallback?.label,
+        containerDomFallback?.label,
+        nextContext?.landmark?.label,
+        nextContext?.container?.label,
+        nextContext?.group?.label,
+        target.structure?.zone,
+    ]);
+    if (fallbackZone) {
+        stats.fallbackUses += 1;
+        return fallbackZone;
+    }
+    return target.structure?.zone;
+}
+function enrichStructure(target, nextContext, surfaceSemantics, surfaceDomFallback, containerSemantics, containerDomFallback, landmarkSemantics, landmarkDomFallback, stats) {
+    if (!target.structure) {
+        return target.structure;
+    }
+    const nextZone = chooseStructuredGridZone(target, nextContext, surfaceSemantics, surfaceDomFallback, containerSemantics, containerDomFallback, landmarkSemantics, landmarkDomFallback, stats);
+    if (nextZone === target.structure.zone) {
+        return target.structure;
+    }
+    return {
+        ...target.structure,
+        zone: nextZone,
+    };
+}
+async function enrichTargetWithAccessibility(page, target, rootCache, snapshotCache, fallbackCache, stats) {
+    const semantics = await readAccessibilitySemantics(page, target, rootCache, snapshotCache, stats);
+    const surfaceSemantics = await readAccessibilitySemantics(page, { selector: target.surfaceSelector, framePath: target.framePath }, rootCache, snapshotCache, stats);
+    const surfaceDomFallback = await readDomFallbackSemantics(page, { selector: target.surfaceSelector, framePath: target.framePath }, rootCache, fallbackCache);
+    const itemSemantics = await readAccessibilitySemantics(page, { selector: target.context?.item?.selector, framePath: target.framePath }, rootCache, snapshotCache, stats);
+    const itemDomFallback = await readDomFallbackSemantics(page, { selector: target.context?.item?.selector, framePath: target.framePath }, rootCache, fallbackCache);
+    const groupSemantics = await readAccessibilitySemantics(page, { selector: target.context?.group?.selector, framePath: target.framePath }, rootCache, snapshotCache, stats);
+    const groupDomFallback = await readDomFallbackSemantics(page, { selector: target.context?.group?.selector, framePath: target.framePath }, rootCache, fallbackCache);
+    const containerSemantics = await readAccessibilitySemantics(page, { selector: target.context?.container?.selector, framePath: target.framePath }, rootCache, snapshotCache, stats);
+    const containerDomFallback = await readDomFallbackSemantics(page, { selector: target.context?.container?.selector, framePath: target.framePath }, rootCache, fallbackCache);
+    const landmarkSemantics = await readAccessibilitySemantics(page, { selector: target.context?.landmark?.selector, framePath: target.framePath }, rootCache, snapshotCache, stats);
+    const landmarkDomFallback = await readDomFallbackSemantics(page, { selector: target.context?.landmark?.selector, framePath: target.framePath }, rootCache, fallbackCache);
+    const targetDomFallback = await readDomFallbackSemantics(page, target, rootCache, fallbackCache);
+    const nextLabel = isMeaningfulName(semantics?.name) &&
+        shouldPreferAccessibilityName({ label: fallbackTargetLabelOf(target), text: target.text }, semantics.name)
+        ? semantics.name
+        : chooseSemanticName(fallbackTargetLabelOf(target), semantics, stats);
+    const nextContext = target.context
+        ? {
+            ...target.context,
+            item: enrichContextNode(target.context.item, itemSemantics, itemDomFallback, stats),
+            group: enrichContextNode(target.context.group, groupSemantics, groupDomFallback, stats),
+            container: enrichContextNode(target.context.container, containerSemantics, containerDomFallback, stats),
+            landmark: enrichContextNode(target.context.landmark, landmarkSemantics, landmarkDomFallback, stats),
+        }
+        : undefined;
+    const nextStructure = enrichStructure(target, nextContext, surfaceSemantics, surfaceDomFallback, containerSemantics, containerDomFallback, landmarkSemantics, landmarkDomFallback, stats);
+    const usedAxSemantics = Boolean((isMeaningfulName(semantics?.name) && nextLabel === semantics?.name) ||
+        (semantics?.role && semantics.role === (target.role ?? semantics.role)) ||
+        (semantics?.states && Object.keys(semantics.states).length > 0) ||
+        isMeaningfulName(surfaceSemantics?.name) ||
+        isMeaningfulName(itemSemantics?.name) ||
+        isMeaningfulName(groupSemantics?.name) ||
+        isMeaningfulName(containerSemantics?.name) ||
+        isMeaningfulName(landmarkSemantics?.name));
+    return {
+        ...target,
+        label: nextLabel,
+        displayLabel: chooseDisplayLabel(target, nextLabel),
+        role: target.role ?? semantics?.role,
+        semanticsSource: usedAxSemantics ? 'aria-snapshot' : (target.semanticsSource ?? 'dom'),
+        states: semantics?.states && Object.keys(semantics.states).length > 0
+            ? { ...(target.states ?? {}), ...(semantics.states ?? {}) }
+            : target.states,
+        surfaceLabel: chooseContainerSemanticName(fallbackSurfaceLabelOf(target) ?? surfaceDomFallback?.label, target.surfaceKind, surfaceSemantics, stats),
+        structure: nextStructure,
+        context: nextContext
+            ? {
+                ...nextContext,
+                hintText: chooseHintText(target.context, nextContext, targetDomFallback?.hintText, stats),
+            }
+            : target.context,
+    };
+}
+async function mapWithConcurrency(items, limit, worker) {
+    const results = new Array(items.length);
+    let cursor = 0;
+    const runWorker = async () => {
+        while (true) {
+            const index = cursor;
+            cursor += 1;
+            if (index >= items.length) {
+                return;
+            }
+            results[index] = await worker(items[index], index);
+        }
+    };
+    const concurrency = Math.max(1, Math.min(limit, items.length));
+    await Promise.all(Array.from({ length: concurrency }, () => runWorker()));
+    return results;
+}
+export async function enrichDomTargetsWithAccessibility(page, targets, options) {
+    const rootCache = new Map();
+    const snapshotCache = new Map();
+    const fallbackCache = new Map();
+    const stats = emptyStats();
+    const enrichedTargets = await mapWithConcurrency(targets, AX_ENRICHMENT_CONCURRENCY, (target) => enrichTargetWithAccessibility(page, target, rootCache, snapshotCache, fallbackCache, stats));
+    options?.onStats?.(stats);
+    return enrichedTargets;
+}
+export const __testObserveAccessibility = {
+    parseObserveAriaSnapshot,
+};

package/dist/commands/observe-display-label.d.ts

@@ -0,0 +1,4 @@
+export declare function normalizeObserveLiveValue(value: string | undefined): string | undefined;
+export declare function observeLabelIncludesValue(label: string, value: string): boolean;
+export declare function buildObserveDisplayLabel(baseLabel: string | undefined, liveValue: string | undefined): string | undefined;
+//# sourceMappingURL=observe-display-label.d.ts.map

package/dist/commands/observe-display-label.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"observe-display-label.d.ts","sourceRoot":"","sources":["../../src/commands/observe-display-label.ts"],"names":[],"mappings":"AAIA,wBAAgB,yBAAyB,CAAC,KAAK,EAAE,MAAM,GAAG,SAAS,GAAG,MAAM,GAAG,SAAS,CAMvF;AAED,wBAAgB,yBAAyB,CAAC,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,OAAO,CAI/E;AAED,wBAAgB,wBAAwB,CACtC,SAAS,EAAE,MAAM,GAAG,SAAS,EAC7B,SAAS,EAAE,MAAM,GAAG,SAAS,GAC5B,MAAM,GAAG,SAAS,CAapB"}

package/dist/commands/observe-display-label.js

@@ -0,0 +1,26 @@
+function normalizeText(value) {
+    return (value ?? '').replace(/\s+/g, ' ').trim();
+}
+export function normalizeObserveLiveValue(value) {
+    const normalized = normalizeText(value);
+    if (!normalized || normalized.length > 80) {
+        return undefined;
+    }
+    return normalized;
+}
+export function observeLabelIncludesValue(label, value) {
+    const normalizedLabel = normalizeText(label).toLowerCase();
+    const normalizedValue = normalizeText(value).toLowerCase();
+    return Boolean(normalizedValue) && normalizedLabel.includes(normalizedValue);
+}
+export function buildObserveDisplayLabel(baseLabel, liveValue) {
+    const normalizedLabel = normalizeText(baseLabel);
+    const normalizedValue = normalizeObserveLiveValue(liveValue);
+    if (!normalizedLabel || !normalizedValue) {
+        return undefined;
+    }
+    if (observeLabelIncludesValue(normalizedLabel, normalizedValue)) {
+        return undefined;
+    }
+    return `${normalizedLabel} — ${normalizedValue}`;
+}

package/dist/commands/observe-dom-label-contract.d.ts

@@ -0,0 +1,2 @@
+export declare const OBSERVE_DOM_LABEL_CONTRACT_HELPER_SCRIPT: string;
+//# sourceMappingURL=observe-dom-label-contract.d.ts.map

package/dist/commands/observe-dom-label-contract.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"observe-dom-label-contract.d.ts","sourceRoot":"","sources":["../../src/commands/observe-dom-label-contract.ts"],"names":[],"mappings":"AAWA,eAAO,MAAM,wCAAwC,QAkgBpD,CAAC"}