@nuanu-ai/agentbrowse 0.2.7 → 0.2.8

package/dist/commands/observe-semantics.js

@@ -0,0 +1,338 @@
+import { inferAcceptancePolicyFromFacts, inferAllowedActionsFromFacts, inferAvailabilityFromFacts, inferControlFamilyFromFacts, } from '../control-semantics.js';
+import { buildSurfaceRef } from './observe-surfaces.js';
+export function normalizeSemanticDuplicateLabel(target) {
+    return (target.displayLabel ??
+        target.label ??
+        target.text ??
+        target.placeholder ??
+        target.title ??
+        '')
+        .replace(/\s+/g, ' ')
+        .trim()
+        .toLowerCase();
+}
+export function isPrimaryFormControlTarget(target) {
+    const kind = (target.kind ?? '').trim().toLowerCase();
+    const role = (target.role ?? '').trim().toLowerCase();
+    const acceptancePolicy = (target.acceptancePolicy ?? '').trim().toLowerCase();
+    if (target.allowedActions?.includes('fill') || target.allowedActions?.includes('type')) {
+        return true;
+    }
+    if (target.allowedActions?.includes('select')) {
+        return true;
+    }
+    if (acceptancePolicy === 'submit' || acceptancePolicy === 'date-selection') {
+        return true;
+    }
+    if (acceptancePolicy === 'disclosure' &&
+        Boolean(formGroupingKeyOf(target)) &&
+        (kind === 'button' || role === 'button')) {
+        return true;
+    }
+    return (kind === 'input' ||
+        kind === 'textarea' ||
+        kind === 'select' ||
+        role === 'textbox' ||
+        role === 'combobox');
+}
+function semanticFormContextKey(context) {
+    for (const node of [context?.landmark, context?.container, context?.group, context?.item]) {
+        const kind = (node?.kind ?? '').trim().toLowerCase();
+        if (kind !== 'form') {
+            continue;
+        }
+        const label = (node?.label ?? node?.text ?? '').replace(/\s+/g, ' ').trim().toLowerCase();
+        return `${kind}|${label}`;
+    }
+    return undefined;
+}
+export function formGroupingKeyOf(target) {
+    const formSelector = target.formSelector?.trim();
+    if (formSelector) {
+        return `selector:${formSelector}`;
+    }
+    const contextKey = semanticFormContextKey(target.context);
+    return contextKey ? `context:${contextKey}` : undefined;
+}
+export function observedTargetKey(target) {
+    return (target.selector ??
+        target.domSignature ??
+        `${target.formSelector ?? 'no-form'}|${target.ordinal ?? 'no-ordinal'}|${target.label ?? ''}`);
+}
+export function expandRerankedFormTargets(targets, selectedTargets) {
+    const selectedFormControlCounts = new Map();
+    for (const target of selectedTargets) {
+        const formKey = formGroupingKeyOf(target);
+        if (!formKey || !isPrimaryFormControlTarget(target)) {
+            continue;
+        }
+        selectedFormControlCounts.set(formKey, (selectedFormControlCounts.get(formKey) ?? 0) + 1);
+    }
+    const expandableFormKeys = new Set([...selectedFormControlCounts.entries()]
+        .filter(([, count]) => count >= 2)
+        .map(([formKey]) => formKey));
+    if (expandableFormKeys.size === 0) {
+        return [...selectedTargets];
+    }
+    const expanded = new Map();
+    for (const target of selectedTargets) {
+        expanded.set(observedTargetKey(target), target);
+    }
+    for (const target of targets) {
+        const formKey = formGroupingKeyOf(target);
+        if (!formKey || !expandableFormKeys.has(formKey) || !isPrimaryFormControlTarget(target)) {
+            continue;
+        }
+        expanded.set(observedTargetKey(target), target);
+    }
+    return [...expanded.values()].sort((left, right) => {
+        const leftOrdinal = left.ordinal ?? Number.MAX_SAFE_INTEGER;
+        const rightOrdinal = right.ordinal ?? Number.MAX_SAFE_INTEGER;
+        if (leftOrdinal !== rightOrdinal) {
+            return leftOrdinal - rightOrdinal;
+        }
+        return (left.label ?? '').localeCompare(right.label ?? '');
+    });
+}
+function semanticCompressionKey(target) {
+    const band = target.context?.layout?.band;
+    if (band !== 'bottom') {
+        return null;
+    }
+    const kind = (target.kind ?? '').trim().toLowerCase();
+    if (kind !== 'link' && kind !== 'button') {
+        return null;
+    }
+    const label = normalizeSemanticDuplicateLabel(target);
+    if (!label) {
+        return null;
+    }
+    return `${kind}|${label}`;
+}
+export function compressSemanticallyDuplicateTargets(targets) {
+    const repeatedKeys = new Map();
+    for (const target of targets) {
+        const key = semanticCompressionKey(target);
+        if (!key) {
+            continue;
+        }
+        repeatedKeys.set(key, (repeatedKeys.get(key) || 0) + 1);
+    }
+    const seen = new Set();
+    return targets.filter((target) => {
+        const key = semanticCompressionKey(target);
+        if (!key || (repeatedKeys.get(key) || 0) <= 1) {
+            return true;
+        }
+        if (seen.has(key)) {
+            return false;
+        }
+        seen.add(key);
+        return true;
+    });
+}
+function inferAllowedActions(target) {
+    return inferAllowedActionsFromFacts({
+        kind: target.kind,
+        role: target.role,
+        label: target.label,
+        displayLabel: target.displayLabel,
+        interactionHint: target.interactionHint,
+        text: target.text,
+        placeholder: target.placeholder,
+        inputName: target.inputName,
+        inputType: target.inputType,
+        autocomplete: target.autocomplete,
+        surfaceKind: target.surfaceKind,
+        controlsSurfaceSelector: target.controlsSurfaceSelector,
+        states: target.states,
+        structure: target.structure,
+    });
+}
+function inferAvailability(target) {
+    const allowedActions = inferAllowedActions(target);
+    const controlFamily = inferControlFamilyFromFacts({
+        kind: target.kind,
+        role: target.role,
+        label: target.label,
+        displayLabel: target.displayLabel,
+        interactionHint: target.interactionHint,
+        text: target.text,
+        placeholder: target.placeholder,
+        inputName: target.inputName,
+        inputType: target.inputType,
+        autocomplete: target.autocomplete,
+        surfaceKind: target.surfaceKind,
+        controlsSurfaceSelector: target.controlsSurfaceSelector,
+        states: target.states,
+        structure: target.structure,
+    }, allowedActions);
+    const acceptancePolicy = inferAcceptancePolicy(target, allowedActions);
+    return inferAvailabilityFromFacts(target.states, target.context?.hintText, {
+        readonlyInteractive: controlFamily === 'select' ||
+            controlFamily === 'datepicker' ||
+            acceptancePolicy === 'selection' ||
+            acceptancePolicy === 'date-selection',
+    });
+}
+function inferAcceptancePolicy(target, allowedActions) {
+    return inferAcceptancePolicyFromFacts({
+        kind: target.kind,
+        role: target.role,
+        label: target.label,
+        displayLabel: target.displayLabel,
+        interactionHint: target.interactionHint,
+        text: target.text,
+        placeholder: target.placeholder,
+        inputName: target.inputName,
+        inputType: target.inputType,
+        autocomplete: target.autocomplete,
+        surfaceKind: target.surfaceKind,
+        controlsSurfaceSelector: target.controlsSurfaceSelector,
+        states: target.states,
+        structure: target.structure,
+    }, allowedActions);
+}
+function inferCapability(target, allowedActions) {
+    const hasExecutionPath = allowedActions.length > 0;
+    if (!hasExecutionPath) {
+        return 'informational';
+    }
+    const label = target.label ?? target.displayLabel ?? target.text ?? '';
+    const descendantInteractiveCount = target.descendantInteractiveCount ?? 0;
+    const giantContainerLike = label.length > 180 && descendantInteractiveCount >= 4 && allowedActions.includes('click');
+    if (giantContainerLike) {
+        return 'scope';
+    }
+    return 'actionable';
+}
+export function annotateDomTargets(targets) {
+    const annotated = targets.map((target) => {
+        const allowedActions = inferAllowedActions(target);
+        const availability = inferAvailability(target);
+        const capability = inferCapability(target, allowedActions);
+        const acceptancePolicy = inferAcceptancePolicy(target, allowedActions);
+        const surfaceRef = buildSurfaceRef(target);
+        return {
+            ...target,
+            capability,
+            availability,
+            allowedActions,
+            acceptancePolicy,
+            surfaceRef,
+        };
+    });
+    return annotated.map((target, index, current) => {
+        if (!target.surfaceRef ||
+            !target.acceptancePolicy ||
+            !(target.acceptancePolicy === 'selection' || target.acceptancePolicy === 'date-selection')) {
+            return target;
+        }
+        const ownerIndex = current.findIndex((candidate, candidateIndex) => {
+            if (candidateIndex === index)
+                return false;
+            const explicitController = [
+                target.controlsSurfaceSelector,
+                target.surfaceSelector,
+                ...(target.surfaceSelectors ?? []),
+                candidate.surfaceSelector,
+            ]
+                .filter((value) => typeof value === 'string' && value.length > 0)
+                .some((selector) => candidate.controlsSurfaceSelector === selector);
+            const sameSurface = candidate.surfaceRef === target.surfaceRef;
+            if (!explicitController && !sameSurface)
+                return false;
+            if (candidate.allowedActions?.includes('fill'))
+                return true;
+            if (candidate.allowedActions?.includes('select'))
+                return true;
+            if (candidate.structure?.family === 'structured-grid')
+                return false;
+            if (candidate.capability === 'informational' && !explicitController)
+                return false;
+            return (candidate.allowedActions?.includes('click') &&
+                ['input', 'select', 'button', 'combobox'].includes((candidate.kind || '').toLowerCase()));
+        });
+        return ownerIndex >= 0 ? { ...target, ownerIndex } : target;
+    });
+}
+export function orderBySurfaceCompetition(targets) {
+    const iframeFieldLike = (target) => {
+        const kind = (target.kind ?? '').toLowerCase();
+        const role = (target.role ?? '').toLowerCase();
+        return (Boolean(target.framePath?.length) &&
+            (target.allowedActions?.includes('fill') ||
+                target.allowedActions?.includes('type') ||
+                target.allowedActions?.includes('select') ||
+                ['input', 'textarea', 'select', 'combobox'].includes(kind) ||
+                ['textbox', 'combobox'].includes(role)));
+    };
+    return [...targets].sort((left, right) => {
+        const leftPriority = left.surfacePriority ?? 0;
+        const rightPriority = right.surfacePriority ?? 0;
+        let leftScore = leftPriority * 10;
+        let rightScore = rightPriority * 10;
+        if (leftPriority >= 80) {
+            leftScore += 3_000;
+        }
+        else if (leftPriority >= 50) {
+            leftScore += 2_000;
+        }
+        else if (left.surfaceRef) {
+            leftScore += 1_000;
+        }
+        if (rightPriority >= 80) {
+            rightScore += 3_000;
+        }
+        else if (rightPriority >= 50) {
+            rightScore += 2_000;
+        }
+        else if (right.surfaceRef) {
+            rightScore += 1_000;
+        }
+        if (left.capability === 'actionable') {
+            leftScore += 150;
+        }
+        else if (left.capability === 'scope') {
+            leftScore += 75;
+        }
+        if (right.capability === 'actionable') {
+            rightScore += 150;
+        }
+        else if (right.capability === 'scope') {
+            rightScore += 75;
+        }
+        if (left.allowedActions?.includes('fill') || left.allowedActions?.includes('select')) {
+            leftScore += 40;
+        }
+        if (right.allowedActions?.includes('fill') || right.allowedActions?.includes('select')) {
+            rightScore += 40;
+        }
+        if (isPrimaryFormControlTarget(left) && formGroupingKeyOf(left)) {
+            leftScore += 900;
+        }
+        if (isPrimaryFormControlTarget(right) && formGroupingKeyOf(right)) {
+            rightScore += 900;
+        }
+        if (iframeFieldLike(left)) {
+            leftScore += 1_200;
+        }
+        if (iframeFieldLike(right)) {
+            rightScore += 1_200;
+        }
+        if (left.ownerIndex !== undefined) {
+            leftScore -= 5;
+        }
+        if (right.ownerIndex !== undefined) {
+            rightScore -= 5;
+        }
+        const scoreDelta = rightScore - leftScore;
+        if (scoreDelta !== 0) {
+            return scoreDelta;
+        }
+        if (leftPriority !== rightPriority) {
+            return rightPriority - leftPriority;
+        }
+        return (left.ordinal ?? 0) - (right.ordinal ?? 0);
+    });
+}

package/dist/commands/observe-stagehand.d.ts

@@ -0,0 +1,48 @@
+import type { Page } from 'playwright-core';
+import { normalizeStagehandSelector, readStagehandLocatorSnapshot } from './observe-inventory.js';
+type StagehandObserveAction = {
+    description?: string;
+    selector?: string;
+    method?: string;
+    arguments?: unknown[];
+};
+export declare function toStagehandDescriptor(pageRef: string, action: StagehandObserveAction, page: Page, pageSignature?: string): Promise<{
+    pageRef: string;
+    kind: string | undefined;
+    label: string;
+    displayLabel: string | undefined;
+    placeholder: string | undefined;
+    inputName: string | undefined;
+    inputType: string | undefined;
+    autocomplete: string | undefined;
+    validation: undefined;
+    capability: "actionable" | "informational";
+    lifecycle: "live";
+    availability: import("../runtime-state.js").TargetAvailabilityState;
+    allowedActions: import("../runtime-state.js").TargetAllowedAction[];
+    controlFamily: import("../runtime-state.js").TargetControlFamily | undefined;
+    acceptancePolicy: import("../runtime-state.js").TargetAcceptancePolicy | undefined;
+    framePath: string[] | undefined;
+    semantics: {
+        name: string;
+        role: string | undefined;
+        states: Record<string, string | number | boolean> | undefined;
+        source: "stagehand";
+    };
+    locatorCandidates: {
+        strategy: "css" | "xpath";
+        value: string;
+        scope: "root";
+    }[];
+    stagehandAction: StagehandObserveAction;
+    createdAt: number;
+    pageSignature: string | undefined;
+    domSignature: string | undefined;
+}>;
+export declare const __testStagehandDescriptor: {
+    toStagehandDescriptor: typeof toStagehandDescriptor;
+    normalizeStagehandSelector: typeof normalizeStagehandSelector;
+    readStagehandLocatorSnapshot: typeof readStagehandLocatorSnapshot;
+};
+export {};
+//# sourceMappingURL=observe-stagehand.d.ts.map

package/dist/commands/observe-stagehand.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"observe-stagehand.d.ts","sourceRoot":"","sources":["../../src/commands/observe-stagehand.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,iBAAiB,CAAC;AAQ5C,OAAO,EAAE,0BAA0B,EAAE,4BAA4B,EAAE,MAAM,wBAAwB,CAAC;AAElG,KAAK,sBAAsB,GAAG;IAC5B,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,SAAS,CAAC,EAAE,OAAO,EAAE,CAAC;CACvB,CAAC;AAgBF,wBAAsB,qBAAqB,CACzC,OAAO,EAAE,MAAM,EACf,MAAM,EAAE,sBAAsB,EAC9B,IAAI,EAAE,IAAI,EACV,aAAa,CAAC,EAAE,MAAM;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA8FvB;AAED,eAAO,MAAM,yBAAyB;;;;CAIrC,CAAC"}

package/dist/commands/observe-stagehand.js

@@ -0,0 +1,105 @@
+import { inferAcceptancePolicyFromFacts, inferAllowedActionsFromFacts, inferAvailabilityFromFacts, inferControlFamilyFromFacts, } from '../control-semantics.js';
+import { buildLocator, resolveLocatorRoot } from './action-fallbacks.js';
+import { normalizeStagehandSelector, readStagehandLocatorSnapshot } from './observe-inventory.js';
+function normalizeStagehandLiveValue(value) {
+    const normalized = value?.replace(/\s+/g, ' ').trim();
+    if (!normalized || normalized.length > 80) {
+        return undefined;
+    }
+    return normalized;
+}
+function stagehandLabelIncludesValue(label, value) {
+    const normalizedLabel = label.replace(/\s+/g, ' ').trim().toLowerCase();
+    const normalizedValue = value.replace(/\s+/g, ' ').trim().toLowerCase();
+    return Boolean(normalizedValue) && normalizedLabel.includes(normalizedValue);
+}
+export async function toStagehandDescriptor(pageRef, action, page, pageSignature) {
+    const normalizedSelector = normalizeStagehandSelector(typeof action.selector === 'string' ? action.selector.trim() : '');
+    const selector = normalizedSelector.selector;
+    const locatorCandidates = selector
+        ? [
+            {
+                strategy: selector.startsWith('xpath=') ? 'xpath' : 'css',
+                value: selector,
+                scope: 'root',
+            },
+        ]
+        : [];
+    const stagehandLocator = selector
+        ? buildLocator(resolveLocatorRoot(page, normalizedSelector.framePath), locatorCandidates[0])
+        : null;
+    const { domSignature, domFacts } = stagehandLocator
+        ? await readStagehandLocatorSnapshot(stagehandLocator).catch(() => ({
+            domSignature: null,
+            domFacts: null,
+        }))
+        : {
+            domSignature: null,
+            domFacts: null,
+        };
+    const method = (action.method ?? '').trim().toLowerCase();
+    const shadowOrUnsupported = method === 'not-supported' || /shadow dom|inside a shadow/i.test(action.description ?? '');
+    const facts = {
+        kind: domFacts?.kind ?? action.method,
+        role: domFacts?.role,
+        label: action.description ?? action.method,
+        placeholder: domFacts?.placeholder,
+        inputName: domFacts?.inputName,
+        inputType: domFacts?.inputType,
+        autocomplete: domFacts?.autocomplete,
+        text: action.description ?? action.method,
+        states: domFacts?.states,
+        legacyMethod: method,
+    };
+    const allowedActions = inferAllowedActionsFromFacts(facts);
+    const acceptancePolicy = inferAcceptancePolicyFromFacts(facts, allowedActions);
+    const controlFamily = inferControlFamilyFromFacts(facts, allowedActions);
+    const liveValue = normalizeStagehandLiveValue(domFacts?.currentValue) ??
+        normalizeStagehandLiveValue(domFacts?.value) ??
+        normalizeStagehandLiveValue(domFacts?.text);
+    const baseLabel = action.description ?? action.method ?? 'observed target';
+    const displayLabel = liveValue && !stagehandLabelIncludesValue(baseLabel, liveValue)
+        ? `${baseLabel} — ${liveValue}`
+        : undefined;
+    const actionable = !shadowOrUnsupported && locatorCandidates.length > 0 && allowedActions.length > 0;
+    const availability = inferAvailabilityFromFacts(domFacts?.states, undefined, {
+        readonlyInteractive: controlFamily === 'select' ||
+            controlFamily === 'datepicker' ||
+            acceptancePolicy === 'selection' ||
+            acceptancePolicy === 'date-selection',
+    });
+    return {
+        pageRef,
+        kind: domFacts?.kind ?? action.method,
+        label: baseLabel,
+        displayLabel,
+        placeholder: domFacts?.placeholder,
+        inputName: domFacts?.inputName,
+        inputType: domFacts?.inputType,
+        autocomplete: domFacts?.autocomplete,
+        validation: undefined,
+        capability: actionable ? 'actionable' : 'informational',
+        lifecycle: 'live',
+        availability,
+        allowedActions,
+        controlFamily,
+        acceptancePolicy,
+        framePath: normalizedSelector.framePath,
+        semantics: {
+            name: action.description ?? action.method ?? 'observed target',
+            role: domFacts?.role,
+            states: domFacts?.states,
+            source: 'stagehand',
+        },
+        locatorCandidates,
+        stagehandAction: action,
+        createdAt: Date.now(),
+        pageSignature,
+        domSignature: domSignature ?? undefined,
+    };
+}
+export const __testStagehandDescriptor = {
+    toStagehandDescriptor,
+    normalizeStagehandSelector,
+    readStagehandLocatorSnapshot,
+};

package/dist/commands/observe-surfaces.d.ts

@@ -0,0 +1,9 @@
+import type { LocatorCandidate, SurfaceDescriptor, TargetContext, TargetDescriptor } from '../runtime-state.js';
+import type { DomObservedTarget } from './observe-inventory.js';
+export declare function selectScopesForOutput(scopes: ReadonlyArray<SurfaceDescriptor>, targets: ReadonlyArray<Pick<TargetDescriptor, 'surfaceRef' | 'allowedActions' | 'acceptancePolicy'>>, preferredScopeRefs?: ReadonlySet<string>): SurfaceDescriptor[];
+export declare function buildSurfaceRef(target: DomObservedTarget): string | undefined;
+export declare function surfaceReplacementKey(surface: Pick<SurfaceDescriptor, 'pageRef' | 'pageSignature' | 'framePath' | 'kind' | 'label'>): string;
+export declare function summarizeContext(context: TargetContext | undefined): string | undefined;
+export declare function pushUniqueLocatorCandidate(acc: LocatorCandidate[], candidate: LocatorCandidate | undefined): void;
+export declare function collectSurfaceDescriptors(pageRef: string, targets: ReadonlyArray<DomObservedTarget>): Omit<SurfaceDescriptor, "ref">[];
+//# sourceMappingURL=observe-surfaces.d.ts.map

package/dist/commands/observe-surfaces.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"observe-surfaces.d.ts","sourceRoot":"","sources":["../../src/commands/observe-surfaces.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EACV,gBAAgB,EAChB,iBAAiB,EACjB,aAAa,EACb,gBAAgB,EACjB,MAAM,qBAAqB,CAAC;AAE7B,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,wBAAwB,CAAC;AAahE,wBAAgB,qBAAqB,CACnC,MAAM,EAAE,aAAa,CAAC,iBAAiB,CAAC,EACxC,OAAO,EAAE,aAAa,CACpB,IAAI,CAAC,gBAAgB,EAAE,YAAY,GAAG,gBAAgB,GAAG,kBAAkB,CAAC,CAC7E,EACD,kBAAkB,CAAC,EAAE,WAAW,CAAC,MAAM,CAAC,GACvC,iBAAiB,EAAE,CA+CrB;AAED,wBAAgB,eAAe,CAAC,MAAM,EAAE,iBAAiB,GAAG,MAAM,GAAG,SAAS,CAa7E;AAED,wBAAgB,qBAAqB,CACnC,OAAO,EAAE,IAAI,CAAC,iBAAiB,EAAE,SAAS,GAAG,eAAe,GAAG,WAAW,GAAG,MAAM,GAAG,OAAO,CAAC,GAC7F,MAAM,CAMR;AAED,wBAAgB,gBAAgB,CAAC,OAAO,EAAE,aAAa,GAAG,SAAS,GAAG,MAAM,GAAG,SAAS,CA4CvF;AAED,wBAAgB,0BAA0B,CACxC,GAAG,EAAE,gBAAgB,EAAE,EACvB,SAAS,EAAE,gBAAgB,GAAG,SAAS,GACtC,IAAI,CAON;AAwDD,wBAAgB,yBAAyB,CACvC,OAAO,EAAE,MAAM,EACf,OAAO,EAAE,aAAa,CAAC,iBAAiB,CAAC,oCAoC1C"}