@mercuryo-ai/agentbrowse 0.2.50

package/dist/commands/observe-surfaces.js

@@ -0,0 +1,290 @@
+import { locatorCandidateKey } from '../runtime-state.js';
+const HIGH_SIGNAL_SCOPE_KINDS = new Set([
+    'form',
+    'dialog',
+    'listbox',
+    'menu',
+    'grid',
+    'tabpanel',
+    'popover',
+    'dropdown',
+    'datepicker',
+    'floating-panel',
+    'sticky-panel',
+]);
+const SNAPSHOT_SCOPE_KINDS = new Set([
+    'dialog',
+    'listbox',
+    'menu',
+    'grid',
+    'tabpanel',
+    'popover',
+    'dropdown',
+    'datepicker',
+    'floating-panel',
+    'sticky-panel',
+]);
+export const OUTPUT_LEADING_TARGET_LIMIT = 20;
+function inferExtractScopeLifetime(kind, locatorCandidates) {
+    const normalizedKind = (kind ?? '').trim().toLowerCase();
+    if (SNAPSHOT_SCOPE_KINDS.has(normalizedKind)) {
+        return 'snapshot';
+    }
+    if (normalizedKind === 'form') {
+        return locatorCandidates.length > 0 ? 'durable' : 'snapshot';
+    }
+    return 'snapshot';
+}
+export function selectScopesForOutput(scopes, targets, preferredScopeRefs) {
+    if (scopes.length === 0) {
+        return [];
+    }
+    const linkedScopeRefs = new Set(targets
+        .slice(0, OUTPUT_LEADING_TARGET_LIMIT)
+        .map((target) => target.surfaceRef)
+        .filter((value) => typeof value === 'string' && value.length > 0));
+    const explicitlyPreferredScopeRefs = new Set([...(preferredScopeRefs ?? [])].filter((value) => typeof value === 'string' && value.length > 0));
+    const scopeByRef = new Map(scopes.map((scope) => [scope.ref, scope]));
+    const includedScopeRefs = new Set([...linkedScopeRefs, ...explicitlyPreferredScopeRefs]);
+    for (const scopeRef of [...includedScopeRefs]) {
+        let parentSurfaceRef = scopeByRef.get(scopeRef)?.parentSurfaceRef;
+        while (parentSurfaceRef) {
+            if (includedScopeRefs.has(parentSurfaceRef)) {
+                break;
+            }
+            includedScopeRefs.add(parentSurfaceRef);
+            parentSurfaceRef = scopeByRef.get(parentSurfaceRef)?.parentSurfaceRef;
+        }
+    }
+    const scored = scopes
+        .map((scope) => {
+        const directlyLinked = linkedScopeRefs.has(scope.ref);
+        const explicitlyPreferred = explicitlyPreferredScopeRefs.has(scope.ref);
+        const disambiguatingAncestor = includedScopeRefs.has(scope.ref);
+        if (!disambiguatingAncestor) {
+            return null;
+        }
+        const scopeTargets = targets.filter((target) => target.surfaceRef === scope.ref);
+        const kind = (scope.kind ?? '').toLowerCase();
+        const overlayLike = HIGH_SIGNAL_SCOPE_KINDS.has(kind);
+        const linkedCount = scopeTargets.length;
+        const interactionRich = scopeTargets.some((target) => target.allowedActions.includes('fill') ||
+            target.allowedActions.includes('select') ||
+            target.acceptancePolicy === 'selection' ||
+            target.acceptancePolicy === 'date-selection');
+        let score = 0;
+        if (explicitlyPreferred)
+            score += 160;
+        if (directlyLinked)
+            score += 100;
+        if (!directlyLinked && !explicitlyPreferred)
+            score += 40;
+        if (overlayLike)
+            score += 60;
+        if (interactionRich)
+            score += 30;
+        score += Math.min(linkedCount, 6) * 5;
+        return { scope, score };
+    })
+        .filter((entry) => entry !== null)
+        .filter(({ score }) => score > 0)
+        .sort((left, right) => right.score - left.score || left.scope.ref.localeCompare(right.scope.ref));
+    if (scored.length > 0) {
+        return scored.slice(0, 12).map(({ scope }) => scope);
+    }
+    return [];
+}
+export function buildSurfaceRef(target) {
+    if (!target.surfaceKind && !target.surfaceSelector && !target.surfaceLabel) {
+        return undefined;
+    }
+    const frameKey = target.framePath?.join('>') ?? 'top';
+    return [
+        target.pageSignature ?? 'unknown-page',
+        frameKey,
+        target.surfaceKind ?? 'surface',
+        target.surfaceSelector ?? '',
+        target.surfaceLabel ?? '',
+    ].join('|');
+}
+function normalizeSurfaceText(value) {
+    const normalized = value?.replace(/\s+/g, ' ').trim();
+    return normalized ? normalized : undefined;
+}
+function semanticFormContextLabel(target) {
+    for (const node of [
+        target.context?.landmark,
+        target.context?.container,
+        target.context?.group,
+        target.context?.item,
+    ]) {
+        const kind = (node?.kind ?? '').trim().toLowerCase();
+        if (kind !== 'form') {
+            continue;
+        }
+        return normalizeSurfaceText(node?.label ?? node?.text);
+    }
+    return undefined;
+}
+export function buildSyntheticFormSurfaceId(target) {
+    const pageKey = normalizeSurfaceText(target.pageSignature);
+    if (!pageKey) {
+        return undefined;
+    }
+    const formKey = normalizeSurfaceText(target.formSelector) ?? semanticFormContextLabel(target)?.toLowerCase();
+    if (!formKey) {
+        return undefined;
+    }
+    const frameKey = target.framePath?.join('>') ??
+        (normalizeSurfaceText(target.frameUrl)
+            ? `url:${normalizeSurfaceText(target.frameUrl)}`
+            : 'top');
+    return `${pageKey}|${frameKey}|form|${formKey}`;
+}
+export function surfaceReplacementKey(surface) {
+    const pageKey = surface.pageSignature ?? surface.pageRef;
+    const frameKey = surface.framePath?.join('>') ?? 'top';
+    const kindKey = (surface.kind ?? 'surface').toLowerCase();
+    const labelKey = (surface.label ?? '').trim().toLowerCase();
+    return `${pageKey}|${frameKey}|${kindKey}|${labelKey}`;
+}
+export function summarizeContext(context) {
+    if (!context)
+        return undefined;
+    const parts = [];
+    const pushIfDistinct = (value) => {
+        const normalized = normalizeSurfaceText(value);
+        if (!normalized || parts.includes(normalized)) {
+            return;
+        }
+        parts.push(normalized);
+    };
+    pushIfDistinct(context.item?.label ?? context.item?.text ?? context.item?.kind);
+    pushIfDistinct(context.group?.label ?? context.group?.text ?? context.group?.kind);
+    if (parts.length === 0) {
+        pushIfDistinct(context.container?.label ?? context.container?.text ?? context.container?.kind);
+    }
+    if (parts.length === 0 || context.landmark?.label || context.landmark?.text) {
+        pushIfDistinct(context.landmark?.label ?? context.landmark?.text ?? context.landmark?.kind);
+    }
+    if (parts.length === 0 && context.hintText) {
+        pushIfDistinct(context.hintText);
+    }
+    if (parts.length === 0)
+        return undefined;
+    return parts.join(' / ');
+}
+export function pushUniqueLocatorCandidate(acc, candidate) {
+    if (!candidate)
+        return;
+    const key = locatorCandidateKey(candidate);
+    if (acc.some((existing) => locatorCandidateKey(existing) === key)) {
+        return;
+    }
+    acc.push(candidate);
+}
+function surfaceRoleFromKind(kind) {
+    const normalized = (kind ?? '').trim().toLowerCase();
+    switch (normalized) {
+        case 'card':
+            return 'article';
+        case 'article':
+        case 'dialog':
+        case 'listbox':
+        case 'menu':
+        case 'grid':
+        case 'tabpanel':
+        case 'group':
+        case 'region':
+        case 'listitem':
+        case 'row':
+        case 'form':
+            return normalized;
+        case 'fieldset':
+            return 'group';
+        case 'section':
+            return 'region';
+        default:
+            return undefined;
+    }
+}
+function collectSurfaceLocatorCandidates(target) {
+    const candidates = [];
+    const selectors = [target.surfaceSelector, ...(target.surfaceSelectors ?? [])].filter((value) => typeof value === 'string' && value.length > 0);
+    for (const selector of selectors) {
+        pushUniqueLocatorCandidate(candidates, {
+            strategy: selector.startsWith('xpath=') ? 'xpath' : 'css',
+            value: selector,
+            scope: 'root',
+        });
+    }
+    const surfaceRole = surfaceRoleFromKind(target.surfaceKind);
+    if (surfaceRole) {
+        pushUniqueLocatorCandidate(candidates, target.surfaceLabel
+            ? { strategy: 'role', value: surfaceRole, name: target.surfaceLabel }
+            : undefined);
+        pushUniqueLocatorCandidate(candidates, { strategy: 'role', value: surfaceRole });
+    }
+    return candidates;
+}
+export function collectSurfaceDescriptors(pageRef, targets) {
+    const descriptors = new Map();
+    for (const target of targets) {
+        const surfaceId = target.surfaceRef;
+        if (!surfaceId) {
+            continue;
+        }
+        const existing = descriptors.get(surfaceId);
+        const locatorCandidates = existing ? [...existing.locatorCandidates] : [];
+        for (const candidate of collectSurfaceLocatorCandidates(target)) {
+            pushUniqueLocatorCandidate(locatorCandidates, candidate);
+        }
+        descriptors.set(surfaceId, {
+            surfaceId,
+            pageRef,
+            framePath: existing?.framePath ?? target.framePath,
+            frameUrl: existing?.frameUrl ?? target.frameUrl,
+            kind: existing?.kind ?? target.surfaceKind ?? 'surface',
+            label: existing?.label && existing.label !== 'Active surface'
+                ? existing.label
+                : (target.surfaceLabel ?? target.surfaceKind ?? existing?.label ?? 'Active surface'),
+            lifecycle: 'live',
+            availability: { state: 'available' },
+            locatorCandidates,
+            createdAt: existing?.createdAt ?? Date.now(),
+            pageSignature: existing?.pageSignature ?? target.pageSignature,
+            extractScopeLifetime: inferExtractScopeLifetime(existing?.kind ?? target.surfaceKind ?? 'surface', locatorCandidates),
+        });
+    }
+    for (const target of targets) {
+        const surfaceId = buildSyntheticFormSurfaceId(target);
+        if (!surfaceId) {
+            continue;
+        }
+        const existing = descriptors.get(surfaceId);
+        const locatorCandidates = existing ? [...existing.locatorCandidates] : [];
+        const formSelector = target.formSelector?.trim();
+        if (formSelector) {
+            pushUniqueLocatorCandidate(locatorCandidates, {
+                strategy: formSelector.startsWith('xpath=') ? 'xpath' : 'css',
+                value: formSelector,
+                scope: 'root',
+            });
+        }
+        descriptors.set(surfaceId, {
+            surfaceId,
+            pageRef,
+            framePath: existing?.framePath ?? target.framePath,
+            frameUrl: existing?.frameUrl ?? target.frameUrl,
+            kind: 'form',
+            label: existing?.label ?? semanticFormContextLabel(target) ?? 'Form',
+            lifecycle: 'live',
+            availability: { state: 'available' },
+            locatorCandidates,
+            createdAt: existing?.createdAt ?? Date.now(),
+            pageSignature: existing?.pageSignature ?? target.pageSignature,
+            extractScopeLifetime: inferExtractScopeLifetime('form', locatorCandidates),
+        });
+    }
+    return [...descriptors.values()];
+}

package/dist/commands/observe.d.ts

@@ -0,0 +1,113 @@
+/**
+ * browse observe ["<instruction>"] — Discover available actions on the page.
+ */
+import type { BrowserCommandSession } from "../browser-session-state.js";
+import { type BrowseResult } from "../output.js";
+import { type CompactObservedFillableForm, type CompactObservedScope, type CompactObservedSignal, type CompactObservedTarget } from "./observe-projection.js";
+import { toStagehandDescriptor } from "./observe-stagehand.js";
+/** Stable top-level error codes returned by `observe(...)`. */
+export declare const OBSERVE_ERROR_CODES: readonly ["browser_connection_failed", "observe_failed", "protected_observe_blocked"];
+/** Stable outcome categories emitted by `observe(...)`. */
+export declare const OBSERVE_OUTCOME_TYPES: readonly ["blocked", "observation_completed", "protected_exposure_active"];
+export type ObserveErrorCode = (typeof OBSERVE_ERROR_CODES)[number];
+export type ObserveOutcomeType = (typeof OBSERVE_OUTCOME_TYPES)[number];
+export type ObserveExecutionMode = "deterministic_dom" | "goal_heuristic_shortlist" | "goal_assistive_rerank" | "goal_assistive_stagehand";
+/** Flat target entry returned from `observe(...)`. */
+export type ObserveTarget = CompactObservedTarget;
+/** Grouped scope entry returned from `observe(...)`. */
+export type ObserveScope = CompactObservedScope;
+/** Protected-fill form binding returned from `observe(...)`. */
+export type ObserveFillableForm = CompactObservedFillableForm;
+/** Page-level signal entry returned from `observe(...)`. */
+export type ObserveSignal = CompactObservedSignal;
+type ObserveSuccessPayload = BrowseResult & {
+    success: true;
+    observationMode: ObserveExecutionMode;
+    scopes?: ObserveScope[];
+    targets?: ObserveTarget[];
+    signals?: ObserveSignal[];
+    fillableForms?: ObserveFillableForm[];
+};
+type ObserveFailurePayload = {
+    error: ObserveErrorCode;
+    outcomeType: Extract<ObserveOutcomeType, "blocked" | "protected_exposure_active">;
+    message: string;
+    reason: string;
+    pageRef?: string;
+    blockedPath?: "goal-rerank" | "stagehand-fallback";
+    fallbackReason?: "deterministic-observe-empty" | "deterministic-observe-failed" | "dom-rerank-empty";
+    deterministicObserveError?: string;
+    fillRef?: string;
+    requestId?: string;
+    activatedAt?: string;
+    exposureReason?: string;
+};
+export type ObserveSuccessResult = ObserveSuccessPayload & {
+    success: true;
+    observationMode: ObserveExecutionMode;
+    scopes: ObserveScope[];
+    targets: ObserveTarget[];
+    signals: ObserveSignal[];
+    fillableForms: ObserveFillableForm[];
+    targetCount: number;
+    scopeCount: number;
+    projectedTargetCount: number;
+};
+/** Failed observe result with a stable top-level error code. */
+export type ObserveFailureResult = {
+    success: false;
+} & ObserveFailurePayload;
+export type ObserveResult = ObserveSuccessResult | ObserveFailureResult;
+export declare const __testDomTargetCollection: {
+    collectDomTargetsFromDocument: (context: {
+        evaluate<T>(pageFunction: string): Promise<T>;
+    }, options?: import("./observe-inventory.js").DomTargetCollectionOptions & {
+        framePath?: string[];
+        frameUrl?: string;
+    }) => Promise<import("./observe-inventory.js").DomObservedTarget[]>;
+    inferStructuredCellVariantFromEvidence: typeof import("./observe-inventory.js").inferStructuredCellVariantFromEvidence;
+    inferDirectionalControlFallbackFromEvidence: typeof import("./observe-inventory.js").inferDirectionalControlFallbackFromEvidence;
+    locatorDomSignatureScript: string;
+};
+export declare const __testStagehandDescriptor: {
+    toStagehandDescriptor: typeof toStagehandDescriptor;
+    normalizeStagehandSelector: typeof import("./observe-inventory.js").normalizeStagehandSelector;
+    readStagehandDomFacts: (locator: import("playwright-core").Locator) => Promise<{
+        kind?: string;
+        role?: string;
+        placeholder?: string;
+        inputName?: string;
+        inputType?: string;
+        autocomplete?: string;
+        ariaAutocomplete?: string;
+        value?: string;
+        text?: string;
+        currentValue?: string;
+        states?: Record<string, string | boolean | number>;
+    } | null>;
+    readStagehandDomFactsInBrowser: (element: Element) => {
+        kind?: string;
+        role?: string;
+        placeholder?: string;
+        inputName?: string;
+        inputType?: string;
+        autocomplete?: string;
+        ariaAutocomplete?: string;
+        value?: string;
+        text?: string;
+        currentValue?: string;
+        states?: Record<string, string | boolean | number>;
+    } | null;
+    stagehandDomFactsScript: string;
+};
+/**
+ * Observes the current page and returns the current target inventory.
+ *
+ * The success payload always includes a flat `targets` array for direct
+ * consumption and grouped `scopes` for richer container-aware workflows.
+ */
+export declare function observeBrowser(session: BrowserCommandSession, instruction?: string): Promise<ObserveResult>;
+/** CLI wrapper for `observeBrowser(...)` that persists the observed runtime state. */
+export declare function observe(session: BrowserCommandSession, instruction?: string): Promise<void>;
+export {};
+//# sourceMappingURL=observe.d.ts.map

package/dist/commands/observe.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"observe.d.ts","sourceRoot":"","sources":["../../src/commands/observe.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,KAAK,EAAE,qBAAqB,EAAE,MAAM,6BAA6B,CAAC;AAwBzE,OAAO,EAIL,KAAK,YAAY,EAClB,MAAM,cAAc,CAAC;AAsCtB,OAAO,EAIL,KAAK,2BAA2B,EAChC,KAAK,oBAAoB,EACzB,KAAK,qBAAqB,EAC1B,KAAK,qBAAqB,EAM3B,MAAM,yBAAyB,CAAC;AAKjC,OAAO,EAAE,qBAAqB,EAAE,MAAM,wBAAwB,CAAC;AAc/D,+DAA+D;AAC/D,eAAO,MAAM,mBAAmB,uFAItB,CAAC;AAEX,2DAA2D;AAC3D,eAAO,MAAM,qBAAqB,4EAIxB,CAAC;AAEX,MAAM,MAAM,gBAAgB,GAAG,CAAC,OAAO,mBAAmB,CAAC,CAAC,MAAM,CAAC,CAAC;AACpE,MAAM,MAAM,kBAAkB,GAAG,CAAC,OAAO,qBAAqB,CAAC,CAAC,MAAM,CAAC,CAAC;AAExE,MAAM,MAAM,oBAAoB,GAC5B,mBAAmB,GACnB,0BAA0B,GAC1B,uBAAuB,GACvB,0BAA0B,CAAC;AAE/B,sDAAsD;AACtD,MAAM,MAAM,aAAa,GAAG,qBAAqB,CAAC;AAElD,wDAAwD;AACxD,MAAM,MAAM,YAAY,GAAG,oBAAoB,CAAC;AAEhD,gEAAgE;AAChE,MAAM,MAAM,mBAAmB,GAAG,2BAA2B,CAAC;AAE9D,4DAA4D;AAC5D,MAAM,MAAM,aAAa,GAAG,qBAAqB,CAAC;AAElD,KAAK,qBAAqB,GAAG,YAAY,GAAG;IAC1C,OAAO,EAAE,IAAI,CAAC;IACd,eAAe,EAAE,oBAAoB,CAAC;IACtC,MAAM,CAAC,EAAE,YAAY,EAAE,CAAC;IACxB,OAAO,CAAC,EAAE,aAAa,EAAE,CAAC;IAC1B,OAAO,CAAC,EAAE,aAAa,EAAE,CAAC;IAC1B,aAAa,CAAC,EAAE,mBAAmB,EAAE,CAAC;CACvC,CAAC;AAEF,KAAK,qBAAqB,GAAG;IAC3B,KAAK,EAAE,gBAAgB,CAAC;IACxB,WAAW,EAAE,OAAO,CAClB,kBAAkB,EAClB,SAAS,GAAG,2BAA2B,CACxC,CAAC;IACF,OAAO,EAAE,MAAM,CAAC;IAChB,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,WAAW,CAAC,EAAE,aAAa,GAAG,oBAAoB,CAAC;IACnD,cAAc,CAAC,EACX,6BAA6B,GAC7B,8BAA8B,GAC9B,kBAAkB,CAAC;IACvB,yBAAyB,CAAC,EAAE,MAAM,CAAC;IACnC,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,cAAc,CAAC,EAAE,MAAM,CAAC;CACzB,CAAC;AAEF,MAAM,MAAM,oBAAoB,GAAG,qBAAqB,GAAG;IACzD,OAAO,EAAE,IAAI,CAAC;IACd,eAAe,EAAE,oBAAoB,CAAC;IACtC,MAAM,EAAE,YAAY,EAAE,CAAC;IACvB,OAAO,EAAE,aAAa,EAAE,CAAC;IACzB,OAAO,EAAE,aAAa,EAAE,CAAC;IACzB,aAAa,EAAE,mBAAmB,EAAE,CAAC;IACrC,WAAW,EAAE,MAAM,CAAC;IACpB,UAAU,EAAE,MAAM,CAAC;IACnB,oBAAoB,EAAE,MAAM,CAAC;CAC9B,CAAC;AAEF,gEAAgE;AAChE,MAAM,MAAM,oBAAoB,GAAG;IAAE,OAAO,EAAE,KAAK,CAAA;CAAE,GAAG,qBAAqB,CAAC;AAE9E,MAAM,MAAM,aAAa,GAAG,oBAAoB,GAAG,oBAAoB,CAAC;AAyKxE,eAAO,MAAM,yBAAyB;;;;iBAgUnB,CAAC;gBACG,CAAC;;;;;CAjU6C,CAAC;AACtE,eAAO,MAAM,yBAAyB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CAGrC,CAAC;AAEF;;;;;GAKG;AACH,wBAAsB,cAAc,CAClC,OAAO,EAAE,qBAAqB,EAC9B,WAAW,CAAC,EAAE,MAAM,GACnB,OAAO,CAAC,aAAa,CAAC,CAwiBxB;AAED,sFAAsF;AACtF,wBAAsB,OAAO,CAC3B,OAAO,EAAE,qBAAqB,EAC9B,WAAW,CAAC,EAAE,MAAM,GACnB,OAAO,CAAC,IAAI,CAAC,CAWf"}