@nuanu-ai/agentbrowse 0.2.29 → 0.2.31

package/dist/commands/semantic-observe.js

@@ -1,6 +1,7 @@
 import { z } from 'zod';
 import { AgentpayStagehandLlmClient } from '../agentpay-stagehand-llm.js';
 import { resolveAgentpayGatewayConfig } from '../agentpay-gateway.js';
+import { recordLlmUsage, recordPayloadBudget } from '../runtime-state.js';
 const rerankSchema = z.object({
     matches: z
         .array(z.object({
@@ -9,6 +10,7 @@ const rerankSchema = z.object({
         .max(8),
 });
 const RERANK_CANDIDATE_LIMIT = 120;
+const GOAL_RETRIEVAL_ENTITY_LIMIT = 64;
 const FORM_BUCKET_RESERVE_LIMIT = 48;
 const FORM_BUCKET_RESERVE_PER_BUCKET = 8;
 const SCOPE_BUCKET_RESERVE_LIMIT = 24;
@@ -27,12 +29,60 @@ const HIGH_SIGNAL_SCOPE_KINDS = new Set([
     'card',
     'form',
 ]);
+const GOAL_TEXT_STOPWORDS = new Set([
+    'a',
+    'an',
+    'and',
+    'at',
+    'for',
+    'from',
+    'in',
+    'of',
+    'on',
+    'or',
+    'the',
+    'to',
+    'with',
+    'в',
+    'для',
+    'и',
+    'или',
+    'на',
+    'найти',
+]);
 function isFieldLikeTarget(target) {
     const kind = (target.kind ?? '').trim().toLowerCase();
     const role = (target.role ?? '').trim().toLowerCase();
     return (['input', 'textarea', 'select', 'combobox'].includes(kind) ||
         ['textbox', 'combobox'].includes(role));
 }
+function isScopeLikeCandidate(target) {
+    return ((target.goalInventoryType ?? '').trim().toLowerCase() === 'scope' ||
+        (target.capability ?? '').trim().toLowerCase() === 'scope');
+}
+function isActionLikeTargetCandidate(target) {
+    if (isScopeLikeCandidate(target) || isFieldLikeTarget(target)) {
+        return false;
+    }
+    const kind = (target.kind ?? '').trim().toLowerCase();
+    const role = (target.role ?? '').trim().toLowerCase();
+    return (target.allowedActions?.includes('click') ||
+        target.allowedActions?.includes('press') ||
+        kind === 'button' ||
+        role === 'button' ||
+        kind === 'link' ||
+        role === 'link');
+}
+function normalizeRetrievalText(value) {
+    const normalized = value?.replace(/\s+/g, ' ').trim().toLowerCase();
+    return normalized ? normalized : undefined;
+}
+function tokenizeRetrievalText(value) {
+    if (!value) {
+        return [];
+    }
+    return (value.match(/[\p{L}\p{N}]+/gu) ?? []).map((token) => token.toLowerCase());
+}
 function semanticFormContextKey(context) {
     for (const node of [context?.landmark, context?.container, context?.group, context?.item]) {
         const kind = (node?.kind ?? '').trim().toLowerCase();
@@ -58,6 +108,17 @@ function candidateBucketKey(target, options = {}) {
     const surfaceKey = (options.preferFormBucket ? formKey : undefined) ?? target.surfaceRef ?? formKey ?? 'page-root';
     return `${frameKey}|${surfaceKey}`;
 }
+function surfaceIdentityOf(target) {
+    const explicitSurfaceId = target.goalSurfaceId?.trim();
+    if (explicitSurfaceId) {
+        return explicitSurfaceId;
+    }
+    const surfaceRef = target.surfaceRef?.trim();
+    if (!surfaceRef) {
+        return undefined;
+    }
+    return surfaceRef.startsWith('scope:') ? surfaceRef.slice('scope:'.length) : surfaceRef;
+}
 function isPrimaryFormControlTarget(target) {
     const kind = (target.kind ?? '').trim().toLowerCase();
     const role = (target.role ?? '').trim().toLowerCase();
@@ -102,6 +163,31 @@ function isHighSignalScopeCandidate(target) {
     const kind = (target.kind ?? '').trim().toLowerCase();
     return HIGH_SIGNAL_SCOPE_KINDS.has(kind);
 }
+function contentItemBucketKey(target) {
+    if (isScopeLikeCandidate(target) ||
+        !isActionLikeTargetCandidate(target) ||
+        Boolean(formBucketKey(target)) ||
+        isPrimaryFormControlTarget(target)) {
+        return undefined;
+    }
+    const surfaceKind = (target.surfaceKind ?? '').trim().toLowerCase();
+    if (['form', 'dialog', 'listbox', 'menu', 'popover', 'dropdown', 'datepicker'].includes(surfaceKind)) {
+        return undefined;
+    }
+    const surfaceIdentity = surfaceIdentityOf(target);
+    if (surfaceIdentity) {
+        return `surface:${surfaceIdentity}`;
+    }
+    const itemKey = normalizeRetrievalText(target.context?.item?.label ?? target.context?.item?.text);
+    if (itemKey) {
+        return `item:${itemKey}`;
+    }
+    const containerKey = normalizeRetrievalText(target.context?.container?.label ?? target.context?.container?.text);
+    if (containerKey) {
+        return `container:${containerKey}`;
+    }
+    return undefined;
+}
 function scopeCandidatePriority(target) {
     const kind = (target.kind ?? '').trim().toLowerCase();
     if (kind === 'dialog' || kind === 'listbox' || kind === 'menu') {
@@ -118,6 +204,343 @@ function scopeCandidatePriority(target) {
     }
     return 4;
 }
+function representativeCandidateScore(target) {
+    const normalizedLabel = normalizeRetrievalText(target.label);
+    const kind = (target.kind ?? '').trim().toLowerCase();
+    const role = (target.role ?? '').trim().toLowerCase();
+    let score = (target.surfacePriority ?? 0) * 10;
+    if (isPrimaryFormControlTarget(target)) {
+        score += 1_500 - primaryFormTargetPriority(target) * 100;
+    }
+    else if (isFieldLikeTarget(target)) {
+        score += 1_250;
+    }
+    else if (isActionLikeTargetCandidate(target)) {
+        score += 1_000;
+    }
+    else if (isScopeLikeCandidate(target)) {
+        score += 700;
+    }
+    if (target.allowedActions?.includes('fill') || target.allowedActions?.includes('select')) {
+        score += 120;
+    }
+    if (target.allowedActions?.includes('click') || target.allowedActions?.includes('press')) {
+        score += 80;
+    }
+    if (target.acceptancePolicy === 'submit') {
+        score += 120;
+    }
+    if (target.acceptancePolicy === 'navigation') {
+        score += 60;
+    }
+    if (kind === 'link' || role === 'link') {
+        score += 40;
+    }
+    if (normalizedLabel) {
+        score += Math.min(normalizedLabel.length, 100);
+        if (normalizedLabel === 'button' || normalizedLabel === 'link') {
+            score -= 300;
+        }
+        if (normalizedLabel.includes('opens in new window')) {
+            score -= 120;
+        }
+        if (normalizedLabel.includes('save this item')) {
+            score -= 80;
+        }
+    }
+    return score;
+}
+function entityMemberPriority(entityKind, target) {
+    if (entityKind === 'form' && isPrimaryFormControlTarget(target)) {
+        return 5_000 - primaryFormTargetPriority(target) * 100;
+    }
+    if (entityKind === 'scope' && isScopeLikeCandidate(target)) {
+        return representativeCandidateScore(target) - 200;
+    }
+    return representativeCandidateScore(target);
+}
+function compareEntityMembers(entityKind, left, right) {
+    const scoreDelta = entityMemberPriority(entityKind, right.target) - entityMemberPriority(entityKind, left.target);
+    if (scoreDelta !== 0) {
+        return scoreDelta;
+    }
+    return left.index - right.index;
+}
+function pickEntityLabel(entityKind, representative) {
+    if (entityKind === 'form') {
+        return (representative.context?.landmark?.label ??
+            representative.context?.container?.label ??
+            representative.context?.group?.label ??
+            representative.surfaceLabel ??
+            representative.label);
+    }
+    return (representative.label ??
+        representative.context?.container?.label ??
+        representative.context?.item?.label ??
+        representative.surfaceLabel ??
+        representative.context?.group?.label ??
+        representative.context?.landmark?.label);
+}
+function collectRepresentativeLabels(targets) {
+    const labels = [];
+    for (const target of targets) {
+        const label = target.label?.replace(/\s+/g, ' ').trim();
+        if (!label || label === 'Button' || label === 'Link' || labels.includes(label)) {
+            continue;
+        }
+        labels.push(label);
+        if (labels.length >= 4) {
+            break;
+        }
+    }
+    return labels;
+}
+function buildEntitySearchText(entityLabel, representative, representativeLabels) {
+    return [
+        entityLabel,
+        representative.label,
+        ...representativeLabels,
+        representative.surfaceLabel,
+        representative.context?.item?.label,
+        representative.context?.item?.text,
+        representative.context?.group?.label,
+        representative.context?.group?.text,
+        representative.context?.container?.label,
+        representative.context?.container?.text,
+        representative.context?.landmark?.label,
+        representative.context?.landmark?.text,
+        representative.context?.hintText,
+    ]
+        .map((value) => normalizeRetrievalText(value))
+        .filter((value) => Boolean(value))
+        .join(' ');
+}
+function buildGoalRetrievalEntity(entityKind, entityKey, memberIndexes, targets) {
+    const orderedMembers = [...memberIndexes]
+        .map((index) => ({ index, target: targets[index] }))
+        .sort((left, right) => compareEntityMembers(entityKind, left, right));
+    const representative = orderedMembers[0].target;
+    const representativeLabels = collectRepresentativeLabels(orderedMembers.map((entry) => entry.target));
+    const label = pickEntityLabel(entityKind, representative);
+    return {
+        entityKind,
+        entityKey,
+        firstIndex: Math.min(...memberIndexes),
+        memberIndexes: orderedMembers.map((entry) => entry.index),
+        representative,
+        label,
+        kind: entityKind === 'form'
+            ? 'form'
+            : entityKind === 'item'
+                ? representative.surfaceKind ?? representative.kind
+                : representative.kind,
+        surfaceKind: representative.surfaceKind,
+        surfaceLabel: representative.surfaceLabel,
+        surfacePriority: representative.surfacePriority,
+        framePath: representative.framePath,
+        frameUrl: representative.frameUrl,
+        context: representative.context,
+        structure: representative.structure,
+        representativeLabels,
+        searchText: buildEntitySearchText(label, representative, representativeLabels),
+    };
+}
+function buildGoalRetrievalEntities(targets) {
+    const entities = [];
+    const groupedTargetIndexes = new Set();
+    const nonScopeTargetsBySurface = new Map();
+    for (const [index, target] of targets.entries()) {
+        if (isScopeLikeCandidate(target)) {
+            continue;
+        }
+        const surfaceIdentity = surfaceIdentityOf(target);
+        if (!surfaceIdentity) {
+            continue;
+        }
+        const linked = nonScopeTargetsBySurface.get(surfaceIdentity) ?? [];
+        linked.push(index);
+        nonScopeTargetsBySurface.set(surfaceIdentity, linked);
+    }
+    const formGroups = new Map();
+    for (const [index, target] of targets.entries()) {
+        if (isScopeLikeCandidate(target)) {
+            continue;
+        }
+        const bucketKey = formBucketKey(target);
+        if (!bucketKey) {
+            continue;
+        }
+        const members = formGroups.get(bucketKey) ?? [];
+        members.push(index);
+        formGroups.set(bucketKey, members);
+    }
+    for (const [key, memberIndexes] of formGroups.entries()) {
+        entities.push(buildGoalRetrievalEntity('form', `form:${key}`, memberIndexes, targets));
+        memberIndexes.forEach((index) => groupedTargetIndexes.add(index));
+    }
+    const itemGroups = new Map();
+    for (const [index, target] of targets.entries()) {
+        const bucketKey = contentItemBucketKey(target);
+        if (!bucketKey) {
+            continue;
+        }
+        const members = itemGroups.get(bucketKey) ?? [];
+        members.push(index);
+        itemGroups.set(bucketKey, members);
+    }
+    for (const [key, memberIndexes] of itemGroups.entries()) {
+        entities.push(buildGoalRetrievalEntity('item', `item:${key}`, memberIndexes, targets));
+        memberIndexes.forEach((index) => groupedTargetIndexes.add(index));
+    }
+    for (const [index, target] of targets.entries()) {
+        if (!isHighSignalScopeCandidate(target)) {
+            continue;
+        }
+        const kind = (target.kind ?? '').trim().toLowerCase();
+        const surfaceIdentity = surfaceIdentityOf(target);
+        if (!surfaceIdentity) {
+            continue;
+        }
+        if (kind === 'form') {
+            continue;
+        }
+        if (kind === 'card' && itemGroups.has(`surface:${surfaceIdentity}`)) {
+            continue;
+        }
+        const linkedTargets = nonScopeTargetsBySurface.get(surfaceIdentity) ?? [];
+        const memberIndexes = [index, ...linkedTargets];
+        entities.push(buildGoalRetrievalEntity('scope', `scope:${surfaceIdentity}`, memberIndexes, targets));
+        linkedTargets.forEach((targetIndex) => groupedTargetIndexes.add(targetIndex));
+    }
+    for (const [index, target] of targets.entries()) {
+        if (isScopeLikeCandidate(target) || groupedTargetIndexes.has(index)) {
+            continue;
+        }
+        if (!isActionLikeTargetCandidate(target) &&
+            !isFieldLikeTarget(target) &&
+            !isPrimaryFormControlTarget(target)) {
+            continue;
+        }
+        entities.push(buildGoalRetrievalEntity('standalone', `target:${index}`, [index], targets));
+    }
+    return entities.sort((left, right) => left.firstIndex - right.firstIndex);
+}
+function retrievalEntityPriority(entity) {
+    let score = (entity.surfacePriority ?? 0) * 10;
+    switch (entity.entityKind) {
+        case 'form':
+            score += 900;
+            break;
+        case 'item':
+            score += 820;
+            break;
+        case 'scope':
+            score += 720;
+            break;
+        case 'standalone':
+            score += 600;
+            break;
+    }
+    score += Math.min(entity.memberIndexes.length, 6) * 25;
+    score += representativeCandidateScore(entity.representative);
+    return score;
+}
+function scoreRetrievalEntityAgainstGoal(goal, entity) {
+    const normalizedGoal = normalizeRetrievalText(goal);
+    if (!normalizedGoal) {
+        return 0;
+    }
+    const entityText = entity.searchText;
+    if (!entityText) {
+        return 0;
+    }
+    let score = 0;
+    const goalTokens = tokenizeRetrievalText(normalizedGoal).filter((token) => token.length >= 2 && !GOAL_TEXT_STOPWORDS.has(token));
+    const entityTokens = new Set(tokenizeRetrievalText(entityText));
+    if (entityText.includes(normalizedGoal)) {
+        score += 220;
+    }
+    const entityLabel = normalizeRetrievalText(entity.label);
+    if (entityLabel) {
+        if (normalizedGoal.includes(entityLabel)) {
+            score += 180;
+        }
+        if (entityLabel.includes(normalizedGoal)) {
+            score += 140;
+        }
+    }
+    let matchedTokenCount = 0;
+    for (const token of goalTokens) {
+        if (entityTokens.has(token)) {
+            matchedTokenCount += 1;
+            score += 8 + Math.min(token.length, 12);
+        }
+    }
+    if (goalTokens.length > 0) {
+        const coverage = matchedTokenCount / goalTokens.length;
+        if (coverage === 1) {
+            score += 120;
+        }
+        else if (coverage >= 0.75) {
+            score += 70;
+        }
+        else if (coverage >= 0.5) {
+            score += 35;
+        }
+    }
+    return score;
+}
+function preselectRetrievalEntities(goal, entities) {
+    if (entities.length <= GOAL_RETRIEVAL_ENTITY_LIMIT) {
+        return [...entities];
+    }
+    const scored = entities.map((entity) => ({
+        entity,
+        lexicalScore: scoreRetrievalEntityAgainstGoal(goal, entity),
+        priority: retrievalEntityPriority(entity),
+    }));
+    const selected = [];
+    const seenKeys = new Set();
+    for (const entry of scored
+        .filter((candidate) => candidate.lexicalScore > 0)
+        .sort((left, right) => right.lexicalScore - left.lexicalScore ||
+        right.priority - left.priority ||
+        left.entity.firstIndex - right.entity.firstIndex)) {
+        if (seenKeys.has(entry.entity.entityKey)) {
+            continue;
+        }
+        seenKeys.add(entry.entity.entityKey);
+        selected.push(entry.entity);
+        if (selected.length >= GOAL_RETRIEVAL_ENTITY_LIMIT) {
+            return selected;
+        }
+    }
+    for (const entry of scored.sort((left, right) => right.priority - left.priority || left.entity.firstIndex - right.entity.firstIndex)) {
+        if (seenKeys.has(entry.entity.entityKey)) {
+            continue;
+        }
+        seenKeys.add(entry.entity.entityKey);
+        selected.push(entry.entity);
+        if (selected.length >= GOAL_RETRIEVAL_ENTITY_LIMIT) {
+            break;
+        }
+    }
+    return selected;
+}
+function expandRetrievalEntitiesToCandidates(targets, entities) {
+    const orderedIndexes = [];
+    const seenIndexes = new Set();
+    for (const entity of entities) {
+        for (const index of entity.memberIndexes) {
+            if (seenIndexes.has(index)) {
+                continue;
+            }
+            seenIndexes.add(index);
+            orderedIndexes.push(index);
+        }
+    }
+    return orderedIndexes.map((index) => targets[index]).filter(Boolean);
+}
 function collectBucketedCandidateIndexes(targets, options) {
     const bucketEntries = new Map();
     for (const [index, target] of targets.entries()) {
@@ -211,124 +634,220 @@ function diversifyCandidates(targets) {
     }
     return orderedIndexes.map((index) => targets[index]).slice(0, RERANK_CANDIDATE_LIMIT);
 }
-function buildCandidateSummary(target, index) {
+function buildCandidateSummary(target, index, surfaceSummaryIdBySurfaceKey) {
+    return buildCompactCandidateSummary(target, index, surfaceSummaryIdBySurfaceKey);
+}
+function compactContextValue(value, maxLength = 80) {
+    const normalized = value?.replace(/\s+/g, ' ').trim();
+    if (!normalized) {
+        return undefined;
+    }
+    return normalized.length > maxLength ? `${normalized.slice(0, maxLength - 1)}…` : normalized;
+}
+function compactCandidateContext(target) {
+    const context = target.context;
+    if (!context) {
+        return undefined;
+    }
+    const compacted = {};
+    const push = (key, value) => {
+        const compactedValue = compactContextValue(value);
+        if (!compactedValue) {
+            return;
+        }
+        compacted[key] = compactedValue;
+    };
+    push('item', context.item?.label ?? context.item?.text);
+    push('group', context.group?.label ?? context.group?.text);
+    push('container', context.container?.label ?? context.container?.text);
+    push('landmark', context.landmark?.label ?? context.landmark?.text);
+    push('hint', context.hintText);
+    return Object.keys(compacted).length > 0 ? compacted : undefined;
+}
+function normalizedGoalText(value) {
+    return value.replace(/\s+/g, ' ').trim().toLowerCase();
+}
+function goalNeedsScopeCandidates(goal) {
+    const normalizedGoal = normalizedGoalText(goal);
+    if (!normalizedGoal) {
+        return true;
+    }
+    const scopeHints = [
+        'form',
+        'forms',
+        'field',
+        'fields',
+        'control',
+        'controls',
+        'dialog',
+        'modal',
+        'popup',
+        'widget',
+        'section',
+        'panel',
+        'menu',
+        'listbox',
+        'dropdown',
+        'calendar',
+        'datepicker',
+        'card',
+        'results',
+        'scope',
+        'region',
+        'surface',
+        'форма',
+        'форму',
+        'поля',
+        'поля ввода',
+        'контрол',
+        'диалог',
+        'модал',
+        'попап',
+        'виджет',
+        'секци',
+        'панел',
+        'меню',
+        'список',
+        'выпада',
+        'календар',
+        'карточ',
+        'результат',
+        'область',
+        'регион',
+    ];
+    return scopeHints.some((hint) => normalizedGoal.includes(hint));
+}
+function relevantTargetsForGoal(goal, targets) {
+    if (goalNeedsScopeCandidates(goal)) {
+        return [...targets];
+    }
+    const directTargets = targets.filter((target) => !isScopeLikeCandidate(target));
+    return directTargets.length > 0 ? directTargets : [...targets];
+}
+function buildSurfaceSummaryLine(target, id) {
+    const parts = [`${id}`];
+    const context = compactCandidateContext(target);
+    const surfaceKind = target.surfaceKind ??
+        target.context?.container?.kind ??
+        target.context?.group?.kind ??
+        target.kind;
+    const surfaceLabel = target.surfaceLabel ??
+        target.context?.container?.label ??
+        target.context?.group?.label ??
+        target.context?.landmark?.label ??
+        target.label;
+    if (surfaceKind) {
+        parts.push(`kind=${JSON.stringify(surfaceKind)}`);
+    }
+    if (surfaceLabel) {
+        parts.push(`label=${JSON.stringify(surfaceLabel)}`);
+    }
+    if (typeof target.surfacePriority === 'number') {
+        parts.push(`priority=${JSON.stringify(target.surfacePriority)}`);
+    }
+    if (target.framePath?.length) {
+        parts.push(`framePath=${JSON.stringify(target.framePath)}`);
+    }
+    if (context) {
+        parts.push(`context=${JSON.stringify(context)}`);
+    }
+    return parts.join(' | ');
+}
+function buildSurfaceSummaries(targets) {
+    const summaries = [];
+    const summaryIdBySurfaceKey = new Map();
+    for (const target of targets) {
+        const surfaceKey = surfaceIdentityOf(target);
+        if (!surfaceKey || summaryIdBySurfaceKey.has(surfaceKey)) {
+            continue;
+        }
+        const id = `s${summaries.length + 1}`;
+        summaryIdBySurfaceKey.set(surfaceKey, id);
+        summaries.push({
+            id,
+            line: buildSurfaceSummaryLine(target, id),
+        });
+    }
+    return { summaries, summaryIdBySurfaceKey };
+}
+function buildCompactCandidateSummary(target, index, surfaceSummaryIdBySurfaceKey) {
     const parts = [`id=c${index + 1}`];
-    const container = target.context?.container;
-    const item = target.context?.item;
-    const group = target.context?.group;
-    const landmark = target.context?.landmark;
-    const layout = target.context?.layout;
-    const visual = target.context?.visual;
+    const compactContext = compactCandidateContext(target);
+    const surfaceSummaryId = surfaceIdentityOf(target)
+        ? surfaceSummaryIdBySurfaceKey?.get(surfaceIdentityOf(target))
+        : undefined;
     if (target.kind)
         parts.push(`kind=${JSON.stringify(target.kind)}`);
     if (target.role)
         parts.push(`role=${JSON.stringify(target.role)}`);
     if (target.label)
         parts.push(`label=${JSON.stringify(target.label)}`);
-    if (target.interactionHint) {
-        parts.push(`interactionHint=${JSON.stringify(target.interactionHint)}`);
-    }
-    if (target.capability)
-        parts.push(`capability=${JSON.stringify(target.capability)}`);
     if (target.allowedActions?.length) {
-        parts.push(`allowedActions=${JSON.stringify(target.allowedActions)}`);
+        parts.push(`actions=${JSON.stringify(target.allowedActions)}`);
     }
     if (target.acceptancePolicy) {
-        parts.push(`acceptancePolicy=${JSON.stringify(target.acceptancePolicy)}`);
+        parts.push(`policy=${JSON.stringify(target.acceptancePolicy)}`);
     }
     if (target.controlFamily) {
-        parts.push(`controlFamily=${JSON.stringify(target.controlFamily)}`);
-    }
-    if (target.surfaceRef)
-        parts.push(`surfaceRef=${JSON.stringify(target.surfaceRef)}`);
-    if (target.surfaceKind)
-        parts.push(`surfaceKind=${JSON.stringify(target.surfaceKind)}`);
-    if (target.surfaceLabel)
-        parts.push(`surfaceLabel=${JSON.stringify(target.surfaceLabel)}`);
-    if (typeof target.surfacePriority === 'number') {
-        parts.push(`surfacePriority=${JSON.stringify(target.surfacePriority)}`);
+        parts.push(`family=${JSON.stringify(target.controlFamily)}`);
+    }
+    if (target.capability && target.capability !== 'actionable') {
+        parts.push(`capability=${JSON.stringify(target.capability)}`);
+    }
+    if (surfaceSummaryId) {
+        parts.push(`surface=${surfaceSummaryId}`);
     }
-    if (target.framePath?.length)
-        parts.push(`framePath=${JSON.stringify(target.framePath)}`);
-    if (target.frameUrl)
-        parts.push(`frameUrl=${JSON.stringify(target.frameUrl)}`);
-    if (target.formSelector)
-        parts.push(`formSelector=${JSON.stringify(target.formSelector)}`);
     if (target.structure)
         parts.push(`structure=${JSON.stringify(target.structure)}`);
     if (target.placeholder)
         parts.push(`placeholder=${JSON.stringify(target.placeholder)}`);
-    if (target.title)
-        parts.push(`title=${JSON.stringify(target.title)}`);
     if (target.states)
         parts.push(`state=${JSON.stringify(target.states)}`);
-    if (item?.kind)
-        parts.push(`itemKind=${JSON.stringify(item.kind)}`);
-    if (item?.label)
-        parts.push(`itemLabel=${JSON.stringify(item.label)}`);
-    if (item?.text)
-        parts.push(`itemText=${JSON.stringify(item.text)}`);
-    if (group?.kind)
-        parts.push(`groupKind=${JSON.stringify(group.kind)}`);
-    if (group?.label)
-        parts.push(`groupLabel=${JSON.stringify(group.label)}`);
-    if (group?.text)
-        parts.push(`groupText=${JSON.stringify(group.text)}`);
-    if (container?.kind)
-        parts.push(`containerKind=${JSON.stringify(container.kind)}`);
-    if (container?.label)
-        parts.push(`containerLabel=${JSON.stringify(container.label)}`);
-    if (container?.text)
-        parts.push(`containerText=${JSON.stringify(container.text)}`);
-    if (landmark?.kind)
-        parts.push(`landmarkKind=${JSON.stringify(landmark.kind)}`);
-    if (landmark?.label)
-        parts.push(`landmarkLabel=${JSON.stringify(landmark.label)}`);
-    if (landmark?.text)
-        parts.push(`landmarkText=${JSON.stringify(landmark.text)}`);
-    if (layout?.lane)
-        parts.push(`lane=${JSON.stringify(layout.lane)}`);
-    if (layout?.band)
-        parts.push(`band=${JSON.stringify(layout.band)}`);
-    if (visual)
-        parts.push(`visual=${JSON.stringify(visual)}`);
-    if (target.context?.hintText)
-        parts.push(`hintText=${JSON.stringify(target.context.hintText)}`);
+    if (target.framePath?.length)
+        parts.push(`framePath=${JSON.stringify(target.framePath)}`);
+    if (compactContext)
+        parts.push(`context=${JSON.stringify(compactContext)}`);
     return parts.join(' | ');
 }
 function normalizeCandidateId(value) {
     return value.trim().toLowerCase();
 }
-export async function rerankDomTargetsForGoal(instruction, targets) {
+export async function rerankDomTargetsForGoal(instruction, targets, options = {}) {
     if (targets.length === 0) {
         return [];
     }
+    const relevantTargets = relevantTargetsForGoal(instruction, targets);
+    options.session &&
+        recordPayloadBudget(options.session, {
+            observeRerankCandidatesSeen: targets.length,
+        });
     const gateway = resolveAgentpayGatewayConfig();
     const client = new AgentpayStagehandLlmClient(gateway);
-    const candidates = diversifyCandidates(targets);
+    const retrievalEntities = buildGoalRetrievalEntities(relevantTargets);
+    const retrievedCandidates = relevantTargets.length > RERANK_CANDIDATE_LIMIT
+        ? expandRetrievalEntitiesToCandidates(relevantTargets, preselectRetrievalEntities(instruction, retrievalEntities))
+        : [...relevantTargets];
+    const candidates = diversifyCandidates(retrievedCandidates.length > 0 ? retrievedCandidates : [...relevantTargets]);
+    const { summaries: surfaceSummaries, summaryIdBySurfaceKey } = buildSurfaceSummaries(candidates);
+    options.session &&
+        recordPayloadBudget(options.session, {
+            observeRerankCandidatesSent: candidates.length,
+        });
     const prompt = [
-        'You are choosing from already discovered visible actionable candidates on a webpage.',
-        'Select only the candidate IDs that directly satisfy the user goal.',
-        'Use container/landmark/layout/state information to disambiguate similar labels.',
-        'Prefer candidates whose surrounding container clearly corroborates the goal.',
-        'Prefer actionable candidates over scope or informational candidates unless the goal explicitly asks for a container or region.',
-        'Prefer direct visible targets over indirect controls when the direct visible target is present.',
-        'Prefer candidates that belong to an active local surface (dialog/listbox/popover/card container) over unrelated page-root candidates when both are plausible matches.',
-        'For date cells, seat cells, and other structured-grid targets, use row/column/zone/cell metadata and state to distinguish plausible matches.',
-        'When the goal refers to an input field or typed value, prefer the directly editable input/textarea/select over a non-editable wrapper such as a combobox shell.',
-        'When the goal refers to a specific field or control, an exact disabled/readonly match is still relevant. Prefer the exact field over nearby prerequisite controls and use state/context to indicate that it is gated.',
-        'Prefer candidates whose explicit state matches the goal and avoid candidates whose state contradicts the goal.',
-        'Use visual cues only as supporting evidence when they help distinguish active vs muted targets in the same group.',
-        'When the goal is about setting or choosing a form/search/filter value, prefer the primary form control or open picker inside a form/dialog/combobox over lower informational charts, summaries, or content collections that merely mention similar values.',
-        'When the goal asks for a form or a set of related controls, return the direct controls from the relevant form instead of nearby navigation links or surrounding section headings.',
-        'If a secondary chart/list mirrors the same options as a primary form control, do not select the secondary surface unless the goal explicitly asks for that chart/list.',
-        'Avoid navigation, filter, and summary controls when the goal refers to a concrete content item unless the goal explicitly asks for those controls.',
+        'You are choosing from already discovered visible webpage candidates.',
+        'Select only candidate IDs that directly satisfy the goal.',
+        'Prefer direct actionable controls over surrounding regions unless the goal explicitly asks for a form, region, widget, or set of controls.',
+        'Use owning surface, compact context, explicit state, and structure metadata to disambiguate similar labels.',
+        'For input/value goals, prefer the directly editable field or primary picker trigger over wrappers or mirrored summary content.',
+        'For structured-grid targets such as dates or seats, use row/column/zone/cell metadata and state.',
+        'An exact disabled or readonly field can still be relevant when the goal refers to that specific field.',
         'Do not invent IDs. Return an empty list if nothing clearly matches.',
         '',
         `Goal: ${instruction}`,
         '',
+        ...(surfaceSummaries.length > 0 ? ['Surfaces:', ...surfaceSummaries.map((entry) => entry.line), ''] : []),
         'Candidates:',
-        ...candidates.map((target, index) => buildCandidateSummary(target, index)),
+        ...candidates.map((target, index) => buildCandidateSummary(target, index, summaryIdBySurfaceKey)),
     ].join('\n');
     const result = await client.createChatCompletion({
         logger: () => { },
@@ -340,6 +859,13 @@ export async function rerankDomTargetsForGoal(instruction, targets) {
             },
         },
     });
+    if (options.session) {
+        recordLlmUsage(options.session, {
+            purpose: 'browse.observe',
+            usage: result.usage,
+            inputChars: prompt.length,
+        });
+    }
     const selectedIds = new Set(result.data.matches.map((match) => normalizeCandidateId(match.candidateId)));
     return candidates.filter((_, index) => selectedIds.has(`c${index + 1}`));
 }