@nuanu-ai/agentbrowse 0.2.36 → 0.2.38

package/dist/commands/semantic-observe.js

@@ -2,6 +2,7 @@ import { z } from 'zod';
 import { AgentpayStagehandLlmClient } from '../agentpay-stagehand-llm.js';
 import { resolveAgentpayGatewayConfig } from '../agentpay-gateway.js';
 import { recordLlmUsage, recordPayloadBudget } from '../runtime-state.js';
+import { analyzeSemanticObserveText, buildSemanticObserveBm25CorpusStats, buildSemanticObserveLexicalDocument, normalizeSemanticObserveText, scoreSemanticObserveBm25, } from './semantic-observe-lexical.js';
 const rerankSchema = z.object({
     matches: z
         .array(z.object({
@@ -17,6 +18,31 @@ const SCOPE_BUCKET_RESERVE_LIMIT = 24;
 const SCOPE_BUCKET_RESERVE_PER_BUCKET = 2;
 const IFRAME_BUCKET_RESERVE_LIMIT = 24;
 const IFRAME_BUCKET_RESERVE_PER_BUCKET = 2;
+const RETRIEVAL_FIELD_WEIGHTS = {
+    entityLabel: 5.5,
+    representativeLabel: 5,
+    representativeLabels: 4,
+    surfaceLabel: 2.25,
+    placeholder: 2.5,
+    title: 1.25,
+    itemLabel: 2.5,
+    itemText: 1.5,
+    groupLabel: 1.75,
+    groupText: 1,
+    containerLabel: 1.5,
+    containerText: 0.9,
+    landmarkLabel: 1.25,
+    landmarkText: 0.75,
+    hintText: 1.5,
+    kind: 0.6,
+    role: 0.6,
+    surfaceKind: 0.75,
+    controlFamily: 0.9,
+    acceptancePolicy: 1,
+    allowedActions: 0.9,
+    structure: 1.25,
+    latentIntent: 1.2,
+};
 const HIGH_SIGNAL_SCOPE_KINDS = new Set([
     'dialog',
     'listbox',
@@ -29,27 +55,6 @@ const HIGH_SIGNAL_SCOPE_KINDS = new Set([
     'card',
     'form',
 ]);
-const GOAL_TEXT_STOPWORDS = new Set([
-    'a',
-    'an',
-    'and',
-    'at',
-    'for',
-    'from',
-    'in',
-    'of',
-    'on',
-    'or',
-    'the',
-    'to',
-    'with',
-    'в',
-    'для',
-    'и',
-    'или',
-    'на',
-    'найти',
-]);
 function isFieldLikeTarget(target) {
     const kind = (target.kind ?? '').trim().toLowerCase();
     const role = (target.role ?? '').trim().toLowerCase();
@@ -73,16 +78,6 @@ function isActionLikeTargetCandidate(target) {
         kind === 'link' ||
         role === 'link');
 }
-function normalizeRetrievalText(value) {
-    const normalized = value?.replace(/\s+/g, ' ').trim().toLowerCase();
-    return normalized ? normalized : undefined;
-}
-function tokenizeRetrievalText(value) {
-    if (!value) {
-        return [];
-    }
-    return (value.match(/[\p{L}\p{N}]+/gu) ?? []).map((token) => token.toLowerCase());
-}
 function semanticFormContextKey(context) {
     for (const node of [context?.landmark, context?.container, context?.group, context?.item]) {
         const kind = (node?.kind ?? '').trim().toLowerCase();
@@ -140,6 +135,13 @@ function isPrimaryFormControlTarget(target) {
         acceptancePolicy === 'disclosure' &&
         (kind === 'button' || role === 'button'));
 }
+function shouldPreserveStandaloneFormActionEntity(target) {
+    return (Boolean(formBucketKey(target)) &&
+        !isScopeLikeCandidate(target) &&
+        !isFieldLikeTarget(target) &&
+        isActionLikeTargetCandidate(target) &&
+        isPrimaryFormControlTarget(target));
+}
 function primaryFormTargetPriority(target) {
     const acceptancePolicy = (target.acceptancePolicy ?? '').trim().toLowerCase();
     if (isFieldLikeTarget(target)) {
@@ -178,11 +180,11 @@ function contentItemBucketKey(target) {
     if (surfaceIdentity) {
         return `surface:${surfaceIdentity}`;
     }
-    const itemKey = normalizeRetrievalText(target.context?.item?.label ?? target.context?.item?.text);
+    const itemKey = normalizeSemanticObserveText(target.context?.item?.label ?? target.context?.item?.text);
     if (itemKey) {
         return `item:${itemKey}`;
     }
-    const containerKey = normalizeRetrievalText(target.context?.container?.label ?? target.context?.container?.text);
+    const containerKey = normalizeSemanticObserveText(target.context?.container?.label ?? target.context?.container?.text);
     if (containerKey) {
         return `container:${containerKey}`;
     }
@@ -205,7 +207,7 @@ function scopeCandidatePriority(target) {
     return 4;
 }
 function representativeCandidateScore(target) {
-    const normalizedLabel = normalizeRetrievalText(target.label);
+    const normalizedLabel = normalizeSemanticObserveText(target.label);
     const kind = (target.kind ?? '').trim().toLowerCase();
     const role = (target.role ?? '').trim().toLowerCase();
     let score = (target.surfacePriority ?? 0) * 10;
@@ -295,26 +297,131 @@ function collectRepresentativeLabels(targets) {
     }
     return labels;
 }
-function buildEntitySearchText(entityLabel, representative, representativeLabels) {
+function structureLexicalValue(structure) {
+    if (!structure) {
+        return undefined;
+    }
     return [
-        entityLabel,
-        representative.label,
-        ...representativeLabels,
-        representative.surfaceLabel,
-        representative.context?.item?.label,
-        representative.context?.item?.text,
-        representative.context?.group?.label,
-        representative.context?.group?.text,
-        representative.context?.container?.label,
-        representative.context?.container?.text,
-        representative.context?.landmark?.label,
-        representative.context?.landmark?.text,
-        representative.context?.hintText,
+        structure.family,
+        structure.variant,
+        structure.row,
+        structure.column,
+        structure.zone,
+        structure.cellLabel,
     ]
-        .map((value) => normalizeRetrievalText(value))
-        .filter((value) => Boolean(value))
+        .filter(Boolean)
         .join(' ');
 }
+function latentActionHintText(target) {
+    const acceptancePolicy = (target.acceptancePolicy ?? '').trim().toLowerCase();
+    const controlFamily = (target.controlFamily ?? '').trim().toLowerCase();
+    const surfaceKind = (target.surfaceKind ?? '').trim().toLowerCase();
+    const kind = (target.kind ?? '').trim().toLowerCase();
+    const role = (target.role ?? '').trim().toLowerCase();
+    const popupBacked = acceptancePolicy === 'disclosure' ||
+        Boolean(target.controlsSurfaceSelector) ||
+        target.states?.expanded !== undefined;
+    if (!popupBacked) {
+        return undefined;
+    }
+    const hints = new Set();
+    hints.add('open menu options choices');
+    if (!isFieldLikeTarget(target)) {
+        hints.add('sort sorting filter filters view switch picker');
+    }
+    if (controlFamily === 'select' ||
+        ['menu', 'listbox', 'dropdown', 'popover'].includes(surfaceKind)) {
+        hints.add('select choose option options menu dropdown listbox');
+    }
+    if (controlFamily === 'datepicker' || surfaceKind === 'datepicker') {
+        hints.add('open calendar datepicker date picker choose date calendar');
+    }
+    const evidenceText = [
+        target.label,
+        target.placeholder,
+        target.title,
+        target.surfaceLabel,
+        target.context?.group?.label,
+        target.context?.container?.label,
+        target.context?.landmark?.label,
+        target.context?.hintText,
+    ]
+        .filter((value) => Boolean(value))
+        .join(' ')
+        .toLowerCase();
+    if (/(?:sort|sorting|show first|ordered by|order by|popular|relevance|recommended|price|cheap|cheapest|expensive|сорт|сначала|популяр|релевант|рекоменд|цене|дешев|дорог)/i.test(evidenceText)) {
+        hints.add('sort sorting order order by relevance popular recommended price cheapest cheapest first');
+    }
+    if (/(?:filter|filters|refine|brand|airline|amenit|фильтр|фильтры|бренд|авиакомпан|удобств)/i.test(evidenceText)) {
+        hints.add('filter filters refine refine results brand airline amenities');
+    }
+    if (/(?:view|layout|grid|list|map|calendar view|вид|список|сетка|карта|раскладк)/i.test(evidenceText)) {
+        hints.add('view layout grid list map switch');
+    }
+    if (kind === 'button' &&
+        role === 'button' &&
+        (acceptancePolicy === 'disclosure' || controlFamily === 'select' || controlFamily === 'datepicker')) {
+        hints.add('open selector choose mode');
+    }
+    return hints.size > 0 ? [...hints].join(' ') : undefined;
+}
+function buildEntityLexicalFields(entityLabel, representative, representativeLabels) {
+    return [
+        { value: entityLabel, weight: RETRIEVAL_FIELD_WEIGHTS.entityLabel },
+        { value: representative.label, weight: RETRIEVAL_FIELD_WEIGHTS.representativeLabel },
+        ...representativeLabels.map((value) => ({
+            value,
+            weight: RETRIEVAL_FIELD_WEIGHTS.representativeLabels,
+        })),
+        { value: representative.surfaceLabel, weight: RETRIEVAL_FIELD_WEIGHTS.surfaceLabel },
+        { value: representative.placeholder, weight: RETRIEVAL_FIELD_WEIGHTS.placeholder },
+        { value: representative.title, weight: RETRIEVAL_FIELD_WEIGHTS.title },
+        { value: representative.context?.item?.label, weight: RETRIEVAL_FIELD_WEIGHTS.itemLabel },
+        { value: representative.context?.item?.text, weight: RETRIEVAL_FIELD_WEIGHTS.itemText },
+        { value: representative.context?.group?.label, weight: RETRIEVAL_FIELD_WEIGHTS.groupLabel },
+        { value: representative.context?.group?.text, weight: RETRIEVAL_FIELD_WEIGHTS.groupText },
+        {
+            value: representative.context?.container?.label,
+            weight: RETRIEVAL_FIELD_WEIGHTS.containerLabel,
+        },
+        {
+            value: representative.context?.container?.text,
+            weight: RETRIEVAL_FIELD_WEIGHTS.containerText,
+        },
+        {
+            value: representative.context?.landmark?.label,
+            weight: RETRIEVAL_FIELD_WEIGHTS.landmarkLabel,
+        },
+        {
+            value: representative.context?.landmark?.text,
+            weight: RETRIEVAL_FIELD_WEIGHTS.landmarkText,
+        },
+        { value: representative.context?.hintText, weight: RETRIEVAL_FIELD_WEIGHTS.hintText },
+        { value: representative.kind, weight: RETRIEVAL_FIELD_WEIGHTS.kind },
+        { value: representative.role, weight: RETRIEVAL_FIELD_WEIGHTS.role },
+        { value: representative.surfaceKind, weight: RETRIEVAL_FIELD_WEIGHTS.surfaceKind },
+        {
+            value: representative.controlFamily,
+            weight: RETRIEVAL_FIELD_WEIGHTS.controlFamily,
+        },
+        {
+            value: representative.acceptancePolicy,
+            weight: RETRIEVAL_FIELD_WEIGHTS.acceptancePolicy,
+        },
+        {
+            value: representative.allowedActions?.join(' '),
+            weight: RETRIEVAL_FIELD_WEIGHTS.allowedActions,
+        },
+        {
+            value: structureLexicalValue(representative.structure),
+            weight: RETRIEVAL_FIELD_WEIGHTS.structure,
+        },
+        {
+            value: latentActionHintText(representative),
+            weight: RETRIEVAL_FIELD_WEIGHTS.latentIntent,
+        },
+    ];
+}
 function buildGoalRetrievalEntity(entityKind, entityKey, memberIndexes, targets) {
     const orderedMembers = [...memberIndexes]
         .map((index) => ({ index, target: targets[index] }))
@@ -322,6 +429,8 @@ function buildGoalRetrievalEntity(entityKind, entityKey, memberIndexes, targets)
     const representative = orderedMembers[0].target;
     const representativeLabels = collectRepresentativeLabels(orderedMembers.map((entry) => entry.target));
     const label = pickEntityLabel(entityKind, representative);
+    const lexicalFields = buildEntityLexicalFields(label, representative, representativeLabels);
+    const lexicalDocument = buildSemanticObserveLexicalDocument(lexicalFields);
     return {
         entityKind,
         entityKey,
@@ -342,7 +451,8 @@ function buildGoalRetrievalEntity(entityKind, entityKey, memberIndexes, targets)
         context: representative.context,
         structure: representative.structure,
         representativeLabels,
-        searchText: buildEntitySearchText(label, representative, representativeLabels),
+        analyzerKey: lexicalDocument.analyzerKey,
+        lexicalDocument,
     };
 }
 function buildGoalRetrievalEntities(targets) {
@@ -376,7 +486,12 @@ function buildGoalRetrievalEntities(targets) {
     }
     for (const [key, memberIndexes] of formGroups.entries()) {
         entities.push(buildGoalRetrievalEntity('form', `form:${key}`, memberIndexes, targets));
-        memberIndexes.forEach((index) => groupedTargetIndexes.add(index));
+        memberIndexes.forEach((index) => {
+            if (shouldPreserveStandaloneFormActionEntity(targets[index])) {
+                return;
+            }
+            groupedTargetIndexes.add(index);
+        });
     }
     const itemGroups = new Map();
     for (const [index, target] of targets.entries()) {
@@ -445,58 +560,28 @@ function retrievalEntityPriority(entity) {
     score += representativeCandidateScore(entity.representative);
     return score;
 }
-function scoreRetrievalEntityAgainstGoal(goal, entity) {
-    const normalizedGoal = normalizeRetrievalText(goal);
-    if (!normalizedGoal) {
+function scoreRetrievalEntityAgainstGoal(analyzedGoal, entity, corpusStats) {
+    if (!analyzedGoal.normalizedText) {
         return 0;
     }
-    const entityText = entity.searchText;
-    if (!entityText) {
+    if (entity.lexicalDocument.weightedLength <= 0) {
         return 0;
     }
-    let score = 0;
-    const goalTokens = tokenizeRetrievalText(normalizedGoal).filter((token) => token.length >= 2 && !GOAL_TEXT_STOPWORDS.has(token));
-    const entityTokens = new Set(tokenizeRetrievalText(entityText));
-    if (entityText.includes(normalizedGoal)) {
-        score += 220;
-    }
-    const entityLabel = normalizeRetrievalText(entity.label);
-    if (entityLabel) {
-        if (normalizedGoal.includes(entityLabel)) {
-            score += 180;
-        }
-        if (entityLabel.includes(normalizedGoal)) {
-            score += 140;
-        }
-    }
-    let matchedTokenCount = 0;
-    for (const token of goalTokens) {
-        if (entityTokens.has(token)) {
-            matchedTokenCount += 1;
-            score += 8 + Math.min(token.length, 12);
-        }
-    }
-    if (goalTokens.length > 0) {
-        const coverage = matchedTokenCount / goalTokens.length;
-        if (coverage === 1) {
-            score += 120;
-        }
-        else if (coverage >= 0.75) {
-            score += 70;
-        }
-        else if (coverage >= 0.5) {
-            score += 35;
-        }
-    }
-    return score;
+    const bm25Score = scoreSemanticObserveBm25(analyzedGoal, entity.lexicalDocument, corpusStats);
+    const analyzerAlignmentScore = analyzedGoal.analyzerKey !== 'neutral' && analyzedGoal.analyzerKey === entity.analyzerKey
+        ? 15
+        : 0;
+    return bm25Score * 100 + analyzerAlignmentScore;
 }
 function preselectRetrievalEntities(goal, entities) {
     if (entities.length <= GOAL_RETRIEVAL_ENTITY_LIMIT) {
         return [...entities];
     }
+    const analyzedGoal = analyzeSemanticObserveText(goal);
+    const corpusStats = buildSemanticObserveBm25CorpusStats(entities.map((entity) => entity.lexicalDocument));
     const scored = entities.map((entity) => ({
         entity,
-        lexicalScore: scoreRetrievalEntityAgainstGoal(goal, entity),
+        lexicalScore: scoreRetrievalEntityAgainstGoal(analyzedGoal, entity, corpusStats),
         priority: retrievalEntityPriority(entity),
     }));
     const selected = [];
@@ -634,9 +719,6 @@ function diversifyCandidates(targets) {
     }
     return orderedIndexes.map((index) => targets[index]).slice(0, RERANK_CANDIDATE_LIMIT);
 }
-function buildCandidateSummary(target, index, surfaceSummaryIdBySurfaceKey) {
-    return buildCompactCandidateSummary(target, index, surfaceSummaryIdBySurfaceKey);
-}
 function compactContextValue(value, maxLength = 80) {
     const normalized = value?.replace(/\s+/g, ' ').trim();
     if (!normalized) {
@@ -791,6 +873,9 @@ function buildCompactCandidateSummary(target, index, surfaceSummaryIdBySurfaceKe
     if (target.controlFamily) {
         parts.push(`family=${JSON.stringify(target.controlFamily)}`);
     }
+    if (target.controlsSurfaceSelector) {
+        parts.push('controlsPopup=true');
+    }
     if (target.capability && target.capability !== 'actionable') {
         parts.push(`capability=${JSON.stringify(target.capability)}`);
     }
@@ -836,6 +921,7 @@ export async function rerankDomTargetsForGoal(instruction, targets, options = {}
     const prompt = [
         'You are choosing from already discovered visible webpage candidates.',
         'Select only candidate IDs that directly satisfy the goal.',
+        'A disclosure trigger with a current-state label can still directly satisfy sort, filter, view-switch, or picker goals when opening it is the immediate next step.',
         'Prefer direct actionable controls over surrounding regions unless the goal explicitly asks for a form, region, widget, or set of controls.',
         'Use owning surface, compact context, explicit state, and structure metadata to disambiguate similar labels.',
         'For input/value goals, prefer the directly editable field or primary picker trigger over wrappers or mirrored summary content.',
@@ -847,7 +933,7 @@ export async function rerankDomTargetsForGoal(instruction, targets, options = {}
         '',
         ...(surfaceSummaries.length > 0 ? ['Surfaces:', ...surfaceSummaries.map((entry) => entry.line), ''] : []),
         'Candidates:',
-        ...candidates.map((target, index) => buildCandidateSummary(target, index, summaryIdBySurfaceKey)),
+        ...candidates.map((target, index) => buildCompactCandidateSummary(target, index, summaryIdBySurfaceKey)),
     ].join('\n');
     const result = await client.createChatCompletion({
         logger: () => { },

package/dist/control-semantics.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"control-semantics.d.ts","sourceRoot":"","sources":["../src/control-semantics.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EACV,sBAAsB,EACtB,mBAAmB,EACnB,uBAAuB,EACvB,mBAAmB,EACnB,eAAe,EAChB,MAAM,oBAAoB,CAAC;AAE5B,MAAM,MAAM,gBAAgB,GAAG,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,GAAG,MAAM,CAAC,CAAC;AAEzE,MAAM,MAAM,oBAAoB,GAAG;IACjC,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,eAAe,CAAC,EAAE,OAAO,CAAC;IAC1B,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,uBAAuB,CAAC,EAAE,MAAM,CAAC;IACjC,MAAM,CAAC,EAAE,gBAAgB,CAAC;IAC1B,SAAS,CAAC,EAAE,eAAe,CAAC;IAC5B,YAAY,CAAC,EAAE,MAAM,CAAC;CACvB,CAAC;AAEF,MAAM,MAAM,mBAAmB,GAAG,OAAO,GAAG,aAAa,GAAG,QAAQ,GAAG,KAAK,GAAG,MAAM,CAAC;AAyCtF,wBAAgB,iCAAiC,CAC/C,KAAK,EAAE,oBAAoB,GAC1B,mBAAmB,GAAG,SAAS,CAuCjC;AAsCD,wBAAgB,qBAAqB,CAAC,KAAK,EAAE,MAAM,GAAG,SAAS,GAAG,OAAO,CAYxE;AAED,wBAAgB,4BAA4B,CAAC,KAAK,EAAE,oBAAoB,GAAG,mBAAmB,EAAE,CAoG/F;AAED,wBAAgB,0BAA0B,CACxC,MAAM,CAAC,EAAE,gBAAgB,EACzB,WAAW,CAAC,EAAE,MAAM,EACpB,OAAO,GAAE;IACP,mBAAmB,CAAC,EAAE,OAAO,CAAC;CAC1B,GACL,uBAAuB,CAyBzB;AAED,wBAAgB,2BAA2B,CACzC,KAAK,EAAE,oBAAoB,EAC3B,cAAc,EAAE,aAAa,CAAC,mBAAmB,CAAC,GACjD,mBAAmB,GAAG,SAAS,CA0EjC;AAED,wBAAgB,8BAA8B,CAC5C,KAAK,EAAE,oBAAoB,EAC3B,cAAc,EAAE,aAAa,CAAC,mBAAmB,CAAC,GACjD,sBAAsB,GAAG,SAAS,CAiDpC"}
+{"version":3,"file":"control-semantics.d.ts","sourceRoot":"","sources":["../src/control-semantics.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EACV,sBAAsB,EACtB,mBAAmB,EACnB,uBAAuB,EACvB,mBAAmB,EACnB,eAAe,EAChB,MAAM,oBAAoB,CAAC;AAE5B,MAAM,MAAM,gBAAgB,GAAG,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,GAAG,MAAM,CAAC,CAAC;AAEzE,MAAM,MAAM,oBAAoB,GAAG;IACjC,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,eAAe,CAAC,EAAE,OAAO,CAAC;IAC1B,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,uBAAuB,CAAC,EAAE,MAAM,CAAC;IACjC,MAAM,CAAC,EAAE,gBAAgB,CAAC;IAC1B,SAAS,CAAC,EAAE,eAAe,CAAC;IAC5B,YAAY,CAAC,EAAE,MAAM,CAAC;CACvB,CAAC;AAEF,MAAM,MAAM,mBAAmB,GAAG,OAAO,GAAG,aAAa,GAAG,QAAQ,GAAG,KAAK,GAAG,MAAM,CAAC;AAwCtF,wBAAgB,iCAAiC,CAC/C,KAAK,EAAE,oBAAoB,GAC1B,mBAAmB,GAAG,SAAS,CAuCjC;AAyID,wBAAgB,qBAAqB,CAAC,KAAK,EAAE,MAAM,GAAG,SAAS,GAAG,OAAO,CAYxE;AAED,wBAAgB,4BAA4B,CAAC,KAAK,EAAE,oBAAoB,GAAG,mBAAmB,EAAE,CAoG/F;AAED,wBAAgB,0BAA0B,CACxC,MAAM,CAAC,EAAE,gBAAgB,EACzB,WAAW,CAAC,EAAE,MAAM,EACpB,OAAO,GAAE;IACP,mBAAmB,CAAC,EAAE,OAAO,CAAC;CAC1B,GACL,uBAAuB,CAyBzB;AAED,wBAAgB,2BAA2B,CACzC,KAAK,EAAE,oBAAoB,EAC3B,cAAc,EAAE,aAAa,CAAC,mBAAmB,CAAC,GACjD,mBAAmB,GAAG,SAAS,CA0EjC;AAED,wBAAgB,8BAA8B,CAC5C,KAAK,EAAE,oBAAoB,EAC3B,cAAc,EAAE,aAAa,CAAC,mBAAmB,CAAC,GACjD,sBAAsB,GAAG,SAAS,CA6CpC"}

package/dist/control-semantics.js

@@ -83,6 +83,76 @@ function isButtonLikeInputFacts(facts) {
     return (normalizeText(facts.kind) === 'input' &&
         ['button', 'submit', 'reset'].includes(normalizeText(facts.inputType)));
 }
+function isSelectionItemLikeFacts(facts) {
+    const kind = normalizeText(facts.kind);
+    const role = normalizeText(facts.role);
+    return (kind === 'option' ||
+        role === 'option' ||
+        role === 'menuitem' ||
+        facts.structure?.family === 'structured-grid');
+}
+function isBinaryToggleControlFacts(facts) {
+    const kind = normalizeText(facts.kind);
+    const role = normalizeText(facts.role);
+    const inputType = normalizeText(facts.inputType);
+    return (kind === 'checkbox' ||
+        kind === 'radio' ||
+        role === 'checkbox' ||
+        role === 'radio' ||
+        role === 'switch' ||
+        kind === 'tab' ||
+        role === 'tab' ||
+        inputType === 'checkbox' ||
+        inputType === 'radio');
+}
+function isDisclosureLikeTriggerFacts(facts, controlFamily) {
+    const kind = normalizeText(facts.kind);
+    const role = normalizeText(facts.role);
+    const inputType = normalizeText(facts.inputType);
+    const popupBacked = Boolean(facts.controlsSurfaceSelector) ||
+        facts.states?.expanded !== undefined;
+    if (!popupBacked) {
+        return false;
+    }
+    if (isSelectionItemLikeFacts(facts) || controlFamily === 'text-input') {
+        return false;
+    }
+    if (controlFamily === 'select' || controlFamily === 'datepicker') {
+        return true;
+    }
+    return (kind === 'button' ||
+        role === 'button' ||
+        kind === 'link' ||
+        role === 'link' ||
+        kind === 'combobox' ||
+        role === 'combobox' ||
+        kind === 'select' ||
+        isButtonLikeInputFacts(facts) ||
+        inputType === 'button' ||
+        isPopupBackedTextEntry(facts));
+}
+function isToggleLikeControlFacts(facts, controlFamily) {
+    const states = facts.states;
+    if (!states) {
+        return false;
+    }
+    if (states.checked !== undefined || isBinaryToggleControlFacts(facts)) {
+        return true;
+    }
+    if (states.selected !== undefined) {
+        if (isSelectionItemLikeFacts(facts) ||
+            controlFamily === 'select' ||
+            controlFamily === 'datepicker' ||
+            isDisclosureLikeTriggerFacts(facts, controlFamily)) {
+            return false;
+        }
+        return true;
+    }
+    if (states.pressed !== undefined) {
+        return !isDisclosureLikeTriggerFacts(facts, controlFamily);
+    }
+    return false;
+}
 export function isLikelyDateLikeLabel(value) {
     const trimmed = (value ?? '').trim();
     if (!trimmed) {
@@ -280,17 +350,15 @@ export function inferAcceptancePolicyFromFacts(facts, allowedActions) {
     if (facts.structure?.family === 'structured-grid') {
         return facts.structure.variant === 'date-cell' ? 'date-selection' : 'selection';
     }
-    if (states?.checked !== undefined ||
-        states?.selected !== undefined ||
-        states?.pressed !== undefined) {
-        return 'toggle';
-    }
     if (controlFamily === 'text-input') {
         return 'value-change';
     }
-    if (states?.expanded !== undefined) {
+    if (isDisclosureLikeTriggerFacts(facts, controlFamily)) {
         return 'disclosure';
     }
+    if (isToggleLikeControlFacts(facts, controlFamily)) {
+        return 'toggle';
+    }
     if (controlFamily === 'datepicker') {
         return 'date-selection';
     }

package/dist/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":";AA2QA,iBAAe,IAAI,CAAC,IAAI,GAAE,MAAM,EAAiB,GAAG,OAAO,CAAC,IAAI,CAAC,CAsKhE;AAED,OAAO,EAAE,IAAI,EAAE,CAAC;AAEhB,wBAAgB,iBAAiB,CAAC,OAAO,EAAE,MAAM,EAAE,IAAI,GAAE,MAAM,EAAiB,GAAG,OAAO,CAWzF"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":";AA4QA,iBAAe,IAAI,CAAC,IAAI,GAAE,MAAM,EAAiB,GAAG,OAAO,CAAC,IAAI,CAAC,CA4KhE;AAED,OAAO,EAAE,IAAI,EAAE,CAAC;AAEhB,wBAAgB,iBAAiB,CAAC,OAAO,EAAE,MAAM,EAAE,IAAI,GAAE,MAAM,EAAiB,GAAG,OAAO,CAWzF"}

package/dist/index.js

@@ -5,6 +5,7 @@ import { fileURLToPath } from 'node:url';
 loadEnv();
 import { preflightAgentpayGateway } from './agentpay-gateway.js';
 import { browseCommand, browseCommandName } from './command-name.js';
+import { assertSemanticObserveRuntimeSupport } from './commands/semantic-observe-lexical.js';
 import { loadSession } from './session.js';
 import { outputError, outputJSON, fatal, info } from './output.js';
 import { applyDemoProxyBootstrap } from './solver/config.js';
@@ -223,6 +224,12 @@ async function main(argv = process.argv) {
     if (!KNOWN_COMMANDS.has(command)) {
         outputError(`Unknown command: ${command}\n\n${usageText()}`);
     }
+    try {
+        assertSemanticObserveRuntimeSupport();
+    }
+    catch (err) {
+        outputError(err instanceof Error ? err.message : String(err));
+    }
     if (command !== 'init') {
         try {
             await preflightAgentpayGateway();

package/dist/owned-process.d.ts

@@ -11,4 +11,7 @@ export declare function cleanupManagedBrowserPids(options?: {
 export declare function listManagedBrowserPids(options?: {
     processTable?: string;
 }): number[];
+export declare function isManagedBrowserPid(pid: number, options?: {
+    processTable?: string;
+}): boolean;
 //# sourceMappingURL=owned-process.d.ts.map

package/dist/owned-process.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"owned-process.d.ts","sourceRoot":"","sources":["../src/owned-process.ts"],"names":[],"mappings":"AAaA,MAAM,MAAM,yBAAyB,GAAG,WAAW,GAAG,YAAY,GAAG,WAAW,GAAG,aAAa,CAAC;AACjG,MAAM,MAAM,2BAA2B,GAAG;IACxC,UAAU,EAAE,MAAM,EAAE,CAAC;IACrB,OAAO,EAAE,MAAM,EAAE,CAAC;CACnB,CAAC;AAEF,wBAAgB,UAAU,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAW/C;AAED,wBAAsB,iBAAiB,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,yBAAyB,CAAC,CAOvF;AAiCD,wBAAsB,yBAAyB,CAC7C,OAAO,GAAE;IAAE,WAAW,CAAC,EAAE,QAAQ,CAAC,MAAM,CAAC,CAAA;CAAO,GAC/C,OAAO,CAAC,2BAA2B,CAAC,CAqBtC;AAED,wBAAgB,sBAAsB,CACpC,OAAO,GAAE;IACP,YAAY,CAAC,EAAE,MAAM,CAAC;CAClB,GACL,MAAM,EAAE,CAuBV"}
+{"version":3,"file":"owned-process.d.ts","sourceRoot":"","sources":["../src/owned-process.ts"],"names":[],"mappings":"AAaA,MAAM,MAAM,yBAAyB,GAAG,WAAW,GAAG,YAAY,GAAG,WAAW,GAAG,aAAa,CAAC;AACjG,MAAM,MAAM,2BAA2B,GAAG;IACxC,UAAU,EAAE,MAAM,EAAE,CAAC;IACrB,OAAO,EAAE,MAAM,EAAE,CAAC;CACnB,CAAC;AAEF,wBAAgB,UAAU,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAW/C;AAED,wBAAsB,iBAAiB,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,yBAAyB,CAAC,CAOvF;AAiCD,wBAAsB,yBAAyB,CAC7C,OAAO,GAAE;IAAE,WAAW,CAAC,EAAE,QAAQ,CAAC,MAAM,CAAC,CAAA;CAAO,GAC/C,OAAO,CAAC,2BAA2B,CAAC,CAqBtC;AAED,wBAAgB,sBAAsB,CACpC,OAAO,GAAE;IACP,YAAY,CAAC,EAAE,MAAM,CAAC;CAClB,GACL,MAAM,EAAE,CAuBV;AAED,wBAAgB,mBAAmB,CACjC,GAAG,EAAE,MAAM,EACX,OAAO,GAAE;IACP,YAAY,CAAC,EAAE,MAAM,CAAC;CAClB,GACL,OAAO,CAET"}

package/dist/owned-process.js

@@ -93,6 +93,9 @@ export function listManagedBrowserPids(options = {}) {
     }
     return Array.from(pids);
 }
+export function isManagedBrowserPid(pid, options = {}) {
+    return listManagedBrowserPids(options).includes(pid);
+}
 async function waitForPidExit(pid, attempts, intervalMs) {
     for (let attempt = 0; attempt < attempts; attempt++) {
         if (!isPidAlive(pid)) {

package/dist/solver/browser-launcher.d.ts

@@ -10,4 +10,5 @@ export type SolverSession = {
     disconnect: () => Promise<void>;
 };
 export declare function launchSolver(profile: ProfileInfo, opts?: LaunchOptions): Promise<SolverSession>;
+export declare function chromeArgsForEnvironment(platform?: NodeJS.Platform, env?: NodeJS.ProcessEnv): string[];
 //# sourceMappingURL=browser-launcher.d.ts.map

package/dist/solver/browser-launcher.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"browser-launcher.d.ts","sourceRoot":"","sources":["../../src/solver/browser-launcher.ts"],"names":[],"mappings":"AAMA,OAAO,KAAK,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AAC/C,OAAO,KAAK,EAAsB,aAAa,EAAE,WAAW,EAAE,MAAM,YAAY,CAAC;AAajF,MAAM,MAAM,aAAa,GAAG;IAC1B,OAAO,EAAE,OAAO,CAAC;IACjB,IAAI,EAAE,IAAI,CAAC;IACX,MAAM,EAAE,MAAM,CAAC;IACf,GAAG,EAAE,MAAM,CAAC;IACZ,OAAO,EAAE,WAAW,CAAC;IACrB,KAAK,EAAE,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC;IAC3B,UAAU,EAAE,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC;CACjC,CAAC;AAiBF,wBAAsB,YAAY,CAChC,OAAO,EAAE,WAAW,EACpB,IAAI,CAAC,EAAE,aAAa,GACnB,OAAO,CAAC,aAAa,CAAC,CAkExB"}
+{"version":3,"file":"browser-launcher.d.ts","sourceRoot":"","sources":["../../src/solver/browser-launcher.ts"],"names":[],"mappings":"AAMA,OAAO,KAAK,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AAC/C,OAAO,KAAK,EAAsB,aAAa,EAAE,WAAW,EAAE,MAAM,YAAY,CAAC;AAgBjF,MAAM,MAAM,aAAa,GAAG;IAC1B,OAAO,EAAE,OAAO,CAAC;IACjB,IAAI,EAAE,IAAI,CAAC;IACX,MAAM,EAAE,MAAM,CAAC;IACf,GAAG,EAAE,MAAM,CAAC;IACZ,OAAO,EAAE,WAAW,CAAC;IACrB,KAAK,EAAE,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC;IAC3B,UAAU,EAAE,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC;CACjC,CAAC;AAuBF,wBAAsB,YAAY,CAChC,OAAO,EAAE,WAAW,EACpB,IAAI,CAAC,EAAE,aAAa,GACnB,OAAO,CAAC,aAAa,CAAC,CAmExB;AA8TD,wBAAgB,wBAAwB,CACtC,QAAQ,GAAE,MAAM,CAAC,QAA2B,EAC5C,GAAG,GAAE,MAAM,CAAC,UAAwB,GACnC,MAAM,EAAE,CAGV"}

package/dist/solver/browser-launcher.js

@@ -6,11 +6,20 @@ import puppeteerExtra from 'puppeteer-extra';
 import StealthPlugin from 'puppeteer-extra-plugin-stealth';
 import { buildTurnstileApiShimSource, isTurnstileApiRequest } from './turnstile-challenge.js';
 const enhancedPuppeteer = puppeteerExtra;
-enhancedPuppeteer.use(StealthPlugin());
+const solverStealthPlugin = StealthPlugin();
+// Some iframe-backed checkout flows break when this evasion runs on a connected real Chrome.
+solverStealthPlugin.enabledEvasions.delete('iframe.contentWindow');
+enhancedPuppeteer.use(solverStealthPlugin);
 const AUTO_CDP_PORT = 0;
 const CDP_DISCOVERY_TIMEOUT_MS = 30_000;
 const CDP_DISCOVERY_INTERVAL_MS = 500;
+const BROWSER_DISCONNECT_TIMEOUT_MS = 1_000;
 const INTERCEPTED_PAGES = new WeakSet();
+const LINUX_CI_CHROME_ARGS = [
+    '--no-sandbox',
+    '--disable-setuid-sandbox',
+    '--disable-dev-shm-usage',
+];
 const CHROME_PATHS = [
     '/Applications/Google Chrome.app/Contents/MacOS/Google Chrome',
     '/Applications/Google Chrome Canary.app/Contents/MacOS/Google Chrome Canary',
@@ -30,6 +39,7 @@ export async function launchSolver(profile, opts) {
         const cdpUrl = await discoverCdpUrl({
             requestedPort: opts?.cdpPort,
             userDataDir: profile.userDataDir,
+            chromeProcess,
         });
         if (!cdpUrl) {
             throw buildCdpDiscoveryError(chromeProcess, opts?.cdpPort, profile.userDataDir);
@@ -63,7 +73,7 @@ export async function launchSolver(profile, opts) {
             profile,
             close: async () => {
                 try {
-                    await browser.disconnect();
+                    await disconnectBrowserWithinTimeout(browser, BROWSER_DISCONNECT_TIMEOUT_MS);
                 }
                 finally {
                     terminateProcessGroup(chromeProcess.pid);
@@ -96,10 +106,25 @@ async function discoverCdpUrl(options) {
                 return discoveredViaActivePort;
             }
         }
+        if (typeof options.chromeProcess?.exitCode === 'number' || options.chromeProcess?.signalCode) {
+            break;
+        }
         await sleep(CDP_DISCOVERY_INTERVAL_MS);
     }
     return null;
 }
+async function disconnectBrowserWithinTimeout(browser, timeoutMs) {
+    let timeoutId;
+    await Promise.race([
+        browser.disconnect().catch(() => { }),
+        new Promise((resolve) => {
+            timeoutId = setTimeout(resolve, timeoutMs);
+        }),
+    ]);
+    if (timeoutId) {
+        clearTimeout(timeoutId);
+    }
+}
 async function discoverCdpUrlOnPort(port) {
     try {
         const res = await fetch(`http://127.0.0.1:${port}/json/version`);
@@ -290,6 +315,7 @@ function spawnChromeProcess(executablePath, profile, fp, cdpPort, windowSize, he
         '--no-first-run',
         '--no-default-browser-check',
     ];
+    args.push(...chromeArgsForEnvironment());
     if (headless) {
         args.push('--headless=new');
     }
@@ -311,6 +337,10 @@ function spawnChromeProcess(executablePath, profile, fp, cdpPort, windowSize, he
     }
     return proc;
 }
+export function chromeArgsForEnvironment(platform = process.platform, env = process.env) {
+    const isLinuxCi = platform === 'linux' && (env.CI === 'true' || env.GITHUB_ACTIONS === 'true');
+    return isLinuxCi ? [...LINUX_CI_CHROME_ARGS] : [];
+}
 function terminateProcessGroup(pid) {
     try {
         process.kill(-pid, 'SIGTERM');

package/dist/solver/captcha-runtime.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"captcha-runtime.d.ts","sourceRoot":"","sources":["../../src/solver/captcha-runtime.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,YAAY,CAAC;AAE9C,MAAM,MAAM,yBAAyB,GAAG;IACtC,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,KAAK,CAAC,EAAE,WAAW,CAAC;IACpB,eAAe,CAAC,EAAE,OAAO,CAAC;IAC1B,UAAU,CAAC,EAAE,CAAC,OAAO,EAAE,MAAM,KAAK,IAAI,CAAC;CACxC,CAAC;AAEF,MAAM,MAAM,wBAAwB,GAAG;IACrC,MAAM,EAAE,MAAM,CAAC;IACf,QAAQ,EAAE,MAAM,CAAC;IACjB,UAAU,EAAE,MAAM,CAAC;IACnB,kBAAkB,EAAE,MAAM,EAAE,CAAC;IAC7B,QAAQ,EAAE,MAAM,CAAC;IACjB,QAAQ,EAAE,OAAO,CAAC;CACnB,CAAC;AAMF,wBAAsB,kBAAkB,CACtC,MAAM,EAAE,MAAM,EACd,MAAM,EAAE,MAAM,EACd,IAAI,CAAC,EAAE,yBAAyB,GAC/B,OAAO,CAAC,wBAAwB,CAAC,CA6HnC"}
+{"version":3,"file":"captcha-runtime.d.ts","sourceRoot":"","sources":["../../src/solver/captcha-runtime.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,YAAY,CAAC;AAE9C,MAAM,MAAM,yBAAyB,GAAG;IACtC,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,KAAK,CAAC,EAAE,WAAW,CAAC;IACpB,eAAe,CAAC,EAAE,OAAO,CAAC;IAC1B,UAAU,CAAC,EAAE,CAAC,OAAO,EAAE,MAAM,KAAK,IAAI,CAAC;CACxC,CAAC;AAEF,MAAM,MAAM,wBAAwB,GAAG;IACrC,MAAM,EAAE,MAAM,CAAC;IACf,QAAQ,EAAE,MAAM,CAAC;IACjB,UAAU,EAAE,MAAM,CAAC;IACnB,kBAAkB,EAAE,MAAM,EAAE,CAAC;IAC7B,QAAQ,EAAE,MAAM,CAAC;IACjB,QAAQ,EAAE,OAAO,CAAC;CACnB,CAAC;AAMF,wBAAsB,kBAAkB,CACtC,MAAM,EAAE,MAAM,EACd,MAAM,EAAE,MAAM,EACd,IAAI,CAAC,EAAE,yBAAyB,GAC/B,OAAO,CAAC,wBAAwB,CAAC,CAgInC"}