@nuanu-ai/agentbrowse 0.2.36 → 0.2.37

package/dist/commands/observe-accessibility.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"observe-accessibility.d.ts","sourceRoot":"","sources":["../../src/commands/observe-accessibility.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,iBAAiB,CAAC;AAS5C,OAAO,KAAK,EAEV,iBAAiB,EAElB,MAAM,wBAAwB,CAAC;AAEhC,KAAK,6BAA6B,GAAG;IACnC,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,MAAM,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,GAAG,MAAM,CAAC,CAAC;CACpD,CAAC;AAOF,MAAM,MAAM,yBAAyB,GAAG;IACtC,UAAU,EAAE,MAAM,CAAC;IACnB,MAAM,EAAE,MAAM,CAAC;IACf,YAAY,EAAE,MAAM,CAAC;CACtB,CAAC;AAEF,KAAK,2BAA2B,GAAG;IACjC,OAAO,CAAC,EAAE,CAAC,KAAK,EAAE,yBAAyB,KAAK,IAAI,CAAC;CACtD,CAAC;AAyDF,wBAAgB,wBAAwB,CACtC,QAAQ,EAAE,MAAM,GAAG,SAAS,GAAG,IAAI,GAClC,6BAA6B,GAAG,IAAI,CAoDtC;AA+nBD,wBAAsB,iCAAiC,CACrD,IAAI,EAAE,IAAI,EACV,OAAO,EAAE,aAAa,CAAC,iBAAiB,CAAC,EACzC,OAAO,CAAC,EAAE,2BAA2B,GACpC,OAAO,CAAC,iBAAiB,EAAE,CAAC,CAY9B;AAED,eAAO,MAAM,0BAA0B;;CAEtC,CAAC"}
+{"version":3,"file":"observe-accessibility.d.ts","sourceRoot":"","sources":["../../src/commands/observe-accessibility.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,iBAAiB,CAAC;AAa5C,OAAO,KAAK,EAEV,iBAAiB,EAElB,MAAM,wBAAwB,CAAC;AAEhC,KAAK,6BAA6B,GAAG;IACnC,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,MAAM,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,GAAG,MAAM,CAAC,CAAC;CACpD,CAAC;AAOF,MAAM,MAAM,yBAAyB,GAAG;IACtC,UAAU,EAAE,MAAM,CAAC;IACnB,MAAM,EAAE,MAAM,CAAC;IACf,YAAY,EAAE,MAAM,CAAC;CACtB,CAAC;AAEF,KAAK,2BAA2B,GAAG;IACjC,OAAO,CAAC,EAAE,CAAC,KAAK,EAAE,yBAAyB,KAAK,IAAI,CAAC;CACtD,CAAC;AAyDF,wBAAgB,wBAAwB,CACtC,QAAQ,EAAE,MAAM,GAAG,SAAS,GAAG,IAAI,GAClC,6BAA6B,GAAG,IAAI,CAoDtC;AAiqBD,wBAAsB,iCAAiC,CACrD,IAAI,EAAE,IAAI,EACV,OAAO,EAAE,aAAa,CAAC,iBAAiB,CAAC,EACzC,OAAO,CAAC,EAAE,2BAA2B,GACpC,OAAO,CAAC,iBAAiB,EAAE,CAAC,CAY9B;AAED,eAAO,MAAM,0BAA0B;;CAEtC,CAAC"}

package/dist/commands/observe-accessibility.js

@@ -1,5 +1,6 @@
 import { resolveLocatorRoot } from './action-fallbacks.js';
 import { buildObserveDisplayLabel } from './observe-display-label.js';
+import { isButtonLikeObservedInput, shouldAllowLooseFieldLabelFallbackForObservedControl, } from './observe-label-policy.js';
 import { fallbackContextNodeLabelOf, fallbackHintTextOf, fallbackSurfaceLabelOf, fallbackTargetLabelOf, } from './observe-fallback-semantics.js';
 const AX_ENRICHMENT_CONCURRENCY = 12;
 const AX_SNAPSHOT_TIMEOUT_MS = 250;
@@ -105,12 +106,39 @@ function shouldPreferAccessibilityName(target, accessibilityName) {
     const currentKey = normalizeLabelKey(currentLabel);
     const nextLabel = normalizeText(accessibilityName);
     const visibleText = normalizeText(target.text);
+    const kind = normalizeText(target.kind)?.toLowerCase();
+    const role = normalizeText(target.role)?.toLowerCase();
+    const looseFieldFallbackAllowed = shouldAllowLooseFieldLabelFallbackForObservedControl({
+        tag: kind,
+        role,
+        inputType: target.inputType,
+    });
+    const primaryAxControl = !looseFieldFallbackAllowed &&
+        (kind === 'button' ||
+            kind === 'link' ||
+            kind === 'checkbox' ||
+            kind === 'radio' ||
+            role === 'button' ||
+            role === 'link' ||
+            role === 'checkbox' ||
+            role === 'radio' ||
+            role === 'menuitem' ||
+            role === 'option' ||
+            role === 'switch' ||
+            role === 'tab' ||
+            isButtonLikeObservedInput({
+                tag: kind,
+                inputType: target.inputType,
+            }));
     if (!isMeaningfulName(nextLabel)) {
         return false;
     }
     if (!isMeaningfulName(currentLabel)) {
         return true;
     }
+    if (primaryAxControl && currentLabel !== nextLabel) {
+        return true;
+    }
     if (GENERIC_FALLBACK_LABELS.has(currentKey)) {
         return true;
     }
@@ -426,7 +454,13 @@ async function enrichTargetWithAccessibility(page, target, rootCache, snapshotCa
     const landmarkDomFallback = await readDomFallbackSemantics(page, { selector: target.context?.landmark?.selector, framePath: target.framePath }, rootCache, fallbackCache);
     const targetDomFallback = await readDomFallbackSemantics(page, target, rootCache, fallbackCache);
     const nextLabel = isMeaningfulName(semantics?.name) &&
-        shouldPreferAccessibilityName({ label: fallbackTargetLabelOf(target), text: target.text }, semantics.name)
+        shouldPreferAccessibilityName({
+            kind: target.kind,
+            role: target.role,
+            inputType: target.inputType,
+            label: fallbackTargetLabelOf(target),
+            text: target.text,
+        }, semantics.name)
         ? semantics.name
         : chooseSemanticName(fallbackTargetLabelOf(target), semantics, stats);
     const nextContext = target.context

package/dist/commands/observe-inventory.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"observe-inventory.d.ts","sourceRoot":"","sources":["../../src/commands/observe-inventory.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,KAAK,EAAE,OAAO,EAAE,MAAM,iBAAiB,CAAC;AACtD,OAAO,KAAK,EACV,sBAAsB,EACtB,mBAAmB,EACnB,uBAAuB,EACvB,gBAAgB,EAChB,aAAa,EACb,iBAAiB,EACjB,mBAAmB,EACnB,eAAe,EACf,wBAAwB,EACzB,MAAM,qBAAqB,CAAC;AAe7B,MAAM,MAAM,sBAAsB,GAAG,iBAAiB,GAAG;IACvD,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,aAAa,CAAC,EAAE,MAAM,CAAC;CACxB,CAAC;AAEF,MAAM,MAAM,wBAAwB,GAAG,IAAI,CACzC,aAAa,EACb,MAAM,GAAG,OAAO,GAAG,WAAW,GAAG,UAAU,CAC5C,GAAG;IACF,IAAI,CAAC,EAAE,sBAAsB,CAAC;IAC9B,KAAK,CAAC,EAAE,sBAAsB,CAAC;IAC/B,SAAS,CAAC,EAAE,sBAAsB,CAAC;IACnC,QAAQ,CAAC,EAAE,sBAAsB,CAAC;IAClC,gBAAgB,CAAC,EAAE,MAAM,CAAC;CAC3B,CAAC;AAEF,MAAM,MAAM,iBAAiB,GAAG;IAC9B,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,eAAe,CAAC,EAAE,OAAO,CAAC;IAC1B,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,UAAU,CAAC,EAAE,wBAAwB,CAAC;IACtC,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,eAAe,CAAC,EAAE,aAAa,GAAG,cAAc,CAAC;IACjD,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,SAAS,CAAC,EAAE,MAAM,EAAE,CAAC;IACrB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,eAAe,CAAC,EAAE,KAAK,GAAG,eAAe,CAAC;IAC1C,MAAM,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,GAAG,MAAM,CAAC,CAAC;IACnD,OAAO,CAAC,EAAE,wBAAwB,CAAC;IACnC,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,oBAAoB,CAAC,EAAE,MAAM,CAAC;IAC9B,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,gBAAgB,CAAC,EAAE,MAAM,EAAE,CAAC;IAC5B,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,0BAA0B,CAAC,EAAE,MAAM,CAAC;IACpC,uBAAuB,CAAC,EAAE,MAAM,CAAC;IACjC,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,UAAU,CAAC,EAAE,gBAAgB,CAAC;IAC9B,YAAY,CAAC,EAAE,uBAAuB,CAAC;IACvC,aAAa,CAAC,EAAE,mBAAmB,CAAC;IACpC,cAAc,CAAC,EAAE,mBAAmB,EAAE,CAAC;IACvC,gBAAgB,CAAC,EAAE,sBAAsB,CAAC;IAC1C,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,uBAAuB,CAAC,EAAE,MAAM,CAAC;IACjC,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,SAAS,CAAC,EAAE,eAAe,CAAC;IAC5B,UAAU,CAAC,EAAE,MAAM,CAAC;CACrB,CAAC;AAEF,KAAK,iBAAiB,GAAG;IACvB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,MAAM,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,GAAG,MAAM,CAAC,CAAC;CACpD,CAAC;AAEF,KAAK,wBAAwB,GAAG;IAC9B,YAAY,EAAE,MAAM,GAAG,IAAI,CAAC;IAC5B,QAAQ,EAAE,iBAAiB,GAAG,IAAI,CAAC;CACpC,CAAC;AAEF,MAAM,MAAM,0BAA0B,GAAG;IACvC,4BAA4B,CAAC,EAAE,OAAO,CAAC;IACvC,aAAa,CAAC,EAAE,MAAM,CAAC;CACxB,CAAC;AAEF,KAAK,0BAA0B,GAAG;IAChC,QAAQ,CAAC,CAAC,EAAE,YAAY,EAAE,MAAM,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC;CAC/C,CAAC;AAEF,MAAM,MAAM,mBAAmB,GAAG;IAChC,QAAQ,EAAE,MAAM,CAAC;IACjB,WAAW,EAAE,OAAO,CAAC;CACtB,CAAC;AAEF,KAAK,6BAA6B,GAAG;IACnC,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,gBAAgB,CAAC,EAAE,OAAO,CAAC;IAC3B,mBAAmB,CAAC,EAAE,OAAO,CAAC;IAC9B,sBAAsB,CAAC,EAAE,OAAO,CAAC;IACjC,eAAe,CAAC,EAAE,OAAO,CAAC;CAC3B,CAAC;AAEF,MAAM,MAAM,kCAAkC,GAAG,SAAS,GAAG,UAAU,GAAG,OAAO,GAAG,OAAO,CAAC;AAE5F,MAAM,MAAM,kCAAkC,GAAG;IAC/C,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,QAAQ,CAAC,EAAE,kCAAkC,CAAC;CAC/C,CAAC;AAIF,wBAAgB,sCAAsC,CACpD,QAAQ,EAAE,6BAA6B,GACtC,WAAW,GAAG,WAAW,GAAG,WAAW,GAAG,SAAS,CAwCrD;AAED,wBAAgB,2CAA2C,CACzD,QAAQ,EAAE,kCAAkC,GAC3C,MAAM,GAAG,SAAS,CAuCpB;AA2KD,wBAAsB,4BAA4B,CAChD,OAAO,EAAE,OAAO,GACf,OAAO,CAAC,wBAAwB,CAAC,CAuBnC;AAiED,iBAAS,8BAA8B,CAAC,OAAO,EAAE,OAAO,GAAG,iBAAiB,GAAG,IAAI,CAElF;AAED,iBAAe,qBAAqB,CAAC,OAAO,EAAE,OAAO,GAAG,OAAO,CAAC,iBAAiB,GAAG,IAAI,CAAC,CAOxF;AAED,wBAAgB,0BAA0B,CAAC,QAAQ,EAAE,MAAM,GAAG;IAC5D,QAAQ,EAAE,MAAM,CAAC;IACjB,SAAS,CAAC,EAAE,MAAM,EAAE,CAAC;CACtB,CAyBA;AAED,iBAAe,6BAA6B,CAC1C,OAAO,EAAE,0BAA0B,EACnC,OAAO,CAAC,EAAE,0BAA0B,GAAG;IACrC,SAAS,CAAC,EAAE,MAAM,EAAE,CAAC;IACrB,QAAQ,CAAC,EAAE,MAAM,CAAC;CACnB,GACA,OAAO,CAAC,iBAAiB,EAAE,CAAC,CAq7E9B;AA6MD,wBAAsB,uBAAuB,CAAC,KAAK,EAAE,KAAK,GAAG,OAAO,CAAC,mBAAmB,GAAG,IAAI,CAAC,CA0B/F;AAED,wBAAsB,iBAAiB,CACrC,IAAI,EAAE;IACJ,QAAQ,CAAC,CAAC,EAAE,YAAY,EAAE,MAAM,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC;IAC9C,SAAS,CAAC,EAAE,MAAM,KAAK,CAAC;IACxB,GAAG,CAAC,EAAE,MAAM,MAAM,CAAC;CACpB,EACD,OAAO,CAAC,EAAE,0BAA0B,GACnC,OAAO,CAAC,iBAAiB,EAAE,CAAC,CAqD9B;AAED,eAAO,MAAM,yBAAyB;;;;;CAKrC,CAAC;AAEF,eAAO,MAAM,yBAAyB;;;;;CAKrC,CAAC"}
+{"version":3,"file":"observe-inventory.d.ts","sourceRoot":"","sources":["../../src/commands/observe-inventory.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,KAAK,EAAE,OAAO,EAAE,MAAM,iBAAiB,CAAC;AACtD,OAAO,KAAK,EACV,sBAAsB,EACtB,mBAAmB,EACnB,uBAAuB,EACvB,gBAAgB,EAChB,aAAa,EACb,iBAAiB,EACjB,mBAAmB,EACnB,eAAe,EACf,wBAAwB,EACzB,MAAM,qBAAqB,CAAC;AAe7B,MAAM,MAAM,sBAAsB,GAAG,iBAAiB,GAAG;IACvD,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,aAAa,CAAC,EAAE,MAAM,CAAC;CACxB,CAAC;AAEF,MAAM,MAAM,wBAAwB,GAAG,IAAI,CACzC,aAAa,EACb,MAAM,GAAG,OAAO,GAAG,WAAW,GAAG,UAAU,CAC5C,GAAG;IACF,IAAI,CAAC,EAAE,sBAAsB,CAAC;IAC9B,KAAK,CAAC,EAAE,sBAAsB,CAAC;IAC/B,SAAS,CAAC,EAAE,sBAAsB,CAAC;IACnC,QAAQ,CAAC,EAAE,sBAAsB,CAAC;IAClC,gBAAgB,CAAC,EAAE,MAAM,CAAC;CAC3B,CAAC;AAEF,MAAM,MAAM,iBAAiB,GAAG;IAC9B,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,eAAe,CAAC,EAAE,OAAO,CAAC;IAC1B,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,UAAU,CAAC,EAAE,wBAAwB,CAAC;IACtC,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,eAAe,CAAC,EAAE,aAAa,GAAG,cAAc,CAAC;IACjD,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,SAAS,CAAC,EAAE,MAAM,EAAE,CAAC;IACrB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,eAAe,CAAC,EAAE,KAAK,GAAG,eAAe,CAAC;IAC1C,MAAM,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,GAAG,MAAM,CAAC,CAAC;IACnD,OAAO,CAAC,EAAE,wBAAwB,CAAC;IACnC,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,oBAAoB,CAAC,EAAE,MAAM,CAAC;IAC9B,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,gBAAgB,CAAC,EAAE,MAAM,EAAE,CAAC;IAC5B,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,0BAA0B,CAAC,EAAE,MAAM,CAAC;IACpC,uBAAuB,CAAC,EAAE,MAAM,CAAC;IACjC,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,UAAU,CAAC,EAAE,gBAAgB,CAAC;IAC9B,YAAY,CAAC,EAAE,uBAAuB,CAAC;IACvC,aAAa,CAAC,EAAE,mBAAmB,CAAC;IACpC,cAAc,CAAC,EAAE,mBAAmB,EAAE,CAAC;IACvC,gBAAgB,CAAC,EAAE,sBAAsB,CAAC;IAC1C,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,uBAAuB,CAAC,EAAE,MAAM,CAAC;IACjC,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,SAAS,CAAC,EAAE,eAAe,CAAC;IAC5B,UAAU,CAAC,EAAE,MAAM,CAAC;CACrB,CAAC;AAEF,KAAK,iBAAiB,GAAG;IACvB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,MAAM,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,GAAG,MAAM,CAAC,CAAC;CACpD,CAAC;AAEF,KAAK,wBAAwB,GAAG;IAC9B,YAAY,EAAE,MAAM,GAAG,IAAI,CAAC;IAC5B,QAAQ,EAAE,iBAAiB,GAAG,IAAI,CAAC;CACpC,CAAC;AAEF,MAAM,MAAM,0BAA0B,GAAG;IACvC,4BAA4B,CAAC,EAAE,OAAO,CAAC;IACvC,aAAa,CAAC,EAAE,MAAM,CAAC;CACxB,CAAC;AAEF,KAAK,0BAA0B,GAAG;IAChC,QAAQ,CAAC,CAAC,EAAE,YAAY,EAAE,MAAM,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC;CAC/C,CAAC;AAEF,MAAM,MAAM,mBAAmB,GAAG;IAChC,QAAQ,EAAE,MAAM,CAAC;IACjB,WAAW,EAAE,OAAO,CAAC;CACtB,CAAC;AAEF,KAAK,6BAA6B,GAAG;IACnC,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,gBAAgB,CAAC,EAAE,OAAO,CAAC;IAC3B,mBAAmB,CAAC,EAAE,OAAO,CAAC;IAC9B,sBAAsB,CAAC,EAAE,OAAO,CAAC;IACjC,eAAe,CAAC,EAAE,OAAO,CAAC;CAC3B,CAAC;AAEF,MAAM,MAAM,kCAAkC,GAAG,SAAS,GAAG,UAAU,GAAG,OAAO,GAAG,OAAO,CAAC;AAE5F,MAAM,MAAM,kCAAkC,GAAG;IAC/C,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,QAAQ,CAAC,EAAE,kCAAkC,CAAC;CAC/C,CAAC;AAIF,wBAAgB,sCAAsC,CACpD,QAAQ,EAAE,6BAA6B,GACtC,WAAW,GAAG,WAAW,GAAG,WAAW,GAAG,SAAS,CAwCrD;AAED,wBAAgB,2CAA2C,CACzD,QAAQ,EAAE,kCAAkC,GAC3C,MAAM,GAAG,SAAS,CAuCpB;AA2KD,wBAAsB,4BAA4B,CAChD,OAAO,EAAE,OAAO,GACf,OAAO,CAAC,wBAAwB,CAAC,CAuBnC;AAiED,iBAAS,8BAA8B,CAAC,OAAO,EAAE,OAAO,GAAG,iBAAiB,GAAG,IAAI,CAElF;AAED,iBAAe,qBAAqB,CAAC,OAAO,EAAE,OAAO,GAAG,OAAO,CAAC,iBAAiB,GAAG,IAAI,CAAC,CAOxF;AAED,wBAAgB,0BAA0B,CAAC,QAAQ,EAAE,MAAM,GAAG;IAC5D,QAAQ,EAAE,MAAM,CAAC;IACjB,SAAS,CAAC,EAAE,MAAM,EAAE,CAAC;CACtB,CAyBA;AAED,iBAAe,6BAA6B,CAC1C,OAAO,EAAE,0BAA0B,EACnC,OAAO,CAAC,EAAE,0BAA0B,GAAG;IACrC,SAAS,CAAC,EAAE,MAAM,EAAE,CAAC;IACrB,QAAQ,CAAC,EAAE,MAAM,CAAC;CACnB,GACA,OAAO,CAAC,iBAAiB,EAAE,CAAC,CAk8E9B;AA6MD,wBAAsB,uBAAuB,CAAC,KAAK,EAAE,KAAK,GAAG,OAAO,CAAC,mBAAmB,GAAG,IAAI,CAAC,CA0B/F;AAED,wBAAsB,iBAAiB,CACrC,IAAI,EAAE;IACJ,QAAQ,CAAC,CAAC,EAAE,YAAY,EAAE,MAAM,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC;IAC9C,SAAS,CAAC,EAAE,MAAM,KAAK,CAAC;IACxB,GAAG,CAAC,EAAE,MAAM,MAAM,CAAC;CACpB,EACD,OAAO,CAAC,EAAE,0BAA0B,GACnC,OAAO,CAAC,iBAAiB,EAAE,CAAC,CAqD9B;AAED,eAAO,MAAM,yBAAyB;;;;;CAKrC,CAAC;AAEF,eAAO,MAAM,yBAAyB;;;;;CAKrC,CAAC"}

package/dist/commands/observe-inventory.js

@@ -2025,6 +2025,14 @@ async function collectDomTargetsFromDocument(context, options) {
         }
       }
 
+      return undefined;
+    };
+
+    const contextTextOf = (element) => {
+      if (!isHTMLElementNode(element)) {
+        return undefined;
+      }
+
       const text = textOf(element, { container: true });
       if (!text) {
         return undefined;
@@ -2036,14 +2044,19 @@ async function collectDomTargetsFromDocument(context, options) {
       if (!element) return undefined;
 
       const kind = element.getAttribute?.('role')?.trim() || element.tagName?.toLowerCase?.();
-      const label = contextLabelOf(element);
+      const fallbackLabel = contextLabelOf(element);
+      const text = contextTextOf(element);
       const selector = buildSelector(element);
-      if (!kind && !label && !selector) return undefined;
+      if (!kind && !fallbackLabel && !text && !selector) return undefined;
       return {
         kind: kind || undefined,
-        label,
-        text: undefined,
+        label: undefined,
+        text:
+          text && (!fallbackLabel || text.toLowerCase() !== fallbackLabel.toLowerCase())
+            ? text
+            : undefined,
         selector,
+        fallbackLabel,
       };
     };
 

package/dist/commands/semantic-observe-lexical.d.ts

@@ -0,0 +1,31 @@
+export type SemanticObserveAnalyzerKey = 'en' | 'ru' | 'neutral';
+export type SemanticObserveLexicalField = {
+    value?: string;
+    weight: number;
+};
+export type SemanticObserveAnalyzedText = {
+    analyzerKey: SemanticObserveAnalyzerKey;
+    normalizedText?: string;
+    terms: string[];
+};
+export type SemanticObserveLexicalDocument = {
+    analyzerKey: SemanticObserveAnalyzerKey;
+    weightedLength: number;
+    weightedTermFrequencies: Map<string, number>;
+};
+export type SemanticObserveBm25CorpusStats = {
+    averageDocumentLength: number;
+    documentCount: number;
+    documentFrequencyByTerm: Map<string, number>;
+};
+export declare function assertSemanticObserveRuntimeSupport(): void;
+export declare function normalizeSemanticObserveText(value: string | undefined): string | undefined;
+export declare function dominantSemanticObserveAnalyzerKey(value: string | undefined): SemanticObserveAnalyzerKey;
+export declare function analyzeSemanticObserveText(value: string | undefined): SemanticObserveAnalyzedText;
+export declare function buildSemanticObserveLexicalDocument(fields: ReadonlyArray<SemanticObserveLexicalField>): SemanticObserveLexicalDocument;
+export declare function buildSemanticObserveBm25CorpusStats(documents: ReadonlyArray<SemanticObserveLexicalDocument>): SemanticObserveBm25CorpusStats;
+export declare function scoreSemanticObserveBm25(query: SemanticObserveAnalyzedText, document: SemanticObserveLexicalDocument, corpus: SemanticObserveBm25CorpusStats, options?: {
+    k1?: number;
+    b?: number;
+}): number;
+//# sourceMappingURL=semantic-observe-lexical.d.ts.map

package/dist/commands/semantic-observe-lexical.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"semantic-observe-lexical.d.ts","sourceRoot":"","sources":["../../src/commands/semantic-observe-lexical.ts"],"names":[],"mappings":"AAEA,MAAM,MAAM,0BAA0B,GAAG,IAAI,GAAG,IAAI,GAAG,SAAS,CAAC;AAEjE,MAAM,MAAM,2BAA2B,GAAG;IACxC,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,MAAM,CAAC;CAChB,CAAC;AAEF,MAAM,MAAM,2BAA2B,GAAG;IACxC,WAAW,EAAE,0BAA0B,CAAC;IACxC,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,KAAK,EAAE,MAAM,EAAE,CAAC;CACjB,CAAC;AAEF,MAAM,MAAM,8BAA8B,GAAG;IAC3C,WAAW,EAAE,0BAA0B,CAAC;IACxC,cAAc,EAAE,MAAM,CAAC;IACvB,uBAAuB,EAAE,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;CAC9C,CAAC;AAEF,MAAM,MAAM,8BAA8B,GAAG;IAC3C,qBAAqB,EAAE,MAAM,CAAC;IAC9B,aAAa,EAAE,MAAM,CAAC;IACtB,uBAAuB,EAAE,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;CAC9C,CAAC;AAMF,wBAAgB,mCAAmC,IAAI,IAAI,CAM1D;AAgDD,wBAAgB,4BAA4B,CAAC,KAAK,EAAE,MAAM,GAAG,SAAS,GAAG,MAAM,GAAG,SAAS,CAO1F;AAED,wBAAgB,kCAAkC,CAChD,KAAK,EAAE,MAAM,GAAG,SAAS,GACxB,0BAA0B,CAwB5B;AAmCD,wBAAgB,0BAA0B,CAAC,KAAK,EAAE,MAAM,GAAG,SAAS,GAAG,2BAA2B,CAqBjG;AAED,wBAAgB,mCAAmC,CACjD,MAAM,EAAE,aAAa,CAAC,2BAA2B,CAAC,GACjD,8BAA8B,CAoChC;AAED,wBAAgB,mCAAmC,CACjD,SAAS,EAAE,aAAa,CAAC,8BAA8B,CAAC,GACvD,8BAA8B,CAiBhC;AAED,wBAAgB,wBAAwB,CACtC,KAAK,EAAE,2BAA2B,EAClC,QAAQ,EAAE,8BAA8B,EACxC,MAAM,EAAE,8BAA8B,EACtC,OAAO,GAAE;IAAE,EAAE,CAAC,EAAE,MAAM,CAAC;IAAC,CAAC,CAAC,EAAE,MAAM,CAAA;CAAO,GACxC,MAAM,CA0BR"}

package/dist/commands/semantic-observe-lexical.js

@@ -0,0 +1,188 @@
+import { newStemmer } from 'snowball-stemmers';
+const TOKENIZER_FALLBACK_RE = /[\p{L}\p{N}]+/gu;
+const segmenterCache = new Map();
+const stemmerCache = new Map();
+export function assertSemanticObserveRuntimeSupport() {
+    if (typeof Intl === 'undefined' || typeof Intl.Segmenter !== 'function') {
+        throw new Error('AgentBrowse requires Intl.Segmenter. Use a modern Node runtime with standard Intl support.');
+    }
+}
+function segmenterFor(locale) {
+    const cached = segmenterCache.get(locale);
+    if (cached !== undefined) {
+        return cached;
+    }
+    assertSemanticObserveRuntimeSupport();
+    const segmenter = new Intl.Segmenter(locale === 'neutral' ? undefined : locale, {
+        granularity: 'word',
+    });
+    segmenterCache.set(locale, segmenter);
+    return segmenter;
+}
+function stemmerFor(locale) {
+    const cached = stemmerCache.get(locale);
+    if (cached !== undefined) {
+        return cached;
+    }
+    const algorithm = locale === 'en' ? 'english' : locale === 'ru' ? 'russian' : undefined;
+    if (!algorithm) {
+        stemmerCache.set(locale, null);
+        return null;
+    }
+    const stemmer = newStemmer(algorithm);
+    stemmerCache.set(locale, stemmer);
+    return stemmer;
+}
+function isNumericToken(token) {
+    return /^[\p{N}]+$/u.test(token);
+}
+function detectAnalyzerKeyForToken(token) {
+    if (/[\p{Script=Cyrillic}]/u.test(token)) {
+        return 'ru';
+    }
+    if (/[\p{Script=Latin}]/u.test(token)) {
+        return 'en';
+    }
+    return 'neutral';
+}
+export function normalizeSemanticObserveText(value) {
+    const normalized = value
+        ?.normalize('NFKC')
+        .replace(/\s+/g, ' ')
+        .trim()
+        .toLowerCase();
+    return normalized ? normalized : undefined;
+}
+export function dominantSemanticObserveAnalyzerKey(value) {
+    const normalized = normalizeSemanticObserveText(value);
+    if (!normalized) {
+        return 'neutral';
+    }
+    let latinCount = 0;
+    let cyrillicCount = 0;
+    for (const token of normalized.match(TOKENIZER_FALLBACK_RE) ?? []) {
+        const key = detectAnalyzerKeyForToken(token);
+        if (key === 'en') {
+            latinCount += 1;
+        }
+        else if (key === 'ru') {
+            cyrillicCount += 1;
+        }
+    }
+    if (cyrillicCount > latinCount) {
+        return 'ru';
+    }
+    if (latinCount > 0) {
+        return 'en';
+    }
+    return 'neutral';
+}
+function segmentSemanticObserveText(normalizedText, locale) {
+    const segmenter = segmenterFor(locale);
+    const tokens = [];
+    for (const segment of segmenter.segment(normalizedText)) {
+        if (segment.isWordLike === false) {
+            continue;
+        }
+        const token = normalizeSemanticObserveText(segment.segment);
+        if (!token) {
+            continue;
+        }
+        if (!/[\p{L}\p{N}]/u.test(token)) {
+            continue;
+        }
+        tokens.push(token);
+    }
+    return tokens;
+}
+function stemSemanticObserveToken(token) {
+    const analyzerKey = detectAnalyzerKeyForToken(token);
+    const stemmer = stemmerFor(analyzerKey);
+    if (!stemmer) {
+        return token;
+    }
+    const stemmed = normalizeSemanticObserveText(stemmer.stem(token));
+    return stemmed && (stemmed.length >= 2 || isNumericToken(stemmed)) ? stemmed : token;
+}
+export function analyzeSemanticObserveText(value) {
+    const normalizedText = normalizeSemanticObserveText(value);
+    if (!normalizedText) {
+        return {
+            analyzerKey: 'neutral',
+            normalizedText: undefined,
+            terms: [],
+        };
+    }
+    const analyzerKey = dominantSemanticObserveAnalyzerKey(normalizedText);
+    const segmented = segmentSemanticObserveText(normalizedText, analyzerKey);
+    const terms = segmented
+        .map((token) => stemSemanticObserveToken(token))
+        .filter((token) => token.length >= 2 || isNumericToken(token));
+    return {
+        analyzerKey,
+        normalizedText,
+        terms,
+    };
+}
+export function buildSemanticObserveLexicalDocument(fields) {
+    const weightedTermFrequencies = new Map();
+    const analyzerVotes = new Map();
+    let weightedLength = 0;
+    for (const field of fields) {
+        if (field.weight <= 0) {
+            continue;
+        }
+        const analyzed = analyzeSemanticObserveText(field.value);
+        if (analyzed.terms.length === 0) {
+            continue;
+        }
+        analyzerVotes.set(analyzed.analyzerKey, (analyzerVotes.get(analyzed.analyzerKey) ?? 0) + analyzed.terms.length);
+        weightedLength += analyzed.terms.length * field.weight;
+        for (const term of analyzed.terms) {
+            weightedTermFrequencies.set(term, (weightedTermFrequencies.get(term) ?? 0) + field.weight);
+        }
+    }
+    const analyzerKey = [...analyzerVotes.entries()].sort((left, right) => right[1] - left[1])[0]?.[0] ?? 'neutral';
+    return {
+        analyzerKey,
+        weightedLength,
+        weightedTermFrequencies,
+    };
+}
+export function buildSemanticObserveBm25CorpusStats(documents) {
+    const documentFrequencyByTerm = new Map();
+    let totalLength = 0;
+    for (const document of documents) {
+        totalLength += document.weightedLength;
+        for (const term of document.weightedTermFrequencies.keys()) {
+            documentFrequencyByTerm.set(term, (documentFrequencyByTerm.get(term) ?? 0) + 1);
+        }
+    }
+    return {
+        averageDocumentLength: documents.length > 0 ? totalLength / documents.length : 0,
+        documentCount: documents.length,
+        documentFrequencyByTerm,
+    };
+}
+export function scoreSemanticObserveBm25(query, document, corpus, options = {}) {
+    if (query.terms.length === 0 || document.weightedTermFrequencies.size === 0 || corpus.documentCount === 0) {
+        return 0;
+    }
+    const k1 = options.k1 ?? 1.2;
+    const b = options.b ?? 0.75;
+    const averageDocumentLength = Math.max(corpus.averageDocumentLength, 1);
+    const documentLength = Math.max(document.weightedLength, 1);
+    const uniqueTerms = [...new Set(query.terms)];
+    let score = 0;
+    for (const term of uniqueTerms) {
+        const termFrequency = document.weightedTermFrequencies.get(term) ?? 0;
+        if (termFrequency <= 0) {
+            continue;
+        }
+        const documentFrequency = corpus.documentFrequencyByTerm.get(term) ?? 0;
+        const idf = Math.log(1 + (corpus.documentCount - documentFrequency + 0.5) / (documentFrequency + 0.5));
+        const denominator = termFrequency + k1 * (1 - b + b * (documentLength / averageDocumentLength));
+        score += idf * ((termFrequency * (k1 + 1)) / denominator);
+    }
+    return score;
+}

package/dist/commands/semantic-observe.d.ts

@@ -24,6 +24,7 @@ type ObserveInventoryTarget = {
     pageSignature?: string;
     goalInventoryType?: 'target' | 'scope';
     goalSurfaceId?: string;
+    controlsSurfaceSelector?: string;
 };
 export declare function rerankDomTargetsForGoal<T extends ObserveInventoryTarget>(instruction: string, targets: ReadonlyArray<T>, options?: {
     session?: BrowseSession;

package/dist/commands/semantic-observe.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"semantic-observe.d.ts","sourceRoot":"","sources":["../../src/commands/semantic-observe.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,eAAe,CAAC;AAEnD,OAAO,KAAK,EACV,sBAAsB,EACtB,mBAAmB,EACnB,gBAAgB,EAChB,aAAa,EACb,mBAAmB,EACnB,eAAe,EAChB,MAAM,qBAAqB,CAAC;AAE7B,KAAK,sBAAsB,GAAG;IAC5B,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,eAAe,CAAC,EAAE,OAAO,CAAC;IAC1B,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,MAAM,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,GAAG,MAAM,CAAC,CAAC;IACnD,OAAO,CAAC,EAAE,aAAa,CAAC;IACxB,UAAU,CAAC,EAAE,gBAAgB,CAAC;IAC9B,cAAc,CAAC,EAAE,mBAAmB,EAAE,CAAC;IACvC,gBAAgB,CAAC,EAAE,sBAAsB,CAAC;IAC1C,aAAa,CAAC,EAAE,mBAAmB,CAAC;IACpC,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,SAAS,CAAC,EAAE,eAAe,CAAC;IAC5B,SAAS,CAAC,EAAE,MAAM,EAAE,CAAC;IACrB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,iBAAiB,CAAC,EAAE,QAAQ,GAAG,OAAO,CAAC;IACvC,aAAa,CAAC,EAAE,MAAM,CAAC;CACxB,CAAC;AAwhCF,wBAAsB,uBAAuB,CAAC,CAAC,SAAS,sBAAsB,EAC5E,WAAW,EAAE,MAAM,EACnB,OAAO,EAAE,aAAa,CAAC,CAAC,CAAC,EACzB,OAAO,GAAE;IAAE,OAAO,CAAC,EAAE,aAAa,CAAA;CAAO,GACxC,OAAO,CAAC,CAAC,EAAE,CAAC,CA0Ed"}
+{"version":3,"file":"semantic-observe.d.ts","sourceRoot":"","sources":["../../src/commands/semantic-observe.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,eAAe,CAAC;AAEnD,OAAO,KAAK,EACV,sBAAsB,EACtB,mBAAmB,EACnB,gBAAgB,EAChB,aAAa,EACb,mBAAmB,EACnB,eAAe,EAChB,MAAM,qBAAqB,CAAC;AAY7B,KAAK,sBAAsB,GAAG;IAC5B,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,eAAe,CAAC,EAAE,OAAO,CAAC;IAC1B,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,MAAM,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,GAAG,MAAM,CAAC,CAAC;IACnD,OAAO,CAAC,EAAE,aAAa,CAAC;IACxB,UAAU,CAAC,EAAE,gBAAgB,CAAC;IAC9B,cAAc,CAAC,EAAE,mBAAmB,EAAE,CAAC;IACvC,gBAAgB,CAAC,EAAE,sBAAsB,CAAC;IAC1C,aAAa,CAAC,EAAE,mBAAmB,CAAC;IACpC,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,SAAS,CAAC,EAAE,eAAe,CAAC;IAC5B,SAAS,CAAC,EAAE,MAAM,EAAE,CAAC;IACrB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,iBAAiB,CAAC,EAAE,QAAQ,GAAG,OAAO,CAAC;IACvC,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,uBAAuB,CAAC,EAAE,MAAM,CAAC;CAClC,CAAC;AAwoCF,wBAAsB,uBAAuB,CAAC,CAAC,SAAS,sBAAsB,EAC5E,WAAW,EAAE,MAAM,EACnB,OAAO,EAAE,aAAa,CAAC,CAAC,CAAC,EACzB,OAAO,GAAE;IAAE,OAAO,CAAC,EAAE,aAAa,CAAA;CAAO,GACxC,OAAO,CAAC,CAAC,EAAE,CAAC,CA2Ed"}

package/dist/commands/semantic-observe.js

@@ -2,6 +2,7 @@ import { z } from 'zod';
 import { AgentpayStagehandLlmClient } from '../agentpay-stagehand-llm.js';
 import { resolveAgentpayGatewayConfig } from '../agentpay-gateway.js';
 import { recordLlmUsage, recordPayloadBudget } from '../runtime-state.js';
+import { analyzeSemanticObserveText, buildSemanticObserveBm25CorpusStats, buildSemanticObserveLexicalDocument, normalizeSemanticObserveText, scoreSemanticObserveBm25, } from './semantic-observe-lexical.js';
 const rerankSchema = z.object({
     matches: z
         .array(z.object({
@@ -17,6 +18,31 @@ const SCOPE_BUCKET_RESERVE_LIMIT = 24;
 const SCOPE_BUCKET_RESERVE_PER_BUCKET = 2;
 const IFRAME_BUCKET_RESERVE_LIMIT = 24;
 const IFRAME_BUCKET_RESERVE_PER_BUCKET = 2;
+const RETRIEVAL_FIELD_WEIGHTS = {
+    entityLabel: 5.5,
+    representativeLabel: 5,
+    representativeLabels: 4,
+    surfaceLabel: 2.25,
+    placeholder: 2.5,
+    title: 1.25,
+    itemLabel: 2.5,
+    itemText: 1.5,
+    groupLabel: 1.75,
+    groupText: 1,
+    containerLabel: 1.5,
+    containerText: 0.9,
+    landmarkLabel: 1.25,
+    landmarkText: 0.75,
+    hintText: 1.5,
+    kind: 0.6,
+    role: 0.6,
+    surfaceKind: 0.75,
+    controlFamily: 0.9,
+    acceptancePolicy: 1,
+    allowedActions: 0.9,
+    structure: 1.25,
+    latentIntent: 1.2,
+};
 const HIGH_SIGNAL_SCOPE_KINDS = new Set([
     'dialog',
     'listbox',
@@ -29,27 +55,6 @@ const HIGH_SIGNAL_SCOPE_KINDS = new Set([
     'card',
     'form',
 ]);
-const GOAL_TEXT_STOPWORDS = new Set([
-    'a',
-    'an',
-    'and',
-    'at',
-    'for',
-    'from',
-    'in',
-    'of',
-    'on',
-    'or',
-    'the',
-    'to',
-    'with',
-    'в',
-    'для',
-    'и',
-    'или',
-    'на',
-    'найти',
-]);
 function isFieldLikeTarget(target) {
     const kind = (target.kind ?? '').trim().toLowerCase();
     const role = (target.role ?? '').trim().toLowerCase();
@@ -73,16 +78,6 @@ function isActionLikeTargetCandidate(target) {
         kind === 'link' ||
         role === 'link');
 }
-function normalizeRetrievalText(value) {
-    const normalized = value?.replace(/\s+/g, ' ').trim().toLowerCase();
-    return normalized ? normalized : undefined;
-}
-function tokenizeRetrievalText(value) {
-    if (!value) {
-        return [];
-    }
-    return (value.match(/[\p{L}\p{N}]+/gu) ?? []).map((token) => token.toLowerCase());
-}
 function semanticFormContextKey(context) {
     for (const node of [context?.landmark, context?.container, context?.group, context?.item]) {
         const kind = (node?.kind ?? '').trim().toLowerCase();
@@ -140,6 +135,13 @@ function isPrimaryFormControlTarget(target) {
         acceptancePolicy === 'disclosure' &&
         (kind === 'button' || role === 'button'));
 }
+function shouldPreserveStandaloneFormActionEntity(target) {
+    return (Boolean(formBucketKey(target)) &&
+        !isScopeLikeCandidate(target) &&
+        !isFieldLikeTarget(target) &&
+        isActionLikeTargetCandidate(target) &&
+        isPrimaryFormControlTarget(target));
+}
 function primaryFormTargetPriority(target) {
     const acceptancePolicy = (target.acceptancePolicy ?? '').trim().toLowerCase();
     if (isFieldLikeTarget(target)) {
@@ -178,11 +180,11 @@ function contentItemBucketKey(target) {
     if (surfaceIdentity) {
         return `surface:${surfaceIdentity}`;
     }
-    const itemKey = normalizeRetrievalText(target.context?.item?.label ?? target.context?.item?.text);
+    const itemKey = normalizeSemanticObserveText(target.context?.item?.label ?? target.context?.item?.text);
     if (itemKey) {
         return `item:${itemKey}`;
     }
-    const containerKey = normalizeRetrievalText(target.context?.container?.label ?? target.context?.container?.text);
+    const containerKey = normalizeSemanticObserveText(target.context?.container?.label ?? target.context?.container?.text);
     if (containerKey) {
         return `container:${containerKey}`;
     }
@@ -205,7 +207,7 @@ function scopeCandidatePriority(target) {
     return 4;
 }
 function representativeCandidateScore(target) {
-    const normalizedLabel = normalizeRetrievalText(target.label);
+    const normalizedLabel = normalizeSemanticObserveText(target.label);
     const kind = (target.kind ?? '').trim().toLowerCase();
     const role = (target.role ?? '').trim().toLowerCase();
     let score = (target.surfacePriority ?? 0) * 10;
@@ -295,26 +297,131 @@ function collectRepresentativeLabels(targets) {
     }
     return labels;
 }
-function buildEntitySearchText(entityLabel, representative, representativeLabels) {
+function structureLexicalValue(structure) {
+    if (!structure) {
+        return undefined;
+    }
     return [
-        entityLabel,
-        representative.label,
-        ...representativeLabels,
-        representative.surfaceLabel,
-        representative.context?.item?.label,
-        representative.context?.item?.text,
-        representative.context?.group?.label,
-        representative.context?.group?.text,
-        representative.context?.container?.label,
-        representative.context?.container?.text,
-        representative.context?.landmark?.label,
-        representative.context?.landmark?.text,
-        representative.context?.hintText,
+        structure.family,
+        structure.variant,
+        structure.row,
+        structure.column,
+        structure.zone,
+        structure.cellLabel,
     ]
-        .map((value) => normalizeRetrievalText(value))
-        .filter((value) => Boolean(value))
+        .filter(Boolean)
         .join(' ');
 }
+function latentActionHintText(target) {
+    const acceptancePolicy = (target.acceptancePolicy ?? '').trim().toLowerCase();
+    const controlFamily = (target.controlFamily ?? '').trim().toLowerCase();
+    const surfaceKind = (target.surfaceKind ?? '').trim().toLowerCase();
+    const kind = (target.kind ?? '').trim().toLowerCase();
+    const role = (target.role ?? '').trim().toLowerCase();
+    const popupBacked = acceptancePolicy === 'disclosure' ||
+        Boolean(target.controlsSurfaceSelector) ||
+        target.states?.expanded !== undefined;
+    if (!popupBacked) {
+        return undefined;
+    }
+    const hints = new Set();
+    hints.add('open menu options choices');
+    if (!isFieldLikeTarget(target)) {
+        hints.add('sort sorting filter filters view switch picker');
+    }
+    if (controlFamily === 'select' ||
+        ['menu', 'listbox', 'dropdown', 'popover'].includes(surfaceKind)) {
+        hints.add('select choose option options menu dropdown listbox');
+    }
+    if (controlFamily === 'datepicker' || surfaceKind === 'datepicker') {
+        hints.add('open calendar datepicker date picker choose date calendar');
+    }
+    const evidenceText = [
+        target.label,
+        target.placeholder,
+        target.title,
+        target.surfaceLabel,
+        target.context?.group?.label,
+        target.context?.container?.label,
+        target.context?.landmark?.label,
+        target.context?.hintText,
+    ]
+        .filter((value) => Boolean(value))
+        .join(' ')
+        .toLowerCase();
+    if (/(?:sort|sorting|show first|ordered by|order by|popular|relevance|recommended|price|cheap|cheapest|expensive|сорт|сначала|популяр|релевант|рекоменд|цене|дешев|дорог)/i.test(evidenceText)) {
+        hints.add('sort sorting order order by relevance popular recommended price cheapest cheapest first');
+    }
+    if (/(?:filter|filters|refine|brand|airline|amenit|фильтр|фильтры|бренд|авиакомпан|удобств)/i.test(evidenceText)) {
+        hints.add('filter filters refine refine results brand airline amenities');
+    }
+    if (/(?:view|layout|grid|list|map|calendar view|вид|список|сетка|карта|раскладк)/i.test(evidenceText)) {
+        hints.add('view layout grid list map switch');
+    }
+    if (kind === 'button' &&
+        role === 'button' &&
+        (acceptancePolicy === 'disclosure' || controlFamily === 'select' || controlFamily === 'datepicker')) {
+        hints.add('open selector choose mode');
+    }
+    return hints.size > 0 ? [...hints].join(' ') : undefined;
+}
+function buildEntityLexicalFields(entityLabel, representative, representativeLabels) {
+    return [
+        { value: entityLabel, weight: RETRIEVAL_FIELD_WEIGHTS.entityLabel },
+        { value: representative.label, weight: RETRIEVAL_FIELD_WEIGHTS.representativeLabel },
+        ...representativeLabels.map((value) => ({
+            value,
+            weight: RETRIEVAL_FIELD_WEIGHTS.representativeLabels,
+        })),
+        { value: representative.surfaceLabel, weight: RETRIEVAL_FIELD_WEIGHTS.surfaceLabel },
+        { value: representative.placeholder, weight: RETRIEVAL_FIELD_WEIGHTS.placeholder },
+        { value: representative.title, weight: RETRIEVAL_FIELD_WEIGHTS.title },
+        { value: representative.context?.item?.label, weight: RETRIEVAL_FIELD_WEIGHTS.itemLabel },
+        { value: representative.context?.item?.text, weight: RETRIEVAL_FIELD_WEIGHTS.itemText },
+        { value: representative.context?.group?.label, weight: RETRIEVAL_FIELD_WEIGHTS.groupLabel },
+        { value: representative.context?.group?.text, weight: RETRIEVAL_FIELD_WEIGHTS.groupText },
+        {
+            value: representative.context?.container?.label,
+            weight: RETRIEVAL_FIELD_WEIGHTS.containerLabel,
+        },
+        {
+            value: representative.context?.container?.text,
+            weight: RETRIEVAL_FIELD_WEIGHTS.containerText,
+        },
+        {
+            value: representative.context?.landmark?.label,
+            weight: RETRIEVAL_FIELD_WEIGHTS.landmarkLabel,
+        },
+        {
+            value: representative.context?.landmark?.text,
+            weight: RETRIEVAL_FIELD_WEIGHTS.landmarkText,
+        },
+        { value: representative.context?.hintText, weight: RETRIEVAL_FIELD_WEIGHTS.hintText },
+        { value: representative.kind, weight: RETRIEVAL_FIELD_WEIGHTS.kind },
+        { value: representative.role, weight: RETRIEVAL_FIELD_WEIGHTS.role },
+        { value: representative.surfaceKind, weight: RETRIEVAL_FIELD_WEIGHTS.surfaceKind },
+        {
+            value: representative.controlFamily,
+            weight: RETRIEVAL_FIELD_WEIGHTS.controlFamily,
+        },
+        {
+            value: representative.acceptancePolicy,
+            weight: RETRIEVAL_FIELD_WEIGHTS.acceptancePolicy,
+        },
+        {
+            value: representative.allowedActions?.join(' '),
+            weight: RETRIEVAL_FIELD_WEIGHTS.allowedActions,
+        },
+        {
+            value: structureLexicalValue(representative.structure),
+            weight: RETRIEVAL_FIELD_WEIGHTS.structure,
+        },
+        {
+            value: latentActionHintText(representative),
+            weight: RETRIEVAL_FIELD_WEIGHTS.latentIntent,
+        },
+    ];
+}
 function buildGoalRetrievalEntity(entityKind, entityKey, memberIndexes, targets) {
     const orderedMembers = [...memberIndexes]
         .map((index) => ({ index, target: targets[index] }))
@@ -322,6 +429,8 @@ function buildGoalRetrievalEntity(entityKind, entityKey, memberIndexes, targets)
     const representative = orderedMembers[0].target;
     const representativeLabels = collectRepresentativeLabels(orderedMembers.map((entry) => entry.target));
     const label = pickEntityLabel(entityKind, representative);
+    const lexicalFields = buildEntityLexicalFields(label, representative, representativeLabels);
+    const lexicalDocument = buildSemanticObserveLexicalDocument(lexicalFields);
     return {
         entityKind,
         entityKey,
@@ -342,7 +451,8 @@ function buildGoalRetrievalEntity(entityKind, entityKey, memberIndexes, targets)
         context: representative.context,
         structure: representative.structure,
         representativeLabels,
-        searchText: buildEntitySearchText(label, representative, representativeLabels),
+        analyzerKey: lexicalDocument.analyzerKey,
+        lexicalDocument,
     };
 }
 function buildGoalRetrievalEntities(targets) {
@@ -376,7 +486,12 @@ function buildGoalRetrievalEntities(targets) {
     }
     for (const [key, memberIndexes] of formGroups.entries()) {
         entities.push(buildGoalRetrievalEntity('form', `form:${key}`, memberIndexes, targets));
-        memberIndexes.forEach((index) => groupedTargetIndexes.add(index));
+        memberIndexes.forEach((index) => {
+            if (shouldPreserveStandaloneFormActionEntity(targets[index])) {
+                return;
+            }
+            groupedTargetIndexes.add(index);
+        });
     }
     const itemGroups = new Map();
     for (const [index, target] of targets.entries()) {
@@ -445,58 +560,28 @@ function retrievalEntityPriority(entity) {
     score += representativeCandidateScore(entity.representative);
     return score;
 }
-function scoreRetrievalEntityAgainstGoal(goal, entity) {
-    const normalizedGoal = normalizeRetrievalText(goal);
-    if (!normalizedGoal) {
+function scoreRetrievalEntityAgainstGoal(analyzedGoal, entity, corpusStats) {
+    if (!analyzedGoal.normalizedText) {
         return 0;
     }
-    const entityText = entity.searchText;
-    if (!entityText) {
+    if (entity.lexicalDocument.weightedLength <= 0) {
         return 0;
     }
-    let score = 0;
-    const goalTokens = tokenizeRetrievalText(normalizedGoal).filter((token) => token.length >= 2 && !GOAL_TEXT_STOPWORDS.has(token));
-    const entityTokens = new Set(tokenizeRetrievalText(entityText));
-    if (entityText.includes(normalizedGoal)) {
-        score += 220;
-    }
-    const entityLabel = normalizeRetrievalText(entity.label);
-    if (entityLabel) {
-        if (normalizedGoal.includes(entityLabel)) {
-            score += 180;
-        }
-        if (entityLabel.includes(normalizedGoal)) {
-            score += 140;
-        }
-    }
-    let matchedTokenCount = 0;
-    for (const token of goalTokens) {
-        if (entityTokens.has(token)) {
-            matchedTokenCount += 1;
-            score += 8 + Math.min(token.length, 12);
-        }
-    }
-    if (goalTokens.length > 0) {
-        const coverage = matchedTokenCount / goalTokens.length;
-        if (coverage === 1) {
-            score += 120;
-        }
-        else if (coverage >= 0.75) {
-            score += 70;
-        }
-        else if (coverage >= 0.5) {
-            score += 35;
-        }
-    }
-    return score;
+    const bm25Score = scoreSemanticObserveBm25(analyzedGoal, entity.lexicalDocument, corpusStats);
+    const analyzerAlignmentScore = analyzedGoal.analyzerKey !== 'neutral' && analyzedGoal.analyzerKey === entity.analyzerKey
+        ? 15
+        : 0;
+    return bm25Score * 100 + analyzerAlignmentScore;
 }
 function preselectRetrievalEntities(goal, entities) {
     if (entities.length <= GOAL_RETRIEVAL_ENTITY_LIMIT) {
         return [...entities];
     }
+    const analyzedGoal = analyzeSemanticObserveText(goal);
+    const corpusStats = buildSemanticObserveBm25CorpusStats(entities.map((entity) => entity.lexicalDocument));
     const scored = entities.map((entity) => ({
         entity,
-        lexicalScore: scoreRetrievalEntityAgainstGoal(goal, entity),
+        lexicalScore: scoreRetrievalEntityAgainstGoal(analyzedGoal, entity, corpusStats),
         priority: retrievalEntityPriority(entity),
     }));
     const selected = [];
@@ -634,9 +719,6 @@ function diversifyCandidates(targets) {
     }
     return orderedIndexes.map((index) => targets[index]).slice(0, RERANK_CANDIDATE_LIMIT);
 }
-function buildCandidateSummary(target, index, surfaceSummaryIdBySurfaceKey) {
-    return buildCompactCandidateSummary(target, index, surfaceSummaryIdBySurfaceKey);
-}
 function compactContextValue(value, maxLength = 80) {
     const normalized = value?.replace(/\s+/g, ' ').trim();
     if (!normalized) {
@@ -791,6 +873,9 @@ function buildCompactCandidateSummary(target, index, surfaceSummaryIdBySurfaceKe
     if (target.controlFamily) {
         parts.push(`family=${JSON.stringify(target.controlFamily)}`);
     }
+    if (target.controlsSurfaceSelector) {
+        parts.push('controlsPopup=true');
+    }
     if (target.capability && target.capability !== 'actionable') {
         parts.push(`capability=${JSON.stringify(target.capability)}`);
     }
@@ -836,6 +921,7 @@ export async function rerankDomTargetsForGoal(instruction, targets, options = {}
     const prompt = [
         'You are choosing from already discovered visible webpage candidates.',
         'Select only candidate IDs that directly satisfy the goal.',
+        'A disclosure trigger with a current-state label can still directly satisfy sort, filter, view-switch, or picker goals when opening it is the immediate next step.',
         'Prefer direct actionable controls over surrounding regions unless the goal explicitly asks for a form, region, widget, or set of controls.',
         'Use owning surface, compact context, explicit state, and structure metadata to disambiguate similar labels.',
         'For input/value goals, prefer the directly editable field or primary picker trigger over wrappers or mirrored summary content.',
@@ -847,7 +933,7 @@ export async function rerankDomTargetsForGoal(instruction, targets, options = {}
         '',
         ...(surfaceSummaries.length > 0 ? ['Surfaces:', ...surfaceSummaries.map((entry) => entry.line), ''] : []),
         'Candidates:',
-        ...candidates.map((target, index) => buildCandidateSummary(target, index, summaryIdBySurfaceKey)),
+        ...candidates.map((target, index) => buildCompactCandidateSummary(target, index, summaryIdBySurfaceKey)),
     ].join('\n');
     const result = await client.createChatCompletion({
         logger: () => { },

package/dist/control-semantics.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"control-semantics.d.ts","sourceRoot":"","sources":["../src/control-semantics.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EACV,sBAAsB,EACtB,mBAAmB,EACnB,uBAAuB,EACvB,mBAAmB,EACnB,eAAe,EAChB,MAAM,oBAAoB,CAAC;AAE5B,MAAM,MAAM,gBAAgB,GAAG,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,GAAG,MAAM,CAAC,CAAC;AAEzE,MAAM,MAAM,oBAAoB,GAAG;IACjC,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,eAAe,CAAC,EAAE,OAAO,CAAC;IAC1B,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,uBAAuB,CAAC,EAAE,MAAM,CAAC;IACjC,MAAM,CAAC,EAAE,gBAAgB,CAAC;IAC1B,SAAS,CAAC,EAAE,eAAe,CAAC;IAC5B,YAAY,CAAC,EAAE,MAAM,CAAC;CACvB,CAAC;AAEF,MAAM,MAAM,mBAAmB,GAAG,OAAO,GAAG,aAAa,GAAG,QAAQ,GAAG,KAAK,GAAG,MAAM,CAAC;AAyCtF,wBAAgB,iCAAiC,CAC/C,KAAK,EAAE,oBAAoB,GAC1B,mBAAmB,GAAG,SAAS,CAuCjC;AAsCD,wBAAgB,qBAAqB,CAAC,KAAK,EAAE,MAAM,GAAG,SAAS,GAAG,OAAO,CAYxE;AAED,wBAAgB,4BAA4B,CAAC,KAAK,EAAE,oBAAoB,GAAG,mBAAmB,EAAE,CAoG/F;AAED,wBAAgB,0BAA0B,CACxC,MAAM,CAAC,EAAE,gBAAgB,EACzB,WAAW,CAAC,EAAE,MAAM,EACpB,OAAO,GAAE;IACP,mBAAmB,CAAC,EAAE,OAAO,CAAC;CAC1B,GACL,uBAAuB,CAyBzB;AAED,wBAAgB,2BAA2B,CACzC,KAAK,EAAE,oBAAoB,EAC3B,cAAc,EAAE,aAAa,CAAC,mBAAmB,CAAC,GACjD,mBAAmB,GAAG,SAAS,CA0EjC;AAED,wBAAgB,8BAA8B,CAC5C,KAAK,EAAE,oBAAoB,EAC3B,cAAc,EAAE,aAAa,CAAC,mBAAmB,CAAC,GACjD,sBAAsB,GAAG,SAAS,CAiDpC"}
+{"version":3,"file":"control-semantics.d.ts","sourceRoot":"","sources":["../src/control-semantics.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EACV,sBAAsB,EACtB,mBAAmB,EACnB,uBAAuB,EACvB,mBAAmB,EACnB,eAAe,EAChB,MAAM,oBAAoB,CAAC;AAE5B,MAAM,MAAM,gBAAgB,GAAG,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,GAAG,MAAM,CAAC,CAAC;AAEzE,MAAM,MAAM,oBAAoB,GAAG;IACjC,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,eAAe,CAAC,EAAE,OAAO,CAAC;IAC1B,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,uBAAuB,CAAC,EAAE,MAAM,CAAC;IACjC,MAAM,CAAC,EAAE,gBAAgB,CAAC;IAC1B,SAAS,CAAC,EAAE,eAAe,CAAC;IAC5B,YAAY,CAAC,EAAE,MAAM,CAAC;CACvB,CAAC;AAEF,MAAM,MAAM,mBAAmB,GAAG,OAAO,GAAG,aAAa,GAAG,QAAQ,GAAG,KAAK,GAAG,MAAM,CAAC;AAiDtF,wBAAgB,iCAAiC,CAC/C,KAAK,EAAE,oBAAoB,GAC1B,mBAAmB,GAAG,SAAS,CAuCjC;AA2ID,wBAAgB,qBAAqB,CAAC,KAAK,EAAE,MAAM,GAAG,SAAS,GAAG,OAAO,CAYxE;AAED,wBAAgB,4BAA4B,CAAC,KAAK,EAAE,oBAAoB,GAAG,mBAAmB,EAAE,CAoG/F;AAED,wBAAgB,0BAA0B,CACxC,MAAM,CAAC,EAAE,gBAAgB,EACzB,WAAW,CAAC,EAAE,MAAM,EACpB,OAAO,GAAE;IACP,mBAAmB,CAAC,EAAE,OAAO,CAAC;CAC1B,GACL,uBAAuB,CAyBzB;AAED,wBAAgB,2BAA2B,CACzC,KAAK,EAAE,oBAAoB,EAC3B,cAAc,EAAE,aAAa,CAAC,mBAAmB,CAAC,GACjD,mBAAmB,GAAG,SAAS,CA0EjC;AAED,wBAAgB,8BAA8B,CAC5C,KAAK,EAAE,oBAAoB,EAC3B,cAAc,EAAE,aAAa,CAAC,mBAAmB,CAAC,GACjD,sBAAsB,GAAG,SAAS,CA6CpC"}

package/dist/control-semantics.js

@@ -7,6 +7,14 @@ const CARD_EXPIRY_FIELD_RE = /\b(exp(?:iry|iration)(?: date)?|cc-exp|valid\s+thr
 const CARD_CVC_FIELD_RE = /\b(cvc|cvv|cc-csc|security code|card cvc)\b/i;
 const DATE_VALUE_MASK_RE = /(?:^|\b)(?:dd|d|дд|д)\s*[./-]\s*(?:mm|m|мм|м)\s*[./-]\s*(?:yyyy|yyy|yy|y|гггг|гг|г)(?:\b|$)/i;
 const NON_CARD_DATE_FIELD_RE = /\b(?:birth|dob|date|issued|expires|expiry|expiration|expir)\b|(?:дата|рожд|срок)/i;
+const POPUP_DISCLOSURE_SURFACE_KINDS = new Set([
+    'dialog',
+    'dropdown',
+    'listbox',
+    'menu',
+    'popover',
+    'datepicker',
+]);
 function normalizeText(value) {
     return (value ?? '').trim().toLowerCase();
 }
@@ -83,6 +91,78 @@ function isButtonLikeInputFacts(facts) {
     return (normalizeText(facts.kind) === 'input' &&
         ['button', 'submit', 'reset'].includes(normalizeText(facts.inputType)));
 }
+function isSelectionItemLikeFacts(facts) {
+    const kind = normalizeText(facts.kind);
+    const role = normalizeText(facts.role);
+    return (kind === 'option' ||
+        role === 'option' ||
+        role === 'menuitem' ||
+        facts.structure?.family === 'structured-grid');
+}
+function isBinaryToggleControlFacts(facts) {
+    const kind = normalizeText(facts.kind);
+    const role = normalizeText(facts.role);
+    const inputType = normalizeText(facts.inputType);
+    return (kind === 'checkbox' ||
+        kind === 'radio' ||
+        role === 'checkbox' ||
+        role === 'radio' ||
+        role === 'switch' ||
+        kind === 'tab' ||
+        role === 'tab' ||
+        inputType === 'checkbox' ||
+        inputType === 'radio');
+}
+function isDisclosureLikeTriggerFacts(facts, controlFamily) {
+    const kind = normalizeText(facts.kind);
+    const role = normalizeText(facts.role);
+    const surfaceKind = normalizeText(facts.surfaceKind);
+    const inputType = normalizeText(facts.inputType);
+    const popupBacked = Boolean(facts.controlsSurfaceSelector) ||
+        facts.states?.expanded !== undefined ||
+        POPUP_DISCLOSURE_SURFACE_KINDS.has(surfaceKind);
+    if (!popupBacked) {
+        return false;
+    }
+    if (isSelectionItemLikeFacts(facts) || controlFamily === 'text-input') {
+        return false;
+    }
+    if (controlFamily === 'select' || controlFamily === 'datepicker') {
+        return true;
+    }
+    return (kind === 'button' ||
+        role === 'button' ||
+        kind === 'link' ||
+        role === 'link' ||
+        kind === 'combobox' ||
+        role === 'combobox' ||
+        kind === 'select' ||
+        isButtonLikeInputFacts(facts) ||
+        inputType === 'button' ||
+        isPopupBackedTextEntry(facts));
+}
+function isToggleLikeControlFacts(facts, controlFamily) {
+    const states = facts.states;
+    if (!states) {
+        return false;
+    }
+    if (states.checked !== undefined || isBinaryToggleControlFacts(facts)) {
+        return true;
+    }
+    if (states.selected !== undefined) {
+        if (isSelectionItemLikeFacts(facts) ||
+            controlFamily === 'select' ||
+            controlFamily === 'datepicker' ||
+            isDisclosureLikeTriggerFacts(facts, controlFamily)) {
+            return false;
+        }
+        return true;
+    }
+    if (states.pressed !== undefined) {
+        return !isDisclosureLikeTriggerFacts(facts, controlFamily);
+    }
+    return false;
+}
 export function isLikelyDateLikeLabel(value) {
     const trimmed = (value ?? '').trim();
     if (!trimmed) {
@@ -280,17 +360,15 @@ export function inferAcceptancePolicyFromFacts(facts, allowedActions) {
     if (facts.structure?.family === 'structured-grid') {
         return facts.structure.variant === 'date-cell' ? 'date-selection' : 'selection';
     }
-    if (states?.checked !== undefined ||
-        states?.selected !== undefined ||
-        states?.pressed !== undefined) {
-        return 'toggle';
-    }
     if (controlFamily === 'text-input') {
         return 'value-change';
     }
-    if (states?.expanded !== undefined) {
+    if (isDisclosureLikeTriggerFacts(facts, controlFamily)) {
         return 'disclosure';
     }
+    if (isToggleLikeControlFacts(facts, controlFamily)) {
+        return 'toggle';
+    }
     if (controlFamily === 'datepicker') {
         return 'date-selection';
     }

package/dist/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":";AA2QA,iBAAe,IAAI,CAAC,IAAI,GAAE,MAAM,EAAiB,GAAG,OAAO,CAAC,IAAI,CAAC,CAsKhE;AAED,OAAO,EAAE,IAAI,EAAE,CAAC;AAEhB,wBAAgB,iBAAiB,CAAC,OAAO,EAAE,MAAM,EAAE,IAAI,GAAE,MAAM,EAAiB,GAAG,OAAO,CAWzF"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":";AA4QA,iBAAe,IAAI,CAAC,IAAI,GAAE,MAAM,EAAiB,GAAG,OAAO,CAAC,IAAI,CAAC,CA4KhE;AAED,OAAO,EAAE,IAAI,EAAE,CAAC;AAEhB,wBAAgB,iBAAiB,CAAC,OAAO,EAAE,MAAM,EAAE,IAAI,GAAE,MAAM,EAAiB,GAAG,OAAO,CAWzF"}

package/dist/index.js

@@ -5,6 +5,7 @@ import { fileURLToPath } from 'node:url';
 loadEnv();
 import { preflightAgentpayGateway } from './agentpay-gateway.js';
 import { browseCommand, browseCommandName } from './command-name.js';
+import { assertSemanticObserveRuntimeSupport } from './commands/semantic-observe-lexical.js';
 import { loadSession } from './session.js';
 import { outputError, outputJSON, fatal, info } from './output.js';
 import { applyDemoProxyBootstrap } from './solver/config.js';
@@ -223,6 +224,12 @@ async function main(argv = process.argv) {
     if (!KNOWN_COMMANDS.has(command)) {
         outputError(`Unknown command: ${command}\n\n${usageText()}`);
     }
+    try {
+        assertSemanticObserveRuntimeSupport();
+    }
+    catch (err) {
+        outputError(err instanceof Error ? err.message : String(err));
+    }
     if (command !== 'init') {
         try {
             await preflightAgentpayGateway();

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@nuanu-ai/agentbrowse",
-  "version": "0.2.36",
+  "version": "0.2.37",
   "type": "module",
   "description": "Browser automation CLI for AI agents: control a CDP browser, observe UI surfaces, act on refs, extract data, capture screenshots, complete protected fills, and solve captchas",
   "keywords": [
@@ -57,6 +57,7 @@
     "puppeteer": "^23.11.1",
     "puppeteer-extra": "^3.3.6",
     "puppeteer-extra-plugin-stealth": "^2.11.2",
+    "snowball-stemmers": "^0.6.0",
     "zod": "^3.24.0"
   },
   "devDependencies": {