@nuanu-ai/agentbrowse 0.2.41 → 0.2.42

package/dist/commands/observe-semantics.d.ts

@@ -4,6 +4,7 @@ export declare function isPrimaryFormControlTarget(target: DomObservedTarget): b
 export declare function formGroupingKeyOf(target: DomObservedTarget): string | undefined;
 export declare function observedTargetKey(target: DomObservedTarget): string;
 export declare function expandRerankedFormTargets(targets: ReadonlyArray<DomObservedTarget>, selectedTargets: ReadonlyArray<DomObservedTarget>): DomObservedTarget[];
+export declare function expandCompanionSubmitTargets(targets: ReadonlyArray<DomObservedTarget>, selectedTargets: ReadonlyArray<DomObservedTarget>): DomObservedTarget[];
 export declare function prioritizeGoalActionTargets(instruction: string, selectedTargets: ReadonlyArray<DomObservedTarget>): DomObservedTarget[];
 export declare function compressSemanticallyDuplicateTargets(targets: ReadonlyArray<DomObservedTarget>): DomObservedTarget[];
 export declare function annotateDomTargets(targets: ReadonlyArray<DomObservedTarget>): DomObservedTarget[];

package/dist/commands/observe-semantics.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"observe-semantics.d.ts","sourceRoot":"","sources":["../../src/commands/observe-semantics.ts"],"names":[],"mappings":"AAaA,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,wBAAwB,CAAC;AAGhE,wBAAgB,+BAA+B,CAAC,MAAM,EAAE,iBAAiB,GAAG,MAAM,CAYjF;AAED,wBAAgB,0BAA0B,CAAC,MAAM,EAAE,iBAAiB,GAAG,OAAO,CA6B7E;AAgBD,wBAAgB,iBAAiB,CAAC,MAAM,EAAE,iBAAiB,GAAG,MAAM,GAAG,SAAS,CAQ/E;AAED,wBAAgB,iBAAiB,CAAC,MAAM,EAAE,iBAAiB,GAAG,MAAM,CAMnE;AAED,wBAAgB,yBAAyB,CACvC,OAAO,EAAE,aAAa,CAAC,iBAAiB,CAAC,EACzC,eAAe,EAAE,aAAa,CAAC,iBAAiB,CAAC,GAChD,iBAAiB,EAAE,CAyCrB;AAkFD,wBAAgB,2BAA2B,CACzC,WAAW,EAAE,MAAM,EACnB,eAAe,EAAE,aAAa,CAAC,iBAAiB,CAAC,GAChD,iBAAiB,EAAE,CA+BrB;AAqBD,wBAAgB,oCAAoC,CAClD,OAAO,EAAE,aAAa,CAAC,iBAAiB,CAAC,GACxC,iBAAiB,EAAE,CAuBrB;AAkGD,wBAAgB,kBAAkB,CAAC,OAAO,EAAE,aAAa,CAAC,iBAAiB,CAAC,GAAG,iBAAiB,EAAE,CAoFjG;AAED,wBAAgB,yBAAyB,CACvC,OAAO,EAAE,aAAa,CAAC,iBAAiB,CAAC,GACxC,iBAAiB,EAAE,CAsFrB"}
+{"version":3,"file":"observe-semantics.d.ts","sourceRoot":"","sources":["../../src/commands/observe-semantics.ts"],"names":[],"mappings":"AAaA,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,wBAAwB,CAAC;AAGhE,wBAAgB,+BAA+B,CAAC,MAAM,EAAE,iBAAiB,GAAG,MAAM,CAYjF;AAED,wBAAgB,0BAA0B,CAAC,MAAM,EAAE,iBAAiB,GAAG,OAAO,CA6B7E;AAgBD,wBAAgB,iBAAiB,CAAC,MAAM,EAAE,iBAAiB,GAAG,MAAM,GAAG,SAAS,CAQ/E;AAED,wBAAgB,iBAAiB,CAAC,MAAM,EAAE,iBAAiB,GAAG,MAAM,CAMnE;AAED,wBAAgB,yBAAyB,CACvC,OAAO,EAAE,aAAa,CAAC,iBAAiB,CAAC,EACzC,eAAe,EAAE,aAAa,CAAC,iBAAiB,CAAC,GAChD,iBAAiB,EAAE,CAyCrB;AAsDD,wBAAgB,4BAA4B,CAC1C,OAAO,EAAE,aAAa,CAAC,iBAAiB,CAAC,EACzC,eAAe,EAAE,aAAa,CAAC,iBAAiB,CAAC,GAChD,iBAAiB,EAAE,CAsFrB;AAkFD,wBAAgB,2BAA2B,CACzC,WAAW,EAAE,MAAM,EACnB,eAAe,EAAE,aAAa,CAAC,iBAAiB,CAAC,GAChD,iBAAiB,EAAE,CA+BrB;AAqBD,wBAAgB,oCAAoC,CAClD,OAAO,EAAE,aAAa,CAAC,iBAAiB,CAAC,GACxC,iBAAiB,EAAE,CAuBrB;AAkGD,wBAAgB,kBAAkB,CAAC,OAAO,EAAE,aAAa,CAAC,iBAAiB,CAAC,GAAG,iBAAiB,EAAE,CAoFjG;AAED,wBAAgB,yBAAyB,CACvC,OAAO,EAAE,aAAa,CAAC,iBAAiB,CAAC,GACxC,iBAAiB,EAAE,CAsFrB"}

package/dist/commands/observe-semantics.js

@@ -94,6 +94,116 @@ export function expandRerankedFormTargets(targets, selectedTargets) {
         return (left.label ?? '').localeCompare(right.label ?? '');
     });
 }
+function isSubmitLikeTarget(target) {
+    const acceptancePolicy = (target.acceptancePolicy ?? '').trim().toLowerCase();
+    const inputType = (target.inputType ?? '').trim().toLowerCase();
+    return acceptancePolicy === 'submit' || inputType === 'submit';
+}
+function selectedFormClusterKeyOf(target) {
+    const formKey = formGroupingKeyOf(target);
+    if (formKey) {
+        return `form:${formKey}`;
+    }
+    const surfaceRef = target.surfaceRef?.trim();
+    return surfaceRef ? `surface:${surfaceRef}` : undefined;
+}
+function matchesSelectedFormCluster(target, clusterKey) {
+    if (!clusterKey) {
+        return false;
+    }
+    if (clusterKey.startsWith('form:')) {
+        return `form:${formGroupingKeyOf(target)}` === clusterKey;
+    }
+    if (!clusterKey.startsWith('surface:')) {
+        return false;
+    }
+    return `surface:${target.surfaceRef?.trim() ?? ''}` === clusterKey;
+}
+function isStandaloneBottomSubmitInputTarget(target) {
+    const inputType = (target.inputType ?? '').trim().toLowerCase();
+    if (inputType !== 'submit') {
+        return false;
+    }
+    if (target.framePath?.length) {
+        return false;
+    }
+    if (target.surfaceRef?.trim() || formGroupingKeyOf(target)) {
+        return false;
+    }
+    return target.context?.layout?.band === 'bottom';
+}
+export function expandCompanionSubmitTargets(targets, selectedTargets) {
+    if (selectedTargets.some(isSubmitLikeTarget)) {
+        return [...selectedTargets];
+    }
+    const primarySelectedTargets = selectedTargets.filter(isPrimaryFormControlTarget);
+    if (primarySelectedTargets.length < 2) {
+        return [...selectedTargets];
+    }
+    const clusterEntries = new Map();
+    for (const target of primarySelectedTargets) {
+        const clusterKey = selectedFormClusterKeyOf(target);
+        if (!clusterKey) {
+            continue;
+        }
+        const existing = clusterEntries.get(clusterKey) ?? {
+            count: 0,
+            priority: clusterKey.startsWith('form:') ? 2 : 1,
+            inFrame: false,
+        };
+        existing.count += 1;
+        existing.inFrame ||= Boolean(target.framePath?.length);
+        clusterEntries.set(clusterKey, existing);
+    }
+    const dominantCluster = [...clusterEntries.entries()]
+        .filter(([, entry]) => entry.count >= 2)
+        .sort((left, right) => {
+        const [, leftEntry] = left;
+        const [, rightEntry] = right;
+        if (leftEntry.count !== rightEntry.count) {
+            return rightEntry.count - leftEntry.count;
+        }
+        if (leftEntry.priority !== rightEntry.priority) {
+            return rightEntry.priority - leftEntry.priority;
+        }
+        return left[0].localeCompare(right[0]);
+    })[0];
+    if (!dominantCluster) {
+        return [...selectedTargets];
+    }
+    const [dominantClusterKey, dominantClusterMeta] = dominantCluster;
+    const expanded = new Map();
+    for (const target of selectedTargets) {
+        expanded.set(observedTargetKey(target), target);
+    }
+    const explicitCompanions = targets.filter((target) => {
+        const key = observedTargetKey(target);
+        if (expanded.has(key) || !isSubmitLikeTarget(target)) {
+            return false;
+        }
+        return matchesSelectedFormCluster(target, dominantClusterKey);
+    });
+    for (const target of explicitCompanions) {
+        expanded.set(observedTargetKey(target), target);
+    }
+    if (explicitCompanions.length === 0 && dominantClusterMeta.inFrame) {
+        const fallbackCandidates = targets.filter((target) => {
+            const key = observedTargetKey(target);
+            return !expanded.has(key) && isStandaloneBottomSubmitInputTarget(target);
+        });
+        if (fallbackCandidates.length === 1) {
+            expanded.set(observedTargetKey(fallbackCandidates[0]), fallbackCandidates[0]);
+        }
+    }
+    return [...expanded.values()].sort((left, right) => {
+        const leftOrdinal = left.ordinal ?? Number.MAX_SAFE_INTEGER;
+        const rightOrdinal = right.ordinal ?? Number.MAX_SAFE_INTEGER;
+        if (leftOrdinal !== rightOrdinal) {
+            return leftOrdinal - rightOrdinal;
+        }
+        return (left.label ?? '').localeCompare(right.label ?? '');
+    });
+}
 function parseNumericAmount(value) {
     const normalized = value.replace(/,/g, '.').trim();
     if (!/^\d+(?:\.\d+)?$/.test(normalized)) {

package/dist/commands/observe.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"observe.d.ts","sourceRoot":"","sources":["../../src/commands/observe.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,eAAe,CAAC;AA2DnD,OAAO,EAAE,qBAAqB,EAAE,MAAM,wBAAwB,CAAC;AAU/D,eAAO,MAAM,yBAAyB;;;;iBAyUghH,CAAC;gBAAwB,CAAC;;;;;CAzU3gH,CAAC;AACtE,eAAO,MAAM,yBAAyB;;;;;;;;;;;;;;;;;;;;;;;;;;;;CAGrC,CAAC;AAEF,wBAAsB,OAAO,CAAC,OAAO,EAAE,aAAa,EAAE,WAAW,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAkUzF"}
+{"version":3,"file":"observe.d.ts","sourceRoot":"","sources":["../../src/commands/observe.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,eAAe,CAAC;AA4DnD,OAAO,EAAE,qBAAqB,EAAE,MAAM,wBAAwB,CAAC;AAU/D,eAAO,MAAM,yBAAyB;;;;iBA4Uk6G,CAAC;gBAAwB,CAAC;;;;;CA5U75G,CAAC;AACtE,eAAO,MAAM,yBAAyB;;;;;;;;;;;;;;;;;;;;;;;;;;;;CAGrC,CAAC;AAEF,wBAAsB,OAAO,CAAC,OAAO,EAAE,aAAa,EAAE,WAAW,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAqUzF"}

package/dist/commands/observe.js

@@ -14,7 +14,7 @@ import { collectPageSignals } from './observe-signals.js';
 import { attachObservedTargetOwners, linkObservedSurfaceGraph, reconcileObservedTargetsForPage, persistObservedSurfacesForPage, toDomDescriptor, } from './observe-persistence.js';
 import { clearProtectedFillableFormsForPage, persistProtectedFillableFormsForPage, } from './observe-protected.js';
 import { classifyObservePageState, shouldSuppressFillableFormsForObserve, } from './observe-page-state.js';
-import { annotateDomTargets, compressSemanticallyDuplicateTargets, expandRerankedFormTargets, orderBySurfaceCompetition, prioritizeGoalActionTargets, } from './observe-semantics.js';
+import { annotateDomTargets, compressSemanticallyDuplicateTargets, expandCompanionSubmitTargets, expandRerankedFormTargets, orderBySurfaceCompetition, prioritizeGoalActionTargets, } from './observe-semantics.js';
 import { buildGroupedObserveScopes, buildExplicitScopeRefs, buildGoalObserveInventoryCandidates, compactFillableForms, compactSignals, projectPersistedTargetsForGoal, selectTargetsForGoalMatches, } from './observe-projection.js';
 import { collectSurfaceDescriptors, selectScopesForOutput } from './observe-surfaces.js';
 import { toStagehandDescriptor } from './observe-stagehand.js';
@@ -126,7 +126,7 @@ export async function observe(session, instruction) {
             if (domTargets.length > 0) {
                 const rerankedCandidates = await rerankDomTargetsForGoal(instruction, buildGoalObserveInventoryCandidates(domTargets, surfaceInputs), { session });
                 const { targets: goalMatchedTargets, selectedSurfaceIds } = selectTargetsForGoalMatches(domTargets, rerankedCandidates);
-                const selectedTargets = prioritizeGoalActionTargets(instruction, expandRerankedFormTargets(domTargets, goalMatchedTargets));
+                const selectedTargets = prioritizeGoalActionTargets(instruction, expandCompanionSubmitTargets(domTargets, expandRerankedFormTargets(domTargets, goalMatchedTargets)));
                 const persisted = persistObservedSurfacesForPage(session, pageRef, domTargets, {
                     allSurfaceInputs: surfaceInputs,
                     explicitSurfaceIds: selectedSurfaceIds,

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@nuanu-ai/agentbrowse",
-  "version": "0.2.41",
+  "version": "0.2.42",
   "type": "module",
   "description": "Browser automation CLI for AI agents: control a CDP browser, observe UI surfaces, act on refs, extract data, capture screenshots, complete protected fills, and solve captchas",
   "keywords": [