@enricai/barnacle 1.12.0 → 1.12.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/scraper/deep-locator-fake.d.ts.map +1 -1
- package/dist/scraper/deep-locator-fake.js +7 -0
- package/dist/scraper/deep-locator-fake.js.map +1 -1
- package/dist/scraper/flow-runner.d.ts +51 -0
- package/dist/scraper/flow-runner.d.ts.map +1 -1
- package/dist/scraper/flow-runner.js +748 -25
- package/dist/scraper/flow-runner.js.map +1 -1
- package/dist/scraper/frame-target.d.ts.map +1 -1
- package/dist/scraper/frame-target.js +124 -22
- package/dist/scraper/frame-target.js.map +1 -1
- package/dist/scripts/recon-browser.d.ts +15 -0
- package/dist/scripts/recon-browser.d.ts.map +1 -1
- package/dist/scripts/recon-browser.js +46 -0
- package/dist/scripts/recon-browser.js.map +1 -1
- package/dist/scripts/recon-generate.d.ts.map +1 -1
- package/dist/scripts/recon-generate.js +104 -29
- package/dist/scripts/recon-generate.js.map +1 -1
- package/package.json +3 -2
|
@@ -66,11 +66,13 @@ exports.parseSelectStep = parseSelectStep;
|
|
|
66
66
|
exports.parseFillStep = parseFillStep;
|
|
67
67
|
exports.parseFillValueIntent = parseFillValueIntent;
|
|
68
68
|
exports.parseRadioStep = parseRadioStep;
|
|
69
|
+
exports.parseCheckStep = parseCheckStep;
|
|
69
70
|
exports.pollEnumerate = pollEnumerate;
|
|
70
71
|
exports.waitForTransitionBody = waitForTransitionBody;
|
|
71
72
|
exports.chooseRequiredSelectOption = chooseRequiredSelectOption;
|
|
72
73
|
exports.buildRadioIdXPath = buildRadioIdXPath;
|
|
73
74
|
exports.selectRadioGroupOption = selectRadioGroupOption;
|
|
75
|
+
exports.hasUnfilledRequiredControlForStep = hasUnfilledRequiredControlForStep;
|
|
74
76
|
exports.simulateDragDropUpload = simulateDragDropUpload;
|
|
75
77
|
exports.dispatchJqueryChangeEvent = dispatchJqueryChangeEvent;
|
|
76
78
|
exports.verifyDomEffect = verifyDomEffect;
|
|
@@ -375,6 +377,151 @@ const RADIO_SETTLE_MS = 400;
|
|
|
375
377
|
* value that a later worklet re-render will wipe.
|
|
376
378
|
*/
|
|
377
379
|
const SELECT_SETTLE_MS = 400;
|
|
380
|
+
/**
|
|
381
|
+
* Same async-revalidation tick as {@link RADIO_SETTLE_MS}/{@link SELECT_SETTLE_MS},
|
|
382
|
+
* for the popup-dropdown widget family {@link tryPromptSelectorPrimitive} handles:
|
|
383
|
+
* after opening the popup, after typing into an in-popup filter, and after
|
|
384
|
+
* clicking an option, the DOM (option-list render, selection-label text,
|
|
385
|
+
* `aria-invalid` marker) settles a beat later than the trusted event that
|
|
386
|
+
* triggered it.
|
|
387
|
+
*/
|
|
388
|
+
const PROMPT_SELECTOR_SETTLE_MS = 400;
|
|
389
|
+
/**
|
|
390
|
+
* Temporary attribute {@link tryPromptSelectorPrimitive} stamps onto each
|
|
391
|
+
* candidate widget during its read-only enumerate pass, so the Node-side click
|
|
392
|
+
* (a real Playwright gesture — these widgets ignore a bare `el.click()`) can
|
|
393
|
+
* address the exact widget chosen by index without re-deriving a CSS/xpath
|
|
394
|
+
* selector for an element with no stable id.
|
|
395
|
+
*/
|
|
396
|
+
const PROMPT_WIDGET_MARK_ATTR = "data-bcl-prompt-idx";
|
|
397
|
+
/** Same role as {@link PROMPT_WIDGET_MARK_ATTR}, but for the popup's rendered option entries. */
|
|
398
|
+
const PROMPT_OPTION_MARK_ATTR = "data-bcl-prompt-opt-idx";
|
|
399
|
+
/**
|
|
400
|
+
* Cross-vendor selector union that identifies the TRIGGER of a native-control-less
|
|
401
|
+
* popup-dropdown widget (a combobox that opens a listbox popup and renders no
|
|
402
|
+
* `<select>`/`<input>` a focused `<select>`/`<input>` probe can see). Ordered as
|
|
403
|
+
* a UNION, standards first, with well-known widget-library markers as co-equal
|
|
404
|
+
* members — the same multi-vendor-union discipline as
|
|
405
|
+
* {@link INVALID_MARKER_CLASS_SOURCE} (`ng-invalid|Mui-error|…`). No member is a
|
|
406
|
+
* per-site branch; each is one recognizable spelling of the same widget shape.
|
|
407
|
+
* Growth is a one-line edit here — never a new hardcoded selector scattered into
|
|
408
|
+
* logic (the site-agnostic guard trips on the latter, allows this curated list).
|
|
409
|
+
*
|
|
410
|
+
* Members: ARIA combobox / listbox-owning triggers (`role=combobox`,
|
|
411
|
+
* `aria-haspopup=listbox`, `aria-controls`/`aria-owns`→listbox); and the
|
|
412
|
+
* `data-uxi-widget-type` / `data-automation-id` prompt family emitted by the
|
|
413
|
+
* Canvas/UXI widget kit some ATS front-ends build on (verified on a real
|
|
414
|
+
* capture: the trigger is an `<input data-uxi-widget-type="selectinput"
|
|
415
|
+
* aria-required aria-invalid>` beside a `data-automation-id="promptIcon"` icon,
|
|
416
|
+
* with NO `role=combobox` — hence ARIA alone under-selects it).
|
|
417
|
+
*/
|
|
418
|
+
const PROMPT_TRIGGER_SELECTORS = [
|
|
419
|
+
"[role='combobox'][aria-haspopup='listbox']",
|
|
420
|
+
"[role='combobox'][aria-controls]",
|
|
421
|
+
"[role='combobox'][aria-owns]",
|
|
422
|
+
"[aria-haspopup='listbox']",
|
|
423
|
+
"[data-uxi-widget-type='selectinput']",
|
|
424
|
+
"[data-uxi-widget-type='multiselect']",
|
|
425
|
+
"[data-automation-id='promptIcon']",
|
|
426
|
+
"[data-automation-id='multiSelectContainer']",
|
|
427
|
+
"[data-automation-id='promptSelectionLabel']",
|
|
428
|
+
].join(",");
|
|
429
|
+
/**
|
|
430
|
+
* Cross-vendor selector union for an OPTION rendered inside the opened popup —
|
|
431
|
+
* standards first (`role=option`), then the widget-kit option markers. Sibling
|
|
432
|
+
* of {@link PROMPT_TRIGGER_SELECTORS}; same union discipline and guard treatment.
|
|
433
|
+
*/
|
|
434
|
+
const PROMPT_OPTION_SELECTORS = [
|
|
435
|
+
"[role='option']",
|
|
436
|
+
"[data-automation-id='promptOption']",
|
|
437
|
+
"[data-uxi-widget-type='selectinputlistitem']",
|
|
438
|
+
].join(",");
|
|
439
|
+
/**
|
|
440
|
+
* Cross-vendor selector union for an in-popup filter/typeahead input — standards
|
|
441
|
+
* first, then the widget-kit's bare filter `<input>` (which carries no
|
|
442
|
+
* `role=searchbox`, only `data-uxi-widget-type='selectinput'` on the input
|
|
443
|
+
* itself). Sibling of {@link PROMPT_TRIGGER_SELECTORS}; same guard treatment.
|
|
444
|
+
*/
|
|
445
|
+
const PROMPT_SEARCH_SELECTORS = [
|
|
446
|
+
"[role='searchbox']",
|
|
447
|
+
"[role='combobox'] input[type='text']",
|
|
448
|
+
"[data-automation-id='searchBox'] input",
|
|
449
|
+
"[data-automation-id='searchBoxInput']",
|
|
450
|
+
"input[data-uxi-widget-type='selectinput']",
|
|
451
|
+
].join(",");
|
|
452
|
+
/**
|
|
453
|
+
* Cross-vendor selector union for the node that carries a widget's CURRENT
|
|
454
|
+
* (committed) value text — standards-poor here (no `aria-activedescendant` on
|
|
455
|
+
* the verified widget), so the widget-kit's selection-label node is a co-equal
|
|
456
|
+
* union member. Sibling of {@link PROMPT_TRIGGER_SELECTORS}.
|
|
457
|
+
*/
|
|
458
|
+
const PROMPT_VALUE_SELECTORS = [
|
|
459
|
+
"[data-automation-id='promptSelectionLabel']",
|
|
460
|
+
"[aria-live='polite'][data-automation-id='promptAriaInstruction']",
|
|
461
|
+
].join(",");
|
|
462
|
+
/**
|
|
463
|
+
* The widget-kit's empty-state value text ("0 items selected") — a filled
|
|
464
|
+
* widget's value node reads the chosen option instead. Matched
|
|
465
|
+
* case-insensitively as a whole phrase so a non-empty selection ("1 item
|
|
466
|
+
* selected, …") is correctly read as filled.
|
|
467
|
+
*/
|
|
468
|
+
const PROMPT_EMPTY_VALUE_RX = /^\s*0\s+items?\s+selected\s*$/i;
|
|
469
|
+
/**
|
|
470
|
+
* {@link PROMPT_EMPTY_VALUE_RX}'s source and flags as plain strings, passed into
|
|
471
|
+
* the browser-evaluated expressions to reconstruct the RegExp there. Kept as
|
|
472
|
+
* separate literals (not a `.toString()` round-trip) so the reconstruction can't
|
|
473
|
+
* be corrupted if the pattern ever gains a `/`.
|
|
474
|
+
*/
|
|
475
|
+
const PROMPT_EMPTY_VALUE_RX_SRC = PROMPT_EMPTY_VALUE_RX.source;
|
|
476
|
+
const PROMPT_EMPTY_VALUE_RX_FLAGS = PROMPT_EMPTY_VALUE_RX.flags;
|
|
477
|
+
/**
|
|
478
|
+
* Browser-side expression reading a `<button>`-trigger's own committed-value
|
|
479
|
+
* text. A button's label may be a direct text node (`<button>Mobile</button>`)
|
|
480
|
+
* or a child element (`<button><span>Mobile</span></button>`), so a recursive
|
|
481
|
+
* read is needed — but it must exclude (a) a popup rendered INSIDE the trigger
|
|
482
|
+
* (`[role='option']`/`[role='listbox']`, which some libraries nest) and (b)
|
|
483
|
+
* decorative descendants (`[aria-hidden='true']`, a required-marker `<abbr>`, an
|
|
484
|
+
* icon `<svg>`) whose text would otherwise make an EMPTY placeholder button read
|
|
485
|
+
* as filled and get skipped as a candidate. Clone, strip those subtrees, read
|
|
486
|
+
* the remaining `textContent`.
|
|
487
|
+
*/
|
|
488
|
+
const BUTTON_VALUE_EXPR = "((el) => { const c = el.cloneNode(true); for (const n of c.querySelectorAll(\"[role='option'],[role='listbox'],[aria-hidden='true'],abbr,svg\")) n.remove(); return c.textContent || \"\"; })";
|
|
489
|
+
/**
|
|
490
|
+
* Browser-side expression resolving the DOM root within which a chosen widget's
|
|
491
|
+
* popup options / filter input live. Popup placement is vendor-split: the ARIA
|
|
492
|
+
* standard and most libraries (MUI/Radix/react-select) render the listbox in a
|
|
493
|
+
* PORTAL at `document.body` linked by `aria-controls`/`aria-owns`, while others
|
|
494
|
+
* (the Canvas/UXI kit) render it INLINE inside the widget. So resolve in order:
|
|
495
|
+
* (1) the `aria-controls`/`aria-owns` target of the widget or its trigger
|
|
496
|
+
* descendant (portal-safe); (2) the widget subtree itself if it already
|
|
497
|
+
* contains the popup (inline); (3) `document` as a last resort. Returns the
|
|
498
|
+
* scope Element/Document; callers query options/search within it, which stops a
|
|
499
|
+
* sibling widget's options from being picked on a multi-widget page.
|
|
500
|
+
*/
|
|
501
|
+
const PROMPT_SCOPE_ROOT_EXPR = `((w) => {
|
|
502
|
+
const refAttr = (el) => (el && (el.getAttribute("aria-controls") || el.getAttribute("aria-owns"))) || "";
|
|
503
|
+
let ids = refAttr(w);
|
|
504
|
+
if (!ids) {
|
|
505
|
+
const inner = w.querySelector("[aria-controls],[aria-owns]");
|
|
506
|
+
if (inner) ids = refAttr(inner);
|
|
507
|
+
}
|
|
508
|
+
if (ids) {
|
|
509
|
+
// aria-controls/aria-owns may reference MULTIPLE ids (e.g. a listbox plus a
|
|
510
|
+
// status region). Prefer the referenced element that IS or CONTAINS a
|
|
511
|
+
// listbox/option (the portaled popup).
|
|
512
|
+
const refs = ids.split(/\\s+/).map((id) => document.getElementById(id)).filter(Boolean);
|
|
513
|
+
const withListbox = refs.find((el) => el.matches("[role='listbox']") || el.querySelector("[role='listbox'],[role='option']"));
|
|
514
|
+
if (withListbox) return withListbox;
|
|
515
|
+
}
|
|
516
|
+
// No referenced listbox: try the inline popup in the widget's own subtree
|
|
517
|
+
// BEFORE falling back to a resolved-but-listbox-less ref, then to document.
|
|
518
|
+
if (w.querySelector("[role='listbox'],[role='option']")) return w;
|
|
519
|
+
if (ids) {
|
|
520
|
+
const refs = ids.split(/\\s+/).map((id) => document.getElementById(id)).filter(Boolean);
|
|
521
|
+
if (refs.length) return refs[0];
|
|
522
|
+
}
|
|
523
|
+
return document;
|
|
524
|
+
})`;
|
|
378
525
|
/**
|
|
379
526
|
* Extra bounded poll window for a network-only advance whose real
|
|
380
527
|
* `TransitionWorklet(type="next")` POST lands AFTER the `STEP_PAUSE_MS`
|
|
@@ -1034,11 +1181,26 @@ function isDomOnlyAdvanceVerified(params) {
|
|
|
1034
1181
|
* `describeAttemptEffectSignals`'s own reflow-vs-reveal boundary). Requiring
|
|
1035
1182
|
* `textChanged` keeps a trivial reflow/tooltip (DOM churn with no new visible
|
|
1036
1183
|
* content) from being credited — those still cascade to failure as before.
|
|
1184
|
+
*
|
|
1185
|
+
* **Blocked-submit veto:** a form-submit/advance click that a wizard-style
|
|
1186
|
+
* client-side validation rejects renders inline error text — exactly the
|
|
1187
|
+
* `textChanged` + DOM-growth shape the reveal-credit branch above rewards —
|
|
1188
|
+
* with zero network, zero URL change. The reported false positive
|
|
1189
|
+
* (`recon-viewswap-false-positive-on-blocked-form-submit.md`): a "Create
|
|
1190
|
+
* Account" click with two empty required fields was scored
|
|
1191
|
+
* `verifiedBy=view-swap` on `network=false url=false dom=false`, and recon
|
|
1192
|
+
* advanced past a step the wizard never left. When `invalidMarkerDelta`
|
|
1193
|
+
* (post-click minus pre-click ng-invalid-style container count, from
|
|
1194
|
+
* {@link countNgInvalidContainers}) is positive, the click revealed NEW
|
|
1195
|
+
* validation errors rather than a legitimate view transition — veto the
|
|
1196
|
+
* credit regardless of which growth branch would otherwise fire. Optional
|
|
1197
|
+
* and defaults to 0 (no veto) so existing callers that don't thread the
|
|
1198
|
+
* marker count are unaffected.
|
|
1037
1199
|
*/
|
|
1038
1200
|
function isClickViewSwapVerified(params) {
|
|
1039
1201
|
const VIEW_SWAP_MIN_BYTES = config_1.config.scraper.viewSwapMinBytesThreshold;
|
|
1040
1202
|
const VIEW_SWAP_REVEAL_MIN_BYTES = config_1.config.scraper.viewSwapRevealMinBytesThreshold;
|
|
1041
|
-
const { resolvedAction, isFinalStep, submitStep, isAdvanceWithPattern, networkDelta, bytesDelta, textChanged, } = params;
|
|
1203
|
+
const { resolvedAction, isFinalStep, submitStep, isAdvanceWithPattern, networkDelta, bytesDelta, textChanged, invalidMarkerDelta = 0, } = params;
|
|
1042
1204
|
if (resolvedAction?.method !== "click")
|
|
1043
1205
|
return false;
|
|
1044
1206
|
if (isFinalStep || submitStep)
|
|
@@ -1047,6 +1209,8 @@ function isClickViewSwapVerified(params) {
|
|
|
1047
1209
|
return false;
|
|
1048
1210
|
if (networkDelta !== 0)
|
|
1049
1211
|
return false;
|
|
1212
|
+
if (invalidMarkerDelta > 0)
|
|
1213
|
+
return false;
|
|
1050
1214
|
if (bytesDelta >= VIEW_SWAP_MIN_BYTES)
|
|
1051
1215
|
return true;
|
|
1052
1216
|
return textChanged && bytesDelta >= VIEW_SWAP_REVEAL_MIN_BYTES;
|
|
@@ -2817,6 +2981,21 @@ async function readElementSelectionFingerprint(target, selector) {
|
|
|
2817
2981
|
* nesting without over-reaching into an outer listbox/group.
|
|
2818
2982
|
*/
|
|
2819
2983
|
const MAX_SELECTION_ANCESTOR_DEPTH = 6;
|
|
2984
|
+
/**
|
|
2985
|
+
* Cross-vendor selector union for a selection-state widget that carries NO
|
|
2986
|
+
* standard selection `role` or `aria-*`/`data-state` marker — a component-kit
|
|
2987
|
+
* container whose selected-ness lives only in the library's own private
|
|
2988
|
+
* attribute. Same multi-vendor-union discipline as {@link INVALID_MARKER_CLASS_SOURCE}
|
|
2989
|
+
* and the `PROMPT_*_SELECTORS` unions: standards are checked FIRST (see
|
|
2990
|
+
* `hasMarker` in {@link selectionAncestorChanged}); this union is the fallback
|
|
2991
|
+
* for widgets that under-annotate ARIA, and no member is a per-site branch —
|
|
2992
|
+
* each is one component library's signature. Grows by a one-line edit.
|
|
2993
|
+
*
|
|
2994
|
+
* Members: `data-baseweb` (Uber Base Web — verified in a real capture to mark
|
|
2995
|
+
* 150 selection elements that expose no role/aria-state, so dropping it loses
|
|
2996
|
+
* real coverage). Add other under-annotating kits here as they surface.
|
|
2997
|
+
*/
|
|
2998
|
+
const WIDGET_KIT_SELECTION_MARKER_SELECTORS = ["[data-baseweb]"].join(",");
|
|
2820
2999
|
/**
|
|
2821
3000
|
* Element-scoped selection read-back for the case the clicked node's OWN
|
|
2822
3001
|
* fingerprint can't credit: a design-system option that wraps its label in a
|
|
@@ -2829,7 +3008,8 @@ const MAX_SELECTION_ANCESTOR_DEPTH = 6;
|
|
|
2829
3008
|
*
|
|
2830
3009
|
* Walks from the leaf up to {@link MAX_SELECTION_ANCESTOR_DEPTH}, and on the
|
|
2831
3010
|
* NEAREST ancestor that (a) carries a selection marker (a non-empty fingerprint
|
|
2832
|
-
* field, a selection `role`, or a
|
|
3011
|
+
* field, a selection `role`, or a component-kit marker from
|
|
3012
|
+
* {@link WIDGET_KIT_SELECTION_MARKER_SELECTORS} — `aria-expanded` is
|
|
2833
3013
|
* deliberately NOT a marker so a bare disclosure/expander is skipped) AND (b) is
|
|
2834
3014
|
* present in the pre-baseline map, diffs that ancestor's current fingerprint
|
|
2835
3015
|
* against its baseline. Nearest-wins and returns even when unchanged, so a
|
|
@@ -2848,10 +3028,13 @@ async function selectionAncestorChanged(target, leafXpath, preSelectionState) {
|
|
|
2848
3028
|
const xpathOf = ${XPATH_OF_FN_SRC};
|
|
2849
3029
|
const fp = (el) => { const ds = el.getAttribute("data-state") || ""; return ${selectionFingerprintObjSrc("el", "ds")}; };
|
|
2850
3030
|
const SELECTION_ROLES = new Set(["option", "tab", "switch", "radio", "checkbox", "menuitemcheckbox"]);
|
|
3031
|
+
const KIT_MARKER_SEL = ${JSON.stringify(WIDGET_KIT_SELECTION_MARKER_SELECTORS)};
|
|
2851
3032
|
const hasMarker = (el, f) => {
|
|
3033
|
+
// Standards first: fingerprint fields / aria-states, then a selection role.
|
|
2852
3034
|
if (f.kind || f.ariaPressed || f.ariaChecked || f.ariaSelected || f.dataState || f.dataSelected || f.dataChecked || f.checked) return true;
|
|
2853
3035
|
if (SELECTION_ROLES.has((el.getAttribute("role") || "").toLowerCase())) return true;
|
|
2854
|
-
|
|
3036
|
+
// Fallback: a component-kit selection widget that exposes no standard marker.
|
|
3037
|
+
if (el.matches(KIT_MARKER_SEL)) return true;
|
|
2855
3038
|
return false;
|
|
2856
3039
|
};
|
|
2857
3040
|
const changed = (a, b) =>
|
|
@@ -3489,6 +3672,49 @@ function parseRadioStep(instruction) {
|
|
|
3489
3672
|
const questionLabel = quoted.find((q) => q.trim() !== option)?.trim() ?? null;
|
|
3490
3673
|
return { option, questionLabel };
|
|
3491
3674
|
}
|
|
3675
|
+
/**
|
|
3676
|
+
* Parse a CHECKBOX flow step into the label of the checkbox it targets.
|
|
3677
|
+
*
|
|
3678
|
+
* Why this exists (sibling of `parseSelectStep`/`parseFillStep`): checkbox
|
|
3679
|
+
* steps are phrased as "Check the '…' checkbox" or "Click the '…' checkbox"
|
|
3680
|
+
* (see `src/recon/fixtures/shipped-ats-flow-steps.json`), neither of which
|
|
3681
|
+
* `parseSelectStep` recognizes (no "select" verb) and neither of which
|
|
3682
|
+
* `parseRadioStep` recognizes (no "answer"/"radio" noun) — so a checkbox
|
|
3683
|
+
* step's label was previously unextractable by any parser.
|
|
3684
|
+
*
|
|
3685
|
+
* Recognizes: "Check the 'Nights' workshift checkbox", "Click the 'No'
|
|
3686
|
+
* checkbox for 'Please indicate if you are Hispanic or Latino'". Returns
|
|
3687
|
+
* null when the step isn't checkbox-shaped or has no quoted label.
|
|
3688
|
+
*/
|
|
3689
|
+
function parseCheckStep(instruction) {
|
|
3690
|
+
const lower = instruction.toLowerCase();
|
|
3691
|
+
if (/\bany\s+remaining\b/.test(lower))
|
|
3692
|
+
return null;
|
|
3693
|
+
const match = instruction.match(/\b(?:check|click)\s+(?:the\s+)?'([^']+)'.*?\bcheckbox\b/i);
|
|
3694
|
+
const label = match?.[1]?.trim();
|
|
3695
|
+
return label ? { label } : null;
|
|
3696
|
+
}
|
|
3697
|
+
/** Strips wrapping single quotes a parser's capture group may include (e.g. when a flow author quotes the field name itself), so a probe label compares plain text against plain text. */
|
|
3698
|
+
function stripQuotedLabel(label) {
|
|
3699
|
+
return label.replace(/^'+|'+$/g, "").trim();
|
|
3700
|
+
}
|
|
3701
|
+
/**
|
|
3702
|
+
* Extracts the DOM label {@link hasUnfilledRequiredControlForStep}'s probe
|
|
3703
|
+
* should match against, widened beyond `parseSelectStep`'s literal "select"
|
|
3704
|
+
* verb (which never matches fill or checkbox phrasing) to also try
|
|
3705
|
+
* `parseFillStep` and `parseCheckStep` — see that function's docblock for
|
|
3706
|
+
* why generalizing the label source, not just the select-step case, matters.
|
|
3707
|
+
*/
|
|
3708
|
+
function extractRequiredControlProbeLabel(instruction) {
|
|
3709
|
+
const selectLabel = parseSelectStep(instruction)?.questionLabel;
|
|
3710
|
+
if (selectLabel)
|
|
3711
|
+
return selectLabel;
|
|
3712
|
+
const fillLabel = parseFillStep(instruction)?.fieldLabel;
|
|
3713
|
+
if (fillLabel)
|
|
3714
|
+
return stripQuotedLabel(fillLabel);
|
|
3715
|
+
const checkLabel = parseCheckStep(instruction)?.label;
|
|
3716
|
+
return checkLabel ? stripQuotedLabel(checkLabel) : null;
|
|
3717
|
+
}
|
|
3492
3718
|
function resolveDeepLocatorActuation(step) {
|
|
3493
3719
|
const selectParsed = parseSelectStep(step);
|
|
3494
3720
|
if (selectParsed)
|
|
@@ -4591,6 +4817,427 @@ async function applyRadioSelection(target, gi, ri, hint) {
|
|
|
4591
4817
|
await target.evaluate(applyExpr).catch(() => ({ ok: false }));
|
|
4592
4818
|
return await readback();
|
|
4593
4819
|
}
|
|
4820
|
+
/**
|
|
4821
|
+
* Answer a native-control-less popup-dropdown widget (a combobox that opens a
|
|
4822
|
+
* listbox popup and renders NO `<select>`/`<input>` a focused `<select>`/
|
|
4823
|
+
* `<input>` probe can see). Detected by a cross-vendor UNION of shape signals
|
|
4824
|
+
* ({@link PROMPT_TRIGGER_SELECTORS}/{@link PROMPT_OPTION_SELECTORS}), not any
|
|
4825
|
+
* one vendor's private attribute — standards (ARIA `combobox`/`listbox`/
|
|
4826
|
+
* `option`) where the widget exposes them, plus the well-known Canvas/UXI
|
|
4827
|
+
* widget-kit markers as co-equal union members for the widgets that under-
|
|
4828
|
+
* annotate ARIA (some emit `role=option` only once the popup is open, and no
|
|
4829
|
+
* `role=combobox` at all).
|
|
4830
|
+
*
|
|
4831
|
+
* Why this exists (parallels `trySelectPrimitive`/`tryRadioPrimitive`): such a
|
|
4832
|
+
* widget renders as neither a native `<select>` nor a control the observe
|
|
4833
|
+
* cascade resolves to a click target, so the focused probe reports "0
|
|
4834
|
+
* candidates" and the step strands. This primitive finds the widget by the
|
|
4835
|
+
* trigger union, opens the popup with a real Playwright click gesture (a bare
|
|
4836
|
+
* `el.click()` does not fire these widgets' open handler — the same failure
|
|
4837
|
+
* mode as the MUI radio/select widgets), types into an in-popup filter input
|
|
4838
|
+
* when one is present (the searchable/typeahead variant renders only a partial
|
|
4839
|
+
* option slice until filtered), and clicks the matching option with a real
|
|
4840
|
+
* click.
|
|
4841
|
+
*
|
|
4842
|
+
* Accepts a SELECT-shaped step (`parseSelectStep`, "select 'X' in the Y
|
|
4843
|
+
* dropdown"), a FILL-shaped step (`parseFillStep`, "Fill in the Y field with
|
|
4844
|
+
* 'X'"), or an ANSWER/RADIO-shaped step (`parseRadioStep`, "click the 'Yes'
|
|
4845
|
+
* answer for the question '…'") — flow/replan generation describes this
|
|
4846
|
+
* widget family's searchable variant as a fill because its filter box renders
|
|
4847
|
+
* a real `<input>`, and describes its Yes/No variant with the same
|
|
4848
|
+
* answer-verb phrasing used for native radio groups, even though committing
|
|
4849
|
+
* either still requires the popup-open/option-click gesture below, not typed
|
|
4850
|
+
* text or a bare click. `tryRadioPrimitive` (called first, see the call site)
|
|
4851
|
+
* already claims answer-verb steps whose target has native
|
|
4852
|
+
* `input[type=radio]` elements; this primitive only ever sees an answer-verb
|
|
4853
|
+
* step after that primitive has fallen through for lack of any, so there is
|
|
4854
|
+
* no double-claim. A fill or radio step's `value`/`option` becomes the option
|
|
4855
|
+
* to match and its `fieldLabel`/`questionLabel` becomes the question label;
|
|
4856
|
+
* downstream matching is identical across all three shapes.
|
|
4857
|
+
*
|
|
4858
|
+
* Matches the target widget by `questionLabel` (when the step carries one) or,
|
|
4859
|
+
* failing that, by there being exactly one unfilled widget on the page —
|
|
4860
|
+
* deliberately conservative: an ambiguous multi-widget page with no question
|
|
4861
|
+
* label falls through to the cascade rather than guessing which widget the
|
|
4862
|
+
* step means. Option matching mirrors `trySelectPrimitive`: an exact
|
|
4863
|
+
* (normalized) text match is applied directly; otherwise, when an LLM client
|
|
4864
|
+
* is present, `judgeSelectOptionWithLLM` picks the best rendered option
|
|
4865
|
+
* (per-requisition option variance).
|
|
4866
|
+
*
|
|
4867
|
+
* Returns the resolved widget's DOM id (or `""`) on success — the step's
|
|
4868
|
+
* `targetId` — or `null` when unhandled (no widget, no unambiguous widget
|
|
4869
|
+
* match, no option match, or the selection didn't commit), so the caller
|
|
4870
|
+
* falls through to the cascade unchanged, matching
|
|
4871
|
+
* `trySelectPrimitive`/`tryRadioPrimitive`'s null-fallthrough contract.
|
|
4872
|
+
*/
|
|
4873
|
+
async function tryPromptSelectorPrimitive(params) {
|
|
4874
|
+
const { page, target, instruction, logger, anthropic, captureFn } = params;
|
|
4875
|
+
// A prompt-selector widget's search box renders a real <input>, so flow/
|
|
4876
|
+
// replan generation routinely describes filling it as a FILL step ("Fill in
|
|
4877
|
+
// the 'How Did You Hear About Us?' field with 'Internet/Online'") rather
|
|
4878
|
+
// than a SELECT step, and its Yes/No variant renders no native radio inputs
|
|
4879
|
+
// at all, so flow/replan generation describes it with the same answer-verb
|
|
4880
|
+
// phrasing used for native radios ("Click the 'Yes' answer for the question
|
|
4881
|
+
// '…'"). Accept all three shapes: parseSelectStep first, then parseFillStep
|
|
4882
|
+
// (fieldLabel -> questionLabel, value -> option), then parseRadioStep
|
|
4883
|
+
// (option/questionLabel already in this primitive's shape) so the widget-
|
|
4884
|
+
// matching/open/readback phases below run unchanged regardless of which
|
|
4885
|
+
// verb the instruction used.
|
|
4886
|
+
const parsedSelect = parseSelectStep(instruction);
|
|
4887
|
+
const parsedFill = parsedSelect ? null : parseFillStep(instruction);
|
|
4888
|
+
const parsedAnswer = parsedSelect || parsedFill ? null : parseRadioStep(instruction);
|
|
4889
|
+
const parsed = parsedSelect
|
|
4890
|
+
? parsedSelect
|
|
4891
|
+
: parsedFill
|
|
4892
|
+
? { option: parsedFill.value, questionLabel: stripQuotedLabel(parsedFill.fieldLabel) }
|
|
4893
|
+
: parsedAnswer;
|
|
4894
|
+
if (!parsed)
|
|
4895
|
+
return null;
|
|
4896
|
+
const { option, questionLabel } = parsed;
|
|
4897
|
+
const optLabel = `option "${option.slice(0, 40)}"${questionLabel ? `, question "${questionLabel.slice(0, 40)}"` : ""}`;
|
|
4898
|
+
const norm = (s) => s.replace(/\s+/g, " ").trim().toLowerCase();
|
|
4899
|
+
// Phase 1 (browser, read-only except for the marker attribute stamped for
|
|
4900
|
+
// Phase 2's click addressing): find candidate widgets — popup-dropdown
|
|
4901
|
+
// triggers with no native <select> — that are still unfilled/invalid. Options
|
|
4902
|
+
// are NOT enumerated here; these widgets only render option entries once the
|
|
4903
|
+
// popup is open.
|
|
4904
|
+
const enumerateWidgetsExpr = `((markAttr, triggerSel, valueSel, emptyRxSrc, emptyRxFlags) => {
|
|
4905
|
+
const norm = (s) => (s || "").replace(/\\s+/g, " ").trim().toLowerCase();
|
|
4906
|
+
const buttonValue = ${BUTTON_VALUE_EXPR};
|
|
4907
|
+
const emptyRx = new RegExp(emptyRxSrc, emptyRxFlags);
|
|
4908
|
+
const triggers = Array.from(document.querySelectorAll(triggerSel));
|
|
4909
|
+
if (triggers.length === 0) return { widgetPresent: false };
|
|
4910
|
+
const seen = new Set();
|
|
4911
|
+
const resolved = [];
|
|
4912
|
+
for (const el of triggers) {
|
|
4913
|
+
// Resolve each union hit to ONE widget container. A single widget often
|
|
4914
|
+
// matches the union more than once (e.g. an outer container AND its inner
|
|
4915
|
+
// filter input both carry data-uxi-widget-type), so prefer a real
|
|
4916
|
+
// interactive ancestor, else the OUTERMOST widget-kit container on the
|
|
4917
|
+
// ancestor chain (so the inner input and its container collapse to the
|
|
4918
|
+
// same node), else the element itself.
|
|
4919
|
+
const interactive = el.closest("button,[role='button'],[role='combobox']");
|
|
4920
|
+
let container = interactive || el;
|
|
4921
|
+
if (!interactive) {
|
|
4922
|
+
const kitSel = "[data-uxi-widget-type='selectinput'],[data-uxi-widget-type='multiselect'],[data-automation-id='multiSelectContainer']";
|
|
4923
|
+
let node = el.closest(kitSel);
|
|
4924
|
+
while (node) {
|
|
4925
|
+
container = node;
|
|
4926
|
+
const up = node.parentElement && node.parentElement.closest(kitSel);
|
|
4927
|
+
if (!up || up === node) break;
|
|
4928
|
+
node = up;
|
|
4929
|
+
}
|
|
4930
|
+
}
|
|
4931
|
+
if (seen.has(container)) continue;
|
|
4932
|
+
seen.add(container);
|
|
4933
|
+
resolved.push(container);
|
|
4934
|
+
}
|
|
4935
|
+
// Drop any candidate contained by another (belt-and-suspenders against a
|
|
4936
|
+
// widget whose union hits resolve to nested containers).
|
|
4937
|
+
const widgets = resolved.filter((w) => !resolved.some((o) => o !== w && o.contains(w)));
|
|
4938
|
+
if (widgets.length === 0) return { widgetPresent: false };
|
|
4939
|
+
const isInvalid = ${INVALID_MARKER_EL_EXPR};
|
|
4940
|
+
const widgetLabel = (w) => {
|
|
4941
|
+
// Standard first: aria-labelledby, aria-label, then a <label for=id>
|
|
4942
|
+
// referencing the widget or a control inside it, then a labelled group
|
|
4943
|
+
// ancestor (role=group[aria-labelledby] / fieldset<legend>), then the
|
|
4944
|
+
// nearest non-empty ancestor text as a last resort.
|
|
4945
|
+
const alb = w.getAttribute("aria-labelledby");
|
|
4946
|
+
if (alb) {
|
|
4947
|
+
const parts = [];
|
|
4948
|
+
for (const id of alb.split(/\\s+/)) { const el = document.getElementById(id); if (el) parts.push(el.textContent); }
|
|
4949
|
+
if (parts.length) return parts.join(" ").replace(/\\s+/g, " ").trim().slice(0, 120);
|
|
4950
|
+
}
|
|
4951
|
+
const al = w.getAttribute("aria-label");
|
|
4952
|
+
if (al) return al.replace(/\\s+/g, " ").trim().slice(0, 120);
|
|
4953
|
+
const labelledIds = [w.id, ...Array.from(w.querySelectorAll("[id]")).map((e) => e.id)].filter(Boolean);
|
|
4954
|
+
for (const id of labelledIds) {
|
|
4955
|
+
const lab = document.querySelector("label[for='" + (window.CSS && CSS.escape ? CSS.escape(id) : id) + "']");
|
|
4956
|
+
if (lab && lab.textContent.trim()) return lab.textContent.replace(/\\s+/g, " ").trim().slice(0, 120);
|
|
4957
|
+
}
|
|
4958
|
+
const grp = w.closest("[role='group'][aria-labelledby],fieldset");
|
|
4959
|
+
if (grp) {
|
|
4960
|
+
const gid = grp.getAttribute("aria-labelledby");
|
|
4961
|
+
const gref = gid ? document.getElementById(gid.split(/\\s+/)[0]) : grp.querySelector("legend");
|
|
4962
|
+
if (gref && gref.textContent.trim()) return gref.textContent.replace(/\\s+/g, " ").trim().slice(0, 120);
|
|
4963
|
+
}
|
|
4964
|
+
let node = w.parentElement;
|
|
4965
|
+
for (let d = 0; d < 5 && node; d++) {
|
|
4966
|
+
const t = (node.textContent || "").trim();
|
|
4967
|
+
if (t) return t.replace(/\\s+/g, " ").trim().slice(0, 120);
|
|
4968
|
+
node = node.parentElement;
|
|
4969
|
+
}
|
|
4970
|
+
return "";
|
|
4971
|
+
};
|
|
4972
|
+
const currentText = (w) => {
|
|
4973
|
+
// Value via the union: aria-activedescendant → option text, the widget's
|
|
4974
|
+
// own value (an <input>'s value, or a <button>-trigger's own label text —
|
|
4975
|
+
// a button that IS the trigger carries the committed choice as its own
|
|
4976
|
+
// text), else a selection-label node — treating the widget-kit empty-state
|
|
4977
|
+
// phrase as no value.
|
|
4978
|
+
const adid = w.getAttribute && w.getAttribute("aria-activedescendant");
|
|
4979
|
+
if (adid) { const opt = document.getElementById(adid); if (opt && opt.textContent.trim()) return norm(opt.textContent); }
|
|
4980
|
+
if (w.tagName === "INPUT" && (w.value || "").trim()) return norm(w.value);
|
|
4981
|
+
if (w.tagName === "BUTTON" && buttonValue(w).trim()) return norm(buttonValue(w));
|
|
4982
|
+
const lbl = w.matches(valueSel) ? w : w.querySelector(valueSel);
|
|
4983
|
+
const raw = lbl ? (lbl.textContent || "") : "";
|
|
4984
|
+
if (raw.trim() && !emptyRx.test(raw.trim())) return norm(raw);
|
|
4985
|
+
return "";
|
|
4986
|
+
};
|
|
4987
|
+
const isUnfilled = (w) => {
|
|
4988
|
+
let node = w;
|
|
4989
|
+
for (let d = 0; d < 6 && node; d++) {
|
|
4990
|
+
if (node.getAttribute && isInvalid(node)) return true;
|
|
4991
|
+
node = node.parentElement;
|
|
4992
|
+
}
|
|
4993
|
+
return currentText(w) === "";
|
|
4994
|
+
};
|
|
4995
|
+
// Clear stale marks from a prior call on this same page (this primitive
|
|
4996
|
+
// runs once per "select 'X'" step, and an application wizard answers several
|
|
4997
|
+
// such steps on the same unreloaded page) — otherwise a widget already
|
|
4998
|
+
// filled by an earlier call keeps its old index and collides with whatever
|
|
4999
|
+
// new widget claims that index this round, and the trigger-click selector's
|
|
5000
|
+
// \`.first()\` can resolve to the stale widget.
|
|
5001
|
+
for (const el of document.querySelectorAll("[" + markAttr + "]")) el.removeAttribute(markAttr);
|
|
5002
|
+
const candidates = [];
|
|
5003
|
+
let idx = 0;
|
|
5004
|
+
for (const w of widgets) {
|
|
5005
|
+
if (!isUnfilled(w)) continue;
|
|
5006
|
+
w.setAttribute(markAttr, String(idx));
|
|
5007
|
+
candidates.push({ wIdx: idx, label: widgetLabel(w) });
|
|
5008
|
+
idx++;
|
|
5009
|
+
}
|
|
5010
|
+
if (candidates.length === 0) return { widgetPresent: true, candidates: [] };
|
|
5011
|
+
return { widgetPresent: true, candidates };
|
|
5012
|
+
})(${JSON.stringify(PROMPT_WIDGET_MARK_ATTR)}, ${JSON.stringify(PROMPT_TRIGGER_SELECTORS)}, ${JSON.stringify(PROMPT_VALUE_SELECTORS)}, ${JSON.stringify(PROMPT_EMPTY_VALUE_RX_SRC)}, ${JSON.stringify(PROMPT_EMPTY_VALUE_RX_FLAGS)})`;
|
|
5013
|
+
try {
|
|
5014
|
+
const enumResult = await pollEnumerate(page, target, enumerateWidgetsExpr, (r) => r?.widgetPresent === true);
|
|
5015
|
+
if (!enumResult?.widgetPresent) {
|
|
5016
|
+
logger.info(`prompt-selector primitive: no prompt widget on page for ${optLabel}; falling through to cascade`);
|
|
5017
|
+
return null;
|
|
5018
|
+
}
|
|
5019
|
+
const candidates = enumResult.candidates ?? [];
|
|
5020
|
+
if (candidates.length === 0) {
|
|
5021
|
+
logger.info(`prompt-selector primitive: no unfilled prompt widget for ${optLabel}; falling through to cascade`);
|
|
5022
|
+
return null;
|
|
5023
|
+
}
|
|
5024
|
+
// Deliberately conservative widget disambiguation (see doc comment): a
|
|
5025
|
+
// labeled match must be UNIQUE, and an unlabeled step only resolves when
|
|
5026
|
+
// there is exactly one unfilled widget on the page.
|
|
5027
|
+
const labelMatches = questionLabel
|
|
5028
|
+
? candidates.filter((c) => c.label !== "" &&
|
|
5029
|
+
(norm(c.label).includes(norm(questionLabel)) ||
|
|
5030
|
+
norm(questionLabel).includes(norm(c.label))))
|
|
5031
|
+
: [];
|
|
5032
|
+
const chosen = labelMatches.length === 1
|
|
5033
|
+
? labelMatches[0]
|
|
5034
|
+
: !questionLabel && candidates.length === 1
|
|
5035
|
+
? candidates[0]
|
|
5036
|
+
: null;
|
|
5037
|
+
if (!chosen) {
|
|
5038
|
+
logger.info(`prompt-selector primitive: no unambiguous widget match for ${optLabel}; falling through to cascade`);
|
|
5039
|
+
return null;
|
|
5040
|
+
}
|
|
5041
|
+
// Phase 2 (real gesture): open the popup. These widgets' trigger requires a
|
|
5042
|
+
// genuine click — a synthetic dispatchEvent does not fire its handler.
|
|
5043
|
+
const triggerSel = `[${PROMPT_WIDGET_MARK_ATTR}="${chosen.wIdx}"]`;
|
|
5044
|
+
try {
|
|
5045
|
+
await target.locator(triggerSel).first().click();
|
|
5046
|
+
}
|
|
5047
|
+
catch (err) {
|
|
5048
|
+
logger.info(`prompt-selector primitive: trigger click failed for ${optLabel}: ${(0, errors_1.toErrorMessage)(err)}; falling through`);
|
|
5049
|
+
return null;
|
|
5050
|
+
}
|
|
5051
|
+
await page.waitForTimeout(PROMPT_SELECTOR_SETTLE_MS);
|
|
5052
|
+
const enumerateOptionsExpr = `((widgetMarkAttr, wIdx, markAttr, optionSel, searchSel) => {
|
|
5053
|
+
const w = document.querySelector("[" + widgetMarkAttr + '="' + wIdx + '"]');
|
|
5054
|
+
if (!w) return { optionsPresent: false };
|
|
5055
|
+
// Clear stale option + filter marks from a PRIOR prompt-selector call on
|
|
5056
|
+
// this same unreloaded page (an application wizard answers several "select
|
|
5057
|
+
// X" steps without a reload). Both the option-click and the filter-fill
|
|
5058
|
+
// address these marks with a document-wide \`.first()\`, so a leftover mark
|
|
5059
|
+
// on an earlier widget's popup would otherwise win in DOM order — the
|
|
5060
|
+
// inter-call sibling collision (mirrors the widget-mark reset).
|
|
5061
|
+
for (const el of document.querySelectorAll("[" + markAttr + "],[" + markAttr + "-search]")) {
|
|
5062
|
+
el.removeAttribute(markAttr);
|
|
5063
|
+
el.removeAttribute(markAttr + "-search");
|
|
5064
|
+
}
|
|
5065
|
+
// Scope options/search to THIS widget's popup (aria-controls portal, or
|
|
5066
|
+
// inline subtree), so a sibling widget's options are never picked. Fall
|
|
5067
|
+
// back to document only when neither a portal nor an inline popup is found.
|
|
5068
|
+
const scope = ${PROMPT_SCOPE_ROOT_EXPR}(w);
|
|
5069
|
+
const all = Array.from(scope.querySelectorAll(optionSel));
|
|
5070
|
+
// A union hit may be an ancestor of another (e.g. role=listbox > li > p):
|
|
5071
|
+
// keep only leaf-most option nodes with their own text so we don't double
|
|
5072
|
+
// count or address a wrapper.
|
|
5073
|
+
const opts = all.filter((el) => !all.some((o) => o !== el && el.contains(o)));
|
|
5074
|
+
const searchInput = scope.querySelector(searchSel);
|
|
5075
|
+
// Mark the scoped filter input so the Node-side fill addresses THIS
|
|
5076
|
+
// widget's input (Playwright's locator can't re-run the scope resolution).
|
|
5077
|
+
if (searchInput) searchInput.setAttribute(markAttr + "-search", "1");
|
|
5078
|
+
// A searchable/typeahead widget renders NO options until its filter input
|
|
5079
|
+
// is typed into — report the popup as present (with searchable=true) so the
|
|
5080
|
+
// caller types the filter first, rather than falling through as "no popup".
|
|
5081
|
+
if (opts.length === 0) {
|
|
5082
|
+
return searchInput ? { optionsPresent: true, searchable: true, options: [] } : { optionsPresent: false };
|
|
5083
|
+
}
|
|
5084
|
+
const options = [];
|
|
5085
|
+
for (let i = 0; i < opts.length; i++) {
|
|
5086
|
+
opts[i].setAttribute(markAttr, String(i));
|
|
5087
|
+
const label = opts[i].getAttribute("data-automation-label") || opts[i].getAttribute("aria-label") || opts[i].textContent || "";
|
|
5088
|
+
options.push({ oIdx: i, text: label.replace(/\\s+/g, " ").trim() });
|
|
5089
|
+
}
|
|
5090
|
+
return { optionsPresent: true, searchable: !!searchInput, options };
|
|
5091
|
+
})(${JSON.stringify(PROMPT_WIDGET_MARK_ATTR)}, ${JSON.stringify(chosen.wIdx)}, ${JSON.stringify(PROMPT_OPTION_MARK_ATTR)}, ${JSON.stringify(PROMPT_OPTION_SELECTORS)}, ${JSON.stringify(PROMPT_SEARCH_SELECTORS)})`;
|
|
5092
|
+
const optionsInitial = await pollEnumerate(page, target, enumerateOptionsExpr, (r) => r?.optionsPresent === true);
|
|
5093
|
+
if (!optionsInitial?.optionsPresent) {
|
|
5094
|
+
logger.info(`prompt-selector primitive: popup for ${optLabel} did not render options; falling through`);
|
|
5095
|
+
return null;
|
|
5096
|
+
}
|
|
5097
|
+
// Searchable/typeahead variant: type the option text to filter before the
|
|
5098
|
+
// matching option is even rendered (a searchable widget may show only a
|
|
5099
|
+
// paginated slice until filtered). Non-searchable widgets (no in-popup
|
|
5100
|
+
// filter input) commit directly from the already-rendered list — a widget
|
|
5101
|
+
// without a search box must NOT fall through to the cascade.
|
|
5102
|
+
if (!optionsInitial.searchable) {
|
|
5103
|
+
const optionsResult = optionsInitial;
|
|
5104
|
+
return await commitPromptOption({
|
|
5105
|
+
page,
|
|
5106
|
+
target,
|
|
5107
|
+
logger,
|
|
5108
|
+
anthropic,
|
|
5109
|
+
captureFn,
|
|
5110
|
+
optLabel,
|
|
5111
|
+
option,
|
|
5112
|
+
questionLabel,
|
|
5113
|
+
chosen,
|
|
5114
|
+
optionsResult,
|
|
5115
|
+
});
|
|
5116
|
+
}
|
|
5117
|
+
try {
|
|
5118
|
+
// Fill the filter input marked for THIS widget during enumeration (scoped
|
|
5119
|
+
// to its popup), falling back to the union only if the mark is absent.
|
|
5120
|
+
const scopedSearchSel = `[${PROMPT_OPTION_MARK_ATTR}-search="1"]`;
|
|
5121
|
+
const marked = await target.locator(scopedSearchSel).count();
|
|
5122
|
+
await target
|
|
5123
|
+
.locator(marked > 0 ? scopedSearchSel : PROMPT_SEARCH_SELECTORS)
|
|
5124
|
+
.first()
|
|
5125
|
+
.fill(option);
|
|
5126
|
+
}
|
|
5127
|
+
catch (err) {
|
|
5128
|
+
logger.info(`prompt-selector primitive: filter-input type failed for ${optLabel}: ${(0, errors_1.toErrorMessage)(err)}; falling through`);
|
|
5129
|
+
return null;
|
|
5130
|
+
}
|
|
5131
|
+
await page.waitForTimeout(PROMPT_SELECTOR_SETTLE_MS);
|
|
5132
|
+
const optionsFiltered = await pollEnumerate(page, target, enumerateOptionsExpr, (r) => r?.optionsPresent === true);
|
|
5133
|
+
if (!optionsFiltered?.optionsPresent) {
|
|
5134
|
+
logger.info(`prompt-selector primitive: filtered popup for ${optLabel} rendered no options; falling through`);
|
|
5135
|
+
return null;
|
|
5136
|
+
}
|
|
5137
|
+
return await commitPromptOption({
|
|
5138
|
+
page,
|
|
5139
|
+
target,
|
|
5140
|
+
logger,
|
|
5141
|
+
anthropic,
|
|
5142
|
+
captureFn,
|
|
5143
|
+
optLabel,
|
|
5144
|
+
option,
|
|
5145
|
+
questionLabel,
|
|
5146
|
+
chosen,
|
|
5147
|
+
optionsResult: optionsFiltered,
|
|
5148
|
+
});
|
|
5149
|
+
}
|
|
5150
|
+
catch (err) {
|
|
5151
|
+
logger.warn(`prompt-selector primitive: evaluate threw: ${(0, errors_1.toErrorMessage)(err)}; falling through`);
|
|
5152
|
+
return null;
|
|
5153
|
+
}
|
|
5154
|
+
}
|
|
5155
|
+
/**
|
|
5156
|
+
* Phase 3 of {@link tryPromptSelectorPrimitive}: given the popup's rendered
|
|
5157
|
+
* (possibly filtered) option list, match the requested option — deterministic
|
|
5158
|
+
* exact-text match first, `judgeSelectOptionWithLLM` fallback for per-req
|
|
5159
|
+
* variance — click it with a real gesture, and verify the selection committed
|
|
5160
|
+
* (the widget's accessible value / selection-label reflects the chosen option
|
|
5161
|
+
* by the value union, or its invalid marker cleared). Split out from the main
|
|
5162
|
+
* function because both the static and searchable-then-filtered branches need
|
|
5163
|
+
* the identical match/click/verify sequence.
|
|
5164
|
+
*/
|
|
5165
|
+
async function commitPromptOption(params) {
|
|
5166
|
+
const { page, target, logger, anthropic, captureFn, optLabel, option, questionLabel, chosen, optionsResult, } = params;
|
|
5167
|
+
const norm = (s) => s.replace(/\s+/g, " ").trim().toLowerCase();
|
|
5168
|
+
const wantOpt = norm(option);
|
|
5169
|
+
const options = optionsResult.options ?? [];
|
|
5170
|
+
const detMatch = options.find((o) => norm(o.text) === wantOpt) ?? null;
|
|
5171
|
+
if (detMatch === null && (anthropic === null || options.length === 0)) {
|
|
5172
|
+
logger.info(`prompt-selector primitive: no option match for ${optLabel}${anthropic === null ? " (no LLM client)" : ""}; falling through to cascade`);
|
|
5173
|
+
return null;
|
|
5174
|
+
}
|
|
5175
|
+
const verdict = detMatch === null
|
|
5176
|
+
? await (0, select_option_1.judgeSelectOptionWithLLM)({
|
|
5177
|
+
client: anthropic,
|
|
5178
|
+
input: {
|
|
5179
|
+
questionLabel,
|
|
5180
|
+
desiredHint: option,
|
|
5181
|
+
candidates: [{ label: chosen.label || null, options: options.map((o) => o.text) }],
|
|
5182
|
+
},
|
|
5183
|
+
captureFn,
|
|
5184
|
+
})
|
|
5185
|
+
: null;
|
|
5186
|
+
if (detMatch === null &&
|
|
5187
|
+
(!verdict || verdict.selectIndex === null || verdict.optionIndex === null)) {
|
|
5188
|
+
logger.info(`prompt-selector primitive: LLM found no matching option for ${optLabel}${verdict ? ` (${verdict.reason})` : ""}; falling through to cascade`);
|
|
5189
|
+
return null;
|
|
5190
|
+
}
|
|
5191
|
+
const chosenOption = detMatch ??
|
|
5192
|
+
(verdict && verdict.optionIndex !== null ? (options[verdict.optionIndex] ?? null) : null);
|
|
5193
|
+
if (!chosenOption) {
|
|
5194
|
+
logger.info(`prompt-selector primitive: option match out of range for ${optLabel}; falling through`);
|
|
5195
|
+
return null;
|
|
5196
|
+
}
|
|
5197
|
+
const matchReason = detMatch ? "deterministic" : `LLM: ${(verdict?.reason ?? "").slice(0, 60)}`;
|
|
5198
|
+
const optionSel = `[${PROMPT_OPTION_MARK_ATTR}="${chosenOption.oIdx}"]`;
|
|
5199
|
+
try {
|
|
5200
|
+
await target.locator(optionSel).first().click();
|
|
5201
|
+
}
|
|
5202
|
+
catch (err) {
|
|
5203
|
+
logger.info(`prompt-selector primitive: option click failed for ${optLabel}: ${(0, errors_1.toErrorMessage)(err)}; falling through`);
|
|
5204
|
+
return null;
|
|
5205
|
+
}
|
|
5206
|
+
await page.waitForTimeout(PROMPT_SELECTOR_SETTLE_MS);
|
|
5207
|
+
const readbackExpr = `((wIdx, markAttr, valueSel, emptyRxSrc, emptyRxFlags, wantText) => {
|
|
5208
|
+
const isInvalid = ${INVALID_MARKER_EL_EXPR};
|
|
5209
|
+
const norm = (s) => (s || "").replace(/\\s+/g, " ").trim().toLowerCase();
|
|
5210
|
+
const buttonValue = ${BUTTON_VALUE_EXPR};
|
|
5211
|
+
const emptyRx = new RegExp(emptyRxSrc, emptyRxFlags);
|
|
5212
|
+
const w = document.querySelector("[" + markAttr + '="' + wIdx + '"]');
|
|
5213
|
+
if (!w) return { ok: false, id: "" };
|
|
5214
|
+
// Value via the union: aria-activedescendant, own input value, a <button>'s
|
|
5215
|
+
// own value text (popup-pollution-safe, see BUTTON_VALUE_EXPR), or a
|
|
5216
|
+
// selection-label node (empty-state phrase treated as no value).
|
|
5217
|
+
const adid = w.getAttribute && w.getAttribute("aria-activedescendant");
|
|
5218
|
+
const adText = adid && document.getElementById(adid) ? norm(document.getElementById(adid).textContent) : "";
|
|
5219
|
+
const own = w.tagName === "INPUT" ? norm(w.value || "") : w.tagName === "BUTTON" ? norm(buttonValue(w)) : "";
|
|
5220
|
+
const lbl = w.matches(valueSel) ? w : w.querySelector(valueSel);
|
|
5221
|
+
const lblRaw = lbl ? (lbl.textContent || "") : "";
|
|
5222
|
+
const lblText = lblRaw.trim() && !emptyRx.test(lblRaw.trim()) ? norm(lblRaw) : "";
|
|
5223
|
+
const text = adText || own || lblText;
|
|
5224
|
+
const textMatches = wantText ? text.includes(wantText) : text !== "";
|
|
5225
|
+
let node = w;
|
|
5226
|
+
let stillInvalid = false;
|
|
5227
|
+
for (let d = 0; d < 6 && node; d++) {
|
|
5228
|
+
if (node.getAttribute && isInvalid(node)) { stillInvalid = true; break; }
|
|
5229
|
+
node = node.parentElement;
|
|
5230
|
+
}
|
|
5231
|
+
return { ok: textMatches || !stillInvalid, id: w.id || "" };
|
|
5232
|
+
})(${JSON.stringify(chosen.wIdx)}, ${JSON.stringify(PROMPT_WIDGET_MARK_ATTR)}, ${JSON.stringify(PROMPT_VALUE_SELECTORS)}, ${JSON.stringify(PROMPT_EMPTY_VALUE_RX_SRC)}, ${JSON.stringify(PROMPT_EMPTY_VALUE_RX_FLAGS)}, ${JSON.stringify(norm(chosenOption.text))})`;
|
|
5233
|
+
const readback = (await target.evaluate(readbackExpr).catch(() => ({ ok: false, id: "" })));
|
|
5234
|
+
if (!readback?.ok) {
|
|
5235
|
+
logger.info(`prompt-selector primitive: selection for ${optLabel} did not commit; falling through to cascade`);
|
|
5236
|
+
return null;
|
|
5237
|
+
}
|
|
5238
|
+
logger.info(`prompt-selector primitive: selected "${chosenOption.text.slice(0, 40)}" for ${optLabel} (${matchReason})`);
|
|
5239
|
+
return readback.id;
|
|
5240
|
+
}
|
|
4594
5241
|
/**
|
|
4595
5242
|
* Guard for the optional-step fast-skip: is there a REQUIRED, still-empty (or
|
|
4596
5243
|
* aria-invalid) form control on the page whose nearby label matches this step's
|
|
@@ -4602,25 +5249,30 @@ async function applyRadioSelection(target, gi, ri, hint) {
|
|
|
4602
5249
|
* should fall through to the cascade/replan instead.
|
|
4603
5250
|
*
|
|
4604
5251
|
* Conservative by construction: returns false unless the step parses as a
|
|
4605
|
-
* select/
|
|
4606
|
-
*
|
|
4607
|
-
* step (e.g. "dismiss modal" on
|
|
4608
|
-
* fast-skip the comments call
|
|
5252
|
+
* select/fill/check step (via {@link extractRequiredControlProbeLabel}) AND a
|
|
5253
|
+
* required-and-unsatisfied control with a label matching that target is
|
|
5254
|
+
* actually present. A genuinely-absent optional step (e.g. "dismiss modal" on
|
|
5255
|
+
* a modal-less page) has no such control, so the fast-skip the comments call
|
|
5256
|
+
* essential is preserved.
|
|
4609
5257
|
*/
|
|
4610
5258
|
async function hasUnfilledRequiredControlForStep(target, instruction) {
|
|
4611
|
-
const
|
|
4612
|
-
if (!
|
|
5259
|
+
const label = extractRequiredControlProbeLabel(instruction);
|
|
5260
|
+
if (!label)
|
|
4613
5261
|
return false;
|
|
4614
|
-
const expr = `((label) => {
|
|
5262
|
+
const expr = `((label, triggerSel) => {
|
|
4615
5263
|
const norm = (s) => (s || "").replace(/\\s+/g, " ").trim().toLowerCase();
|
|
4616
5264
|
const want = norm(label);
|
|
4617
5265
|
if (!want) return false;
|
|
4618
5266
|
const isInvalid = ${INVALID_MARKER_EL_EXPR};
|
|
4619
|
-
// Required markers: the control itself,
|
|
5267
|
+
// Required markers: the control itself, a required-asterisk label nearby,
|
|
5268
|
+
// or (the UXI widget-kit shape) a trailing "Required" suffix baked into the
|
|
5269
|
+
// accessible name rather than exposed as aria-required at all.
|
|
4620
5270
|
const isRequired = (el) => {
|
|
4621
5271
|
if (!el || !el.getAttribute) return false;
|
|
4622
5272
|
if (el.hasAttribute("required")) return true;
|
|
4623
5273
|
if (el.getAttribute("aria-required") === "true") return true;
|
|
5274
|
+
const al = el.getAttribute("aria-label");
|
|
5275
|
+
if (al && /required\\s*$/i.test(al.trim())) return true;
|
|
4624
5276
|
return false;
|
|
4625
5277
|
};
|
|
4626
5278
|
const isEmptyish = (el) => {
|
|
@@ -4630,11 +5282,38 @@ async function hasUnfilledRequiredControlForStep(target, instruction) {
|
|
|
4630
5282
|
const v = ("value" in el) ? el.value : "";
|
|
4631
5283
|
return !v || String(v).trim() === "";
|
|
4632
5284
|
};
|
|
4633
|
-
//
|
|
4634
|
-
// label
|
|
4635
|
-
//
|
|
5285
|
+
// Standards-first, depth-independent label lookup: aria-labelledby, then
|
|
5286
|
+
// a <label for=id> referencing the control or a control inside it, then
|
|
5287
|
+
// the depth-capped ancestor label/legend walk as a last resort (real
|
|
5288
|
+
// markup can nest a control several wrapper divs below its <label>,
|
|
5289
|
+
// deeper than any fixed ancestor cap can safely assume).
|
|
5290
|
+
const labelFor = (el) => {
|
|
5291
|
+
const alb = el.getAttribute && el.getAttribute("aria-labelledby");
|
|
5292
|
+
if (alb) {
|
|
5293
|
+
const parts = [];
|
|
5294
|
+
for (const id of alb.split(/\\s+/)) { const ref = document.getElementById(id); if (ref) parts.push(ref.textContent); }
|
|
5295
|
+
if (parts.length) return norm(parts.join(" "));
|
|
5296
|
+
}
|
|
5297
|
+
const ids = [el.id, ...Array.from(el.querySelectorAll ? el.querySelectorAll("[id]") : []).map((e) => e.id)].filter(Boolean);
|
|
5298
|
+
for (const id of ids) {
|
|
5299
|
+
const lab = document.querySelector("label[for='" + (window.CSS && CSS.escape ? CSS.escape(id) : id) + "']");
|
|
5300
|
+
if (lab && lab.textContent && lab.textContent.trim()) return norm(lab.textContent);
|
|
5301
|
+
}
|
|
5302
|
+
let node = el;
|
|
5303
|
+
for (let d = 0; d < 6 && node; d++) {
|
|
5304
|
+
const lbl = node.querySelector && node.querySelector("label,legend");
|
|
5305
|
+
const txt = lbl && lbl.textContent ? norm(lbl.textContent) : "";
|
|
5306
|
+
if (txt) return txt;
|
|
5307
|
+
node = node.parentElement;
|
|
5308
|
+
}
|
|
5309
|
+
return "";
|
|
5310
|
+
};
|
|
5311
|
+
// Scan form controls; for each required+empty one, check whether its
|
|
5312
|
+
// label matches the question. Adds the prompt-selector trigger union
|
|
5313
|
+
// (button/aria-haspopup shapes with no native input/select at all) to
|
|
5314
|
+
// the container-widget union already covered.
|
|
4636
5315
|
const controls = Array.from(document.querySelectorAll(
|
|
4637
|
-
"input,select,textarea,[role=combobox],[role=listbox],.bb-custom-select-container,[class*='MultiCheckboxInput']"
|
|
5316
|
+
"input,select,textarea,[role=combobox],[role=listbox],.bb-custom-select-container,[class*='MultiCheckboxInput']," + triggerSel
|
|
4638
5317
|
));
|
|
4639
5318
|
for (const el of controls) {
|
|
4640
5319
|
if (!isRequired(el) && !(el.querySelector && el.querySelector("[required],[aria-required=true]"))) {
|
|
@@ -4644,17 +5323,11 @@ async function hasUnfilledRequiredControlForStep(target, instruction) {
|
|
|
4644
5323
|
// is it empty/invalid?
|
|
4645
5324
|
const emptyOrInvalid = isEmptyish(el) || (el.querySelector && !!el.querySelector("[aria-invalid=true]"));
|
|
4646
5325
|
if (!emptyOrInvalid) continue;
|
|
4647
|
-
|
|
4648
|
-
|
|
4649
|
-
for (let d = 0; d < 6 && node; d++) {
|
|
4650
|
-
const lbl = node.querySelector && node.querySelector("label,legend");
|
|
4651
|
-
const txt = lbl && lbl.textContent ? norm(lbl.textContent) : "";
|
|
4652
|
-
if (txt && (txt.includes(want) || want.includes(txt))) return true;
|
|
4653
|
-
node = node.parentElement;
|
|
4654
|
-
}
|
|
5326
|
+
const txt = labelFor(el);
|
|
5327
|
+
if (txt && (txt.includes(want) || want.includes(txt))) return true;
|
|
4655
5328
|
}
|
|
4656
5329
|
return false;
|
|
4657
|
-
})(${JSON.stringify(
|
|
5330
|
+
})(${JSON.stringify(label)}, ${JSON.stringify(PROMPT_TRIGGER_SELECTORS)})`;
|
|
4658
5331
|
try {
|
|
4659
5332
|
return (await target.evaluate(expr)) === true;
|
|
4660
5333
|
}
|
|
@@ -5624,6 +6297,30 @@ async function executeStepWithHealing(params) {
|
|
|
5624
6297
|
trajectory?.push({ stepIndex, verifiedBy: "dom", targetId: radioTargetId });
|
|
5625
6298
|
return "completed";
|
|
5626
6299
|
}
|
|
6300
|
+
// When the step is a "select 'X'" OR a "Fill in the Y field with 'X'" step
|
|
6301
|
+
// (the latter shape is how flow/replan generation describes this widget's
|
|
6302
|
+
// searchable filter box, which renders a real <input>) whose only matching
|
|
6303
|
+
// control is a native-control-less popup-dropdown widget (a combobox that
|
|
6304
|
+
// opens a listbox popup, options rendered on-demand — see
|
|
6305
|
+
// PROMPT_TRIGGER_SELECTORS) rather than a <select> or MUI radio group,
|
|
6306
|
+
// answer it directly. Runs AFTER select/checkbox/radio (which own their own
|
|
6307
|
+
// widget shapes and would already have claimed the step) and BEFORE the
|
|
6308
|
+
// cascade, since the widget's trigger exposes no native control the observe
|
|
6309
|
+
// cascade resolves. No-op (falls through) when there's no unfilled/
|
|
6310
|
+
// unambiguous prompt widget or no confident option match.
|
|
6311
|
+
const promptSelectorTargetId = await tryPromptSelectorPrimitive({
|
|
6312
|
+
page,
|
|
6313
|
+
target: selectFrameTarget,
|
|
6314
|
+
instruction: step,
|
|
6315
|
+
logger,
|
|
6316
|
+
anthropic,
|
|
6317
|
+
captureFn,
|
|
6318
|
+
});
|
|
6319
|
+
if (promptSelectorTargetId !== null) {
|
|
6320
|
+
logger.info(`${formatStepPrefix(stepIndex, totalSteps)} resolved by prompt-selector primitive`);
|
|
6321
|
+
trajectory?.push({ stepIndex, verifiedBy: "dom", targetId: promptSelectorTargetId });
|
|
6322
|
+
return "completed";
|
|
6323
|
+
}
|
|
5627
6324
|
// On a CATCH-ALL step ("for any remaining … question"), fill every
|
|
5628
6325
|
// required-but-empty native <select> — including MuiNativeSelect dropdowns
|
|
5629
6326
|
// (tabindex=-1) that Stagehand observe can't see and that no concrete flow
|
|
@@ -5991,6 +6688,12 @@ async function executeStepWithHealing(params) {
|
|
|
5991
6688
|
// disk for entries indexed after this point, which is eviction-proof when
|
|
5992
6689
|
// >RECENT_CAPTURES_WINDOW captures flood during the step.
|
|
5993
6690
|
const preCaptureIdx = latestCaptureIndex(recentCaptures);
|
|
6691
|
+
// Pre-click ng-invalid baseline for the view-swap veto below
|
|
6692
|
+
// (isClickViewSwapVerified's invalidMarkerDelta). Read unconditionally —
|
|
6693
|
+
// cheap DOM query, and unlike `preSubmitInvalidCount` (final/submit-only)
|
|
6694
|
+
// this must cover ANY click step: the reported false positive was an
|
|
6695
|
+
// interior "Create Account" click, not the flow's final step.
|
|
6696
|
+
const preInvalidMarkerCount = await countNgInvalidContainers(frameTarget ?? (0, frame_target_1.mainFrameTarget)(page)).catch(() => 0);
|
|
5994
6697
|
const record = {
|
|
5995
6698
|
attempt,
|
|
5996
6699
|
technique: "act-string",
|
|
@@ -6903,13 +7606,24 @@ async function executeStepWithHealing(params) {
|
|
|
6903
7606
|
if (domVerified && !domVerifiedForStep) {
|
|
6904
7607
|
logger.info(`${formatStepPrefix(stepIndex, totalSteps)} advance step succeeded only via DOM state change (field toggle / non-advancing POST), not a real transition; not treating as verified`);
|
|
6905
7608
|
}
|
|
7609
|
+
// Blocked-submit veto input for the view-swap gate below. Read post-click
|
|
7610
|
+
// ng-invalid count only for a resolved click — the veto is meaningless
|
|
7611
|
+
// for fill/select attempts, and this avoids a page.evaluate on every
|
|
7612
|
+
// non-click attempt. See recon-viewswap-false-positive-on-blocked-form-submit.md:
|
|
7613
|
+
// a "Create Account" click blocked by required-field validation grew the
|
|
7614
|
+
// DOM (inline error text) with network=false url=false, which the reveal
|
|
7615
|
+
// branch below alone would credit as verified.
|
|
7616
|
+
const postInvalidMarkerCount = isClick
|
|
7617
|
+
? await countNgInvalidContainers(frameTarget ?? (0, frame_target_1.mainFrameTarget)(page)).catch(() => 0)
|
|
7618
|
+
: preInvalidMarkerCount;
|
|
6906
7619
|
// Client-side view-swap gate: credit a click that produces substantial
|
|
6907
7620
|
// DOM growth (≥5KB) with zero network when it's NOT a submit/final step
|
|
6908
7621
|
// and NOT an advance-pattern step. Fixes the top-window site "Manual Application"
|
|
6909
7622
|
// case where a +49KB DOM-only view swap was scored as "no observable effect".
|
|
6910
7623
|
// Below that threshold, also credits a smaller text-changing reveal (≥500B) —
|
|
6911
7624
|
// fixes the top-window site Work-History gate-message reveal (+789B) that used to
|
|
6912
|
-
// cascade to a 5-attempt failure and global replan.
|
|
7625
|
+
// cascade to a 5-attempt failure and global replan. Vetoed when the click's
|
|
7626
|
+
// ng-invalid marker count grew (see isClickViewSwapVerified's doc comment).
|
|
6913
7627
|
const clickViewSwapVerified = isClickViewSwapVerified({
|
|
6914
7628
|
resolvedAction,
|
|
6915
7629
|
isFinalStep,
|
|
@@ -6918,7 +7632,16 @@ async function executeStepWithHealing(params) {
|
|
|
6918
7632
|
networkDelta: post.networkCount - pre.networkCount,
|
|
6919
7633
|
bytesDelta: post.bodyHtmlLength - pre.bodyHtmlLength,
|
|
6920
7634
|
textChanged: post.visibleTextSignature !== pre.visibleTextSignature,
|
|
7635
|
+
invalidMarkerDelta: postInvalidMarkerCount - preInvalidMarkerCount,
|
|
6921
7636
|
});
|
|
7637
|
+
if (clickViewSwapVerified === false &&
|
|
7638
|
+
isClick &&
|
|
7639
|
+
postInvalidMarkerCount > preInvalidMarkerCount &&
|
|
7640
|
+
post.networkCount - pre.networkCount === 0 &&
|
|
7641
|
+
post.url === pre.url &&
|
|
7642
|
+
post.bodyHtmlLength - pre.bodyHtmlLength >= config_1.config.scraper.viewSwapRevealMinBytesThreshold) {
|
|
7643
|
+
logger.info(`${formatStepPrefix(stepIndex, totalSteps)} click grew the DOM (view-swap shape) but ng-invalid markers grew from ${preInvalidMarkerCount} to ${postInvalidMarkerCount} — a blocked form submit, not a real view transition; not treating as verified`);
|
|
7644
|
+
}
|
|
6922
7645
|
// A network-free selection/option toggle — including a Base Web `kind`
|
|
6923
7646
|
// flip or hashed-class swap that exposes no aria/data-state marker and
|
|
6924
7647
|
// moves a trivial or NEGATIVE byte delta — is now credited authoritatively
|