@enricai/barnacle 1.12.1 → 1.12.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/dist/scraper/flow-runner.d.ts +71 -19
- package/dist/scraper/flow-runner.d.ts.map +1 -1
- package/dist/scraper/flow-runner.js +873 -43
- package/dist/scraper/flow-runner.js.map +1 -1
- package/dist/scripts/recon-browser.d.ts.map +1 -1
- package/dist/scripts/recon-browser.js +5 -0
- package/dist/scripts/recon-browser.js.map +1 -1
- package/dist/scripts/recon-generate.d.ts +27 -2
- package/dist/scripts/recon-generate.d.ts.map +1 -1
- package/dist/scripts/recon-generate.js +116 -5
- package/dist/scripts/recon-generate.js.map +1 -1
- package/package.json +2 -1
|
@@ -24,6 +24,7 @@ exports.findRecentPageTransition = findRecentPageTransition;
|
|
|
24
24
|
exports.isWizardExitAction = isWizardExitAction;
|
|
25
25
|
exports.isAdvanceStep = isAdvanceStep;
|
|
26
26
|
exports.shouldCaptureSelectionState = shouldCaptureSelectionState;
|
|
27
|
+
exports.flowHasSubmitSemantics = flowHasSubmitSemantics;
|
|
27
28
|
exports.shouldWarnMissingAdvancePattern = shouldWarnMissingAdvancePattern;
|
|
28
29
|
exports.parseCaptureIndex = parseCaptureIndex;
|
|
29
30
|
exports.latestCaptureIndex = latestCaptureIndex;
|
|
@@ -58,6 +59,7 @@ exports.normalizeDateValue = normalizeDateValue;
|
|
|
58
59
|
exports.fillHtml5DateTimeInput = fillHtml5DateTimeInput;
|
|
59
60
|
exports.fillTextDatepickerInput = fillTextDatepickerInput;
|
|
60
61
|
exports.verifyFillReadback = verifyFillReadback;
|
|
62
|
+
exports.verifyPromptSelectorCommitted = verifyPromptSelectorCommitted;
|
|
61
63
|
exports.isUploadAffordanceLabel = isUploadAffordanceLabel;
|
|
62
64
|
exports.writeFixtureToTempFile = writeFixtureToTempFile;
|
|
63
65
|
exports.attachToSurfacedInput = attachToSurfacedInput;
|
|
@@ -72,6 +74,7 @@ exports.waitForTransitionBody = waitForTransitionBody;
|
|
|
72
74
|
exports.chooseRequiredSelectOption = chooseRequiredSelectOption;
|
|
73
75
|
exports.buildRadioIdXPath = buildRadioIdXPath;
|
|
74
76
|
exports.selectRadioGroupOption = selectRadioGroupOption;
|
|
77
|
+
exports.hasUnfilledRequiredControlForStep = hasUnfilledRequiredControlForStep;
|
|
75
78
|
exports.simulateDragDropUpload = simulateDragDropUpload;
|
|
76
79
|
exports.dispatchJqueryChangeEvent = dispatchJqueryChangeEvent;
|
|
77
80
|
exports.verifyDomEffect = verifyDomEffect;
|
|
@@ -376,6 +379,187 @@ const RADIO_SETTLE_MS = 400;
|
|
|
376
379
|
* value that a later worklet re-render will wipe.
|
|
377
380
|
*/
|
|
378
381
|
const SELECT_SETTLE_MS = 400;
|
|
382
|
+
/**
|
|
383
|
+
* Same async-revalidation tick as {@link RADIO_SETTLE_MS}/{@link SELECT_SETTLE_MS},
|
|
384
|
+
* for the popup-dropdown widget family {@link tryPromptSelectorPrimitive} handles:
|
|
385
|
+
* after opening the popup, after typing into an in-popup filter, and after
|
|
386
|
+
* clicking an option, the DOM (option-list render, selection-label text,
|
|
387
|
+
* `aria-invalid` marker) settles a beat later than the trusted event that
|
|
388
|
+
* triggered it.
|
|
389
|
+
*/
|
|
390
|
+
const PROMPT_SELECTOR_SETTLE_MS = 400;
|
|
391
|
+
/**
|
|
392
|
+
* Temporary attribute {@link tryPromptSelectorPrimitive} stamps onto each
|
|
393
|
+
* candidate widget during its read-only enumerate pass, so the Node-side click
|
|
394
|
+
* (a real Playwright gesture — these widgets ignore a bare `el.click()`) can
|
|
395
|
+
* address the exact widget chosen by index without re-deriving a CSS/xpath
|
|
396
|
+
* selector for an element with no stable id.
|
|
397
|
+
*/
|
|
398
|
+
const PROMPT_WIDGET_MARK_ATTR = "data-bcl-prompt-idx";
|
|
399
|
+
/** Same role as {@link PROMPT_WIDGET_MARK_ATTR}, but for the popup's rendered option entries. */
|
|
400
|
+
const PROMPT_OPTION_MARK_ATTR = "data-bcl-prompt-opt-idx";
|
|
401
|
+
/**
|
|
402
|
+
* Cross-vendor selector for a genuine focusable form control — the only kind
|
|
403
|
+
* of element a native click event reliably opens. Phase 1's container walk
|
|
404
|
+
* marks the OUTERMOST widget-kit container so one widget resolves to one
|
|
405
|
+
* candidate, but that container is often a layout wrapper with no click
|
|
406
|
+
* handler of its own (e.g. a typeahead/chip widget's real open+filter control
|
|
407
|
+
* is an `<input>` nested a level deeper, inside its own `*InputContainer`
|
|
408
|
+
* wrapper). Phase 2 prefers clicking this interactive descendant over the
|
|
409
|
+
* marked container itself, falling back to the container only when it has
|
|
410
|
+
* none — no vendor branch, just "click the real control if one exists".
|
|
411
|
+
*/
|
|
412
|
+
const PROMPT_INTERACTIVE_CONTROL_SELECTORS = "button,[role='combobox'],[role='button'],input";
|
|
413
|
+
/**
|
|
414
|
+
* Cross-vendor selector union that identifies the TRIGGER of a native-control-less
|
|
415
|
+
* popup-dropdown widget (a combobox that opens a listbox popup and renders no
|
|
416
|
+
* `<select>`/`<input>` a focused `<select>`/`<input>` probe can see). Ordered as
|
|
417
|
+
* a UNION, standards first, with well-known widget-library markers as co-equal
|
|
418
|
+
* members — the same multi-vendor-union discipline as
|
|
419
|
+
* {@link INVALID_MARKER_CLASS_SOURCE} (`ng-invalid|Mui-error|…`). No member is a
|
|
420
|
+
* per-site branch; each is one recognizable spelling of the same widget shape.
|
|
421
|
+
* Growth is a one-line edit here — never a new hardcoded selector scattered into
|
|
422
|
+
* logic (the site-agnostic guard trips on the latter, allows this curated list).
|
|
423
|
+
*
|
|
424
|
+
* Members: ARIA combobox / listbox-owning triggers (`role=combobox`,
|
|
425
|
+
* `aria-haspopup=listbox`, `aria-controls`/`aria-owns`→listbox); and the
|
|
426
|
+
* `data-uxi-widget-type` / `data-automation-id` prompt family emitted by the
|
|
427
|
+
* Canvas/UXI widget kit some ATS front-ends build on (verified on a real
|
|
428
|
+
* capture: the trigger is an `<input data-uxi-widget-type="selectinput"
|
|
429
|
+
* aria-required aria-invalid>` beside a `data-automation-id="promptIcon"` icon,
|
|
430
|
+
* with NO `role=combobox` — hence ARIA alone under-selects it).
|
|
431
|
+
*/
|
|
432
|
+
const PROMPT_TRIGGER_SELECTORS = [
|
|
433
|
+
"[role='combobox'][aria-haspopup='listbox']",
|
|
434
|
+
"[role='combobox'][aria-controls]",
|
|
435
|
+
"[role='combobox'][aria-owns]",
|
|
436
|
+
"[aria-haspopup='listbox']",
|
|
437
|
+
"[data-uxi-widget-type='selectinput']",
|
|
438
|
+
"[data-uxi-widget-type='multiselect']",
|
|
439
|
+
"[data-automation-id='promptIcon']",
|
|
440
|
+
"[data-automation-id='multiSelectContainer']",
|
|
441
|
+
"[data-automation-id='promptSelectionLabel']",
|
|
442
|
+
].join(",");
|
|
443
|
+
/**
|
|
444
|
+
* Cross-vendor selector union for an OPTION rendered inside the opened popup —
|
|
445
|
+
* standards first (`role=option`), then the widget-kit option markers. Sibling
|
|
446
|
+
* of {@link PROMPT_TRIGGER_SELECTORS}; same union discipline and guard treatment.
|
|
447
|
+
*/
|
|
448
|
+
const PROMPT_OPTION_SELECTORS = [
|
|
449
|
+
"[role='option']",
|
|
450
|
+
"[data-automation-id='promptOption']",
|
|
451
|
+
"[data-uxi-widget-type='selectinputlistitem']",
|
|
452
|
+
].join(",");
|
|
453
|
+
/**
|
|
454
|
+
* Cross-vendor selector union for an in-popup filter/typeahead input — standards
|
|
455
|
+
* first, then the widget-kit's bare filter `<input>` (which carries no
|
|
456
|
+
* `role=searchbox`, only `data-uxi-widget-type='selectinput'` on the input
|
|
457
|
+
* itself). Sibling of {@link PROMPT_TRIGGER_SELECTORS}; same guard treatment.
|
|
458
|
+
*/
|
|
459
|
+
const PROMPT_SEARCH_SELECTORS = [
|
|
460
|
+
"[role='searchbox']",
|
|
461
|
+
"[role='combobox'] input[type='text']",
|
|
462
|
+
"[data-automation-id='searchBox'] input",
|
|
463
|
+
"[data-automation-id='searchBoxInput']",
|
|
464
|
+
"input[data-uxi-widget-type='selectinput']",
|
|
465
|
+
].join(",");
|
|
466
|
+
/**
|
|
467
|
+
* Cross-vendor selector union for the node that carries a widget's CURRENT
|
|
468
|
+
* (committed) value text — standards-poor here (no `aria-activedescendant` on
|
|
469
|
+
* the verified widget), so the widget-kit's selection-label node is a co-equal
|
|
470
|
+
* union member. Sibling of {@link PROMPT_TRIGGER_SELECTORS}.
|
|
471
|
+
*/
|
|
472
|
+
const PROMPT_VALUE_SELECTORS = [
|
|
473
|
+
"[data-automation-id='promptSelectionLabel']",
|
|
474
|
+
"[aria-live='polite'][data-automation-id='promptAriaInstruction']",
|
|
475
|
+
].join(",");
|
|
476
|
+
/**
|
|
477
|
+
* The widget-kit's empty-state value text ("0 items selected") — a filled
|
|
478
|
+
* widget's value node reads the chosen option instead. Matched
|
|
479
|
+
* case-insensitively as a whole phrase so a non-empty selection ("1 item
|
|
480
|
+
* selected, …") is correctly read as filled.
|
|
481
|
+
*/
|
|
482
|
+
const PROMPT_EMPTY_VALUE_RX = /^\s*0\s+items?\s+selected\s*$/i;
|
|
483
|
+
/**
|
|
484
|
+
* {@link PROMPT_EMPTY_VALUE_RX}'s source and flags as plain strings, passed into
|
|
485
|
+
* the browser-evaluated expressions to reconstruct the RegExp there. Kept as
|
|
486
|
+
* separate literals (not a `.toString()` round-trip) so the reconstruction can't
|
|
487
|
+
* be corrupted if the pattern ever gains a `/`.
|
|
488
|
+
*/
|
|
489
|
+
const PROMPT_EMPTY_VALUE_RX_SRC = PROMPT_EMPTY_VALUE_RX.source;
|
|
490
|
+
const PROMPT_EMPTY_VALUE_RX_FLAGS = PROMPT_EMPTY_VALUE_RX.flags;
|
|
491
|
+
/**
|
|
492
|
+
* Browser-side expression reading a `<button>`-trigger's own committed-value
|
|
493
|
+
* text. A button's label may be a direct text node (`<button>Mobile</button>`)
|
|
494
|
+
* or a child element (`<button><span>Mobile</span></button>`), so a recursive
|
|
495
|
+
* read is needed — but it must exclude (a) a popup rendered INSIDE the trigger
|
|
496
|
+
* (`[role='option']`/`[role='listbox']`, which some libraries nest) and (b)
|
|
497
|
+
* decorative descendants (`[aria-hidden='true']`, a required-marker `<abbr>`, an
|
|
498
|
+
* icon `<svg>`) whose text would otherwise make an EMPTY placeholder button read
|
|
499
|
+
* as filled and get skipped as a candidate. Clone, strip those subtrees, read
|
|
500
|
+
* the remaining `textContent`.
|
|
501
|
+
*/
|
|
502
|
+
const BUTTON_VALUE_EXPR = "((el) => { const c = el.cloneNode(true); for (const n of c.querySelectorAll(\"[role='option'],[role='listbox'],[aria-hidden='true'],abbr,svg\")) n.remove(); return c.textContent || \"\"; })";
|
|
503
|
+
/**
|
|
504
|
+
* Browser-side expression reading a prompt-selector widget's OWN committed-value
|
|
505
|
+
* text — the single source of truth for "is this widget filled?", shared by the
|
|
506
|
+
* widget-enumeration phase ({@link tryPromptSelectorPrimitive}'s
|
|
507
|
+
* `enumerateWidgetsExpr`) and the act-success committed-value gate
|
|
508
|
+
* ({@link verifyPromptSelectorCommitted}) so the two never drift into separate
|
|
509
|
+
* ideas of "committed". Value via the union: `aria-activedescendant` → the
|
|
510
|
+
* referenced option's text, else the widget's own value (an `<input>`'s
|
|
511
|
+
* `value`, or a `<button>`-trigger's own label text via {@link BUTTON_VALUE_EXPR}),
|
|
512
|
+
* else a selection-label node ({@link PROMPT_VALUE_SELECTORS}) — treating the
|
|
513
|
+
* widget-kit empty-state phrase ({@link PROMPT_EMPTY_VALUE_RX}) as no value.
|
|
514
|
+
*/
|
|
515
|
+
const PROMPT_CURRENT_TEXT_EXPR = `((w, valueSel, emptyRx) => {
|
|
516
|
+
const norm = (s) => (s || "").replace(/\\s+/g, " ").trim().toLowerCase();
|
|
517
|
+
const buttonValue = ${BUTTON_VALUE_EXPR};
|
|
518
|
+
const adid = w.getAttribute && w.getAttribute("aria-activedescendant");
|
|
519
|
+
if (adid) { const opt = document.getElementById(adid); if (opt && opt.textContent.trim()) return norm(opt.textContent); }
|
|
520
|
+
if (w.tagName === "INPUT" && (w.value || "").trim()) return norm(w.value);
|
|
521
|
+
if (w.tagName === "BUTTON" && buttonValue(w).trim()) return norm(buttonValue(w));
|
|
522
|
+
const lbl = w.matches(valueSel) ? w : w.querySelector(valueSel);
|
|
523
|
+
const raw = lbl ? (lbl.textContent || "") : "";
|
|
524
|
+
if (raw.trim() && !emptyRx.test(raw.trim())) return norm(raw);
|
|
525
|
+
return "";
|
|
526
|
+
})`;
|
|
527
|
+
/**
|
|
528
|
+
* Browser-side expression resolving the DOM root within which a chosen widget's
|
|
529
|
+
* popup options / filter input live. Popup placement is vendor-split: the ARIA
|
|
530
|
+
* standard and most libraries (MUI/Radix/react-select) render the listbox in a
|
|
531
|
+
* PORTAL at `document.body` linked by `aria-controls`/`aria-owns`, while others
|
|
532
|
+
* (the Canvas/UXI kit) render it INLINE inside the widget. So resolve in order:
|
|
533
|
+
* (1) the `aria-controls`/`aria-owns` target of the widget or its trigger
|
|
534
|
+
* descendant (portal-safe); (2) the widget subtree itself if it already
|
|
535
|
+
* contains the popup (inline); (3) `document` as a last resort. Returns the
|
|
536
|
+
* scope Element/Document; callers query options/search within it, which stops a
|
|
537
|
+
* sibling widget's options from being picked on a multi-widget page.
|
|
538
|
+
*/
|
|
539
|
+
const PROMPT_SCOPE_ROOT_EXPR = `((w) => {
|
|
540
|
+
const refAttr = (el) => (el && (el.getAttribute("aria-controls") || el.getAttribute("aria-owns"))) || "";
|
|
541
|
+
let ids = refAttr(w);
|
|
542
|
+
if (!ids) {
|
|
543
|
+
const inner = w.querySelector("[aria-controls],[aria-owns]");
|
|
544
|
+
if (inner) ids = refAttr(inner);
|
|
545
|
+
}
|
|
546
|
+
if (ids) {
|
|
547
|
+
// aria-controls/aria-owns may reference MULTIPLE ids (e.g. a listbox plus a
|
|
548
|
+
// status region). Prefer the referenced element that IS or CONTAINS a
|
|
549
|
+
// listbox/option (the portaled popup).
|
|
550
|
+
const refs = ids.split(/\\s+/).map((id) => document.getElementById(id)).filter(Boolean);
|
|
551
|
+
const withListbox = refs.find((el) => el.matches("[role='listbox']") || el.querySelector("[role='listbox'],[role='option']"));
|
|
552
|
+
if (withListbox) return withListbox;
|
|
553
|
+
}
|
|
554
|
+
// No referenced listbox: try the inline popup in the widget's own subtree
|
|
555
|
+
// BEFORE falling back to a resolved-but-listbox-less ref, then to document.
|
|
556
|
+
if (w.querySelector("[role='listbox'],[role='option']")) return w;
|
|
557
|
+
if (ids) {
|
|
558
|
+
const refs = ids.split(/\\s+/).map((id) => document.getElementById(id)).filter(Boolean);
|
|
559
|
+
if (refs.length) return refs[0];
|
|
560
|
+
}
|
|
561
|
+
return document;
|
|
562
|
+
})`;
|
|
379
563
|
/**
|
|
380
564
|
* Extra bounded poll window for a network-only advance whose real
|
|
381
565
|
* `TransitionWorklet(type="next")` POST lands AFTER the `STEP_PAUSE_MS`
|
|
@@ -777,8 +961,20 @@ function isAdvanceStep(instruction) {
|
|
|
777
961
|
* cascade depends on is unit-testable, not buried in `executeStepWithHealing`.
|
|
778
962
|
*/
|
|
779
963
|
function shouldCaptureSelectionState(params) {
|
|
780
|
-
const { step, isFinalStep, submitStep } = params;
|
|
781
|
-
return !(
|
|
964
|
+
const { step, isFinalStep, submitStep, flowHasSubmitSemantics } = params;
|
|
965
|
+
return !(submitStep || (isFinalStep && flowHasSubmitSemantics) || isAdvanceStep(step));
|
|
966
|
+
}
|
|
967
|
+
/**
|
|
968
|
+
* Whether a flow has ANY submit semantics at all — a step flagged
|
|
969
|
+
* `submitStep: true`, a `submitEndpointPattern`, or `requireSubmitEndpointMatch`.
|
|
970
|
+
* A read-only flow (none of the three) has no submit shape anywhere, so its
|
|
971
|
+
* final step is an ordinary read/click, not a submit. Pure + exported so
|
|
972
|
+
* callers can stop inferring submit-shape from `isFinalStep` alone on flows
|
|
973
|
+
* that never declared a submit.
|
|
974
|
+
*/
|
|
975
|
+
function flowHasSubmitSemantics(params) {
|
|
976
|
+
const { steps, submitEndpointPattern, requireSubmitEndpointMatch } = params;
|
|
977
|
+
return (steps.some((s) => s.submitStep) || submitEndpointPattern !== null || requireSubmitEndpointMatch);
|
|
782
978
|
}
|
|
783
979
|
/**
|
|
784
980
|
* Whether a flow should be WARNed that its DOM-only advance guard is disarmed.
|
|
@@ -1054,10 +1250,10 @@ function isDomOnlyAdvanceVerified(params) {
|
|
|
1054
1250
|
function isClickViewSwapVerified(params) {
|
|
1055
1251
|
const VIEW_SWAP_MIN_BYTES = config_1.config.scraper.viewSwapMinBytesThreshold;
|
|
1056
1252
|
const VIEW_SWAP_REVEAL_MIN_BYTES = config_1.config.scraper.viewSwapRevealMinBytesThreshold;
|
|
1057
|
-
const { resolvedAction, isFinalStep, submitStep, isAdvanceWithPattern, networkDelta, bytesDelta, textChanged, invalidMarkerDelta = 0, } = params;
|
|
1253
|
+
const { resolvedAction, isFinalStep, submitStep, flowHasSubmitSemantics, isAdvanceWithPattern, networkDelta, bytesDelta, textChanged, invalidMarkerDelta = 0, } = params;
|
|
1058
1254
|
if (resolvedAction?.method !== "click")
|
|
1059
1255
|
return false;
|
|
1060
|
-
if (
|
|
1256
|
+
if (submitStep || (isFinalStep && flowHasSubmitSemantics))
|
|
1061
1257
|
return false;
|
|
1062
1258
|
if (isAdvanceWithPattern)
|
|
1063
1259
|
return false;
|
|
@@ -2569,6 +2765,50 @@ async function verifyFillReadback(target, selector, expectedValue) {
|
|
|
2569
2765
|
return null;
|
|
2570
2766
|
}
|
|
2571
2767
|
}
|
|
2768
|
+
/**
|
|
2769
|
+
* Committed-value guard for the prompt-selector widget family
|
|
2770
|
+
* (`data-uxi-widget-type='multiselect'`/`selectinput`, `role=combobox`, …),
|
|
2771
|
+
* mirroring {@link verifyFillReadback}'s role for plain inputs but reading the
|
|
2772
|
+
* widget's OWN committed-value node ({@link PROMPT_CURRENT_TEXT_EXPR}) instead
|
|
2773
|
+
* of a bare `<input>.value` — a chip multiselect's committed state lives on
|
|
2774
|
+
* `aria-activedescendant`/a selection-label node, not the trigger's own value.
|
|
2775
|
+
* Walks UP from the resolved element to the nearest {@link PROMPT_TRIGGER_SELECTORS}
|
|
2776
|
+
* ancestor (the resolved element is often the inner filter input or icon, not
|
|
2777
|
+
* the widget container itself — same resolution discipline as
|
|
2778
|
+
* `tryPromptSelectorPrimitive`'s widget-container walk). Returns
|
|
2779
|
+
* `isPromptWidget: false` for a resolved element that isn't part of this
|
|
2780
|
+
* widget family at all, so a non-prompt click/fill is untouched.
|
|
2781
|
+
*/
|
|
2782
|
+
async function verifyPromptSelectorCommitted(target, selector) {
|
|
2783
|
+
const xpath = xpathBodyForEvaluate(selector);
|
|
2784
|
+
if (xpath === null)
|
|
2785
|
+
return null;
|
|
2786
|
+
const expr = `(() => {
|
|
2787
|
+
const triggerSel = ${JSON.stringify(PROMPT_TRIGGER_SELECTORS)};
|
|
2788
|
+
const valueSel = ${JSON.stringify(PROMPT_VALUE_SELECTORS)};
|
|
2789
|
+
const emptyRx = new RegExp(${JSON.stringify(PROMPT_EMPTY_VALUE_RX_SRC)}, ${JSON.stringify(PROMPT_EMPTY_VALUE_RX_FLAGS)});
|
|
2790
|
+
const currentText = ${PROMPT_CURRENT_TEXT_EXPR};
|
|
2791
|
+
const xpath = ${JSON.stringify(xpath)};
|
|
2792
|
+
const result = document.evaluate(xpath, document, null, XPathResult.FIRST_ORDERED_NODE_TYPE, null);
|
|
2793
|
+
const el = result.singleNodeValue;
|
|
2794
|
+
if (!el) return { isPromptWidget: false, committed: false };
|
|
2795
|
+
const widget = el.matches && el.matches(triggerSel) ? el : el.closest(triggerSel);
|
|
2796
|
+
if (!widget) return { isPromptWidget: false, committed: false };
|
|
2797
|
+
return { isPromptWidget: true, committed: currentText(widget, valueSel, emptyRx) !== "" };
|
|
2798
|
+
})()`;
|
|
2799
|
+
try {
|
|
2800
|
+
const raw = await target.evaluate(expr);
|
|
2801
|
+
if (raw === null || typeof raw !== "object")
|
|
2802
|
+
return null;
|
|
2803
|
+
const r = raw;
|
|
2804
|
+
if (typeof r.isPromptWidget !== "boolean" || typeof r.committed !== "boolean")
|
|
2805
|
+
return null;
|
|
2806
|
+
return { isPromptWidget: r.isPromptWidget, committed: r.committed };
|
|
2807
|
+
}
|
|
2808
|
+
catch {
|
|
2809
|
+
return null;
|
|
2810
|
+
}
|
|
2811
|
+
}
|
|
2572
2812
|
/**
|
|
2573
2813
|
* Pull field-level errors out of an arbitrary JSON response body. Walks a
|
|
2574
2814
|
* few of the conventional ATS shapes; falls through to `[]` so the caller
|
|
@@ -2835,6 +3075,21 @@ async function readElementSelectionFingerprint(target, selector) {
|
|
|
2835
3075
|
* nesting without over-reaching into an outer listbox/group.
|
|
2836
3076
|
*/
|
|
2837
3077
|
const MAX_SELECTION_ANCESTOR_DEPTH = 6;
|
|
3078
|
+
/**
|
|
3079
|
+
* Cross-vendor selector union for a selection-state widget that carries NO
|
|
3080
|
+
* standard selection `role` or `aria-*`/`data-state` marker — a component-kit
|
|
3081
|
+
* container whose selected-ness lives only in the library's own private
|
|
3082
|
+
* attribute. Same multi-vendor-union discipline as {@link INVALID_MARKER_CLASS_SOURCE}
|
|
3083
|
+
* and the `PROMPT_*_SELECTORS` unions: standards are checked FIRST (see
|
|
3084
|
+
* `hasMarker` in {@link selectionAncestorChanged}); this union is the fallback
|
|
3085
|
+
* for widgets that under-annotate ARIA, and no member is a per-site branch —
|
|
3086
|
+
* each is one component library's signature. Grows by a one-line edit.
|
|
3087
|
+
*
|
|
3088
|
+
* Members: `data-baseweb` (Uber Base Web — verified in a real capture to mark
|
|
3089
|
+
* 150 selection elements that expose no role/aria-state, so dropping it loses
|
|
3090
|
+
* real coverage). Add other under-annotating kits here as they surface.
|
|
3091
|
+
*/
|
|
3092
|
+
const WIDGET_KIT_SELECTION_MARKER_SELECTORS = ["[data-baseweb]"].join(",");
|
|
2838
3093
|
/**
|
|
2839
3094
|
* Element-scoped selection read-back for the case the clicked node's OWN
|
|
2840
3095
|
* fingerprint can't credit: a design-system option that wraps its label in a
|
|
@@ -2847,7 +3102,8 @@ const MAX_SELECTION_ANCESTOR_DEPTH = 6;
|
|
|
2847
3102
|
*
|
|
2848
3103
|
* Walks from the leaf up to {@link MAX_SELECTION_ANCESTOR_DEPTH}, and on the
|
|
2849
3104
|
* NEAREST ancestor that (a) carries a selection marker (a non-empty fingerprint
|
|
2850
|
-
* field, a selection `role`, or a
|
|
3105
|
+
* field, a selection `role`, or a component-kit marker from
|
|
3106
|
+
* {@link WIDGET_KIT_SELECTION_MARKER_SELECTORS} — `aria-expanded` is
|
|
2851
3107
|
* deliberately NOT a marker so a bare disclosure/expander is skipped) AND (b) is
|
|
2852
3108
|
* present in the pre-baseline map, diffs that ancestor's current fingerprint
|
|
2853
3109
|
* against its baseline. Nearest-wins and returns even when unchanged, so a
|
|
@@ -2866,10 +3122,13 @@ async function selectionAncestorChanged(target, leafXpath, preSelectionState) {
|
|
|
2866
3122
|
const xpathOf = ${XPATH_OF_FN_SRC};
|
|
2867
3123
|
const fp = (el) => { const ds = el.getAttribute("data-state") || ""; return ${selectionFingerprintObjSrc("el", "ds")}; };
|
|
2868
3124
|
const SELECTION_ROLES = new Set(["option", "tab", "switch", "radio", "checkbox", "menuitemcheckbox"]);
|
|
3125
|
+
const KIT_MARKER_SEL = ${JSON.stringify(WIDGET_KIT_SELECTION_MARKER_SELECTORS)};
|
|
2869
3126
|
const hasMarker = (el, f) => {
|
|
3127
|
+
// Standards first: fingerprint fields / aria-states, then a selection role.
|
|
2870
3128
|
if (f.kind || f.ariaPressed || f.ariaChecked || f.ariaSelected || f.dataState || f.dataSelected || f.dataChecked || f.checked) return true;
|
|
2871
3129
|
if (SELECTION_ROLES.has((el.getAttribute("role") || "").toLowerCase())) return true;
|
|
2872
|
-
|
|
3130
|
+
// Fallback: a component-kit selection widget that exposes no standard marker.
|
|
3131
|
+
if (el.matches(KIT_MARKER_SEL)) return true;
|
|
2873
3132
|
return false;
|
|
2874
3133
|
};
|
|
2875
3134
|
const changed = (a, b) =>
|
|
@@ -3356,11 +3615,54 @@ async function setFilesViaCdp(params) {
|
|
|
3356
3615
|
* Recognizes the flow's conventional phrasings, all quoted:
|
|
3357
3616
|
* "select 'Yes'", "select or check 'BLS'",
|
|
3358
3617
|
* "for 'What is your highest level…?' select 'BSN completed'",
|
|
3359
|
-
* "select 'Texas' in the State/Region dropdown"
|
|
3618
|
+
* "select 'Texas' in the State/Region dropdown",
|
|
3619
|
+
* "…then select the option 'Job Boards' from the popup list" (a compound
|
|
3620
|
+
* step that opens the widget in one clause and selects in another —
|
|
3621
|
+
* "select" is followed by a noun phrase like "the option"/"the value"
|
|
3622
|
+
* before the quoted option itself, rather than the quote directly).
|
|
3360
3623
|
* Returns null when the step is not a single-dropdown select (e.g. generic
|
|
3361
3624
|
* "for any remaining question…" catch-alls, or radio/checkbox-only steps) so
|
|
3362
3625
|
* the caller falls through to the normal cascade.
|
|
3363
3626
|
*/
|
|
3627
|
+
// Nouns that name a form widget/question, used by `pickQuestionLabel` to tell
|
|
3628
|
+
// a genuine field label ("'How Did You Hear About Us?' prompt selector")
|
|
3629
|
+
// apart from an unrelated leading quoted phrase that just happens to precede
|
|
3630
|
+
// it in the instruction (e.g. a page/step-context quote like "'My
|
|
3631
|
+
// Information' step").
|
|
3632
|
+
const WIDGET_NOUN_RE = /\b(prompt\s+selector|multiselect|typeahead|dropdown|field|question|checkbox|radio\s+button|radio)\b/i;
|
|
3633
|
+
/**
|
|
3634
|
+
* Pick the QUESTION LABEL out of an instruction's quoted phrases, given the
|
|
3635
|
+
* already-extracted OPTION.
|
|
3636
|
+
*
|
|
3637
|
+
* Why this exists: an instruction can carry more than one quoted phrase that
|
|
3638
|
+
* is not the option — e.g. a page/step-context phrase ("On the authenticated
|
|
3639
|
+
* 'My Information' step, open the 'How Did You Hear About Us?' prompt
|
|
3640
|
+
* selector…") — so naively taking the first non-option quote picks the
|
|
3641
|
+
* context phrase instead of the actual widget label. This prefers a quote
|
|
3642
|
+
* that sits immediately next to a widget noun (`multiselect`/`typeahead`/
|
|
3643
|
+
* `dropdown`/`field`/`question`/`checkbox`/`radio button`/`prompt selector`)
|
|
3644
|
+
* or is introduced
|
|
3645
|
+
* by "for"/"for the"/"for the question"/"for the answer" (e.g. "click the
|
|
3646
|
+
* 'Yes' answer for the question '…'"), before falling back to the first
|
|
3647
|
+
* non-option quote so existing un-adorned phrasings keep working.
|
|
3648
|
+
*/
|
|
3649
|
+
function pickQuestionLabel(instruction, option) {
|
|
3650
|
+
// biome-ignore lint/style/noNonNullAssertion: capture group 1 is required by the pattern, so it is present on every match
|
|
3651
|
+
const candidates = [...instruction.matchAll(/'([^']+)'/g)].filter((m) => m[1].trim() !== option);
|
|
3652
|
+
if (candidates.length === 0)
|
|
3653
|
+
return null;
|
|
3654
|
+
const adjacentToWidgetNoun = candidates.find((m) => {
|
|
3655
|
+
const start = m.index ?? 0;
|
|
3656
|
+
const end = start + m[0].length;
|
|
3657
|
+
const before = instruction.slice(Math.max(0, start - 25), start);
|
|
3658
|
+
const after = instruction.slice(end, end + 40);
|
|
3659
|
+
return (/\bfor(?:\s+the)?(?:\s+(?:question|answer))?\s*$/i.test(before) || WIDGET_NOUN_RE.test(after));
|
|
3660
|
+
});
|
|
3661
|
+
// biome-ignore lint/style/noNonNullAssertion: candidates is guarded non-empty above, so the fallback element is always present
|
|
3662
|
+
const picked = (adjacentToWidgetNoun ?? candidates[0]);
|
|
3663
|
+
// biome-ignore lint/style/noNonNullAssertion: capture group 1 is required by the pattern, so it is present on every match
|
|
3664
|
+
return picked[1].trim();
|
|
3665
|
+
}
|
|
3364
3666
|
function parseSelectStep(instruction) {
|
|
3365
3667
|
const lower = instruction.toLowerCase();
|
|
3366
3668
|
// Must look like a dropdown selection, not a radio/checkbox click. "select
|
|
@@ -3376,16 +3678,20 @@ function parseSelectStep(instruction) {
|
|
|
3376
3678
|
const quoted = [...instruction.matchAll(/'([^']+)'/g)].map((m) => m[1]);
|
|
3377
3679
|
if (quoted.length === 0)
|
|
3378
3680
|
return null;
|
|
3379
|
-
// The OPTION is the quoted string
|
|
3380
|
-
|
|
3681
|
+
// The OPTION is the quoted string following the word "select", allowing a
|
|
3682
|
+
// short filler ("the option", "the answer") between the verb and the
|
|
3683
|
+
// quote — real flow phrasing like "select the option 'Job Boards'" or
|
|
3684
|
+
// "select the answer 'X'" otherwise fails to parse and the caller
|
|
3685
|
+
// silently no-ops before touching the DOM.
|
|
3686
|
+
const selMatch = instruction.match(/\bselect(?:\s+or\s+check)?\s+(?:the\s+\S+\s+)?'([^']+)'/i);
|
|
3381
3687
|
if (!selMatch)
|
|
3382
3688
|
return null;
|
|
3383
3689
|
// biome-ignore lint/style/noNonNullAssertion: guarded by the !selMatch early-return; group 1 is required by the pattern
|
|
3384
3690
|
const option = selMatch[1].trim();
|
|
3385
3691
|
// The QUESTION LABEL, when present, is a DIFFERENT quoted string — the one
|
|
3386
|
-
// introduced by "for '…'" or
|
|
3387
|
-
//
|
|
3388
|
-
const questionLabel =
|
|
3692
|
+
// introduced by "for '…'" or adjacent to a widget noun like "'…' dropdown"
|
|
3693
|
+
// or "'…' prompt selector". See `pickQuestionLabel`.
|
|
3694
|
+
const questionLabel = pickQuestionLabel(instruction, option);
|
|
3389
3695
|
return { option, questionLabel };
|
|
3390
3696
|
}
|
|
3391
3697
|
/**
|
|
@@ -3503,8 +3809,9 @@ function parseRadioStep(instruction) {
|
|
|
3503
3809
|
// introduced by "for the question '…'" / "for the '…' question". Some steps
|
|
3504
3810
|
// phrase the question un-quoted ("…about requiring visa sponsorship"); in
|
|
3505
3811
|
// that case there is no second quoted string and questionLabel stays null,
|
|
3506
|
-
// which the primitive handles via LLM group-matching.
|
|
3507
|
-
|
|
3812
|
+
// which the primitive handles via LLM group-matching. See
|
|
3813
|
+
// `pickQuestionLabel`.
|
|
3814
|
+
const questionLabel = pickQuestionLabel(instruction, option);
|
|
3508
3815
|
return { option, questionLabel };
|
|
3509
3816
|
}
|
|
3510
3817
|
/**
|
|
@@ -4652,6 +4959,430 @@ async function applyRadioSelection(target, gi, ri, hint) {
|
|
|
4652
4959
|
await target.evaluate(applyExpr).catch(() => ({ ok: false }));
|
|
4653
4960
|
return await readback();
|
|
4654
4961
|
}
|
|
4962
|
+
/**
|
|
4963
|
+
* Answer a native-control-less popup-dropdown widget (a combobox that opens a
|
|
4964
|
+
* listbox popup and renders NO `<select>`/`<input>` a focused `<select>`/
|
|
4965
|
+
* `<input>` probe can see). Detected by a cross-vendor UNION of shape signals
|
|
4966
|
+
* ({@link PROMPT_TRIGGER_SELECTORS}/{@link PROMPT_OPTION_SELECTORS}), not any
|
|
4967
|
+
* one vendor's private attribute — standards (ARIA `combobox`/`listbox`/
|
|
4968
|
+
* `option`) where the widget exposes them, plus the well-known Canvas/UXI
|
|
4969
|
+
* widget-kit markers as co-equal union members for the widgets that under-
|
|
4970
|
+
* annotate ARIA (some emit `role=option` only once the popup is open, and no
|
|
4971
|
+
* `role=combobox` at all).
|
|
4972
|
+
*
|
|
4973
|
+
* Why this exists (parallels `trySelectPrimitive`/`tryRadioPrimitive`): such a
|
|
4974
|
+
* widget renders as neither a native `<select>` nor a control the observe
|
|
4975
|
+
* cascade resolves to a click target, so the focused probe reports "0
|
|
4976
|
+
* candidates" and the step strands. This primitive finds the widget by the
|
|
4977
|
+
* trigger union, opens the popup with a real Playwright click gesture (a bare
|
|
4978
|
+
* `el.click()` does not fire these widgets' open handler — the same failure
|
|
4979
|
+
* mode as the MUI radio/select widgets), types into an in-popup filter input
|
|
4980
|
+
* when one is present (the searchable/typeahead variant renders only a partial
|
|
4981
|
+
* option slice until filtered), and clicks the matching option with a real
|
|
4982
|
+
* click.
|
|
4983
|
+
*
|
|
4984
|
+
* Accepts a SELECT-shaped step (`parseSelectStep`, "select 'X' in the Y
|
|
4985
|
+
* dropdown"), a FILL-shaped step (`parseFillStep`, "Fill in the Y field with
|
|
4986
|
+
* 'X'"), or an ANSWER/RADIO-shaped step (`parseRadioStep`, "click the 'Yes'
|
|
4987
|
+
* answer for the question '…'") — flow/replan generation describes this
|
|
4988
|
+
* widget family's searchable variant as a fill because its filter box renders
|
|
4989
|
+
* a real `<input>`, and describes its Yes/No variant with the same
|
|
4990
|
+
* answer-verb phrasing used for native radio groups, even though committing
|
|
4991
|
+
* either still requires the popup-open/option-click gesture below, not typed
|
|
4992
|
+
* text or a bare click. `tryRadioPrimitive` (called first, see the call site)
|
|
4993
|
+
* already claims answer-verb steps whose target has native
|
|
4994
|
+
* `input[type=radio]` elements; this primitive only ever sees an answer-verb
|
|
4995
|
+
* step after that primitive has fallen through for lack of any, so there is
|
|
4996
|
+
* no double-claim. A fill or radio step's `value`/`option` becomes the option
|
|
4997
|
+
* to match and its `fieldLabel`/`questionLabel` becomes the question label;
|
|
4998
|
+
* downstream matching is identical across all three shapes.
|
|
4999
|
+
*
|
|
5000
|
+
* Matches the target widget by `questionLabel` (when the step carries one) or,
|
|
5001
|
+
* failing that, by there being exactly one unfilled widget on the page —
|
|
5002
|
+
* deliberately conservative: an ambiguous multi-widget page with no question
|
|
5003
|
+
* label falls through to the cascade rather than guessing which widget the
|
|
5004
|
+
* step means. Option matching mirrors `trySelectPrimitive`: an exact
|
|
5005
|
+
* (normalized) text match is applied directly; otherwise, when an LLM client
|
|
5006
|
+
* is present, `judgeSelectOptionWithLLM` picks the best rendered option
|
|
5007
|
+
* (per-requisition option variance).
|
|
5008
|
+
*
|
|
5009
|
+
* Returns the resolved widget's DOM id (or `""`) on success — the step's
|
|
5010
|
+
* `targetId` — or `null` when unhandled (no widget, no unambiguous widget
|
|
5011
|
+
* match, no option match, or the selection didn't commit), so the caller
|
|
5012
|
+
* falls through to the cascade unchanged, matching
|
|
5013
|
+
* `trySelectPrimitive`/`tryRadioPrimitive`'s null-fallthrough contract.
|
|
5014
|
+
*/
|
|
5015
|
+
async function tryPromptSelectorPrimitive(params) {
|
|
5016
|
+
const { page, target, instruction, logger, anthropic, captureFn } = params;
|
|
5017
|
+
// A prompt-selector widget's search box renders a real <input>, so flow/
|
|
5018
|
+
// replan generation routinely describes filling it as a FILL step ("Fill in
|
|
5019
|
+
// the 'How Did You Hear About Us?' field with 'Internet/Online'") rather
|
|
5020
|
+
// than a SELECT step, and its Yes/No variant renders no native radio inputs
|
|
5021
|
+
// at all, so flow/replan generation describes it with the same answer-verb
|
|
5022
|
+
// phrasing used for native radios ("Click the 'Yes' answer for the question
|
|
5023
|
+
// '…'"). Accept all three shapes: parseSelectStep first, then parseFillStep
|
|
5024
|
+
// (fieldLabel -> questionLabel, value -> option), then parseRadioStep
|
|
5025
|
+
// (option/questionLabel already in this primitive's shape) so the widget-
|
|
5026
|
+
// matching/open/readback phases below run unchanged regardless of which
|
|
5027
|
+
// verb the instruction used.
|
|
5028
|
+
const parsedSelect = parseSelectStep(instruction);
|
|
5029
|
+
const parsedFill = parsedSelect ? null : parseFillStep(instruction);
|
|
5030
|
+
const parsedAnswer = parsedSelect || parsedFill ? null : parseRadioStep(instruction);
|
|
5031
|
+
const parsed = parsedSelect
|
|
5032
|
+
? parsedSelect
|
|
5033
|
+
: parsedFill
|
|
5034
|
+
? { option: parsedFill.value, questionLabel: stripQuotedLabel(parsedFill.fieldLabel) }
|
|
5035
|
+
: parsedAnswer;
|
|
5036
|
+
if (!parsed)
|
|
5037
|
+
return null;
|
|
5038
|
+
const { option, questionLabel } = parsed;
|
|
5039
|
+
const optLabel = `option "${option.slice(0, 40)}"${questionLabel ? `, question "${questionLabel.slice(0, 40)}"` : ""}`;
|
|
5040
|
+
const norm = (s) => s.replace(/\s+/g, " ").trim().toLowerCase();
|
|
5041
|
+
// Phase 1 (browser, read-only except for the marker attribute stamped for
|
|
5042
|
+
// Phase 2's click addressing): find candidate widgets — popup-dropdown
|
|
5043
|
+
// triggers with no native <select> — that are still unfilled/invalid. Options
|
|
5044
|
+
// are NOT enumerated here; these widgets only render option entries once the
|
|
5045
|
+
// popup is open.
|
|
5046
|
+
const enumerateWidgetsExpr = `((markAttr, triggerSel, valueSel, emptyRxSrc, emptyRxFlags) => {
|
|
5047
|
+
const norm = (s) => (s || "").replace(/\\s+/g, " ").trim().toLowerCase();
|
|
5048
|
+
const buttonValue = ${BUTTON_VALUE_EXPR};
|
|
5049
|
+
const emptyRx = new RegExp(emptyRxSrc, emptyRxFlags);
|
|
5050
|
+
const triggers = Array.from(document.querySelectorAll(triggerSel));
|
|
5051
|
+
if (triggers.length === 0) return { widgetPresent: false };
|
|
5052
|
+
const seen = new Set();
|
|
5053
|
+
const resolved = [];
|
|
5054
|
+
for (const el of triggers) {
|
|
5055
|
+
// Resolve each union hit to ONE widget container. A single widget often
|
|
5056
|
+
// matches the union more than once (e.g. an outer container AND its inner
|
|
5057
|
+
// filter input both carry data-uxi-widget-type), so prefer a real
|
|
5058
|
+
// interactive ancestor, else the OUTERMOST widget-kit container on the
|
|
5059
|
+
// ancestor chain (so the inner input and its container collapse to the
|
|
5060
|
+
// same node), else the element itself.
|
|
5061
|
+
const interactive = el.closest("button,[role='button'],[role='combobox']");
|
|
5062
|
+
let container = interactive || el;
|
|
5063
|
+
if (!interactive) {
|
|
5064
|
+
const kitSel = "[data-uxi-widget-type='selectinput'],[data-uxi-widget-type='multiselect'],[data-automation-id='multiSelectContainer']";
|
|
5065
|
+
let node = el.closest(kitSel);
|
|
5066
|
+
while (node) {
|
|
5067
|
+
container = node;
|
|
5068
|
+
const up = node.parentElement && node.parentElement.closest(kitSel);
|
|
5069
|
+
if (!up || up === node) break;
|
|
5070
|
+
node = up;
|
|
5071
|
+
}
|
|
5072
|
+
}
|
|
5073
|
+
if (seen.has(container)) continue;
|
|
5074
|
+
seen.add(container);
|
|
5075
|
+
resolved.push(container);
|
|
5076
|
+
}
|
|
5077
|
+
// Drop any candidate contained by another (belt-and-suspenders against a
|
|
5078
|
+
// widget whose union hits resolve to nested containers).
|
|
5079
|
+
const widgets = resolved.filter((w) => !resolved.some((o) => o !== w && o.contains(w)));
|
|
5080
|
+
if (widgets.length === 0) return { widgetPresent: false };
|
|
5081
|
+
const isInvalid = ${INVALID_MARKER_EL_EXPR};
|
|
5082
|
+
const widgetLabel = (w) => {
|
|
5083
|
+
// Standard first: aria-labelledby, aria-label, then a <label for=id>
|
|
5084
|
+
// referencing the widget or a control inside it, then a labelled group
|
|
5085
|
+
// ancestor (role=group[aria-labelledby] / fieldset<legend>), then the
|
|
5086
|
+
// nearest non-empty ancestor text as a last resort.
|
|
5087
|
+
const alb = w.getAttribute("aria-labelledby");
|
|
5088
|
+
if (alb) {
|
|
5089
|
+
const parts = [];
|
|
5090
|
+
for (const id of alb.split(/\\s+/)) { const el = document.getElementById(id); if (el) parts.push(el.textContent); }
|
|
5091
|
+
if (parts.length) return parts.join(" ").replace(/\\s+/g, " ").trim().slice(0, 120);
|
|
5092
|
+
}
|
|
5093
|
+
const al = w.getAttribute("aria-label");
|
|
5094
|
+
if (al) return al.replace(/\\s+/g, " ").trim().slice(0, 120);
|
|
5095
|
+
const labelledIds = [w.id, ...Array.from(w.querySelectorAll("[id]")).map((e) => e.id)].filter(Boolean);
|
|
5096
|
+
for (const id of labelledIds) {
|
|
5097
|
+
const lab = document.querySelector("label[for='" + (window.CSS && CSS.escape ? CSS.escape(id) : id) + "']");
|
|
5098
|
+
if (lab && lab.textContent.trim()) return lab.textContent.replace(/\\s+/g, " ").trim().slice(0, 120);
|
|
5099
|
+
}
|
|
5100
|
+
const grp = w.closest("[role='group'][aria-labelledby],fieldset");
|
|
5101
|
+
if (grp) {
|
|
5102
|
+
const gid = grp.getAttribute("aria-labelledby");
|
|
5103
|
+
const gref = gid ? document.getElementById(gid.split(/\\s+/)[0]) : grp.querySelector("legend");
|
|
5104
|
+
if (gref && gref.textContent.trim()) return gref.textContent.replace(/\\s+/g, " ").trim().slice(0, 120);
|
|
5105
|
+
}
|
|
5106
|
+
let node = w.parentElement;
|
|
5107
|
+
for (let d = 0; d < 5 && node; d++) {
|
|
5108
|
+
const t = (node.textContent || "").trim();
|
|
5109
|
+
if (t) return t.replace(/\\s+/g, " ").trim().slice(0, 120);
|
|
5110
|
+
node = node.parentElement;
|
|
5111
|
+
}
|
|
5112
|
+
return "";
|
|
5113
|
+
};
|
|
5114
|
+
const currentText = ${PROMPT_CURRENT_TEXT_EXPR};
|
|
5115
|
+
const currentTextOf = (w) => currentText(w, valueSel, emptyRx);
|
|
5116
|
+
const isUnfilled = (w) => {
|
|
5117
|
+
let node = w;
|
|
5118
|
+
for (let d = 0; d < 6 && node; d++) {
|
|
5119
|
+
if (node.getAttribute && isInvalid(node)) return true;
|
|
5120
|
+
node = node.parentElement;
|
|
5121
|
+
}
|
|
5122
|
+
return currentTextOf(w) === "";
|
|
5123
|
+
};
|
|
5124
|
+
// Clear stale marks from a prior call on this same page (this primitive
|
|
5125
|
+
// runs once per "select 'X'" step, and an application wizard answers several
|
|
5126
|
+
// such steps on the same unreloaded page) — otherwise a widget already
|
|
5127
|
+
// filled by an earlier call keeps its old index and collides with whatever
|
|
5128
|
+
// new widget claims that index this round, and the trigger-click selector's
|
|
5129
|
+
// \`.first()\` can resolve to the stale widget.
|
|
5130
|
+
for (const el of document.querySelectorAll("[" + markAttr + "]")) el.removeAttribute(markAttr);
|
|
5131
|
+
const candidates = [];
|
|
5132
|
+
let idx = 0;
|
|
5133
|
+
for (const w of widgets) {
|
|
5134
|
+
if (!isUnfilled(w)) continue;
|
|
5135
|
+
w.setAttribute(markAttr, String(idx));
|
|
5136
|
+
candidates.push({ wIdx: idx, label: widgetLabel(w) });
|
|
5137
|
+
idx++;
|
|
5138
|
+
}
|
|
5139
|
+
if (candidates.length === 0) return { widgetPresent: true, candidates: [] };
|
|
5140
|
+
return { widgetPresent: true, candidates };
|
|
5141
|
+
})(${JSON.stringify(PROMPT_WIDGET_MARK_ATTR)}, ${JSON.stringify(PROMPT_TRIGGER_SELECTORS)}, ${JSON.stringify(PROMPT_VALUE_SELECTORS)}, ${JSON.stringify(PROMPT_EMPTY_VALUE_RX_SRC)}, ${JSON.stringify(PROMPT_EMPTY_VALUE_RX_FLAGS)})`;
|
|
5142
|
+
try {
|
|
5143
|
+
const enumResult = await pollEnumerate(page, target, enumerateWidgetsExpr, (r) => r?.widgetPresent === true);
|
|
5144
|
+
if (!enumResult?.widgetPresent) {
|
|
5145
|
+
logger.info(`prompt-selector primitive: no prompt widget on page for ${optLabel}; falling through to cascade`);
|
|
5146
|
+
return null;
|
|
5147
|
+
}
|
|
5148
|
+
const candidates = enumResult.candidates ?? [];
|
|
5149
|
+
if (candidates.length === 0) {
|
|
5150
|
+
logger.info(`prompt-selector primitive: no unfilled prompt widget for ${optLabel}; falling through to cascade`);
|
|
5151
|
+
return null;
|
|
5152
|
+
}
|
|
5153
|
+
// Deliberately conservative widget disambiguation (see doc comment): a
|
|
5154
|
+
// labeled match must be UNIQUE, and an unlabeled step only resolves when
|
|
5155
|
+
// there is exactly one unfilled widget on the page.
|
|
5156
|
+
const labelMatches = questionLabel
|
|
5157
|
+
? candidates.filter((c) => c.label !== "" &&
|
|
5158
|
+
(norm(c.label).includes(norm(questionLabel)) ||
|
|
5159
|
+
norm(questionLabel).includes(norm(c.label))))
|
|
5160
|
+
: [];
|
|
5161
|
+
const chosen = labelMatches.length === 1
|
|
5162
|
+
? labelMatches[0]
|
|
5163
|
+
: !questionLabel && candidates.length === 1
|
|
5164
|
+
? candidates[0]
|
|
5165
|
+
: null;
|
|
5166
|
+
if (!chosen) {
|
|
5167
|
+
logger.info(`prompt-selector primitive: no unambiguous widget match for ${optLabel}; falling through to cascade`);
|
|
5168
|
+
return null;
|
|
5169
|
+
}
|
|
5170
|
+
// Phase 2 (real gesture): open the popup. These widgets' trigger requires a
|
|
5171
|
+
// genuine click — a synthetic dispatchEvent does not fire its handler.
|
|
5172
|
+
// Prefer the marked widget's own interactive descendant (a real form
|
|
5173
|
+
// control) over the marked container itself, since the container is
|
|
5174
|
+
// frequently a non-interactive layout wrapper for widget shapes whose
|
|
5175
|
+
// actual trigger sits a level deeper (see PROMPT_INTERACTIVE_CONTROL_SELECTORS).
|
|
5176
|
+
const containerTriggerSel = `[${PROMPT_WIDGET_MARK_ATTR}="${chosen.wIdx}"]`;
|
|
5177
|
+
const innerTriggerSel = PROMPT_INTERACTIVE_CONTROL_SELECTORS.split(",")
|
|
5178
|
+
.map((s) => `${containerTriggerSel} ${s}`)
|
|
5179
|
+
.join(",");
|
|
5180
|
+
let triggerSel = containerTriggerSel;
|
|
5181
|
+
try {
|
|
5182
|
+
const innerCount = await target.locator(innerTriggerSel).count();
|
|
5183
|
+
if (innerCount > 0)
|
|
5184
|
+
triggerSel = innerTriggerSel;
|
|
5185
|
+
}
|
|
5186
|
+
catch {
|
|
5187
|
+
// Fall back to the container selector.
|
|
5188
|
+
}
|
|
5189
|
+
try {
|
|
5190
|
+
await target.locator(triggerSel).first().click();
|
|
5191
|
+
}
|
|
5192
|
+
catch (err) {
|
|
5193
|
+
logger.info(`prompt-selector primitive: trigger click failed for ${optLabel}: ${(0, errors_1.toErrorMessage)(err)}; falling through`);
|
|
5194
|
+
return null;
|
|
5195
|
+
}
|
|
5196
|
+
await page.waitForTimeout(PROMPT_SELECTOR_SETTLE_MS);
|
|
5197
|
+
const enumerateOptionsExpr = `((widgetMarkAttr, wIdx, markAttr, optionSel, searchSel) => {
|
|
5198
|
+
const w = document.querySelector("[" + widgetMarkAttr + '="' + wIdx + '"]');
|
|
5199
|
+
if (!w) return { optionsPresent: false };
|
|
5200
|
+
// Clear stale option + filter marks from a PRIOR prompt-selector call on
|
|
5201
|
+
// this same unreloaded page (an application wizard answers several "select
|
|
5202
|
+
// X" steps without a reload). Both the option-click and the filter-fill
|
|
5203
|
+
// address these marks with a document-wide \`.first()\`, so a leftover mark
|
|
5204
|
+
// on an earlier widget's popup would otherwise win in DOM order — the
|
|
5205
|
+
// inter-call sibling collision (mirrors the widget-mark reset).
|
|
5206
|
+
for (const el of document.querySelectorAll("[" + markAttr + "],[" + markAttr + "-search]")) {
|
|
5207
|
+
el.removeAttribute(markAttr);
|
|
5208
|
+
el.removeAttribute(markAttr + "-search");
|
|
5209
|
+
}
|
|
5210
|
+
// Scope options/search to THIS widget's popup (aria-controls portal, or
|
|
5211
|
+
// inline subtree), so a sibling widget's options are never picked. Fall
|
|
5212
|
+
// back to document only when neither a portal nor an inline popup is found.
|
|
5213
|
+
const scope = ${PROMPT_SCOPE_ROOT_EXPR}(w);
|
|
5214
|
+
const all = Array.from(scope.querySelectorAll(optionSel));
|
|
5215
|
+
// A union hit may be an ancestor of another (e.g. role=listbox > li > p):
|
|
5216
|
+
// keep only leaf-most option nodes with their own text so we don't double
|
|
5217
|
+
// count or address a wrapper.
|
|
5218
|
+
const opts = all.filter((el) => !all.some((o) => o !== el && el.contains(o)));
|
|
5219
|
+
const searchInput = scope.querySelector(searchSel);
|
|
5220
|
+
// Mark the scoped filter input so the Node-side fill addresses THIS
|
|
5221
|
+
// widget's input (Playwright's locator can't re-run the scope resolution).
|
|
5222
|
+
if (searchInput) searchInput.setAttribute(markAttr + "-search", "1");
|
|
5223
|
+
// A searchable/typeahead widget renders NO options until its filter input
|
|
5224
|
+
// is typed into — report the popup as present (with searchable=true) so the
|
|
5225
|
+
// caller types the filter first, rather than falling through as "no popup".
|
|
5226
|
+
if (opts.length === 0) {
|
|
5227
|
+
return searchInput ? { optionsPresent: true, searchable: true, options: [] } : { optionsPresent: false };
|
|
5228
|
+
}
|
|
5229
|
+
const options = [];
|
|
5230
|
+
for (let i = 0; i < opts.length; i++) {
|
|
5231
|
+
opts[i].setAttribute(markAttr, String(i));
|
|
5232
|
+
const label = opts[i].getAttribute("data-automation-label") || opts[i].getAttribute("aria-label") || opts[i].textContent || "";
|
|
5233
|
+
options.push({ oIdx: i, text: label.replace(/\\s+/g, " ").trim() });
|
|
5234
|
+
}
|
|
5235
|
+
return { optionsPresent: true, searchable: !!searchInput, options };
|
|
5236
|
+
})(${JSON.stringify(PROMPT_WIDGET_MARK_ATTR)}, ${JSON.stringify(chosen.wIdx)}, ${JSON.stringify(PROMPT_OPTION_MARK_ATTR)}, ${JSON.stringify(PROMPT_OPTION_SELECTORS)}, ${JSON.stringify(PROMPT_SEARCH_SELECTORS)})`;
|
|
5237
|
+
const optionsInitial = await pollEnumerate(page, target, enumerateOptionsExpr, (r) => r?.optionsPresent === true);
|
|
5238
|
+
if (!optionsInitial?.optionsPresent) {
|
|
5239
|
+
logger.info(`prompt-selector primitive: popup for ${optLabel} did not render options; falling through`);
|
|
5240
|
+
return null;
|
|
5241
|
+
}
|
|
5242
|
+
// Searchable/typeahead variant: type the option text to filter before the
|
|
5243
|
+
// matching option is even rendered (a searchable widget may show only a
|
|
5244
|
+
// paginated slice until filtered). Non-searchable widgets (no in-popup
|
|
5245
|
+
// filter input) commit directly from the already-rendered list — a widget
|
|
5246
|
+
// without a search box must NOT fall through to the cascade.
|
|
5247
|
+
if (!optionsInitial.searchable) {
|
|
5248
|
+
const optionsResult = optionsInitial;
|
|
5249
|
+
return await commitPromptOption({
|
|
5250
|
+
page,
|
|
5251
|
+
target,
|
|
5252
|
+
logger,
|
|
5253
|
+
anthropic,
|
|
5254
|
+
captureFn,
|
|
5255
|
+
optLabel,
|
|
5256
|
+
option,
|
|
5257
|
+
questionLabel,
|
|
5258
|
+
chosen,
|
|
5259
|
+
optionsResult,
|
|
5260
|
+
});
|
|
5261
|
+
}
|
|
5262
|
+
try {
|
|
5263
|
+
// Fill the filter input marked for THIS widget during enumeration (scoped
|
|
5264
|
+
// to its popup), falling back to the union only if the mark is absent.
|
|
5265
|
+
const scopedSearchSel = `[${PROMPT_OPTION_MARK_ATTR}-search="1"]`;
|
|
5266
|
+
const marked = await target.locator(scopedSearchSel).count();
|
|
5267
|
+
await target
|
|
5268
|
+
.locator(marked > 0 ? scopedSearchSel : PROMPT_SEARCH_SELECTORS)
|
|
5269
|
+
.first()
|
|
5270
|
+
.fill(option);
|
|
5271
|
+
}
|
|
5272
|
+
catch (err) {
|
|
5273
|
+
logger.info(`prompt-selector primitive: filter-input type failed for ${optLabel}: ${(0, errors_1.toErrorMessage)(err)}; falling through`);
|
|
5274
|
+
return null;
|
|
5275
|
+
}
|
|
5276
|
+
await page.waitForTimeout(PROMPT_SELECTOR_SETTLE_MS);
|
|
5277
|
+
const optionsFiltered = await pollEnumerate(page, target, enumerateOptionsExpr, (r) => r?.optionsPresent === true);
|
|
5278
|
+
if (!optionsFiltered?.optionsPresent) {
|
|
5279
|
+
logger.info(`prompt-selector primitive: filtered popup for ${optLabel} rendered no options; falling through`);
|
|
5280
|
+
return null;
|
|
5281
|
+
}
|
|
5282
|
+
return await commitPromptOption({
|
|
5283
|
+
page,
|
|
5284
|
+
target,
|
|
5285
|
+
logger,
|
|
5286
|
+
anthropic,
|
|
5287
|
+
captureFn,
|
|
5288
|
+
optLabel,
|
|
5289
|
+
option,
|
|
5290
|
+
questionLabel,
|
|
5291
|
+
chosen,
|
|
5292
|
+
optionsResult: optionsFiltered,
|
|
5293
|
+
});
|
|
5294
|
+
}
|
|
5295
|
+
catch (err) {
|
|
5296
|
+
logger.warn(`prompt-selector primitive: evaluate threw: ${(0, errors_1.toErrorMessage)(err)}; falling through`);
|
|
5297
|
+
return null;
|
|
5298
|
+
}
|
|
5299
|
+
}
|
|
5300
|
+
/**
|
|
5301
|
+
* Phase 3 of {@link tryPromptSelectorPrimitive}: given the popup's rendered
|
|
5302
|
+
* (possibly filtered) option list, match the requested option — deterministic
|
|
5303
|
+
* exact-text match first, `judgeSelectOptionWithLLM` fallback for per-req
|
|
5304
|
+
* variance — click it with a real gesture, and verify the selection committed
|
|
5305
|
+
* (the widget's accessible value / selection-label reflects the chosen option
|
|
5306
|
+
* by the value union, or its invalid marker cleared). Split out from the main
|
|
5307
|
+
* function because both the static and searchable-then-filtered branches need
|
|
5308
|
+
* the identical match/click/verify sequence.
|
|
5309
|
+
*/
|
|
5310
|
+
async function commitPromptOption(params) {
|
|
5311
|
+
const { page, target, logger, anthropic, captureFn, optLabel, option, questionLabel, chosen, optionsResult, } = params;
|
|
5312
|
+
const norm = (s) => s.replace(/\s+/g, " ").trim().toLowerCase();
|
|
5313
|
+
const wantOpt = norm(option);
|
|
5314
|
+
const options = optionsResult.options ?? [];
|
|
5315
|
+
const detMatch = options.find((o) => norm(o.text) === wantOpt) ?? null;
|
|
5316
|
+
if (detMatch === null && (anthropic === null || options.length === 0)) {
|
|
5317
|
+
logger.info(`prompt-selector primitive: no option match for ${optLabel}${anthropic === null ? " (no LLM client)" : ""}; falling through to cascade`);
|
|
5318
|
+
return null;
|
|
5319
|
+
}
|
|
5320
|
+
const verdict = detMatch === null
|
|
5321
|
+
? await (0, select_option_1.judgeSelectOptionWithLLM)({
|
|
5322
|
+
client: anthropic,
|
|
5323
|
+
input: {
|
|
5324
|
+
questionLabel,
|
|
5325
|
+
desiredHint: option,
|
|
5326
|
+
candidates: [{ label: chosen.label || null, options: options.map((o) => o.text) }],
|
|
5327
|
+
},
|
|
5328
|
+
captureFn,
|
|
5329
|
+
})
|
|
5330
|
+
: null;
|
|
5331
|
+
if (detMatch === null &&
|
|
5332
|
+
(!verdict || verdict.selectIndex === null || verdict.optionIndex === null)) {
|
|
5333
|
+
logger.info(`prompt-selector primitive: LLM found no matching option for ${optLabel}${verdict ? ` (${verdict.reason})` : ""}; falling through to cascade`);
|
|
5334
|
+
return null;
|
|
5335
|
+
}
|
|
5336
|
+
const chosenOption = detMatch ??
|
|
5337
|
+
(verdict && verdict.optionIndex !== null ? (options[verdict.optionIndex] ?? null) : null);
|
|
5338
|
+
if (!chosenOption) {
|
|
5339
|
+
logger.info(`prompt-selector primitive: option match out of range for ${optLabel}; falling through`);
|
|
5340
|
+
return null;
|
|
5341
|
+
}
|
|
5342
|
+
const matchReason = detMatch ? "deterministic" : `LLM: ${(verdict?.reason ?? "").slice(0, 60)}`;
|
|
5343
|
+
const optionSel = `[${PROMPT_OPTION_MARK_ATTR}="${chosenOption.oIdx}"]`;
|
|
5344
|
+
try {
|
|
5345
|
+
await target.locator(optionSel).first().click();
|
|
5346
|
+
}
|
|
5347
|
+
catch (err) {
|
|
5348
|
+
logger.info(`prompt-selector primitive: option click failed for ${optLabel}: ${(0, errors_1.toErrorMessage)(err)}; falling through`);
|
|
5349
|
+
return null;
|
|
5350
|
+
}
|
|
5351
|
+
await page.waitForTimeout(PROMPT_SELECTOR_SETTLE_MS);
|
|
5352
|
+
const readbackExpr = `((wIdx, markAttr, valueSel, emptyRxSrc, emptyRxFlags, wantText) => {
|
|
5353
|
+
const isInvalid = ${INVALID_MARKER_EL_EXPR};
|
|
5354
|
+
const norm = (s) => (s || "").replace(/\\s+/g, " ").trim().toLowerCase();
|
|
5355
|
+
const buttonValue = ${BUTTON_VALUE_EXPR};
|
|
5356
|
+
const emptyRx = new RegExp(emptyRxSrc, emptyRxFlags);
|
|
5357
|
+
const w = document.querySelector("[" + markAttr + '="' + wIdx + '"]');
|
|
5358
|
+
if (!w) return { ok: false, id: "" };
|
|
5359
|
+
// Value via the union: aria-activedescendant, own input value, a <button>'s
|
|
5360
|
+
// own value text (popup-pollution-safe, see BUTTON_VALUE_EXPR), or a
|
|
5361
|
+
// selection-label node (empty-state phrase treated as no value).
|
|
5362
|
+
const adid = w.getAttribute && w.getAttribute("aria-activedescendant");
|
|
5363
|
+
const adText = adid && document.getElementById(adid) ? norm(document.getElementById(adid).textContent) : "";
|
|
5364
|
+
const own = w.tagName === "INPUT" ? norm(w.value || "") : w.tagName === "BUTTON" ? norm(buttonValue(w)) : "";
|
|
5365
|
+
const lbl = w.matches(valueSel) ? w : w.querySelector(valueSel);
|
|
5366
|
+
const lblRaw = lbl ? (lbl.textContent || "") : "";
|
|
5367
|
+
const lblText = lblRaw.trim() && !emptyRx.test(lblRaw.trim()) ? norm(lblRaw) : "";
|
|
5368
|
+
const text = adText || own || lblText;
|
|
5369
|
+
const textMatches = wantText ? text.includes(wantText) : text !== "";
|
|
5370
|
+
let node = w;
|
|
5371
|
+
let stillInvalid = false;
|
|
5372
|
+
for (let d = 0; d < 6 && node; d++) {
|
|
5373
|
+
if (node.getAttribute && isInvalid(node)) { stillInvalid = true; break; }
|
|
5374
|
+
node = node.parentElement;
|
|
5375
|
+
}
|
|
5376
|
+
return { ok: textMatches || !stillInvalid, id: w.id || "" };
|
|
5377
|
+
})(${JSON.stringify(chosen.wIdx)}, ${JSON.stringify(PROMPT_WIDGET_MARK_ATTR)}, ${JSON.stringify(PROMPT_VALUE_SELECTORS)}, ${JSON.stringify(PROMPT_EMPTY_VALUE_RX_SRC)}, ${JSON.stringify(PROMPT_EMPTY_VALUE_RX_FLAGS)}, ${JSON.stringify(norm(chosenOption.text))})`;
|
|
5378
|
+
const readback = (await target.evaluate(readbackExpr).catch(() => ({ ok: false, id: "" })));
|
|
5379
|
+
if (!readback?.ok) {
|
|
5380
|
+
logger.info(`prompt-selector primitive: selection for ${optLabel} did not commit; falling through to cascade`);
|
|
5381
|
+
return null;
|
|
5382
|
+
}
|
|
5383
|
+
logger.info(`prompt-selector primitive: selected "${chosenOption.text.slice(0, 40)}" for ${optLabel} (${matchReason})`);
|
|
5384
|
+
return readback.id;
|
|
5385
|
+
}
|
|
4655
5386
|
/**
|
|
4656
5387
|
* Guard for the optional-step fast-skip: is there a REQUIRED, still-empty (or
|
|
4657
5388
|
* aria-invalid) form control on the page whose nearby label matches this step's
|
|
@@ -4673,16 +5404,20 @@ async function hasUnfilledRequiredControlForStep(target, instruction) {
|
|
|
4673
5404
|
const label = extractRequiredControlProbeLabel(instruction);
|
|
4674
5405
|
if (!label)
|
|
4675
5406
|
return false;
|
|
4676
|
-
const expr = `((label) => {
|
|
5407
|
+
const expr = `((label, triggerSel) => {
|
|
4677
5408
|
const norm = (s) => (s || "").replace(/\\s+/g, " ").trim().toLowerCase();
|
|
4678
5409
|
const want = norm(label);
|
|
4679
5410
|
if (!want) return false;
|
|
4680
5411
|
const isInvalid = ${INVALID_MARKER_EL_EXPR};
|
|
4681
|
-
// Required markers: the control itself,
|
|
5412
|
+
// Required markers: the control itself, a required-asterisk label nearby,
|
|
5413
|
+
// or (the UXI widget-kit shape) a trailing "Required" suffix baked into the
|
|
5414
|
+
// accessible name rather than exposed as aria-required at all.
|
|
4682
5415
|
const isRequired = (el) => {
|
|
4683
5416
|
if (!el || !el.getAttribute) return false;
|
|
4684
5417
|
if (el.hasAttribute("required")) return true;
|
|
4685
5418
|
if (el.getAttribute("aria-required") === "true") return true;
|
|
5419
|
+
const al = el.getAttribute("aria-label");
|
|
5420
|
+
if (al && /required\\s*$/i.test(al.trim())) return true;
|
|
4686
5421
|
return false;
|
|
4687
5422
|
};
|
|
4688
5423
|
const isEmptyish = (el) => {
|
|
@@ -4692,11 +5427,38 @@ async function hasUnfilledRequiredControlForStep(target, instruction) {
|
|
|
4692
5427
|
const v = ("value" in el) ? el.value : "";
|
|
4693
5428
|
return !v || String(v).trim() === "";
|
|
4694
5429
|
};
|
|
4695
|
-
//
|
|
4696
|
-
// label
|
|
4697
|
-
//
|
|
5430
|
+
// Standards-first, depth-independent label lookup: aria-labelledby, then
|
|
5431
|
+
// a <label for=id> referencing the control or a control inside it, then
|
|
5432
|
+
// the depth-capped ancestor label/legend walk as a last resort (real
|
|
5433
|
+
// markup can nest a control several wrapper divs below its <label>,
|
|
5434
|
+
// deeper than any fixed ancestor cap can safely assume).
|
|
5435
|
+
const labelFor = (el) => {
|
|
5436
|
+
const alb = el.getAttribute && el.getAttribute("aria-labelledby");
|
|
5437
|
+
if (alb) {
|
|
5438
|
+
const parts = [];
|
|
5439
|
+
for (const id of alb.split(/\\s+/)) { const ref = document.getElementById(id); if (ref) parts.push(ref.textContent); }
|
|
5440
|
+
if (parts.length) return norm(parts.join(" "));
|
|
5441
|
+
}
|
|
5442
|
+
const ids = [el.id, ...Array.from(el.querySelectorAll ? el.querySelectorAll("[id]") : []).map((e) => e.id)].filter(Boolean);
|
|
5443
|
+
for (const id of ids) {
|
|
5444
|
+
const lab = document.querySelector("label[for='" + (window.CSS && CSS.escape ? CSS.escape(id) : id) + "']");
|
|
5445
|
+
if (lab && lab.textContent && lab.textContent.trim()) return norm(lab.textContent);
|
|
5446
|
+
}
|
|
5447
|
+
let node = el;
|
|
5448
|
+
for (let d = 0; d < 6 && node; d++) {
|
|
5449
|
+
const lbl = node.querySelector && node.querySelector("label,legend");
|
|
5450
|
+
const txt = lbl && lbl.textContent ? norm(lbl.textContent) : "";
|
|
5451
|
+
if (txt) return txt;
|
|
5452
|
+
node = node.parentElement;
|
|
5453
|
+
}
|
|
5454
|
+
return "";
|
|
5455
|
+
};
|
|
5456
|
+
// Scan form controls; for each required+empty one, check whether its
|
|
5457
|
+
// label matches the question. Adds the prompt-selector trigger union
|
|
5458
|
+
// (button/aria-haspopup shapes with no native input/select at all) to
|
|
5459
|
+
// the container-widget union already covered.
|
|
4698
5460
|
const controls = Array.from(document.querySelectorAll(
|
|
4699
|
-
"input,select,textarea,[role=combobox],[role=listbox],.bb-custom-select-container,[class*='MultiCheckboxInput']"
|
|
5461
|
+
"input,select,textarea,[role=combobox],[role=listbox],.bb-custom-select-container,[class*='MultiCheckboxInput']," + triggerSel
|
|
4700
5462
|
));
|
|
4701
5463
|
for (const el of controls) {
|
|
4702
5464
|
if (!isRequired(el) && !(el.querySelector && el.querySelector("[required],[aria-required=true]"))) {
|
|
@@ -4706,17 +5468,11 @@ async function hasUnfilledRequiredControlForStep(target, instruction) {
|
|
|
4706
5468
|
// is it empty/invalid?
|
|
4707
5469
|
const emptyOrInvalid = isEmptyish(el) || (el.querySelector && !!el.querySelector("[aria-invalid=true]"));
|
|
4708
5470
|
if (!emptyOrInvalid) continue;
|
|
4709
|
-
|
|
4710
|
-
|
|
4711
|
-
for (let d = 0; d < 6 && node; d++) {
|
|
4712
|
-
const lbl = node.querySelector && node.querySelector("label,legend");
|
|
4713
|
-
const txt = lbl && lbl.textContent ? norm(lbl.textContent) : "";
|
|
4714
|
-
if (txt && (txt.includes(want) || want.includes(txt))) return true;
|
|
4715
|
-
node = node.parentElement;
|
|
4716
|
-
}
|
|
5471
|
+
const txt = labelFor(el);
|
|
5472
|
+
if (txt && (txt.includes(want) || want.includes(txt))) return true;
|
|
4717
5473
|
}
|
|
4718
5474
|
return false;
|
|
4719
|
-
})(${JSON.stringify(label)})`;
|
|
5475
|
+
})(${JSON.stringify(label)}, ${JSON.stringify(PROMPT_TRIGGER_SELECTORS)})`;
|
|
4720
5476
|
try {
|
|
4721
5477
|
return (await target.evaluate(expr)) === true;
|
|
4722
5478
|
}
|
|
@@ -5535,7 +6291,7 @@ async function probeStepBeforeAttempts(params) {
|
|
|
5535
6291
|
}
|
|
5536
6292
|
}
|
|
5537
6293
|
async function executeStepWithHealing(params) {
|
|
5538
|
-
const { stagehand, page, step, optional, upload, submitStep, stepIndex, totalSteps, phase, signalCounter, recentCaptures, recentCaptureMeta, anthropic, rephraseModel, logger, captureFn, uploadFixture, isFinalStep, submitEndpointPattern, submittedStateSelectors, requireSubmitEndpointMatch, advanceTransitionBodyPattern, successUrlFragments, successPageTitleHints, ownBackendHostnames, knownErrorClassPrefixes, wizardExitButtonLabels, getSuppressedAisdkElementIdErrorCount, trajectory, onStepFailure, } = params;
|
|
6294
|
+
const { stagehand, page, step, optional, upload, submitStep, flowHasSubmitSemantics: flowHasSubmitSemanticsFlag, stepIndex, totalSteps, phase, signalCounter, recentCaptures, recentCaptureMeta, anthropic, rephraseModel, logger, captureFn, uploadFixture, isFinalStep, submitEndpointPattern, submittedStateSelectors, requireSubmitEndpointMatch, advanceTransitionBodyPattern, successUrlFragments, successPageTitleHints, ownBackendHostnames, knownErrorClassPrefixes, wizardExitButtonLabels, getSuppressedAisdkElementIdErrorCount, trajectory, onStepFailure, } = params;
|
|
5539
6295
|
// Mutable (not the destructured const above) so a lost frame-attach race
|
|
5540
6296
|
// can be upgraded in place once the OOPIF attaches later in the cascade —
|
|
5541
6297
|
// see reresolveFrameTargetIfLost below. Every existing reference in this
|
|
@@ -5603,7 +6359,12 @@ async function executeStepWithHealing(params) {
|
|
|
5603
6359
|
// false-credit the step, and whose advance/submit verdicts require a real
|
|
5604
6360
|
// network/URL transition. Also keeps the extra full-DOM evaluate off the
|
|
5605
6361
|
// submit/advance path. Step-level intent (available before the attempt loop).
|
|
5606
|
-
const captureSelectionState = shouldCaptureSelectionState({
|
|
6362
|
+
const captureSelectionState = shouldCaptureSelectionState({
|
|
6363
|
+
step,
|
|
6364
|
+
isFinalStep,
|
|
6365
|
+
submitStep,
|
|
6366
|
+
flowHasSubmitSemantics: flowHasSubmitSemanticsFlag,
|
|
6367
|
+
});
|
|
5607
6368
|
const attempts = [];
|
|
5608
6369
|
const triedSelectors = [];
|
|
5609
6370
|
const failureReasons = [];
|
|
@@ -5686,6 +6447,30 @@ async function executeStepWithHealing(params) {
|
|
|
5686
6447
|
trajectory?.push({ stepIndex, verifiedBy: "dom", targetId: radioTargetId });
|
|
5687
6448
|
return "completed";
|
|
5688
6449
|
}
|
|
6450
|
+
// When the step is a "select 'X'" OR a "Fill in the Y field with 'X'" step
|
|
6451
|
+
// (the latter shape is how flow/replan generation describes this widget's
|
|
6452
|
+
// searchable filter box, which renders a real <input>) whose only matching
|
|
6453
|
+
// control is a native-control-less popup-dropdown widget (a combobox that
|
|
6454
|
+
// opens a listbox popup, options rendered on-demand — see
|
|
6455
|
+
// PROMPT_TRIGGER_SELECTORS) rather than a <select> or MUI radio group,
|
|
6456
|
+
// answer it directly. Runs AFTER select/checkbox/radio (which own their own
|
|
6457
|
+
// widget shapes and would already have claimed the step) and BEFORE the
|
|
6458
|
+
// cascade, since the widget's trigger exposes no native control the observe
|
|
6459
|
+
// cascade resolves. No-op (falls through) when there's no unfilled/
|
|
6460
|
+
// unambiguous prompt widget or no confident option match.
|
|
6461
|
+
const promptSelectorTargetId = await tryPromptSelectorPrimitive({
|
|
6462
|
+
page,
|
|
6463
|
+
target: selectFrameTarget,
|
|
6464
|
+
instruction: step,
|
|
6465
|
+
logger,
|
|
6466
|
+
anthropic,
|
|
6467
|
+
captureFn,
|
|
6468
|
+
});
|
|
6469
|
+
if (promptSelectorTargetId !== null) {
|
|
6470
|
+
logger.info(`${formatStepPrefix(stepIndex, totalSteps)} resolved by prompt-selector primitive`);
|
|
6471
|
+
trajectory?.push({ stepIndex, verifiedBy: "dom", targetId: promptSelectorTargetId });
|
|
6472
|
+
return "completed";
|
|
6473
|
+
}
|
|
5689
6474
|
// On a CATCH-ALL step ("for any remaining … question"), fill every
|
|
5690
6475
|
// required-but-empty native <select> — including MuiNativeSelect dropdowns
|
|
5691
6476
|
// (tabindex=-1) that Stagehand observe can't see and that no concrete flow
|
|
@@ -6013,7 +6798,7 @@ async function executeStepWithHealing(params) {
|
|
|
6013
6798
|
// techniques faster.
|
|
6014
6799
|
if (attempt > 1) {
|
|
6015
6800
|
const wouldBeTechnique = attempt === 2
|
|
6016
|
-
? phantomClickAfterAttempt1 && (
|
|
6801
|
+
? phantomClickAfterAttempt1 && (submitStep || (isFinalStep && flowHasSubmitSemanticsFlag))
|
|
6017
6802
|
? "deep-submit-locator"
|
|
6018
6803
|
: phantomClickAfterAttempt1
|
|
6019
6804
|
? "trusted-click-retry"
|
|
@@ -6032,7 +6817,7 @@ async function executeStepWithHealing(params) {
|
|
|
6032
6817
|
})),
|
|
6033
6818
|
advanceUnmovedAfterAttempt1,
|
|
6034
6819
|
phantomClickAfterAttempt1,
|
|
6035
|
-
submitShapedStep:
|
|
6820
|
+
submitShapedStep: submitStep || (isFinalStep && flowHasSubmitSemanticsFlag),
|
|
6036
6821
|
});
|
|
6037
6822
|
if (decision.skip) {
|
|
6038
6823
|
logger.info(`${formatStepPrefix(stepIndex, totalSteps)} attempt ${attempt} (${wouldBeTechnique}) skipped: ${decision.reason}`);
|
|
@@ -6160,7 +6945,9 @@ async function executeStepWithHealing(params) {
|
|
|
6160
6945
|
}
|
|
6161
6946
|
}
|
|
6162
6947
|
}
|
|
6163
|
-
else if (attempt === 2 &&
|
|
6948
|
+
else if (attempt === 2 &&
|
|
6949
|
+
phantomClickAfterAttempt1 &&
|
|
6950
|
+
(submitStep || (isFinalStep && flowHasSubmitSemanticsFlag))) {
|
|
6164
6951
|
// Deep submit-control locator: attempt 1 phantom-clicked (Stagehand
|
|
6165
6952
|
// reported success but pre/post showed zero effect), so the target is
|
|
6166
6953
|
// almost certainly unreachable via document.querySelectorAll — most
|
|
@@ -6282,7 +7069,9 @@ async function executeStepWithHealing(params) {
|
|
|
6282
7069
|
}
|
|
6283
7070
|
}
|
|
6284
7071
|
}
|
|
6285
|
-
else if (attempt === 2 &&
|
|
7072
|
+
else if (attempt === 2 &&
|
|
7073
|
+
phantomClickAfterAttempt1 &&
|
|
7074
|
+
!(submitStep || (isFinalStep && flowHasSubmitSemanticsFlag))) {
|
|
6286
7075
|
// Trusted-click retry: attempt 1 phantom-clicked a NON-submit control —
|
|
6287
7076
|
// Stagehand reported success but pre/post showed zero effect. On a
|
|
6288
7077
|
// design-system widget (React synthetic-event delegation, custom
|
|
@@ -6960,7 +7749,7 @@ async function executeStepWithHealing(params) {
|
|
|
6960
7749
|
// `advanceTransitionBodyPattern` are unaffected.
|
|
6961
7750
|
const domVerifiedForStep = isDomOnlyAdvanceVerified({
|
|
6962
7751
|
hasPattern: advanceTransitionBodyPattern !== null,
|
|
6963
|
-
isFinalOrSubmit:
|
|
7752
|
+
isFinalOrSubmit: submitStep || (isFinalStep && flowHasSubmitSemanticsFlag),
|
|
6964
7753
|
isAdvance: isAdvanceStep(step),
|
|
6965
7754
|
domVerified,
|
|
6966
7755
|
networkIsRealAdvance,
|
|
@@ -6993,6 +7782,7 @@ async function executeStepWithHealing(params) {
|
|
|
6993
7782
|
resolvedAction,
|
|
6994
7783
|
isFinalStep,
|
|
6995
7784
|
submitStep,
|
|
7785
|
+
flowHasSubmitSemantics: flowHasSubmitSemanticsFlag,
|
|
6996
7786
|
isAdvanceWithPattern: isAdvanceStep(step) && advanceTransitionBodyPattern !== null,
|
|
6997
7787
|
networkDelta: post.networkCount - pre.networkCount,
|
|
6998
7788
|
bytesDelta: post.bodyHtmlLength - pre.bodyHtmlLength,
|
|
@@ -7079,11 +7869,44 @@ async function executeStepWithHealing(params) {
|
|
|
7079
7869
|
}
|
|
7080
7870
|
}
|
|
7081
7871
|
}
|
|
7872
|
+
// Committed-value guard for the prompt-selector widget family (popup
|
|
7873
|
+
// multiselect/typeahead trigger widgets). The primitive's own picking
|
|
7874
|
+
// gesture ({@link tryPromptSelectorPrimitive}) only runs on the observe-act
|
|
7875
|
+
// fallback path; a resolved-as-click act against the SAME widget shape can
|
|
7876
|
+
// open its popup, report success, and — since opening a popup mutates the
|
|
7877
|
+
// DOM (bytes changed) — ride `clickViewSwapVerified` to a phantom success
|
|
7878
|
+
// without ever committing an option. Gate it the same way as the datepicker
|
|
7879
|
+
// guard above: readback the widget's own committed-value node
|
|
7880
|
+
// ({@link verifyPromptSelectorCommitted}) and, when the resolved element IS
|
|
7881
|
+
// this widget shape but nothing committed, suppress the weak signals.
|
|
7882
|
+
// `isPromptWidget: false` for anything outside this widget family, so a
|
|
7883
|
+
// plain click/fill is untouched.
|
|
7884
|
+
let promptSelectorRejected = false;
|
|
7885
|
+
// Exempt only on a genuine network-advance or URL-change signal, not the
|
|
7886
|
+
// broader `hasStrongSignal` (which the sibling datepicker gate above
|
|
7887
|
+
// legitimately relies on) — `domVerifiedForStep` is true here simply
|
|
7888
|
+
// because opening the widget's popup mutates the DOM, so reusing it would
|
|
7889
|
+
// let that popup-open alone skip the readback below.
|
|
7890
|
+
const hasNetworkOrUrlSignal = networkIsRealAdvance || urlChanged;
|
|
7891
|
+
if (record.actResultSuccess === true && !hasNetworkOrUrlSignal) {
|
|
7892
|
+
const readbackSelector = resolvedAction?.selector ?? triedSelectors[triedSelectors.length - 1] ?? null;
|
|
7893
|
+
if (readbackSelector) {
|
|
7894
|
+
const readbackTarget = frameTarget ?? (0, frame_target_1.mainFrameTarget)(page);
|
|
7895
|
+
const promptReadback = await verifyPromptSelectorCommitted(readbackTarget, readbackSelector);
|
|
7896
|
+
if (promptReadback?.isPromptWidget === true && promptReadback.committed === false) {
|
|
7897
|
+
promptSelectorRejected = true;
|
|
7898
|
+
record.actResultSuccess = false;
|
|
7899
|
+
record.errorMessage = `prompt-selector-fill-rejected: resolved element <${readbackSelector}> is a prompt-selector-shaped widget but no committed value landed`;
|
|
7900
|
+
}
|
|
7901
|
+
}
|
|
7902
|
+
}
|
|
7082
7903
|
let verified = networkIsRealAdvance ||
|
|
7083
7904
|
urlChanged ||
|
|
7084
7905
|
domVerifiedForStep ||
|
|
7085
7906
|
datepickerCommitted ||
|
|
7086
|
-
(!datepickerRejected &&
|
|
7907
|
+
(!datepickerRejected &&
|
|
7908
|
+
!promptSelectorRejected &&
|
|
7909
|
+
(clickViewSwapVerified || formValueVerified));
|
|
7087
7910
|
// Final-step submit-verification gate. Replaces the deterministic
|
|
7088
7911
|
// submitEndpointPattern regex with a Haiku 4.5 LLM judgment over
|
|
7089
7912
|
// multi-signal evidence (network captures, page URL/title, DOM
|
|
@@ -7369,7 +8192,7 @@ async function executeStepWithHealing(params) {
|
|
|
7369
8192
|
}));
|
|
7370
8193
|
const fallbackDomOnlyAdvance = shouldVetoFallbackAdvance({
|
|
7371
8194
|
hasPattern: advanceTransitionBodyPattern !== null,
|
|
7372
|
-
isFinalOrSubmit:
|
|
8195
|
+
isFinalOrSubmit: submitStep || (isFinalStep && flowHasSubmitSemanticsFlag),
|
|
7373
8196
|
isAdvance: isAdvanceStep(step),
|
|
7374
8197
|
retryUrlChanged,
|
|
7375
8198
|
retryNetworkIsRealAdvance,
|
|
@@ -7524,7 +8347,7 @@ async function executeStepWithHealing(params) {
|
|
|
7524
8347
|
// checked) into `domVerified`. A registered selection toggle no longer
|
|
7525
8348
|
// reads as a phantom just because it moved no network/URL/bytes.
|
|
7526
8349
|
elementStateChanged: domVerified,
|
|
7527
|
-
isSubmitShapedStep:
|
|
8350
|
+
isSubmitShapedStep: submitStep || (isFinalStep && flowHasSubmitSemanticsFlag),
|
|
7528
8351
|
});
|
|
7529
8352
|
const reason = record.errorMessage
|
|
7530
8353
|
? effectSignals
|
|
@@ -7543,7 +8366,8 @@ async function executeStepWithHealing(params) {
|
|
|
7543
8366
|
// Empirically grounded: 22 of 22 JSON-envelope ATS Continue/Submit step-failure
|
|
7544
8367
|
// dumps in a 2026-06-10 survey had the paired touched+dirty + visible
|
|
7545
8368
|
// error text pattern with 3 distinct rejection messages.
|
|
7546
|
-
if (record.resolvedMethod === "click" &&
|
|
8369
|
+
if (record.resolvedMethod === "click" &&
|
|
8370
|
+
(submitStep || (isFinalStep && flowHasSubmitSemanticsFlag))) {
|
|
7547
8371
|
const live = await extractLivePageFormEvidence(page, frameTarget ?? (0, frame_target_1.mainFrameTarget)(page), {
|
|
7548
8372
|
client: anthropic,
|
|
7549
8373
|
knownErrorClassPrefixes,
|
|
@@ -7588,7 +8412,7 @@ async function executeStepWithHealing(params) {
|
|
|
7588
8412
|
phantomClickAfterAttempt1 = record.phantomClickVerdict === "phantom";
|
|
7589
8413
|
if (phantomClickAfterAttempt1) {
|
|
7590
8414
|
const suppressedCount = getSuppressedAisdkElementIdErrorCount?.();
|
|
7591
|
-
const escalationTarget =
|
|
8415
|
+
const escalationTarget = submitStep || (isFinalStep && flowHasSubmitSemanticsFlag)
|
|
7592
8416
|
? "escalating attempt 2 to deep-submit-locator"
|
|
7593
8417
|
: "non-submit step — escalating attempt 2 to trusted-click-retry (trusted CDP click on the resolved target)";
|
|
7594
8418
|
logger.warn(`${formatStepPrefix(stepIndex, totalSteps)} phantom click detected on attempt 1 (${record.technique}): reported success with no network/url/dom change${suppressedCount !== undefined ? `; ${suppressedCount} AISDK elementId errors suppressed this session (corroborating, not causal)` : ""} — ${escalationTarget}`);
|
|
@@ -7598,7 +8422,7 @@ async function executeStepWithHealing(params) {
|
|
|
7598
8422
|
// Treat the canonical submit click as "final" for this predicate
|
|
7599
8423
|
// even when it lives mid-flow. See requireSubmitEndpoint derivation
|
|
7600
8424
|
// above for the same gate-widening rationale.
|
|
7601
|
-
isFinalStep:
|
|
8425
|
+
isFinalStep: submitStep || (isFinalStep && flowHasSubmitSemanticsFlag),
|
|
7602
8426
|
requireSubmitEndpoint,
|
|
7603
8427
|
resolvedMethod: record.resolvedMethod,
|
|
7604
8428
|
effectSignals,
|
|
@@ -7618,7 +8442,7 @@ async function executeStepWithHealing(params) {
|
|
|
7618
8442
|
// can reorder a later step forward — instead of burning the cascade.
|
|
7619
8443
|
const advanceStalled = isAdvanceStalled({
|
|
7620
8444
|
isAdvance: isAdvanceStep(step),
|
|
7621
|
-
isFinalOrSubmit:
|
|
8445
|
+
isFinalOrSubmit: submitStep || (isFinalStep && flowHasSubmitSemanticsFlag),
|
|
7622
8446
|
hasPattern: advanceTransitionBodyPattern !== null,
|
|
7623
8447
|
clickFired: record.resolvedMethod === "click" && record.actResultSuccess === true,
|
|
7624
8448
|
networkFired,
|
|
@@ -7764,6 +8588,11 @@ async function runHealingFlow(deps) {
|
|
|
7764
8588
|
}
|
|
7765
8589
|
},
|
|
7766
8590
|
});
|
|
8591
|
+
const flowHasSubmitSemanticsFlag = flowHasSubmitSemantics({
|
|
8592
|
+
steps,
|
|
8593
|
+
submitEndpointPattern: deps.submitEndpointPattern ?? null,
|
|
8594
|
+
requireSubmitEndpointMatch: deps.requireSubmitEndpointMatch ?? false,
|
|
8595
|
+
});
|
|
7767
8596
|
if (shouldWarnMissingAdvancePattern(steps.map((s) => s.instruction), deps.advanceTransitionBodyPattern ?? null)) {
|
|
7768
8597
|
logger.warn("flow has advance steps but no advanceTransitionBodyPattern — DOM-only advance guard is disarmed; pointer/page desync is possible on validation-re-render wizards");
|
|
7769
8598
|
}
|
|
@@ -7788,6 +8617,7 @@ async function runHealingFlow(deps) {
|
|
|
7788
8617
|
optional: s.optional,
|
|
7789
8618
|
upload: s.upload,
|
|
7790
8619
|
submitStep: s.submitStep,
|
|
8620
|
+
flowHasSubmitSemantics: flowHasSubmitSemanticsFlag,
|
|
7791
8621
|
stepIndex: i,
|
|
7792
8622
|
totalSteps: () => steps.length,
|
|
7793
8623
|
phase: "flow",
|