@enricai/barnacle 1.12.32 → 1.12.34

package/dist/scraper/flow-runner.js

@@ -85,6 +85,8 @@ exports.verifyDomEffect = verifyDomEffect;
 exports.narrowInvalidFormControl = narrowInvalidFormControl;
 exports.formatStepPrefix = formatStepPrefix;
 exports.probeStepBeforeAttempts = probeStepBeforeAttempts;
+exports.extractLinkFromMessage = extractLinkFromMessage;
+exports.extractCodeFromMessage = extractCodeFromMessage;
 exports.executeStepWithHealing = executeStepWithHealing;
 exports.waitForSpaReady = waitForSpaReady;
 exports.runHealingFlow = runHealingFlow;
@@ -105,6 +107,7 @@ const schemas_1 = require("../lib/llm/schemas");
 const logging_1 = require("../lib/logging");
 const call_capture_1 = require("../lib/telemetry/call-capture");
 const call_types_1 = require("../lib/telemetry/call-types");
+const capture_filters_1 = require("../recon/capture-filters");
 const browser_click_expr_1 = require("../scraper/browser-click-expr");
 const captcha_solver_1 = require("../scraper/captcha-solver");
 const deep_locator_actuate_1 = require("../scraper/deep-locator-actuate");
@@ -119,6 +122,7 @@ const stagehand_guard_1 = require("../scraper/stagehand-guard");
 const submit_control_1 = require("../scraper/submit-control");
 const watchdog_1 = require("../scraper/watchdog");
 const recon_shared_1 = require("../scripts/recon-shared");
+const client_1 = require("../testmail/client");
 const logger = (0, logging_1.getLogger)({ name: "scraper/flow-runner" });
 /** Cap on the rolling capture-filename window held in memory for failure dumps. */
 const RECENT_CAPTURES_WINDOW = 20;
@@ -4449,12 +4453,12 @@ async function applySelectValue(page, target, selIdx, value) {
  * step's stable `targetId` for cross-run convergence.
  */
 async function trySelectPrimitive(params) {
-    const { page, target, instruction, logger, anthropic, captureFn } = params;
+    const { page, target, instruction, logger, anthropic, captureFn, redactValue } = params;
     const parsed = parseSelectStep(instruction);
     if (!parsed)
         return null;
     const { option, questionLabel } = parsed;
-    const optLabel = `option "${option.slice(0, 40)}"${questionLabel ? `, question "${questionLabel.slice(0, 40)}"` : ""}`;
+    const optLabel = `option "${redactIfSensitive(option, redactValue).slice(0, 40)}"${questionLabel ? `, question "${questionLabel.slice(0, 40)}"` : ""}`;
     // Phase 1 (browser, no mutation): find the target select — the one whose
     // nearby text matches the question label (or, when no label, any select).
     // Try a deterministic option match first; if it hits, apply immediately (no
@@ -4783,12 +4787,12 @@ async function tryFillRequiredSelectsPrimitive(params) {
  * for the id-as-targetId rationale.
  */
 async function tryCheckboxPrimitive(params) {
-    const { page, target, instruction, logger, anthropic, captureFn } = params;
+    const { page, target, instruction, logger, anthropic, captureFn, redactValue } = params;
     const parsed = parseSelectStep(instruction);
     if (!parsed)
         return null;
     const { option, questionLabel } = parsed;
-    const optLabel = `option "${option.slice(0, 40)}"${questionLabel ? `, question "${questionLabel.slice(0, 40)}"` : ""}`;
+    const optLabel = `option "${redactIfSensitive(option, redactValue).slice(0, 40)}"${questionLabel ? `, question "${questionLabel.slice(0, 40)}"` : ""}`;
     // Phase 1 (browser, no mutation): find checkbox GROUPS and their options.
     // A group is a `c-MultiCheckboxInput` container or a `<fieldset>` containing
     // checkboxes. Question label = the group's legend / associated label; each
@@ -5110,12 +5114,12 @@ function selectRadioGroupOption(params) {
  * (checkbox/select/absent) falls through to the cascade.
  */
 async function tryRadioPrimitive(params) {
-    const { page, target, instruction, logger, anthropic, captureFn } = params;
+    const { page, target, instruction, logger, anthropic, captureFn, redactValue } = params;
     const parsed = parseRadioStep(instruction);
     if (!parsed)
         return null;
     const { option, questionLabel } = parsed;
-    const optLabel = `option "${option.slice(0, 40)}"${questionLabel ? `, question "${questionLabel.slice(0, 40)}"` : ""}`;
+    const optLabel = `option "${redactIfSensitive(option, redactValue).slice(0, 40)}"${questionLabel ? `, question "${questionLabel.slice(0, 40)}"` : ""}`;
     // Phase 1 (browser): find radio GROUPS and their options. A group is a
     // `<fieldset>` / `[role=radiogroup]` / `[role=group]` / `[class*='RadioGroup']`
     // container with radios. Question label = the group's legend / associated label; each option
@@ -5399,7 +5403,7 @@ async function applyRadioSelection(target, gi, ri, hint) {
  * `trySelectPrimitive`/`tryRadioPrimitive`'s null-fallthrough contract.
  */
 async function tryPromptSelectorPrimitive(params) {
-    const { page, target, instruction, logger, anthropic, captureFn } = params;
+    const { page, target, instruction, logger, anthropic, captureFn, redactValue } = params;
     // A prompt-selector widget's search box renders a real <input>, so flow/
     // replan generation routinely describes filling it as a FILL step ("Fill in
     // the 'How Did You Hear About Us?' field with 'Internet/Online'") rather
@@ -5432,7 +5436,7 @@ async function tryPromptSelectorPrimitive(params) {
     if (!parsed)
         return null;
     const { option, questionLabel } = parsed;
-    const optLabel = `option "${option.slice(0, 40)}"${questionLabel ? `, question "${questionLabel.slice(0, 40)}"` : ""}`;
+    const optLabel = `option "${redactIfSensitive(option, redactValue).slice(0, 40)}"${questionLabel ? `, question "${questionLabel.slice(0, 40)}"` : ""}`;
     const norm = (s) => s.replace(/\s+/g, " ").trim().toLowerCase();
     // Phase 1 (browser, read-only except for the marker attribute stamped for
     // Phase 2's click addressing): find candidate widgets — popup-dropdown
@@ -6795,8 +6799,92 @@ function assertProbeSessionAlive(page, stepIndex, totalSteps) {
         throw new errors_2.SessionTimeoutError(`${formatStepPrefix(stepIndex, totalSteps)} session appears closed/dead (page.url() threw: ${(0, errors_1.toErrorMessage)(err)}) during probe`);
     }
 }
+/** Matches `re` against `text`, falling back to `html`; returns the first capture group, or the full match when the pattern has none. Shared by {@link extractLinkFromMessage}'s pattern branch and {@link extractCodeFromMessage}. */
+function firstRegexMatch(text, html, re) {
+    const match = text?.match(re) ?? html?.match(re) ?? null;
+    if (!match)
+        return null;
+    return match[1] ?? match[0] ?? null;
+}
+/**
+ * Extracts the verification URL from a testmail message. Why this exists: the
+ * `emailStep` hook must resolve a link out of arbitrary inbox HTML/text
+ * without a flow author having to hand-write a regex for every ATS's email
+ * template — a caller-supplied `linkPattern` covers the templates that need
+ * precision, and the default (first http(s) URL sharing the current page's
+ * registrable domain) covers the rest. Returns `null` — never a guess — when
+ * nothing matches, so the caller fails loudly instead of navigating blind.
+ */
+function extractLinkFromMessage(msg, pattern, currentPageUrl) {
+    if (pattern) {
+        return firstRegexMatch(msg.text, msg.html, new RegExp(pattern));
+    }
+    const fallbackDomain = (() => {
+        try {
+            return (0, capture_filters_1.registrableDomain)(new URL(currentPageUrl).hostname);
+        }
+        catch {
+            return null;
+        }
+    })();
+    if (!fallbackDomain)
+        return null;
+    const candidateUrls = [msg.text, msg.html]
+        .filter((body) => typeof body === "string")
+        .flatMap((body) => body.match(/https?:\/\/\S+/g) ?? []);
+    return (candidateUrls.find((url) => {
+        try {
+            return (0, capture_filters_1.registrableDomain)(new URL(url).hostname) === fallbackDomain;
+        }
+        catch {
+            return false;
+        }
+    }) ?? null);
+}
+/**
+ * Extracts the OTP/code from a testmail message, defaulting to the first
+ * 4-8 digit run (the common OTP shape) so most flows need no `codePattern`.
+ * Sibling of {@link extractLinkFromMessage} — see that function's docblock
+ * for why the default-vs-pattern split exists.
+ */
+function extractCodeFromMessage(msg, pattern) {
+    return firstRegexMatch(msg.text, msg.html, pattern ? new RegExp(pattern) : /\b\d{4,8}\b/);
+}
+/**
+ * Splices an `emailStep` code-extraction result into a fill step's quoted
+ * value so the pre-existing fill cascade (`parseFillStep`/`parseFillValueIntent`,
+ * both of which read the value out of the instruction's own quoted text)
+ * picks it up as the value to type without any of those parsers needing to
+ * know the value came from an inbox rather than the flow's payload.
+ */
+function spliceEmailStepFillValue(step, code) {
+    return /with\s+'[^']*'/i.test(step)
+        ? step.replace(/with\s+'[^']*'/i, `with '${code}'`)
+        : `${step} with '${code}'`;
+}
+/**
+ * Masks `text` when it equals `sensitiveValue` (the emailStep-extracted code
+ * spliced into the step's fill value, see {@link spliceEmailStepFillValue}).
+ * The select/checkbox/radio/prompt-selector primitives all re-parse the
+ * (possibly code-bearing) instruction independently of the fill primitive,
+ * so each one's own "option"/diagnostic logging must redact it too — the
+ * code is a single-use credential and must never reach the logs verbatim.
+ */
+function redactIfSensitive(text, sensitiveValue) {
+    return sensitiveValue && text === sensitiveValue ? "[redacted]" : text;
+}
 async function executeStepWithHealing(params) {
-    const { stagehand, page, step, optional, upload, submitStep, captchaGated = false, flowHasSubmitSemantics: flowHasSubmitSemanticsFlag, stepIndex, totalSteps, phase, signalCounter, recentCaptures, recentCaptureMeta, anthropic, rephraseModel, logger, captureFn, uploadFixture, isFinalStep, submitEndpointPattern, submittedStateSelectors, requireSubmitEndpointMatch, advanceTransitionBodyPattern, successUrlFragments, successPageTitleHints, ownBackendHostnames, knownErrorClassPrefixes, wizardExitButtonLabels, getSuppressedAisdkElementIdErrorCount, trajectory, onStepFailure, } = params;
+    const { stagehand, page, optional, upload, submitStep, captchaGated = false, emailStep = false, emailStepConfig, allocatedInbox = null, flowHasSubmitSemantics: flowHasSubmitSemanticsFlag, stepIndex, totalSteps, phase, signalCounter, recentCaptures, recentCaptureMeta, anthropic, rephraseModel, logger, captureFn, uploadFixture, isFinalStep, submitEndpointPattern, submittedStateSelectors, requireSubmitEndpointMatch, advanceTransitionBodyPattern, successUrlFragments, successPageTitleHints, ownBackendHostnames, knownErrorClassPrefixes, wizardExitButtonLabels, getSuppressedAisdkElementIdErrorCount, trajectory, onStepFailure, } = params;
+    // Mutable: the `emailStep` code-extract path splices the extracted code
+    // into this instruction (see the emailStep hook block below) so the
+    // pre-existing fill cascade downstream picks it up as the value to type
+    // without re-running Stagehand's act/observe resolution a second time.
+    let step = params.step;
+    // Set only by the emailStep code-extract path below. Threaded into every
+    // select/checkbox/radio/prompt-selector primitive call further down so
+    // each one's own re-parse of `step` (now code-bearing) redacts the code
+    // out of its diagnostic logging too, not just the fill primitive's.
+    let emailStepCode;
     // Mutable (not the destructured const above) so a lost frame-attach race
     // can be upgraded in place once the OOPIF attaches later in the cascade —
     // see reresolveFrameTargetIfLost below. Every existing reference in this
@@ -6804,6 +6892,111 @@ async function executeStepWithHealing(params) {
     // moved so it can be reassigned.
     let frameTarget = params.frameTarget;
     let frameReresolveAttempted = false;
+    // Emailed-verification hook. Runs FIRST — before upload/select/checkbox/
+    // radio/prompt-selector/captcha or any other cascade primitive — so an
+    // `emailStep: true` step with no allocated inbox fails loud instead of
+    // being silently claimed by an unrelated primitive that happens to match
+    // the step's instruction text. Polls the run's allocated testmail inbox,
+    // extracts a link or code from the matched message, and either navigates
+    // to the link (gating "completed" on the same transition poll the captcha
+    // hook reuses) or splices the code into this step's fill value and falls
+    // through to the normal cascade below. Never logs the extracted link/code
+    // — either can be a single-use credential — and never navigates off the
+    // current page's registrable domain, so a poisoned inbox message can't
+    // redirect the session.
+    if (emailStep) {
+        if (!allocatedInbox) {
+            // No inbox to poll — fail loud, exactly like the captcha no-key path.
+            throw new errors_2.EmailStepInboxUnavailableError("emailStep set but no testmail inbox allocated (pass --allocate-email)");
+        }
+        const cfg = emailStepConfig ?? {};
+        logger.info(`${formatStepPrefix(stepIndex, totalSteps)} emailStep: polling inbox ${allocatedInbox.address} (subjectContains=${cfg.subjectContains ?? "*"})`);
+        const msg = await (0, client_1.pollTestmailInbox)({
+            inbox: allocatedInbox,
+            subjectContains: cfg.subjectContains,
+            timeoutMs: cfg.timeoutMs ?? 120_000,
+        }).catch((err) => {
+            logger.error(`${formatStepPrefix(stepIndex, totalSteps)} emailStep: no matching email within budget (${(0, errors_1.toErrorMessage)(err)}); failing the step`);
+            throw err;
+        });
+        if ((cfg.extract ?? "link") === "link") {
+            const emailStepTarget = frameTarget ?? (0, frame_target_1.mainFrameTarget)(page);
+            const currentPageUrl = await emailStepTarget.url();
+            const url = extractLinkFromMessage(msg, cfg.linkPattern, currentPageUrl);
+            if (!url) {
+                throw new errors_2.EmailStepExtractError("no link matched in the verification email");
+            }
+            // Allowlist gate: the link's host must share the current page origin's
+            // registrable domain (or be a declared `ownBackendHostnames` entry) —
+            // never follow a link from untrusted inbox content off-domain.
+            const fallbackDomain = (() => {
+                try {
+                    return (0, capture_filters_1.registrableDomain)(new URL(currentPageUrl).hostname);
+                }
+                catch {
+                    return null;
+                }
+            })();
+            const linkHost = (() => {
+                try {
+                    return new URL(url).hostname;
+                }
+                catch {
+                    return null;
+                }
+            })();
+            if (!linkHost || !(0, capture_filters_1.isAllowedFixtureHost)(linkHost, ownBackendHostnames, fallbackDomain)) {
+                throw new errors_2.EmailStepExtractError("extracted link's host is outside the current page's registrable domain; refusing to navigate");
+            }
+            if (cfg.action === "fill") {
+                // extract:"link" + action:"fill" — splice the link into this step's
+                // fill value instead of navigating, same as the extract:"code" path
+                // below, for flows whose verification step is a paste-the-link field
+                // rather than a page transition.
+                logger.info(`${formatStepPrefix(stepIndex, totalSteps)} emailStep: link extracted`);
+                emailStepCode = url;
+                step = spliceEmailStepFillValue(step, url);
+            }
+            else {
+                logger.info(`${formatStepPrefix(stepIndex, totalSteps)} emailStep: navigating to extracted link`);
+                const preIdx = latestCaptureIndex(recentCaptures);
+                await page.goto(url); // NEVER log the URL body — it can be a single-use credential
+                // Gate advance on the same transition poll the captcha hook reuses.
+                if (advanceTransitionBodyPattern) {
+                    const confirmed = await waitForTransitionBody({
+                        page,
+                        preIdx,
+                        advanceTransitionBodyPattern,
+                        timeoutMs: CAPTCHA_TRANSITION_POLL_MS,
+                        intervalMs: ADVANCE_TRANSITION_POLL_INTERVAL_MS,
+                    });
+                    logger.info(`${formatStepPrefix(stepIndex, totalSteps)} emailStep: post-navigate transition poll confirmed=${confirmed}`);
+                    if (confirmed) {
+                        trajectory?.push({ stepIndex, verifiedBy: "network" });
+                    }
+                }
+                return "completed";
+            }
+        }
+        if (cfg.extract === "link") {
+            // extract:"link" + action:"fill" already spliced `url` into `step`
+            // above; fall through to the normal fill cascade below, same as the
+            // extract:"code" path.
+        }
+        else {
+            // extract:"code" — splice the extracted code into this step's fill
+            // value and fall through to the pre-existing fill primitive below,
+            // rather than returning early or re-running Stagehand's act/observe
+            // resolution against the code.
+            const code = extractCodeFromMessage(msg, cfg.codePattern);
+            if (!code) {
+                throw new errors_2.EmailStepExtractError("no code matched in the verification email");
+            }
+            logger.info(`${formatStepPrefix(stepIndex, totalSteps)} emailStep: code extracted (len ${code.length})`);
+            emailStepCode = code;
+            step = spliceEmailStepFillValue(step, code);
+        }
+    }
     /**
      * Re-resolves a frame that lost the attach race at step entry —
      * `resolveFrameTarget`'s per-step poll (called once in the runner's step
@@ -6907,6 +7100,7 @@ async function executeStepWithHealing(params) {
         logger,
         anthropic,
         captureFn,
+        redactValue: emailStepCode,
     });
     if (selectTargetId !== null) {
         logger.info(`${formatStepPrefix(stepIndex, totalSteps)} resolved by select primitive`);
@@ -6925,6 +7119,7 @@ async function executeStepWithHealing(params) {
         logger,
         anthropic,
         captureFn,
+        redactValue: emailStepCode,
     });
     if (checkboxTargetId !== null) {
         logger.info(`${formatStepPrefix(stepIndex, totalSteps)} resolved by checkbox primitive`);
@@ -6944,6 +7139,7 @@ async function executeStepWithHealing(params) {
         logger,
         anthropic,
         captureFn,
+        redactValue: emailStepCode,
     });
     if (radioTargetId !== null) {
         logger.info(`${formatStepPrefix(stepIndex, totalSteps)} resolved by radio primitive`);
@@ -6968,6 +7164,7 @@ async function executeStepWithHealing(params) {
         logger,
         anthropic,
         captureFn,
+        redactValue: emailStepCode,
     });
     if (promptSelectorTargetId !== null) {
         logger.info(`${formatStepPrefix(stepIndex, totalSteps)} resolved by prompt-selector primitive`);
@@ -7974,8 +8171,8 @@ async function executeStepWithHealing(params) {
                             const dateFill = await fillHtml5DateTimeInput(dateFillTarget, overriddenTarget.selector, fillValue);
                             if (dateFill !== null) {
                                 record.errorMessage = dateFill.filled
-                                    ? `html5-date-fallback: filled ${dateFill.inputType}="${dateFill.postValue}"`
-                                    : `html5-date-fallback: failed to fill ${dateFill.inputType} (post=${dateFill.postValue})`;
+                                    ? `html5-date-fallback: filled ${dateFill.inputType}="${redactIfSensitive(dateFill.postValue, emailStepCode)}"`
+                                    : `html5-date-fallback: failed to fill ${dateFill.inputType} (post=${redactIfSensitive(dateFill.postValue, emailStepCode)})`;
                                 // Override the act result based on the deterministic fill
                                 // outcome — the helper bypasses Stagehand's schema-error
                                 // failure mode by writing directly via the native setter.
@@ -7995,8 +8192,8 @@ async function executeStepWithHealing(params) {
                                 const datepickerFill = await fillTextDatepickerInput(dateFillTarget, overriddenTarget.selector, fillValue);
                                 if (datepickerFill !== null) {
                                     record.errorMessage = datepickerFill.filled
-                                        ? `text-datepicker-fill: filled via ${datepickerFill.strategy} "${datepickerFill.postValue}"`
-                                        : `text-datepicker-fill: failed to commit "${fillValue.slice(0, 60)}" (post=${datepickerFill.postValue})`;
+                                        ? `text-datepicker-fill: filled via ${datepickerFill.strategy} "${redactIfSensitive(datepickerFill.postValue, emailStepCode)}"`
+                                        : `text-datepicker-fill: failed to commit "${redactIfSensitive(fillValue.slice(0, 60), emailStepCode)}" (post=${redactIfSensitive(datepickerFill.postValue, emailStepCode)})`;
                                     if (datepickerFill.filled) {
                                         record.actResultSuccess = true;
                                         resolvedAction = overriddenTarget;
@@ -8015,11 +8212,11 @@ async function executeStepWithHealing(params) {
                                     const readback = await verifyFillReadback(dateFillTarget, overriddenTarget.selector, fillValue);
                                     if (readback !== null) {
                                         if (readback.outcome === "rejected") {
-                                            record.errorMessage = `fill-value-rejected: tried "${fillValue.slice(0, 60)}" on <${readback.tag}>; element value remains empty (silent rejection — HTML5 type validation, framework controlled-component, or masked-input library)`;
+                                            record.errorMessage = `fill-value-rejected: tried "${redactIfSensitive(fillValue.slice(0, 60), emailStepCode)}" on <${readback.tag}>; element value remains empty (silent rejection — HTML5 type validation, framework controlled-component, or masked-input library)`;
                                             record.actResultSuccess = false;
                                         }
                                         else if (readback.outcome === "differs") {
-                                            logger.info(`${formatStepPrefix(stepIndex, totalSteps)} fill-value-differs: tried "${fillValue.slice(0, 60)}" got "${readback.postValue.slice(0, 60)}" (framework reformatted)`);
+                                            logger.info(`${formatStepPrefix(stepIndex, totalSteps)} fill-value-differs: tried "${redactIfSensitive(fillValue.slice(0, 60), emailStepCode)}" got "${redactIfSensitive(readback.postValue.slice(0, 60), emailStepCode)}" (framework reformatted)`);
                                         }
                                     }
                                 }
@@ -8428,7 +8625,7 @@ async function executeStepWithHealing(params) {
                     if (datepickerFill !== null) {
                         if (datepickerFill.filled) {
                             datepickerCommitted = true;
-                            record.errorMessage = `text-datepicker-fill: filled via ${datepickerFill.strategy} "${datepickerFill.postValue}"`;
+                            record.errorMessage = `text-datepicker-fill: filled via ${datepickerFill.strategy} "${redactIfSensitive(datepickerFill.postValue, emailStepCode)}"`;
                         }
                         else {
                             // A gated-in datepicker that neither gesture commits is a real fill
@@ -8436,7 +8633,7 @@ async function executeStepWithHealing(params) {
                             // so suppress the weak view-swap/form-value acceptance and escalate.
                             datepickerRejected = true;
                             record.actResultSuccess = false;
-                            record.errorMessage = `text-datepicker-fill: failed to commit "${fillIntent.value.slice(0, 60)}" (post=${datepickerFill.postValue})`;
+                            record.errorMessage = `text-datepicker-fill: failed to commit "${redactIfSensitive(fillIntent.value.slice(0, 60), emailStepCode)}" (post=${redactIfSensitive(datepickerFill.postValue, emailStepCode)})`;
                         }
                     }
                     else {
@@ -8445,7 +8642,7 @@ async function executeStepWithHealing(params) {
                         // failure dump reads identically regardless of which path caught it.
                         datepickerRejected = true;
                         record.actResultSuccess = false;
-                        record.errorMessage = `fill-value-rejected: tried "${fillIntent.value.slice(0, 60)}" on <${readback.tag}>; element value remains empty (silent rejection — HTML5 type validation, framework controlled-component, or masked-input library)`;
+                        record.errorMessage = `fill-value-rejected: tried "${redactIfSensitive(fillIntent.value.slice(0, 60), emailStepCode)}" on <${readback.tag}>; element value remains empty (silent rejection — HTML5 type validation, framework controlled-component, or masked-input library)`;
                     }
                 }
             }
@@ -9274,6 +9471,9 @@ async function runHealingFlow(deps) {
                 upload: s.upload,
                 submitStep: s.submitStep,
                 captchaGated: s.captchaGated === true,
+                emailStep: s.emailStep === true,
+                emailStepConfig: s.emailStepConfig,
+                allocatedInbox: deps.allocatedInbox ?? null,
                 flowHasSubmitSemantics: flowHasSubmitSemanticsFlag,
                 stepIndex: i,
                 totalSteps: () => steps.length,