@enricai/barnacle 1.12.34 → 1.12.36

package/dist/scraper/flow-runner.js

@@ -88,7 +88,7 @@ exports.probeStepBeforeAttempts = probeStepBeforeAttempts;
 exports.extractLinkFromMessage = extractLinkFromMessage;
 exports.extractCodeFromMessage = extractCodeFromMessage;
 exports.executeStepWithHealing = executeStepWithHealing;
-exports.waitForSpaReady = waitForSpaReady;
+exports.executeNavigateStep = executeNavigateStep;
 exports.runHealingFlow = runHealingFlow;
 const node_crypto_1 = require("node:crypto");
 const node_fs_1 = require("node:fs");
@@ -118,6 +118,7 @@ const errors_2 = require("../scraper/errors");
 const frame_target_1 = require("../scraper/frame-target");
 const phantom_click_1 = require("../scraper/phantom-click");
 const session_teardown_1 = require("../scraper/session-teardown");
+const spa_readiness_1 = require("../scraper/spa-readiness");
 const stagehand_guard_1 = require("../scraper/stagehand-guard");
 const submit_control_1 = require("../scraper/submit-control");
 const watchdog_1 = require("../scraper/watchdog");
@@ -4268,18 +4269,25 @@ async function waitForTransitionBody(params) {
     return false;
 }
 /**
- * Site-agnostic captcha-solve hand-off: given an already-solved token, commit
- * it into the page's hidden response field and dispatch the framework-visible
- * `change` event so the field's own value listener observes it.
+ * Site-agnostic captcha-solve hand-off: given an already-solved token,
+ * discover the widget's registered `data-callback` and invoke it with the
+ * token so the site assembles its own submit (its own hidden fields, its own
+ * submit path); only when no callback is discoverable does this fall back to
+ * setting the hidden response field directly and dispatching the
+ * framework-visible `change` event so the field's own value listener
+ * observes it.
  *
- * Widgets of this shape create a hidden `<input name="{responseField}">`
- * carrying the solved token, and some widgets' own render callback submits
- * the enclosing form as soon as a token lands — assigning the field's value
- * can therefore itself navigate the frame synchronously, mid-evaluate. No
- * eval this function issues after the precheck is depended on for a return
- * value: only the precheck (a pure DOM query, which cannot navigate) is
- * awaited for its result; the value-set and change-dispatch evals are fired
- * and their rejections discarded, exactly like a navigating `form.submit()`
+ * Widgets of this shape anchor a `[data-sitekey]` element whose
+ * `data-callback` attribute names a function on `window`; that callback (not
+ * a bare field assignment) is what some sites depend on to append companion
+ * hidden fields before submitting. Invoking it — or, absent one, assigning
+ * the response field's value — can itself navigate the frame synchronously,
+ * mid-evaluate, on widgets whose callback (or value-set side effect) submits
+ * the enclosing form as soon as a token lands. No eval this function issues
+ * after the precheck is depended on for a return value: only the precheck (a
+ * pure DOM query, which cannot navigate) is awaited for its result; the
+ * callback-invoke and the value-set/change-dispatch evals are fired and
+ * their rejections discarded, exactly like a navigating `form.submit()`
  * would be.
  *
  * Deliberately narrow: no polling, no verification, no wait budget, and no
@@ -4299,18 +4307,35 @@ async function injectCaptchaTokenAndSubmit(target, token, responseField = "h-cap
     const precheckExpr = `(() => {
     const responseField = ${JSON.stringify(responseField)};
     const field = document.querySelector('[name="' + responseField + '"]');
-    if (field) return { fieldExists: true, hasForm: Boolean(field.closest("form")) };
+    const sitekeyEl = document.querySelector("[data-sitekey]");
+    const callbackName = sitekeyEl ? sitekeyEl.getAttribute("data-callback") : null;
+    const callbackExists = Boolean(callbackName) && typeof window[callbackName] === "function";
+    const callback = callbackExists ? callbackName : null;
+    if (field) return { fieldExists: true, hasForm: Boolean(field.closest("form")), callback };
     const forms = Array.from(document.querySelectorAll("form"));
     const form = forms.find((candidate) => candidate.querySelector("[data-sitekey]")) ?? forms[0];
-    return { fieldExists: false, hasForm: Boolean(form) };
+    return { fieldExists: false, hasForm: Boolean(form), callback };
   })()`;
     // Purely a DOM query — reads but never mutates the page, so it cannot
     // itself trigger navigation. This is the only evaluate call in this
     // function whose return value is depended on.
-    const { fieldExists, hasForm } = await target.evaluate(precheckExpr);
+    const { fieldExists, hasForm, callback } = await target.evaluate(precheckExpr);
     const injected = fieldExists || hasForm;
     if (!injected)
         return { injected, hasForm };
+    if (callback) {
+        const invokeCallbackExpr = `(() => {
+      const token = ${JSON.stringify(token)};
+      window[${JSON.stringify(callback)}](token);
+    })()`;
+        // Invoking the widget's own callback can navigate the frame synchronously
+        // from within this call (the callback is free to build its own fields and
+        // submit), tearing down the execution context before a return value
+        // marshals — that rejection is the expected outcome, not a real failure,
+        // so it's discarded here rather than depended on.
+        await target.evaluate(invokeCallbackExpr).catch(() => undefined);
+        return { injected, hasForm };
+    }
     const setValueExpr = `(() => {
     const responseField = ${JSON.stringify(responseField)};
     const token = ${JSON.stringify(token)};
@@ -4357,14 +4382,18 @@ async function injectCaptchaTokenAndSubmit(target, token, responseField = "h-cap
  * used only when the caller's own transition poll observed no advance after
  * the inject, so the widget's callback (if any) evidently didn't submit for
  * us. Kept separate from the inject primitive so the caller can gate its use
- * on an observed transition rather than firing it unconditionally.
+ * on an observed transition rather than firing it unconditionally. Prefers
+ * `form.requestSubmit()` so any submit-event listener (including one that
+ * calls `preventDefault()` and drives its own submit logic) and native form
+ * validation still run, matching real-browser submit semantics; falls back
+ * to the bare `form.submit()` only when `requestSubmit` isn't available.
  */
 async function submitCaptchaGatedForm(target, responseField = "h-captcha-response") {
     const submitExpr = `(() => {
     const responseField = ${JSON.stringify(responseField)};
     const field = document.querySelector('[name="' + responseField + '"]');
     const form = field ? field.closest("form") : null;
-    if (form) form.submit();
+    if (form) (form.requestSubmit ? form.requestSubmit() : form.submit());
   })()`;
     // form.submit() navigates the frame synchronously, tearing down the execution
     // context before Runtime.evaluate can marshal a return value for this call —
@@ -9331,57 +9360,32 @@ async function executeStepWithHealing(params) {
     }
     throw new errors_2.StepVerificationError(`${formatStepPrefix(stepIndex, totalSteps)} (${step.slice(0, 60)}) failed verification after ${MAX_STEP_ATTEMPTS} attempts${dumpPath ? `; see ${dumpPath}` : ""}`, phantomClickAfterAttempt1 ? "phantom-click-exhausted" : "cascade-exhausted");
 }
-/** SPA-readiness gate defaults — match the recon CLI's post-navigation wait. */
-const SPA_READINESS_TIMEOUT_MS = 15_000;
-const SPA_READINESS_POLL_MS = 500;
-const SPA_MIN_BODY_LENGTH = 5_000;
-/**
- * Block until a just-navigated SPA has actually hydrated, so the flow does not
- * begin stepping against a shell page. `page.goto(..., "networkidle")` on a
- * bot-managed/CDN-fronted single-page app resolves during the challenge/redirect —
- * before the client framework renders the real DOM — so the first steps would
- * otherwise probe an empty page, find no candidates, and (being optional) skip
- * the entire flow. The recon CLI has this gate inline; generated plugins call it
- * here so they inherit the same behavior. Polls `document.body.outerHTML.length`
- * up to a threshold, then proceeds regardless (best-effort, never throws).
- *
- * Each `document.body.outerHTML.length` read is itself bounded by a watchdog
- * (capped to one poll interval) and treated as "still 0 chars" on timeout —
- * same pattern as `waitForChildFrameReady`'s `isReady()` probe. Without this,
- * a single wedged CDP round-trip inside `readBodyLength` would pend forever
- * and the `while (Date.now() < deadline)` loop below would never be
- * re-entered, defeating the very deadline it exists to enforce.
- * `page.waitForTimeout(pollMs)` is left unguarded: it is a plain delay (no
- * DOM/network read whose response can be lost), so it does not carry the
- * "wedged read" failure mode this fix targets.
- */
-async function waitForSpaReady(page, logger, opts = {}) {
-    const timeoutMs = opts.timeoutMs ?? SPA_READINESS_TIMEOUT_MS;
-    const pollMs = opts.pollMs ?? SPA_READINESS_POLL_MS;
-    const minBodyLength = opts.minBodyLength ?? SPA_MIN_BODY_LENGTH;
-    const bodyLengthExpr = "document.body ? document.body.outerHTML.length : 0";
-    const readBodyLength = async () => {
-        const raw = await (0, watchdog_1.withWatchdog)(() => page.evaluate(bodyLengthExpr), {
-            timeoutMs: pollMs,
-            label: "flow-runner: spa readiness body-length probe",
-        }).catch(() => 0);
-        return typeof raw === "number" ? raw : 0;
-    };
-    let bodyLength = await readBodyLength();
-    if (bodyLength >= minBodyLength) {
-        return;
+/**
+ * Navigates `page` directly to `url` and waits for the SPA to hydrate,
+ * bypassing the self-heal cascade entirely — a multi-page capture flow's
+ * page transitions are a known URL, not an element to locate and click, so
+ * there is nothing for the cascade's DOM-healing attempts to add. Matches
+ * {@link executeStepWithHealing}'s `"completed"|"skipped"` outcome contract
+ * so `runHealingFlow`'s step loop (stuck-detection, `submitStep` gating,
+ * trajectory) needs zero changes to also handle navigate steps.
+ */
+async function executeNavigateStep(params) {
+    const { page, url, optional, logger: log, waitUntil, timeoutMs } = params;
+    try {
+        await page.goto(url, {
+            waitUntil: waitUntil ?? "domcontentloaded",
+            timeoutMs: timeoutMs ?? exports.GOTO_TIMEOUT_MS,
+        });
     }
-    logger.info(`spa readiness: body ${bodyLength} chars < ${minBodyLength} threshold — waiting for SPA to render`);
-    const deadline = Date.now() + timeoutMs;
-    while (Date.now() < deadline) {
-        await page.waitForTimeout(pollMs);
-        bodyLength = await readBodyLength();
-        if (bodyLength >= minBodyLength) {
-            logger.info(`spa readiness: body grew to ${bodyLength} chars — SPA rendered`);
-            return;
+    catch (err) {
+        if (optional) {
+            log.warn(`navigate step: goto failed for optional step, skipping: ${(0, errors_1.toErrorMessage)(err)}`);
+            return "skipped";
         }
+        throw new errors_2.StepVerificationError(`navigate step failed: goto to ${url} threw: ${(0, errors_1.toErrorMessage)(err)}`, "navigate-failed");
     }
-    logger.warn(`spa readiness: body still ${bodyLength} chars after ${timeoutMs}ms — proceeding with possibly incomplete page`);
+    await (0, spa_readiness_1.waitForSpaReady)(page, log);
+    return "completed";
 }
 /**
  * Plugin-facing wrapper that drives a recon flow's steps through the SAME
@@ -9455,48 +9459,62 @@ async function runHealingFlow(deps) {
                 throw new errors_2.SessionTimeoutError(`${formatStepPrefix(i, () => steps.length)} session appears closed/dead (page.url() threw: ${(0, errors_1.toErrorMessage)(err)}) — aborting after ${i} of ${steps.length} steps completed`);
             }
             lastStepIndex = i;
-            // Resolved fresh per step (not cached across the run) so a cross-origin
-            // iframe that attaches mid-flow (e.g. after an "Apply" click reveals a
-            // wizard embedded later in the DOM) is picked up as soon as it's
-            // reachable, paralleling the recon CLI's per-step resolution. `resolveFrameTarget`
-            // falls back to the main-frame target when `frameSelector` is null/unresolvable,
-            // so this is a no-op for every flow that doesn't declare one.
-            const frameTarget = await (0, frame_target_1.resolveFrameTarget)(page, deps.frameSelector);
-            await (0, frame_target_1.waitForChildFrameReady)(frameTarget);
-            const stepPromise = executeStepWithHealing({
-                stagehand,
-                page,
-                step: s.instruction,
-                optional: s.optional,
-                upload: s.upload,
-                submitStep: s.submitStep,
-                captchaGated: s.captchaGated === true,
-                emailStep: s.emailStep === true,
-                emailStepConfig: s.emailStepConfig,
-                allocatedInbox: deps.allocatedInbox ?? null,
-                flowHasSubmitSemantics: flowHasSubmitSemanticsFlag,
-                stepIndex: i,
-                totalSteps: () => steps.length,
-                phase: "flow",
-                signalCounter,
-                recentCaptures,
-                recentCaptureMeta,
-                anthropic,
-                rephraseModel,
-                logger,
-                uploadFixture,
-                frameTarget,
-                isFinalStep: i === steps.length - 1,
-                submitEndpointPattern: deps.submitEndpointPattern ?? null,
-                submittedStateSelectors: deps.submittedStateSelectors ?? [],
-                requireSubmitEndpointMatch: deps.requireSubmitEndpointMatch ?? false,
-                advanceTransitionBodyPattern: deps.advanceTransitionBodyPattern ?? null,
-                successUrlFragments: deps.successUrlFragments ?? [],
-                successPageTitleHints: deps.successPageTitleHints ?? [],
-                ownBackendHostnames: deps.ownBackendHostnames ?? [],
-                knownErrorClassPrefixes: deps.knownErrorClassPrefixes ?? [],
-                wizardExitButtonLabels: deps.wizardExitButtonLabels ?? [],
-            });
+            // A navigateTo step is a direct page.goto, not an act/observe against
+            // the current DOM, so it has no target frame to resolve and skips
+            // straight to executeNavigateStep instead of the self-heal cascade.
+            let stepPromise;
+            if (s.navigateTo !== undefined) {
+                stepPromise = executeNavigateStep({
+                    page,
+                    url: s.navigateTo,
+                    optional: s.optional,
+                    logger,
+                });
+            }
+            else {
+                // Resolved fresh per step (not cached across the run) so a cross-origin
+                // iframe that attaches mid-flow (e.g. after an "Apply" click reveals a
+                // wizard embedded later in the DOM) is picked up as soon as it's
+                // reachable, paralleling the recon CLI's per-step resolution. `resolveFrameTarget`
+                // falls back to the main-frame target when `frameSelector` is null/unresolvable,
+                // so this is a no-op for every flow that doesn't declare one.
+                const frameTarget = await (0, frame_target_1.resolveFrameTarget)(page, deps.frameSelector);
+                await (0, frame_target_1.waitForChildFrameReady)(frameTarget);
+                stepPromise = executeStepWithHealing({
+                    stagehand,
+                    page,
+                    step: s.instruction,
+                    optional: s.optional,
+                    upload: s.upload,
+                    submitStep: s.submitStep,
+                    captchaGated: s.captchaGated === true,
+                    emailStep: s.emailStep === true,
+                    emailStepConfig: s.emailStepConfig,
+                    allocatedInbox: deps.allocatedInbox ?? null,
+                    flowHasSubmitSemantics: flowHasSubmitSemanticsFlag,
+                    stepIndex: i,
+                    totalSteps: () => steps.length,
+                    phase: "flow",
+                    signalCounter,
+                    recentCaptures,
+                    recentCaptureMeta,
+                    anthropic,
+                    rephraseModel,
+                    logger,
+                    uploadFixture,
+                    frameTarget,
+                    isFinalStep: i === steps.length - 1,
+                    submitEndpointPattern: deps.submitEndpointPattern ?? null,
+                    submittedStateSelectors: deps.submittedStateSelectors ?? [],
+                    requireSubmitEndpointMatch: deps.requireSubmitEndpointMatch ?? false,
+                    advanceTransitionBodyPattern: deps.advanceTransitionBodyPattern ?? null,
+                    successUrlFragments: deps.successUrlFragments ?? [],
+                    successPageTitleHints: deps.successPageTitleHints ?? [],
+                    ownBackendHostnames: deps.ownBackendHostnames ?? [],
+                    knownErrorClassPrefixes: deps.knownErrorClassPrefixes ?? [],
+                    wizardExitButtonLabels: deps.wizardExitButtonLabels ?? [],
+                });
+            }
             const outcome = deps.deathSignal
                 ? await (0, session_teardown_1.raceAgainstTeardown)(stepPromise, deps.deathSignal)
                 : await stepPromise;