@enricai/barnacle 1.12.40 → 1.12.42

package/dist/scraper/flow-runner.js

@@ -17,8 +17,10 @@ Object.defineProperty(exports, "__esModule", { value: true });
 exports.TRAILING_GRACE_WINDOW = exports.STEP_WATCHDOG_MS = exports.OPENER_PAIRED_HIDDEN_SELECT_EL_EXPR = exports.GOTO_TIMEOUT_MS = exports.waitForSpaReady = void 0;
 exports.mergeResponseHeaders = mergeResponseHeaders;
 exports.wireSignalCapture = wireSignalCapture;
+exports.shouldRetryCaptchaRegistry = shouldRetryCaptchaRegistry;
 exports.prepareFailureDumpBody = prepareFailureDumpBody;
 exports.snapshotPage = snapshotPage;
+exports.resolveDumpPageIdentity = resolveDumpPageIdentity;
 exports.findRecentBackendError = findRecentBackendError;
 exports.findRecentPageTransition = findRecentPageTransition;
 exports.isWizardExitAction = isWizardExitAction;
@@ -73,6 +75,7 @@ exports.parseWidgetOptionClickStep = parseWidgetOptionClickStep;
 exports.parseCheckStep = parseCheckStep;
 exports.pollEnumerate = pollEnumerate;
 exports.waitForTransitionBody = waitForTransitionBody;
+exports.hasOriginOrPathChanged = hasOriginOrPathChanged;
 exports.injectCaptchaTokenAndSubmit = injectCaptchaTokenAndSubmit;
 exports.submitCaptchaGatedForm = submitCaptchaGatedForm;
 exports.trySelectPrimitive = trySelectPrimitive;
@@ -110,24 +113,24 @@ const logging_1 = require("../lib/logging");
 const call_capture_1 = require("../lib/telemetry/call-capture");
 const call_types_1 = require("../lib/telemetry/call-types");
 const capture_filters_1 = require("../recon/capture-filters");
-const browser_click_expr_1 = require("../scraper/browser-click-expr");
-const captcha_callback_capture_1 = require("../scraper/captcha-callback-capture");
-const captcha_solver_1 = require("../scraper/captcha-solver");
-const deep_locator_actuate_1 = require("../scraper/deep-locator-actuate");
-const deep_locator_candidates_1 = require("../scraper/deep-locator-candidates");
-const deep_locator_click_1 = require("../scraper/deep-locator-click");
-const deep_locator_scan_1 = require("../scraper/deep-locator-scan");
-const errors_2 = require("../scraper/errors");
-const frame_target_1 = require("../scraper/frame-target");
-const phantom_click_1 = require("../scraper/phantom-click");
-const session_teardown_1 = require("../scraper/session-teardown");
-const spa_readiness_1 = require("../scraper/spa-readiness");
-const stagehand_guard_1 = require("../scraper/stagehand-guard");
-const submit_control_1 = require("../scraper/submit-control");
-const watchdog_1 = require("../scraper/watchdog");
+const browser_click_expr_1 = require("./browser-click-expr");
+const captcha_callback_capture_1 = require("./captcha-callback-capture");
+const captcha_solver_1 = require("./captcha-solver");
+const deep_locator_actuate_1 = require("./deep-locator-actuate");
+const deep_locator_candidates_1 = require("./deep-locator-candidates");
+const deep_locator_click_1 = require("./deep-locator-click");
+const deep_locator_scan_1 = require("./deep-locator-scan");
+const errors_2 = require("./errors");
+const frame_target_1 = require("./frame-target");
+const phantom_click_1 = require("./phantom-click");
+const session_teardown_1 = require("./session-teardown");
+const spa_readiness_1 = require("./spa-readiness");
+const stagehand_guard_1 = require("./stagehand-guard");
+const submit_control_1 = require("./submit-control");
+const watchdog_1 = require("./watchdog");
 const recon_shared_1 = require("../scripts/recon-shared");
 const client_1 = require("../testmail/client");
-var spa_readiness_2 = require("../scraper/spa-readiness");
+var spa_readiness_2 = require("./spa-readiness");
 Object.defineProperty(exports, "waitForSpaReady", { enumerable: true, get: function () { return spa_readiness_2.waitForSpaReady; } });
 const logger = (0, logging_1.getLogger)({ name: "scraper/flow-runner" });
 /** Cap on the rolling capture-filename window held in memory for failure dumps. */
@@ -716,6 +719,31 @@ const ADVANCE_TRANSITION_POLL_INTERVAL_MS = 350;
  * {@link ADVANCE_TRANSITION_POLL_MS} unchanged.
  */
 const CAPTCHA_TRANSITION_POLL_MS = 45_000;
+/**
+ * Bounded retry budget for the captchaGated solve+inject+registryState-check
+ * sequence. The registry-empty/absent condition (the hCaptcha render callback
+ * hasn't attached to `HCAPTCHA_CALLBACK_REGISTRY_GLOBAL` yet when the solve
+ * finishes) is an intermittent attach-timing race, not a deterministic
+ * failure, so a single attempt is too eager to give up. Mirrors the
+ * MAX_STEP_ATTEMPTS/PRIMITIVE_ENUMERATE_ATTEMPTS=5 naming precedent, but a
+ * captcha solve is expensive (10-30s+ round trip via a real solver service),
+ * so the budget is kept small.
+ */
+const CAPTCHA_REGISTRY_RETRY_ATTEMPTS = 3;
+/**
+ * Pure decision gate for the captchaGated registry-retry loop: isolates the
+ * retry-vs-give-up call from the solve+inject+poll I/O so the attach-timing
+ * race (registry empty/absent, or a callback that fired without a confirmed
+ * transition) can be pinned with a unit test independent of any browser
+ * evaluate() call.
+ */
+function shouldRetryCaptchaRegistry(attemptNumber, maxAttempts, registryState, callbackDiscovered, confirmed) {
+    if (confirmed)
+        return false;
+    const registryRace = registryState === "empty" || registryState === "absent";
+    const callbackFiredButNoNav = callbackDiscovered && !confirmed;
+    return (registryRace || callbackFiredButNoNav) && attemptNumber < maxAttempts;
+}
 /**
  * Attempts to decode opaque request parameters: tries JSON parse, then
  * URL-decode, then base64. Returns the decoded value or null if none worked.
@@ -939,10 +967,11 @@ async function snapshotPage(target, signalCounter, page, captureSelectionState =
     // A resolved child FrameTarget's url() reads location.href off the CDP
     // frame session (frame-target.ts's childFrameTarget), which rejects (or
     // trips its watchdog) once the OOPIF detaches — most commonly right after
-    // a submit click tears down the wizard iframe. Falling back to page.url()
-    // lets the post-submit navigation still register as a urlChanged signal
-    // instead of throwing the whole step out of the cascade.
-    const url = page ? await target.url().catch(() => page.url()) : await target.url();
+    // a submit click tears down the wizard iframe. readCurrentFrameUrl
+    // re-resolves against the declared frame selector before ever falling
+    // back to page.url(), so a same-origin in-frame navigation still reports
+    // its own scope's URL instead of masquerading as the top wrapper page.
+    const url = page ? await (0, frame_target_1.readCurrentFrameUrl)(page, target) : await target.url();
     return {
         networkCount: signalCounter.n,
         url,
@@ -980,7 +1009,7 @@ async function snapshotPage(target, signalCounter, page, captureSelectionState =
  */
 async function resolveDumpPageIdentity(page, frameTarget) {
     const pageTitle = await (frameTarget ?? (0, frame_target_1.mainFrameTarget)(page)).title().catch(() => "");
-    const pageUrl = await (frameTarget ? frameTarget.url() : Promise.resolve(page.url())).catch(() => page.url());
+    const pageUrl = frameTarget ? await (0, frame_target_1.readCurrentFrameUrl)(page, frameTarget) : page.url();
     return { pageTitle, pageUrl };
 }
 /**
@@ -4454,6 +4483,52 @@ async function waitForTransitionBody(params) {
     }
     return false;
 }
+/**
+ * Reduces a URL to origin+pathname (query/hash ignored) so a captcha
+ * target's post-submit `url()` can be compared to its pre-submit baseline
+ * without a same-page query-string change (e.g. a step counter) registering
+ * as a false advance. Unparseable URLs compare equal only to themselves.
+ */
+function originAndPathOf(url) {
+    try {
+        const parsed = new URL(url);
+        return `${parsed.origin}${parsed.pathname}`;
+    }
+    catch {
+        return url;
+    }
+}
+/**
+ * Credits a captchaGated advance from an observed navigation: true only
+ * when origin or pathname actually changed, so a same-page query-string
+ * mutation (e.g. a step counter) never counts as an advance.
+ */
+function hasOriginOrPathChanged(preUrl, postUrl) {
+    return originAndPathOf(preUrl) !== originAndPathOf(postUrl);
+}
+/**
+ * Polls the captcha target's URL for an origin/path change after a
+ * multipart form submit, which navigates the page without producing any
+ * matchable XHR body for {@link waitForTransitionBody} to catch — the
+ * regex-based poll alone silently misses a full-page submit advance. Runs
+ * independently of whether `advanceTransitionBodyPattern` is configured.
+ */
+async function waitForCaptchaNavigation(params) {
+    const { page, captchaTarget, baselineUrl, timeoutMs, intervalMs } = params;
+    const check = async () => {
+        const currentUrl = await (0, frame_target_1.readCurrentFrameUrl)(page, captchaTarget);
+        return hasOriginOrPathChanged(baselineUrl, currentUrl);
+    };
+    if (await check())
+        return true;
+    const deadline = performance.now() + timeoutMs;
+    while (performance.now() < deadline) {
+        await page.waitForTimeout(intervalMs);
+        if (await check())
+            return true;
+    }
+    return false;
+}
 /**
  * Shared in-page discovery of the widget's callback, used identically by
  * `injectCaptchaTokenAndSubmit`'s precheck (to decide the branch) and its
@@ -7333,7 +7408,7 @@ async function executeStepWithHealing(params) {
         });
         if ((cfg.extract ?? "link") === "link") {
             const emailStepTarget = frameTarget ?? (0, frame_target_1.mainFrameTarget)(page);
-            const currentPageUrl = await emailStepTarget.url();
+            const currentPageUrl = await (0, frame_target_1.readCurrentFrameUrl)(page, emailStepTarget);
             const url = extractLinkFromMessage(msg, cfg.linkPattern, currentPageUrl);
             if (!url) {
                 throw new errors_2.EmailStepExtractError("no link matched in the verification email");
@@ -7629,52 +7704,51 @@ async function executeStepWithHealing(params) {
         }
         else {
             logger.info(`${formatStepPrefix(stepIndex, totalSteps)} captchaGated step: sitekey found, requesting solve`);
-            const pageUrl = await captchaTarget.url();
+            const pageUrl = await (0, frame_target_1.readCurrentFrameUrl)(page, captchaTarget);
             const uaRaw = await page.evaluate("navigator.userAgent").catch(() => null);
             const userAgent = typeof uaRaw === "string" ? uaRaw : undefined;
-            const solved = await (0, captcha_solver_1.solveCaptcha)({
-                type: "hcaptcha",
-                siteKey,
-                pageUrl,
-                isInvisible,
-                userAgent,
-            }).catch((err) => {
-                logger.error(`${formatStepPrefix(stepIndex, totalSteps)} captchaGated step: solve failed (${(0, errors_1.toErrorMessage)(err)}); failing the step rather than silently proceeding`);
-                throw err;
-            });
-            const preCaptchaCaptureIdx = latestCaptureIndex(recentCaptures);
-            const injectResult = await injectCaptchaTokenAndSubmit(captchaTarget, solved.token);
-            const registryState = await captchaTarget.evaluate(`(() => {
+            // Bounded retry around the solve+inject+registryState-check+poll unit:
+            // the registry-empty/absent condition is an intermittent attach-timing
+            // race (the render callback hasn't registered by the time the solve
+            // finishes), not a deterministic failure, so a single attempt is too
+            // eager to give up and route straight to a global replan. Also retries
+            // when the widget's own callback fired (callbackDiscovered) but neither
+            // transition poll below confirmed an advance — the callback-fired-but-
+            // no-nav case is the same underlying race from the other side.
+            for (let captchaAttempt = 1; captchaAttempt <= CAPTCHA_REGISTRY_RETRY_ATTEMPTS; captchaAttempt++) {
+                if (captchaAttempt > 1) {
+                    // Re-assert the capture install in the target frame before
+                    // retrying: a late-attaching registry may not exist yet on the
+                    // first attempt, and re-running this idempotent install gives a
+                    // widget that renders on demand another chance to be caught.
+                    await captchaTarget.evaluate((0, captcha_callback_capture_1.buildHcaptchaCallbackCaptureScript)()).catch(() => undefined);
+                }
+                const solved = await (0, captcha_solver_1.solveCaptcha)({
+                    type: "hcaptcha",
+                    siteKey,
+                    pageUrl,
+                    isInvisible,
+                    userAgent,
+                }).catch((err) => {
+                    logger.error(`${formatStepPrefix(stepIndex, totalSteps)} captchaGated step: solve failed (${(0, errors_1.toErrorMessage)(err)}); failing the step rather than silently proceeding`);
+                    throw err;
+                });
+                const preCaptchaCaptureIdx = latestCaptureIndex(recentCaptures);
+                const injectResult = await injectCaptchaTokenAndSubmit(captchaTarget, solved.token);
+                const registryState = await captchaTarget.evaluate(`(() => {
           const registry = window[${JSON.stringify(captcha_callback_capture_1.HCAPTCHA_CALLBACK_REGISTRY_GLOBAL)}];
           if (!registry) return "absent";
           return Object.keys(registry).length === 0 ? "empty" : "populated";
         })()`);
-            logger.info(`${formatStepPrefix(stepIndex, totalSteps)} captchaGated step: token injected=${injectResult.injected} hasForm=${injectResult.hasForm} callbackDiscovered=${injectResult.callbackDiscovered} registryState=${registryState}`);
-            // Reuse the EXISTING waitForTransitionBody poll (no new captcha-specific
-            // poll loop), just with a widened budget so the solve+submit round trip
-            // doesn't eat into the plain ADVANCE_TRANSITION_POLL_MS window. A
-            // pattern-confirmed transition here is treated as verified the same way
-            // the probe-absent branch below treats a detected transition — the
-            // normal cascade/verifier still runs when the pattern doesn't confirm
-            // (or none is configured), so this never replaces that verifier.
-            let confirmed = false;
-            if (advanceTransitionBodyPattern) {
-                confirmed = await waitForTransitionBody({
-                    page,
-                    preIdx: preCaptchaCaptureIdx,
-                    advanceTransitionBodyPattern,
-                    timeoutMs: CAPTCHA_TRANSITION_POLL_MS,
-                    intervalMs: ADVANCE_TRANSITION_POLL_INTERVAL_MS,
-                });
-                logger.info(`${formatStepPrefix(stepIndex, totalSteps)} captchaGated step: post-submit transition poll confirmed=${confirmed}`);
-            }
-            // Some widgets' own render callback already submitted the form as a
-            // side effect of the inject (the poll above would have confirmed it);
-            // only issue an explicit submit when no transition was observed AND a
-            // form is known to exist to submit — otherwise this would double-submit
-            // a form the widget's own callback already advanced.
-            if (!confirmed && injectResult.hasForm) {
-                await submitCaptchaGatedForm(captchaTarget);
+                logger.info(`${formatStepPrefix(stepIndex, totalSteps)} captchaGated step: attempt=${captchaAttempt}/${CAPTCHA_REGISTRY_RETRY_ATTEMPTS} token injected=${injectResult.injected} hasForm=${injectResult.hasForm} callbackDiscovered=${injectResult.callbackDiscovered} registryState=${registryState}`);
+                // Reuse the EXISTING waitForTransitionBody poll (no new captcha-specific
+                // poll loop), just with a widened budget so the solve+submit round trip
+                // doesn't eat into the plain ADVANCE_TRANSITION_POLL_MS window. A
+                // pattern-confirmed transition here is treated as verified the same way
+                // the probe-absent branch below treats a detected transition — the
+                // normal cascade/verifier still runs when the pattern doesn't confirm
+                // (or none is configured), so this never replaces that verifier.
+                let confirmed = false;
                 if (advanceTransitionBodyPattern) {
                     confirmed = await waitForTransitionBody({
                         page,
@@ -7683,24 +7757,71 @@ async function executeStepWithHealing(params) {
                         timeoutMs: CAPTCHA_TRANSITION_POLL_MS,
                         intervalMs: ADVANCE_TRANSITION_POLL_INTERVAL_MS,
                     });
-                    logger.info(`${formatStepPrefix(stepIndex, totalSteps)} captchaGated step: post-fallback-submit transition poll confirmed=${confirmed}`);
+                    logger.info(`${formatStepPrefix(stepIndex, totalSteps)} captchaGated step: post-submit transition poll confirmed=${confirmed}`);
                 }
-            }
-            if (confirmed) {
-                trajectory?.push({ stepIndex, verifiedBy: "network" });
-                return "completed";
-            }
-            // No render-config callback was discoverable AND neither the inject's
-            // own submit path nor the explicit fallback submit produced a
-            // confirmed transition: the token was never actually delivered
-            // through the site's own submit machinery. Fail loudly instead of
-            // falling through to the normal cascade as if the solve had worked —
-            // that's exactly the misleading "token injected=true" silent no-op
-            // this hook must not produce. Only applies when a pattern is actually
-            // configured to poll for — with none configured, no poll ever ran, so
-            // there's nothing here to contradict the normal cascade/verifier.
-            if (advanceTransitionBodyPattern && !injectResult.callbackDiscovered) {
-                throw new errors_2.CaptchaError(`${formatStepPrefix(stepIndex, totalSteps)} captchaGated step: no render-config callback could be found and delivered, and no transition was confirmed after the solve`);
+                // Some widgets' own render callback already submitted the form as a
+                // side effect of the inject (the poll above would have confirmed it);
+                // only issue an explicit submit when no transition was observed AND a
+                // form is known to exist to submit — otherwise this would double-submit
+                // a form the widget's own callback already advanced.
+                if (!confirmed && injectResult.hasForm) {
+                    await submitCaptchaGatedForm(captchaTarget);
+                    if (advanceTransitionBodyPattern) {
+                        confirmed = await waitForTransitionBody({
+                            page,
+                            preIdx: preCaptchaCaptureIdx,
+                            advanceTransitionBodyPattern,
+                            timeoutMs: CAPTCHA_TRANSITION_POLL_MS,
+                            intervalMs: ADVANCE_TRANSITION_POLL_INTERVAL_MS,
+                        });
+                        logger.info(`${formatStepPrefix(stepIndex, totalSteps)} captchaGated step: post-fallback-submit transition poll confirmed=${confirmed}`);
+                    }
+                }
+                if (confirmed) {
+                    trajectory?.push({ stepIndex, verifiedBy: "network" });
+                    return "completed";
+                }
+                // A full-page multipart form submit (no matching XHR body, or no
+                // pattern configured at all) still navigates the page — that's a
+                // real advance the requestPostData-based poll above can never see.
+                // This poll runs regardless of whether advanceTransitionBodyPattern
+                // is set: it's an additional signal, not a replacement for the
+                // pattern-based one.
+                const navigationConfirmed = await waitForCaptchaNavigation({
+                    page,
+                    captchaTarget,
+                    baselineUrl: pageUrl,
+                    timeoutMs: CAPTCHA_TRANSITION_POLL_MS,
+                    intervalMs: ADVANCE_TRANSITION_POLL_INTERVAL_MS,
+                });
+                if (navigationConfirmed) {
+                    logger.info(`${formatStepPrefix(stepIndex, totalSteps)} captchaGated step: post-submit navigation to a new origin/path confirmed the advance`);
+                    trajectory?.push({ stepIndex, verifiedBy: "url" });
+                    return "completed";
+                }
+                // The intermittent race this loop exists for: a still-empty/absent
+                // registry means the callback attach hadn't landed when this attempt
+                // solved+injected, and a discovered-but-unconfirmed callback means it
+                // fired without a poll seeing the resulting transition. Both are
+                // worth another attempt within budget rather than an immediate throw.
+                if (shouldRetryCaptchaRegistry(captchaAttempt, CAPTCHA_REGISTRY_RETRY_ATTEMPTS, registryState, injectResult.callbackDiscovered, confirmed)) {
+                    logger.info(`${formatStepPrefix(stepIndex, totalSteps)} captchaGated step: registryState=${registryState} callbackDiscovered=${injectResult.callbackDiscovered} with no confirmed transition on attempt ${captchaAttempt}; retrying`);
+                    continue;
+                }
+                // No render-config callback was discoverable AND neither the inject's
+                // own submit path nor the explicit fallback submit produced a
+                // confirmed transition, network or navigation: the token was never
+                // actually delivered through the site's own submit machinery. Fail
+                // loudly instead of falling through to the normal cascade as if the
+                // solve had worked — that's exactly the misleading "token
+                // injected=true" silent no-op this hook must not produce. Only
+                // applies when a pattern is actually configured to poll for — with
+                // none configured, no network poll ever ran, so there's nothing here
+                // to contradict the normal cascade/verifier.
+                if (advanceTransitionBodyPattern && !injectResult.callbackDiscovered) {
+                    throw new errors_2.CaptchaError(`${formatStepPrefix(stepIndex, totalSteps)} captchaGated step: no render-config callback could be found and delivered, and no transition was confirmed after the solve`);
+                }
+                break;
             }
         }
     }