@enricai/barnacle 1.6.8 → 1.6.10

package/dist/scraper/flow-runner.js

@@ -87,6 +87,7 @@ const schemas_1 = require("../lib/llm/schemas");
 const logging_1 = require("../lib/logging");
 const call_capture_1 = require("../lib/telemetry/call-capture");
 const call_types_1 = require("../lib/telemetry/call-types");
+const deep_locator_candidates_1 = require("../scraper/deep-locator-candidates");
 const errors_2 = require("../scraper/errors");
 const frame_target_1 = require("../scraper/frame-target");
 const phantom_click_1 = require("../scraper/phantom-click");
@@ -4408,6 +4409,28 @@ async function probeFormValidityBeforeSubmit(params) {
         return [];
     }
 }
+/**
+ * Adapts `resolveDeepLocatorCandidates` results into `Action`-shaped
+ * evidence for `rephraseWithLLM`, which only knows about Stagehand's
+ * `Action` type. Never throws: `resolveDeepLocatorCandidates` itself
+ * degrades to `[]` on a resolver failure, so this only feeds the rephrase
+ * prompt richer evidence when available — a frame-scoped step whose
+ * observe AND deepLocator both come back empty just gets the same `[]`
+ * evidence it would have gotten before this fix.
+ *
+ * `instruction` is forwarded to `resolveDeepLocatorCandidates` so the
+ * evidence list is ranked by relevance to the step, same as the act path —
+ * an unranked `[]`-then-DOM-order list would feed the rephrase LLM its
+ * worst evidence first instead of its best.
+ */
+async function deepLocatorCandidatesAsActions(page, frameSelector, instruction) {
+    const candidates = await (0, deep_locator_candidates_1.resolveDeepLocatorCandidates)(page, frameSelector, "*", instruction);
+    return candidates.map((c) => ({
+        selector: c.selector,
+        description: c.accessibleText || "(no accessible text)",
+        method: "click",
+    }));
+}
 /**
  * Cheap pre-cascade reachability gate. Runs before the 5-attempt healing cascade
  * (and any global replan) so a step aimed at the wrong page state fails fast
@@ -4419,10 +4442,16 @@ async function probeFormValidityBeforeSubmit(params) {
  *
  * `frameTarget` scopes both observe calls to a resolved cross-origin child
  * frame when the flow declared `frameSelector`; omitted (main frame) is
- * byte-identical to today's unscoped calls.
+ * byte-identical to today's unscoped calls. When `frameTarget.frame` is a
+ * resolved child frame, `observe()` is blind to it (measured against a
+ * cross-origin OOPIF — see `deep-locator-candidates.ts`'s module docblock),
+ * so a 0-candidate focused+unfocused observe pair additionally falls back to
+ * `resolveDeepLocatorCandidates` before declaring the step "absent" — a
+ * frame-scoped step must not short-circuit to replan before the cascade
+ * (which itself now routes through the same resolver) ever runs.
  */
 async function probeStepBeforeAttempts(params) {
-    const { stagehand, step, stepIndex, totalSteps, logger, captureFn, frameTarget } = params;
+    const { stagehand, page, step, stepIndex, totalSteps, logger, captureFn, frameTarget } = params;
     try {
         const candidates = await (0, stagehand_guard_1.guardedObserve)(stagehand, step, { timeout: exports.STEP_WATCHDOG_MS }, captureFn, frameTarget);
         if (candidates.length === 0) {
@@ -4442,6 +4471,13 @@ async function probeStepBeforeAttempts(params) {
                 logger.info(`${formatStepPrefix(stepIndex, totalSteps)}: focused probe found 0 candidates but unfocused observe found ${unfocused.length} — treating as present (let cascade resolve)`);
                 return "present";
             }
+            if (frameTarget?.frame) {
+                const deepLocatorCandidates = await (0, deep_locator_candidates_1.resolveDeepLocatorCandidates)(page, frameTarget.frameSelector, "*");
+                if (deepLocatorCandidates.length > 0) {
+                    logger.info(`${formatStepPrefix(stepIndex, totalSteps)}: observe found 0 candidates (focused and unfocused) but deepLocator found ${deepLocatorCandidates.length} — treating as present (let cascade resolve)`);
+                    return "present";
+                }
+            }
             logger.info(`${formatStepPrefix(stepIndex, totalSteps)}: probe found 0 candidates (focused and unfocused) — treating as absent (skip cascade, route to replan if required)`);
             return "absent";
         }
@@ -4490,7 +4526,7 @@ async function executeStepWithHealing(params) {
     // the existing cascade.
     if (await tryUploadPrimitive({
         page,
-        target: await (0, frame_target_1.resolveFrameTarget)(page),
+        target: frameTarget ?? (0, frame_target_1.mainFrameTarget)(page),
         isUploadStep: upload,
         fixture: resumeFixture,
         logger,
@@ -4508,11 +4544,10 @@ async function executeStepWithHealing(params) {
     // question unanswered. No-op (returns false → falls through) when the step
     // isn't a single-dropdown select or no option matches.
     //
-    // resolveFrameTarget(page) resolves synchronously to the main-frame target
-    // when no frameSelector is set, so this bridge is behavior-identical for
-    // every existing site until the sibling subtask threads a resolved target
-    // through end-to-end.
-    const selectFrameTarget = await (0, frame_target_1.resolveFrameTarget)(page);
+    // Reuse the already-resolved ambient frameTarget rather than re-resolving:
+    // falls back to the main-frame target when no frameSelector is set, so this
+    // bridge stays behavior-identical for every existing site.
+    const selectFrameTarget = frameTarget ?? (0, frame_target_1.mainFrameTarget)(page);
     if (await trySelectPrimitive({
         page,
         target: selectFrameTarget,
@@ -4588,6 +4623,7 @@ async function executeStepWithHealing(params) {
     // burning 4 attempts on a page that clearly isn't the right one.
     const probeResult = await probeStepBeforeAttempts({
         stagehand,
+        page,
         step,
         stepIndex,
         totalSteps,
@@ -4651,7 +4687,15 @@ async function executeStepWithHealing(params) {
             .evaluate("document.body ? document.body.outerHTML : null")
             .catch(() => null);
         const bodyOuterHtml = typeof bodyOuterHtmlRaw === "string" ? bodyOuterHtmlRaw.slice(0, 100_000) : null;
-        const unfocusedObserve = await (0, stagehand_guard_1.guardedObserve)(stagehand, undefined, { timeout: exports.STEP_WATCHDOG_MS }, captureFn, frameTarget).catch(() => []);
+        const probeAbsentObservedUnfocused = await (0, stagehand_guard_1.guardedObserve)(stagehand, undefined, { timeout: exports.STEP_WATCHDOG_MS }, captureFn, frameTarget).catch(() => []);
+        // observe() is blind to a cross-origin OOPIF, so a frame-scoped empty
+        // result degrades to the deep-locator resolver for dump evidence — this
+        // dump feeds replanRemainingFlow's diagnostic prompt, and an empty
+        // candidate list there returns "repeat the failed step", burning the
+        // replan budget on every frame-scoped probe-absent failure.
+        const unfocusedObserve = probeAbsentObservedUnfocused.length === 0 && frameTarget?.frame
+            ? await deepLocatorCandidatesAsActions(page, frameTarget.frameSelector)
+            : probeAbsentObservedUnfocused;
         const dumpPath = onStepFailure?.({
             stepIndex,
             phase,
@@ -4689,7 +4733,7 @@ async function executeStepWithHealing(params) {
         : 0;
     if (requireSubmitEndpoint) {
         const invalidControls = await probeFormValidityBeforeSubmit({
-            target: await (0, frame_target_1.resolveFrameTarget)(page),
+            target: frameTarget ?? (0, frame_target_1.mainFrameTarget)(page),
             stepIndex,
             totalSteps,
             logger,
@@ -4974,7 +5018,58 @@ async function executeStepWithHealing(params) {
                     ? { ignoreSelectors: [...triedSelectors], timeout: exports.STEP_WATCHDOG_MS }
                     : { timeout: exports.STEP_WATCHDOG_MS };
                 const candidates = await (0, stagehand_guard_1.guardedObserve)(stagehand, step, observeOptions, captureFn, frameTarget);
-                if (candidates.length === 0) {
+                // observe() is blind to a cross-origin OOPIF (see
+                // deep-locator-candidates.ts's module docblock) — when the step is
+                // frame-scoped and observe came back empty, fall back to the deep
+                // locator resolver before declaring the attempt candidate-less.
+                // Passing `step` ranks candidates by relevance to the instruction
+                // (see resolveDeepLocatorCandidates) so the top pick below is the
+                // element the step actually names, not just DOM order.
+                // Attempt 4's ignoreSelectors has no deepLocator equivalent, so the
+                // exclusion is applied here by filtering resolved (already-ranked)
+                // candidates against triedSelectors instead — otherwise attempt 4
+                // would re-pick the same failed element and burn the attempt.
+                const deepLocatorCandidates = candidates.length === 0 && frameTarget?.frame
+                    ? (await (0, deep_locator_candidates_1.resolveDeepLocatorCandidates)(page, frameTarget.frameSelector, "*", step)).filter((c) => !triedSelectors.includes(c.selector))
+                    : [];
+                if (candidates.length === 0 && deepLocatorCandidates.length > 0) {
+                    const top = deepLocatorCandidates[0];
+                    if (top) {
+                        record.instruction = `deepLocator: ${top.accessibleText || "(no accessible text)"}`;
+                        // Deny-list guard: mirrors the observe branch's refusal below —
+                        // never act on a wizard-exit control regardless of which
+                        // candidate source (observe vs. deepLocator) surfaced it.
+                        if (isWizardExitAction(top.accessibleText, wizardExitButtonLabels)) {
+                            record.errorMessage = `refused wizard-exit control: "${top.accessibleText.slice(0, 60)}"`;
+                            triedSelectors.push(top.selector);
+                            record.triedSelectors = [top.selector];
+                            attempts.push(record);
+                            failureReasons.push(record.errorMessage);
+                            logger.info(`${formatStepPrefix(stepIndex, totalSteps)} attempt ${attempt}: ${record.errorMessage}`);
+                            continue;
+                        }
+                        triedSelectors.push(top.selector);
+                        record.triedSelectors = [top.selector];
+                        try {
+                            await (0, deep_locator_candidates_1.clickDeepLocatorCandidate)(page, frameTarget?.frameSelector, "*", top.index);
+                            record.actResultSuccess = true;
+                            record.actResultDescription = `deepLocator clicked "${top.accessibleText || top.selector}"`;
+                            // Synthesize a click action so downstream verification (network
+                            // / url / dom) treats this exactly like any other resolved
+                            // click — same idiom as deep-submit-locator/structured-click.
+                            resolvedAction = {
+                                selector: top.selector,
+                                description: record.actResultDescription,
+                                method: "click",
+                            };
+                        }
+                        catch (err) {
+                            record.actResultSuccess = false;
+                            record.errorMessage = `deepLocator: click threw ${(0, errors_1.toErrorMessage)(err)}`;
+                        }
+                    }
+                }
+                else if (candidates.length === 0) {
                     record.errorMessage = "observe returned no candidates";
                     // Optional-step short-circuit: when attempt 2 confirms no candidates
                     // match AND the step was marked optional in the flow, skip cleanly.
@@ -5051,8 +5146,8 @@ async function executeStepWithHealing(params) {
                         target.arguments.length > 0) {
                         const fillValue = target.arguments[0];
                         if (typeof fillValue === "string") {
-                            const frameTarget = await (0, frame_target_1.resolveFrameTarget)(page);
-                            const dateFill = await fillHtml5DateTimeInput(frameTarget, target.selector, fillValue);
+                            const dateFillTarget = frameTarget ?? (0, frame_target_1.mainFrameTarget)(page);
+                            const dateFill = await fillHtml5DateTimeInput(dateFillTarget, target.selector, fillValue);
                             if (dateFill !== null) {
                                 record.errorMessage = dateFill.filled
                                     ? `html5-date-fallback: filled ${dateFill.inputType}="${dateFill.postValue}"`
@@ -5072,7 +5167,7 @@ async function executeStepWithHealing(params) {
                                 // component rejection, masked-input library reformatting).
                                 // Generic primitive that the verifier's existing signals
                                 // (network/url/dom/htmlDelta/textChanged) miss.
-                                const readback = await verifyFillReadback(frameTarget, target.selector, fillValue);
+                                const readback = await verifyFillReadback(dateFillTarget, target.selector, fillValue);
                                 if (readback !== null) {
                                     if (readback.outcome === "rejected") {
                                         record.errorMessage = `fill-value-rejected: tried "${fillValue.slice(0, 60)}" on <${readback.tag}>; element value remains empty (silent rejection — HTML5 type validation, framework controlled-component, or masked-input library)`;
@@ -5190,7 +5285,7 @@ async function executeStepWithHealing(params) {
             return { resolved: true, isCheckable: true, checked: false, strategyUsed: null };
           })()`;
                     try {
-                        const structuredClickTarget = await (0, frame_target_1.resolveFrameTarget)(page);
+                        const structuredClickTarget = frameTarget ?? (0, frame_target_1.mainFrameTarget)(page);
                         const result = await structuredClickTarget.evaluate(probeExpr);
                         if (result !== null && typeof result === "object" && "resolved" in result) {
                             const probe = result;
@@ -5241,7 +5336,15 @@ async function executeStepWithHealing(params) {
                         "anthropic billing exhausted (FATAL_BILLING already logged); skipping rephrase";
                 }
                 else {
-                    const candidates = await (0, stagehand_guard_1.guardedObserve)(stagehand, step, { timeout: exports.STEP_WATCHDOG_MS }, captureFn).catch(() => []);
+                    const observedCandidates = await (0, stagehand_guard_1.guardedObserve)(stagehand, step, { timeout: exports.STEP_WATCHDOG_MS }, captureFn, frameTarget).catch(() => []);
+                    // observe() is blind to a cross-origin OOPIF, so a frame-scoped
+                    // empty result degrades to the deep-locator resolver for prompt
+                    // evidence rather than hard-failing — lower stakes than the
+                    // attempt-2/4 click path since this only feeds the rephrase
+                    // prompt, so a resolver error/empty result is fine to swallow.
+                    const candidates = observedCandidates.length === 0 && frameTarget?.frame
+                        ? await deepLocatorCandidatesAsActions(page, frameTarget.frameSelector, step)
+                        : observedCandidates;
                     // Fetch live-page evidence so the rephrase prompt can reason about
                     // form state, not just observe candidates. Mirrors the same
                     // extraction the cascade-exhaust dump path already does.
@@ -5252,8 +5355,14 @@ async function executeStepWithHealing(params) {
                     });
                     // Unfocused observe so the rephrase prompt can see ambient UI
                     // like modal Save/Close buttons that the focused candidates
-                    // (filtered by the failed step's instruction) would hide.
-                    const unfocused = await (0, stagehand_guard_1.guardedObserve)(stagehand, undefined, { timeout: exports.STEP_WATCHDOG_MS }, captureFn).catch(() => []);
+                    // (filtered by the failed step's instruction) would hide. Scoped to
+                    // frameTarget: for a frame-scoped flow the ambient UI that matters
+                    // (the wizard's own Save/Close controls) lives inside the iframe
+                    // alongside the failed step, not in the top document.
+                    const observedUnfocused = await (0, stagehand_guard_1.guardedObserve)(stagehand, undefined, { timeout: exports.STEP_WATCHDOG_MS }, captureFn, frameTarget).catch(() => []);
+                    const unfocused = observedUnfocused.length === 0 && frameTarget?.frame
+                        ? await deepLocatorCandidatesAsActions(page, frameTarget.frameSelector)
+                        : observedUnfocused;
                     const submitFailureList = extractSubmitFailureEvidence(recentCaptures, ownBackendHostnames);
                     const gaEventList = extractGaEventEvidence(recentCaptures);
                     const priorAttemptsForPrompt = attempts.map((a, i) => ({
@@ -5322,7 +5431,7 @@ async function executeStepWithHealing(params) {
         // false for non-radio/non-checkbox clicks so the network/URL signal still
         // decides those. Radios/checkboxes are click-but-no-network just like fills.
         const domVerified = resolvedAction !== null && (isStateClass || isClick)
-            ? await verifyDomEffect(await (0, frame_target_1.resolveFrameTarget)(page), resolvedAction)
+            ? await verifyDomEffect(frameTarget ?? (0, frame_target_1.mainFrameTarget)(page), resolvedAction)
             : false;
         // Interior-advance transition gate (opt-in). On SPAs where a page advance
         // and a mere field-edit share one endpoint URL (the wizard ATS's `/gq`:
@@ -5415,7 +5524,7 @@ async function executeStepWithHealing(params) {
             return null;
           })()`;
                 try {
-                    const submittedStateTarget = await (0, frame_target_1.resolveFrameTarget)(page);
+                    const submittedStateTarget = frameTarget ?? (0, frame_target_1.mainFrameTarget)(page);
                     domSubmittedMatch = (await submittedStateTarget.evaluate(probeExpr));
                 }
                 catch (err) {
@@ -5424,7 +5533,7 @@ async function executeStepWithHealing(params) {
             }
             // Build the unfocused-observe evidence list. Used by the judge to
             // assess whether the page transitioned to a success state.
-            const unfocusedForJudge = await (0, stagehand_guard_1.guardedObserve)(stagehand, undefined, { timeout: exports.STEP_WATCHDOG_MS }, captureFn).catch(() => []);
+            const unfocusedForJudge = await (0, stagehand_guard_1.guardedObserve)(stagehand, undefined, { timeout: exports.STEP_WATCHDOG_MS }, captureFn, frameTarget).catch(() => []);
             // Quick invalid-marker count (deterministic DOM querying — counting
             // structural ng-invalid containers is not fuzzy matching, just
             // observing existence).
@@ -5540,7 +5649,7 @@ async function executeStepWithHealing(params) {
                     // reliably trigger that default action — same gap N+42 documented
                     // for direct checkbox/radio clicks.
                     const clickExpr = `(() => { const r = document.evaluate(${JSON.stringify(xpath)}, document, null, XPathResult.FIRST_ORDERED_NODE_TYPE, null); let el = r.singleNodeValue; if (!el || typeof el.click !== "function") return { fired: false }; if (el.tagName === "LABEL") { const wrapped = el.querySelector("input[type=checkbox], input[type=radio]"); if (wrapped) el = wrapped; } if (el.type === "checkbox" || el.type === "radio") { el.checked = true; el.dispatchEvent(new Event("click", { bubbles: true })); el.dispatchEvent(new Event("change", { bubbles: true })); return { fired: true, kind: "checkbox", checked: el.checked }; } el.click(); return { fired: true, kind: "click" }; })()`;
-                    const n16FallbackTarget = await (0, frame_target_1.resolveFrameTarget)(page);
+                    const n16FallbackTarget = frameTarget ?? (0, frame_target_1.mainFrameTarget)(page);
                     const probeResult = (await n16FallbackTarget.evaluate(clickExpr));
                     const fired = probeResult.fired;
                     // Vacuous-click guard for the n+16 fallback. Same rationale as
@@ -5860,7 +5969,15 @@ async function executeStepWithHealing(params) {
             }
         }
     }
-    const finalObserve = await (0, stagehand_guard_1.guardedObserve)(stagehand, step, { timeout: exports.STEP_WATCHDOG_MS }, captureFn, frameTarget).catch(() => []);
+    const cascadeExhaustObservedFinal = await (0, stagehand_guard_1.guardedObserve)(stagehand, step, { timeout: exports.STEP_WATCHDOG_MS }, captureFn, frameTarget).catch(() => []);
+    // observe() is blind to a cross-origin OOPIF, so a frame-scoped empty
+    // result degrades to the deep-locator resolver for dump evidence — this
+    // dump feeds replanRemainingFlow's diagnostic prompt, and an empty
+    // candidate list there returns "repeat the failed step", burning the
+    // replan budget on every frame-scoped cascade-exhaust failure.
+    const finalObserve = cascadeExhaustObservedFinal.length === 0 && frameTarget?.frame
+        ? await deepLocatorCandidatesAsActions(page, frameTarget.frameSelector, step)
+        : cascadeExhaustObservedFinal;
     const { pageTitle, pageUrl } = await resolveDumpPageIdentity(page, frameTarget);
     // Discriminator data for "Stagehand sees nothing" failures: capture the raw
     // DOM and an unfocused observe so a triager can tell empty-page from
@@ -5869,7 +5986,10 @@ async function executeStepWithHealing(params) {
         .evaluate("document.body ? document.body.outerHTML : null")
         .catch(() => null);
     const bodyOuterHtml = typeof bodyOuterHtmlRaw === "string" ? bodyOuterHtmlRaw.slice(0, 100_000) : null;
-    const unfocusedObserve = await (0, stagehand_guard_1.guardedObserve)(stagehand, undefined, { timeout: exports.STEP_WATCHDOG_MS }, captureFn, frameTarget).catch(() => []);
+    const cascadeExhaustObservedUnfocused = await (0, stagehand_guard_1.guardedObserve)(stagehand, undefined, { timeout: exports.STEP_WATCHDOG_MS }, captureFn, frameTarget).catch(() => []);
+    const unfocusedObserve = cascadeExhaustObservedUnfocused.length === 0 && frameTarget?.frame
+        ? await deepLocatorCandidatesAsActions(page, frameTarget.frameSelector)
+        : cascadeExhaustObservedUnfocused;
     const dumpPath = onStepFailure?.({
         stepIndex,
         phase,
@@ -5951,8 +6071,6 @@ async function runHealingFlow(deps) {
     let submitVerified = false;
     let submitStepSkipped = false;
     let lastStepIndex = -1;
-    const frameTarget = await (0, frame_target_1.resolveFrameTarget)(page, deps.frameSelector);
-    await (0, frame_target_1.waitForChildFrameReady)(frameTarget);
     const stopCapture = wireSignalCapture(page, {
         counter,
         signalCounter,
@@ -5974,6 +6092,14 @@ async function runHealingFlow(deps) {
                 throw new errors_2.StepVerificationError(`${formatStepPrefix(i, () => steps.length)} flow exceeded its maxFlowMs budget (${maxFlowMs}ms)`, "flow-timeout");
             }
             lastStepIndex = i;
+            // Resolved fresh per step (not cached across the run) so a cross-origin
+            // iframe that attaches mid-flow (e.g. after an "Apply" click reveals a
+            // wizard embedded later in the DOM) is picked up as soon as it's
+            // reachable, mirroring the recon CLI's per-step resolution. `resolveFrameTarget`
+            // falls back to the main-frame target when `frameSelector` is null/unresolvable,
+            // so this is a no-op for every flow that doesn't declare one.
+            const frameTarget = await (0, frame_target_1.resolveFrameTarget)(page, deps.frameSelector);
+            await (0, frame_target_1.waitForChildFrameReady)(frameTarget);
             const outcome = await executeStepWithHealing({
                 stagehand,
                 page,