@enricai/barnacle 1.6.9 → 1.6.10

package/dist/scraper/flow-runner.js

@@ -87,6 +87,7 @@ const schemas_1 = require("../lib/llm/schemas");
 const logging_1 = require("../lib/logging");
 const call_capture_1 = require("../lib/telemetry/call-capture");
 const call_types_1 = require("../lib/telemetry/call-types");
+const deep_locator_candidates_1 = require("../scraper/deep-locator-candidates");
 const errors_2 = require("../scraper/errors");
 const frame_target_1 = require("../scraper/frame-target");
 const phantom_click_1 = require("../scraper/phantom-click");
@@ -4408,6 +4409,28 @@ async function probeFormValidityBeforeSubmit(params) {
         return [];
     }
 }
+/**
+ * Adapts `resolveDeepLocatorCandidates` results into `Action`-shaped
+ * evidence for `rephraseWithLLM`, which only knows about Stagehand's
+ * `Action` type. Never throws: `resolveDeepLocatorCandidates` itself
+ * degrades to `[]` on a resolver failure, so this only feeds the rephrase
+ * prompt richer evidence when available — a frame-scoped step whose
+ * observe AND deepLocator both come back empty just gets the same `[]`
+ * evidence it would have gotten before this fix.
+ *
+ * `instruction` is forwarded to `resolveDeepLocatorCandidates` so the
+ * evidence list is ranked by relevance to the step, same as the act path —
+ * an unranked `[]`-then-DOM-order list would feed the rephrase LLM its
+ * worst evidence first instead of its best.
+ */
+async function deepLocatorCandidatesAsActions(page, frameSelector, instruction) {
+    const candidates = await (0, deep_locator_candidates_1.resolveDeepLocatorCandidates)(page, frameSelector, "*", instruction);
+    return candidates.map((c) => ({
+        selector: c.selector,
+        description: c.accessibleText || "(no accessible text)",
+        method: "click",
+    }));
+}
 /**
  * Cheap pre-cascade reachability gate. Runs before the 5-attempt healing cascade
  * (and any global replan) so a step aimed at the wrong page state fails fast
@@ -4419,10 +4442,16 @@ async function probeFormValidityBeforeSubmit(params) {
  *
  * `frameTarget` scopes both observe calls to a resolved cross-origin child
  * frame when the flow declared `frameSelector`; omitted (main frame) is
- * byte-identical to today's unscoped calls.
+ * byte-identical to today's unscoped calls. When `frameTarget.frame` is a
+ * resolved child frame, `observe()` is blind to it (measured against a
+ * cross-origin OOPIF — see `deep-locator-candidates.ts`'s module docblock),
+ * so a 0-candidate focused+unfocused observe pair additionally falls back to
+ * `resolveDeepLocatorCandidates` before declaring the step "absent" — a
+ * frame-scoped step must not short-circuit to replan before the cascade
+ * (which itself now routes through the same resolver) ever runs.
  */
 async function probeStepBeforeAttempts(params) {
-    const { stagehand, step, stepIndex, totalSteps, logger, captureFn, frameTarget } = params;
+    const { stagehand, page, step, stepIndex, totalSteps, logger, captureFn, frameTarget } = params;
     try {
         const candidates = await (0, stagehand_guard_1.guardedObserve)(stagehand, step, { timeout: exports.STEP_WATCHDOG_MS }, captureFn, frameTarget);
         if (candidates.length === 0) {
@@ -4442,6 +4471,13 @@ async function probeStepBeforeAttempts(params) {
                 logger.info(`${formatStepPrefix(stepIndex, totalSteps)}: focused probe found 0 candidates but unfocused observe found ${unfocused.length} — treating as present (let cascade resolve)`);
                 return "present";
             }
+            if (frameTarget?.frame) {
+                const deepLocatorCandidates = await (0, deep_locator_candidates_1.resolveDeepLocatorCandidates)(page, frameTarget.frameSelector, "*");
+                if (deepLocatorCandidates.length > 0) {
+                    logger.info(`${formatStepPrefix(stepIndex, totalSteps)}: observe found 0 candidates (focused and unfocused) but deepLocator found ${deepLocatorCandidates.length} — treating as present (let cascade resolve)`);
+                    return "present";
+                }
+            }
             logger.info(`${formatStepPrefix(stepIndex, totalSteps)}: probe found 0 candidates (focused and unfocused) — treating as absent (skip cascade, route to replan if required)`);
             return "absent";
         }
@@ -4587,6 +4623,7 @@ async function executeStepWithHealing(params) {
     // burning 4 attempts on a page that clearly isn't the right one.
     const probeResult = await probeStepBeforeAttempts({
         stagehand,
+        page,
         step,
         stepIndex,
         totalSteps,
@@ -4650,7 +4687,15 @@ async function executeStepWithHealing(params) {
             .evaluate("document.body ? document.body.outerHTML : null")
             .catch(() => null);
         const bodyOuterHtml = typeof bodyOuterHtmlRaw === "string" ? bodyOuterHtmlRaw.slice(0, 100_000) : null;
-        const unfocusedObserve = await (0, stagehand_guard_1.guardedObserve)(stagehand, undefined, { timeout: exports.STEP_WATCHDOG_MS }, captureFn, frameTarget).catch(() => []);
+        const probeAbsentObservedUnfocused = await (0, stagehand_guard_1.guardedObserve)(stagehand, undefined, { timeout: exports.STEP_WATCHDOG_MS }, captureFn, frameTarget).catch(() => []);
+        // observe() is blind to a cross-origin OOPIF, so a frame-scoped empty
+        // result degrades to the deep-locator resolver for dump evidence — this
+        // dump feeds replanRemainingFlow's diagnostic prompt, and an empty
+        // candidate list there returns "repeat the failed step", burning the
+        // replan budget on every frame-scoped probe-absent failure.
+        const unfocusedObserve = probeAbsentObservedUnfocused.length === 0 && frameTarget?.frame
+            ? await deepLocatorCandidatesAsActions(page, frameTarget.frameSelector)
+            : probeAbsentObservedUnfocused;
         const dumpPath = onStepFailure?.({
             stepIndex,
             phase,
@@ -4973,7 +5018,58 @@ async function executeStepWithHealing(params) {
                     ? { ignoreSelectors: [...triedSelectors], timeout: exports.STEP_WATCHDOG_MS }
                     : { timeout: exports.STEP_WATCHDOG_MS };
                 const candidates = await (0, stagehand_guard_1.guardedObserve)(stagehand, step, observeOptions, captureFn, frameTarget);
-                if (candidates.length === 0) {
+                // observe() is blind to a cross-origin OOPIF (see
+                // deep-locator-candidates.ts's module docblock) — when the step is
+                // frame-scoped and observe came back empty, fall back to the deep
+                // locator resolver before declaring the attempt candidate-less.
+                // Passing `step` ranks candidates by relevance to the instruction
+                // (see resolveDeepLocatorCandidates) so the top pick below is the
+                // element the step actually names, not just DOM order.
+                // Attempt 4's ignoreSelectors has no deepLocator equivalent, so the
+                // exclusion is applied here by filtering resolved (already-ranked)
+                // candidates against triedSelectors instead — otherwise attempt 4
+                // would re-pick the same failed element and burn the attempt.
+                const deepLocatorCandidates = candidates.length === 0 && frameTarget?.frame
+                    ? (await (0, deep_locator_candidates_1.resolveDeepLocatorCandidates)(page, frameTarget.frameSelector, "*", step)).filter((c) => !triedSelectors.includes(c.selector))
+                    : [];
+                if (candidates.length === 0 && deepLocatorCandidates.length > 0) {
+                    const top = deepLocatorCandidates[0];
+                    if (top) {
+                        record.instruction = `deepLocator: ${top.accessibleText || "(no accessible text)"}`;
+                        // Deny-list guard: mirrors the observe branch's refusal below —
+                        // never act on a wizard-exit control regardless of which
+                        // candidate source (observe vs. deepLocator) surfaced it.
+                        if (isWizardExitAction(top.accessibleText, wizardExitButtonLabels)) {
+                            record.errorMessage = `refused wizard-exit control: "${top.accessibleText.slice(0, 60)}"`;
+                            triedSelectors.push(top.selector);
+                            record.triedSelectors = [top.selector];
+                            attempts.push(record);
+                            failureReasons.push(record.errorMessage);
+                            logger.info(`${formatStepPrefix(stepIndex, totalSteps)} attempt ${attempt}: ${record.errorMessage}`);
+                            continue;
+                        }
+                        triedSelectors.push(top.selector);
+                        record.triedSelectors = [top.selector];
+                        try {
+                            await (0, deep_locator_candidates_1.clickDeepLocatorCandidate)(page, frameTarget?.frameSelector, "*", top.index);
+                            record.actResultSuccess = true;
+                            record.actResultDescription = `deepLocator clicked "${top.accessibleText || top.selector}"`;
+                            // Synthesize a click action so downstream verification (network
+                            // / url / dom) treats this exactly like any other resolved
+                            // click — same idiom as deep-submit-locator/structured-click.
+                            resolvedAction = {
+                                selector: top.selector,
+                                description: record.actResultDescription,
+                                method: "click",
+                            };
+                        }
+                        catch (err) {
+                            record.actResultSuccess = false;
+                            record.errorMessage = `deepLocator: click threw ${(0, errors_1.toErrorMessage)(err)}`;
+                        }
+                    }
+                }
+                else if (candidates.length === 0) {
                     record.errorMessage = "observe returned no candidates";
                     // Optional-step short-circuit: when attempt 2 confirms no candidates
                     // match AND the step was marked optional in the flow, skip cleanly.
@@ -5240,7 +5336,15 @@ async function executeStepWithHealing(params) {
                         "anthropic billing exhausted (FATAL_BILLING already logged); skipping rephrase";
                 }
                 else {
-                    const candidates = await (0, stagehand_guard_1.guardedObserve)(stagehand, step, { timeout: exports.STEP_WATCHDOG_MS }, captureFn, frameTarget).catch(() => []);
+                    const observedCandidates = await (0, stagehand_guard_1.guardedObserve)(stagehand, step, { timeout: exports.STEP_WATCHDOG_MS }, captureFn, frameTarget).catch(() => []);
+                    // observe() is blind to a cross-origin OOPIF, so a frame-scoped
+                    // empty result degrades to the deep-locator resolver for prompt
+                    // evidence rather than hard-failing — lower stakes than the
+                    // attempt-2/4 click path since this only feeds the rephrase
+                    // prompt, so a resolver error/empty result is fine to swallow.
+                    const candidates = observedCandidates.length === 0 && frameTarget?.frame
+                        ? await deepLocatorCandidatesAsActions(page, frameTarget.frameSelector, step)
+                        : observedCandidates;
                     // Fetch live-page evidence so the rephrase prompt can reason about
                     // form state, not just observe candidates. Mirrors the same
                     // extraction the cascade-exhaust dump path already does.
@@ -5255,7 +5359,10 @@ async function executeStepWithHealing(params) {
                     // frameTarget: for a frame-scoped flow the ambient UI that matters
                     // (the wizard's own Save/Close controls) lives inside the iframe
                     // alongside the failed step, not in the top document.
-                    const unfocused = await (0, stagehand_guard_1.guardedObserve)(stagehand, undefined, { timeout: exports.STEP_WATCHDOG_MS }, captureFn, frameTarget).catch(() => []);
+                    const observedUnfocused = await (0, stagehand_guard_1.guardedObserve)(stagehand, undefined, { timeout: exports.STEP_WATCHDOG_MS }, captureFn, frameTarget).catch(() => []);
+                    const unfocused = observedUnfocused.length === 0 && frameTarget?.frame
+                        ? await deepLocatorCandidatesAsActions(page, frameTarget.frameSelector)
+                        : observedUnfocused;
                     const submitFailureList = extractSubmitFailureEvidence(recentCaptures, ownBackendHostnames);
                     const gaEventList = extractGaEventEvidence(recentCaptures);
                     const priorAttemptsForPrompt = attempts.map((a, i) => ({
@@ -5862,7 +5969,15 @@ async function executeStepWithHealing(params) {
             }
         }
     }
-    const finalObserve = await (0, stagehand_guard_1.guardedObserve)(stagehand, step, { timeout: exports.STEP_WATCHDOG_MS }, captureFn, frameTarget).catch(() => []);
+    const cascadeExhaustObservedFinal = await (0, stagehand_guard_1.guardedObserve)(stagehand, step, { timeout: exports.STEP_WATCHDOG_MS }, captureFn, frameTarget).catch(() => []);
+    // observe() is blind to a cross-origin OOPIF, so a frame-scoped empty
+    // result degrades to the deep-locator resolver for dump evidence — this
+    // dump feeds replanRemainingFlow's diagnostic prompt, and an empty
+    // candidate list there returns "repeat the failed step", burning the
+    // replan budget on every frame-scoped cascade-exhaust failure.
+    const finalObserve = cascadeExhaustObservedFinal.length === 0 && frameTarget?.frame
+        ? await deepLocatorCandidatesAsActions(page, frameTarget.frameSelector, step)
+        : cascadeExhaustObservedFinal;
     const { pageTitle, pageUrl } = await resolveDumpPageIdentity(page, frameTarget);
     // Discriminator data for "Stagehand sees nothing" failures: capture the raw
     // DOM and an unfocused observe so a triager can tell empty-page from
@@ -5871,7 +5986,10 @@ async function executeStepWithHealing(params) {
         .evaluate("document.body ? document.body.outerHTML : null")
         .catch(() => null);
     const bodyOuterHtml = typeof bodyOuterHtmlRaw === "string" ? bodyOuterHtmlRaw.slice(0, 100_000) : null;
-    const unfocusedObserve = await (0, stagehand_guard_1.guardedObserve)(stagehand, undefined, { timeout: exports.STEP_WATCHDOG_MS }, captureFn, frameTarget).catch(() => []);
+    const cascadeExhaustObservedUnfocused = await (0, stagehand_guard_1.guardedObserve)(stagehand, undefined, { timeout: exports.STEP_WATCHDOG_MS }, captureFn, frameTarget).catch(() => []);
+    const unfocusedObserve = cascadeExhaustObservedUnfocused.length === 0 && frameTarget?.frame
+        ? await deepLocatorCandidatesAsActions(page, frameTarget.frameSelector)
+        : cascadeExhaustObservedUnfocused;
     const dumpPath = onStepFailure?.({
         stepIndex,
         phase,