@enricai/barnacle 1.12.8 → 1.12.10
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +3 -1
- package/dist/config.d.ts +7 -0
- package/dist/config.d.ts.map +1 -1
- package/dist/config.js +1 -0
- package/dist/config.js.map +1 -1
- package/dist/scraper/retry.d.ts +1 -1
- package/dist/scraper/retry.d.ts.map +1 -1
- package/dist/scraper/retry.js +3 -2
- package/dist/scraper/retry.js.map +1 -1
- package/dist/scripts/recon-browser.d.ts.map +1 -1
- package/dist/scripts/recon-browser.js +628 -581
- package/dist/scripts/recon-browser.js.map +1 -1
- package/dist/scripts/recon-generate-repeated-section-fixture.d.ts +14 -0
- package/dist/scripts/recon-generate-repeated-section-fixture.d.ts.map +1 -0
- package/dist/scripts/recon-generate-repeated-section-fixture.js +52 -0
- package/dist/scripts/recon-generate-repeated-section-fixture.js.map +1 -0
- package/dist/scripts/recon-generate.d.ts +11 -6
- package/dist/scripts/recon-generate.d.ts.map +1 -1
- package/dist/scripts/recon-generate.js +42 -73
- package/dist/scripts/recon-generate.js.map +1 -1
- package/package.json +1 -1
|
@@ -94,6 +94,7 @@ const cookie_jar_1 = require("../scraper/cookie-jar");
|
|
|
94
94
|
const errors_2 = require("../scraper/errors");
|
|
95
95
|
const flow_runner_1 = require("../scraper/flow-runner");
|
|
96
96
|
const frame_target_1 = require("../scraper/frame-target");
|
|
97
|
+
const retry_1 = require("../scraper/retry");
|
|
97
98
|
const session_1 = require("../scraper/session");
|
|
98
99
|
const session_teardown_1 = require("../scraper/session-teardown");
|
|
99
100
|
const stagehand_guard_1 = require("../scraper/stagehand-guard");
|
|
@@ -1703,626 +1704,672 @@ async function main() {
|
|
|
1703
1704
|
? (input) => (0, call_capture_1.captureLlmCall)(input, { sinkPath: callsNdjsonPath })
|
|
1704
1705
|
: async () => { };
|
|
1705
1706
|
logger.info(`recon-browser: target=${url} flow_steps=${flow.length} provider=${provider ?? "(config-default)"} advancedStealth=${advancedStealth} upload_fixture=${uploadFixture ? `${uploadFixturePath} (${uploadFixture.buffer.length}b)` : "(missing)"} runId=${runDir.runId} out=${runDir.root}`);
|
|
1706
|
-
|
|
1707
|
-
//
|
|
1708
|
-
//
|
|
1709
|
-
//
|
|
1710
|
-
//
|
|
1711
|
-
//
|
|
1712
|
-
|
|
1713
|
-
|
|
1714
|
-
//
|
|
1715
|
-
|
|
1716
|
-
|
|
1717
|
-
|
|
1718
|
-
|
|
1719
|
-
|
|
1720
|
-
|
|
1721
|
-
|
|
1722
|
-
|
|
1723
|
-
|
|
1724
|
-
|
|
1725
|
-
|
|
1726
|
-
|
|
1727
|
-
|
|
1728
|
-
|
|
1729
|
-
|
|
1730
|
-
|
|
1731
|
-
|
|
1732
|
-
|
|
1733
|
-
|
|
1734
|
-
|
|
1735
|
-
|
|
1736
|
-
|
|
1737
|
-
const
|
|
1738
|
-
|
|
1739
|
-
//
|
|
1740
|
-
//
|
|
1741
|
-
|
|
1742
|
-
|
|
1743
|
-
|
|
1744
|
-
|
|
1745
|
-
|
|
1746
|
-
|
|
1747
|
-
|
|
1748
|
-
return new URL(page.url()).origin;
|
|
1749
|
-
}
|
|
1750
|
-
catch {
|
|
1751
|
-
return new URL(url).origin;
|
|
1752
|
-
}
|
|
1753
|
-
});
|
|
1754
|
-
logger.info(`navigating to ${url} (waitUntil: ${GOTO_WAIT_UNTIL})`);
|
|
1755
|
-
// A navigation wait that never resolves must not discard the run. Ad-heavy
|
|
1756
|
-
// commercial sites (analytics/session-replay beacons on timers) never reach
|
|
1757
|
-
// `networkidle`, so the wait burns GOTO_TIMEOUT_MS and throws — after the
|
|
1758
|
-
// captures we came for are already on disk. Warn and press on: readiness is
|
|
1759
|
-
// established by the SPA probe below, and the flow steps fail loudly on
|
|
1760
|
-
// their own if the page really is unusable.
|
|
1707
|
+
// Runs one whole-flow attempt on a brand-new session: create session ->
|
|
1708
|
+
// run the step loop -> post-loop truncation/CDP-teardown checks -> replan
|
|
1709
|
+
// write-back. Stagehand can tear its own CDP transport down mid-flow while
|
|
1710
|
+
// traffic is still active (not idle — see docs/recon-1128-heartbeat-
|
|
1711
|
+
// ineffective-1006-fires-under-active-traffic-not-idle.md), which is
|
|
1712
|
+
// recoverable by retrying on a fresh session, so this throws
|
|
1713
|
+
// CdpTransportClosedError instead of exiting so the caller can retry it
|
|
1714
|
+
// through withScraperRetry. Every other failure mode keeps exiting/
|
|
1715
|
+
// rejecting immediately, exactly as a single-attempt run always has.
|
|
1716
|
+
async function runFlowAttempt() {
|
|
1717
|
+
const session = await (0, session_1.createBrowserSession)({ provider, advancedStealth });
|
|
1718
|
+
// `counter` indexes captures on disk (filenames must stay unique).
|
|
1719
|
+
// `signalCounter` drives the verifier — only non-GET methods increment
|
|
1720
|
+
// it so coincident polling/page-load GETs don't falsely "verify" a
|
|
1721
|
+
// click that produced no real effect. See the onFinished comment in
|
|
1722
|
+
// wireNetworkCapture for the rationale.
|
|
1723
|
+
const counter = { n: 0 };
|
|
1724
|
+
const signalCounter = { n: 0 };
|
|
1725
|
+
// Indexes cookie-jar snapshot filenames chronologically, separate from
|
|
1726
|
+
// `counter` (network captures) so a phase with zero network activity still
|
|
1727
|
+
// gets a snapshot without skipping capture indices.
|
|
1728
|
+
const jarCounter = { n: 0 };
|
|
1729
|
+
const recentCaptures = [];
|
|
1730
|
+
// Parallel tracker of recent non-GET captures' method + status. Used by
|
|
1731
|
+
// the Tier 1 trailing-optional-step grace: a verification failure on an
|
|
1732
|
+
// optional trailing step is treated as a benign no-op when a recent
|
|
1733
|
+
// mutation returned 2xx (i.e. the SPA's "real work" already completed
|
|
1734
|
+
// server-side and the trailing step is a redundant tail that the flow
|
|
1735
|
+
// file may have included for sites where it's actually needed). GETs are
|
|
1736
|
+
// filtered at the push site so the window isn't washed out by SPA chunk
|
|
1737
|
+
// loads — see the comment in wireNetworkCapture's onFinished.
|
|
1738
|
+
const recentCaptureMeta = [];
|
|
1739
|
+
// Hoisted out of the try block so the finally can run the replan
|
|
1740
|
+
// write-back even when the cascade throws — replan-discovered steps
|
|
1741
|
+
// accumulated up to the failure point should survive a cascade-exhausted
|
|
1742
|
+
// exit so the user can review them and the next run starts where this
|
|
1743
|
+
// one left off. The "only on success" gate before was the reason most
|
|
1744
|
+
// recon discoveries got thrown away on failed runs. Local to this
|
|
1745
|
+
// attempt so a retried attempt starts from the originally authored flow
|
|
1746
|
+
// rather than a previous attempt's partially-replanned plan.
|
|
1747
|
+
const plan = [];
|
|
1748
|
+
const replanEvents = [];
|
|
1761
1749
|
try {
|
|
1762
|
-
|
|
1763
|
-
|
|
1764
|
-
|
|
1765
|
-
|
|
1766
|
-
|
|
1767
|
-
|
|
1768
|
-
|
|
1769
|
-
|
|
1770
|
-
|
|
1771
|
-
|
|
1772
|
-
|
|
1773
|
-
|
|
1774
|
-
|
|
1775
|
-
|
|
1776
|
-
|
|
1777
|
-
while (Date.now() < spaDeadline) {
|
|
1778
|
-
await new Promise((r) => setTimeout(r, SPA_READINESS_POLL_MS));
|
|
1779
|
-
bodyLength = await page
|
|
1780
|
-
.evaluate("document.body ? document.body.outerHTML.length : 0")
|
|
1781
|
-
.catch(() => 0);
|
|
1782
|
-
if (typeof bodyLength === "number" && bodyLength >= SPA_MIN_BODY_LENGTH) {
|
|
1783
|
-
logger.info(`spa readiness: body grew to ${bodyLength} chars — SPA rendered`);
|
|
1784
|
-
break;
|
|
1750
|
+
const stagehand = session.stagehand;
|
|
1751
|
+
const page = await stagehand.context.awaitActivePage();
|
|
1752
|
+
// Phase label is mutated between flow steps so the single CDP listener
|
|
1753
|
+
// always tags captures with the currently active phase.
|
|
1754
|
+
let currentPhase = "home";
|
|
1755
|
+
const stopCapture = wireNetworkCapture(page, counter, signalCounter, recentCaptures, recentCaptureMeta, () => currentPhase, () => {
|
|
1756
|
+
// Live read so SPA history navigation between captures stays accurate.
|
|
1757
|
+
// Fallback to the initial url on parse failure (e.g. about:blank early
|
|
1758
|
+
// in the goto cycle) so we don't accidentally mark a capture as
|
|
1759
|
+
// cross-origin and miss the user-action signal.
|
|
1760
|
+
try {
|
|
1761
|
+
return new URL(page.url()).origin;
|
|
1762
|
+
}
|
|
1763
|
+
catch {
|
|
1764
|
+
return new URL(url).origin;
|
|
1785
1765
|
}
|
|
1766
|
+
});
|
|
1767
|
+
logger.info(`navigating to ${url} (waitUntil: ${GOTO_WAIT_UNTIL})`);
|
|
1768
|
+
// A navigation wait that never resolves must not discard the run. Ad-heavy
|
|
1769
|
+
// commercial sites (analytics/session-replay beacons on timers) never reach
|
|
1770
|
+
// `networkidle`, so the wait burns GOTO_TIMEOUT_MS and throws — after the
|
|
1771
|
+
// captures we came for are already on disk. Warn and press on: readiness is
|
|
1772
|
+
// established by the SPA probe below, and the flow steps fail loudly on
|
|
1773
|
+
// their own if the page really is unusable.
|
|
1774
|
+
try {
|
|
1775
|
+
await page.goto(url, { waitUntil: GOTO_WAIT_UNTIL, timeoutMs: flow_runner_1.GOTO_TIMEOUT_MS });
|
|
1786
1776
|
}
|
|
1787
|
-
|
|
1788
|
-
logger.warn(`
|
|
1777
|
+
catch (err) {
|
|
1778
|
+
logger.warn(`navigation wait (${GOTO_WAIT_UNTIL}) did not settle: ${(0, errors_1.toErrorMessage)(err)} — continuing; the SPA readiness probe below decides whether the page is usable`);
|
|
1789
1779
|
}
|
|
1790
|
-
|
|
1791
|
-
|
|
1792
|
-
|
|
1793
|
-
|
|
1794
|
-
|
|
1795
|
-
|
|
1796
|
-
|
|
1797
|
-
|
|
1798
|
-
|
|
1799
|
-
|
|
1800
|
-
|
|
1801
|
-
|
|
1802
|
-
|
|
1803
|
-
|
|
1804
|
-
|
|
1805
|
-
|
|
1806
|
-
|
|
1807
|
-
|
|
1808
|
-
|
|
1809
|
-
|
|
1810
|
-
|
|
1811
|
-
|
|
1812
|
-
|
|
1780
|
+
await snapshotAndPersistCookieJar(page, jarCounter, "goto", currentPhase, -1);
|
|
1781
|
+
const SPA_READINESS_TIMEOUT_MS = 15_000;
|
|
1782
|
+
const SPA_READINESS_POLL_MS = 500;
|
|
1783
|
+
const SPA_MIN_BODY_LENGTH = 5_000;
|
|
1784
|
+
const spaDeadline = Date.now() + SPA_READINESS_TIMEOUT_MS;
|
|
1785
|
+
let bodyLength = await page
|
|
1786
|
+
.evaluate("document.body ? document.body.outerHTML.length : 0")
|
|
1787
|
+
.catch(() => 0);
|
|
1788
|
+
if (typeof bodyLength === "number" && bodyLength < SPA_MIN_BODY_LENGTH) {
|
|
1789
|
+
logger.info(`spa readiness: body ${bodyLength} chars < ${SPA_MIN_BODY_LENGTH} threshold — waiting for SPA to render`);
|
|
1790
|
+
while (Date.now() < spaDeadline) {
|
|
1791
|
+
await new Promise((r) => setTimeout(r, SPA_READINESS_POLL_MS));
|
|
1792
|
+
bodyLength = await page
|
|
1793
|
+
.evaluate("document.body ? document.body.outerHTML.length : 0")
|
|
1794
|
+
.catch(() => 0);
|
|
1795
|
+
if (typeof bodyLength === "number" && bodyLength >= SPA_MIN_BODY_LENGTH) {
|
|
1796
|
+
logger.info(`spa readiness: body grew to ${bodyLength} chars — SPA rendered`);
|
|
1797
|
+
break;
|
|
1798
|
+
}
|
|
1799
|
+
}
|
|
1800
|
+
if (typeof bodyLength === "number" && bodyLength < SPA_MIN_BODY_LENGTH) {
|
|
1801
|
+
logger.warn(`spa readiness: body still ${bodyLength} chars after ${SPA_READINESS_TIMEOUT_MS}ms — proceeding with possibly incomplete page`);
|
|
1802
|
+
}
|
|
1813
1803
|
}
|
|
1814
|
-
|
|
1815
|
-
|
|
1804
|
+
const anthropic = (0, anthropic_client_1.buildAnthropicClient)();
|
|
1805
|
+
const rephraseModel = (0, anthropic_client_1.buildRephraseModel)();
|
|
1806
|
+
if (!anthropic) {
|
|
1807
|
+
logger.warn("bedrock-only deployment: global replan will be skipped on step failures (attempt-5 llm rephrase still runs on the Bedrock-backed model)");
|
|
1816
1808
|
}
|
|
1817
|
-
|
|
1818
|
-
|
|
1819
|
-
|
|
1820
|
-
|
|
1821
|
-
|
|
1822
|
-
|
|
1823
|
-
|
|
1824
|
-
|
|
1825
|
-
|
|
1826
|
-
//
|
|
1827
|
-
//
|
|
1828
|
-
//
|
|
1829
|
-
|
|
1809
|
+
plan.push(...flow);
|
|
1810
|
+
const completedSteps = [];
|
|
1811
|
+
const trajectory = [];
|
|
1812
|
+
let probeReplansUsed = 0;
|
|
1813
|
+
let cascadeReplansUsed = 0;
|
|
1814
|
+
const STUCK_SKIP_THRESHOLD = 5;
|
|
1815
|
+
let consecutiveStaleSkips = 0;
|
|
1816
|
+
let lastSuccessNetworkCount = signalCounter.n;
|
|
1817
|
+
let lastSuccessUrl = page.url();
|
|
1818
|
+
// Track the page origin so a cross-origin navigation mid-flow (e.g. the
|
|
1819
|
+
// Apply click taking careers.<employer>.com → apply.<ats>.com) can
|
|
1820
|
+
// re-gate on SPA hydration. The initial goto's readiness gate only covers
|
|
1821
|
+
// the landing page; the wizard app boots on a DIFFERENT origin with no gate,
|
|
1822
|
+
// so its first steps would otherwise probe an un-hydrated shell and skip.
|
|
1823
|
+
const originOf = (u) => {
|
|
1830
1824
|
try {
|
|
1831
|
-
return
|
|
1825
|
+
return new URL(u).origin;
|
|
1832
1826
|
}
|
|
1833
|
-
catch
|
|
1834
|
-
|
|
1827
|
+
catch {
|
|
1828
|
+
return "";
|
|
1835
1829
|
}
|
|
1836
1830
|
};
|
|
1837
|
-
|
|
1838
|
-
|
|
1839
|
-
|
|
1840
|
-
|
|
1841
|
-
|
|
1842
|
-
|
|
1843
|
-
|
|
1844
|
-
|
|
1845
|
-
|
|
1846
|
-
|
|
1847
|
-
|
|
1848
|
-
|
|
1849
|
-
|
|
1850
|
-
|
|
1851
|
-
if (currentOrigin !== "" && currentOrigin !== lastOrigin) {
|
|
1852
|
-
logger.info(`origin changed ${lastOrigin || "(none)"} → ${currentOrigin}; re-gating on SPA hydration`);
|
|
1853
|
-
await (0, flow_runner_1.waitForSpaReady)(page, logger);
|
|
1854
|
-
lastOrigin = currentOrigin;
|
|
1855
|
-
}
|
|
1856
|
-
// Debug: dump the full DOM right before this step's cascade runs. Lets
|
|
1857
|
-
// a triager see the page state exactly as the cascade sees it, without
|
|
1858
|
-
// re-running. One-shot per recon run via --dump-dom-before-step.
|
|
1859
|
-
if (dumpDomBeforeStep !== null && i + 1 === dumpDomBeforeStep) {
|
|
1860
|
-
try {
|
|
1861
|
-
const html = await (0, watchdog_1.withWatchdog)(() => page.evaluate("document.documentElement ? document.documentElement.outerHTML : ''"), {
|
|
1862
|
-
timeoutMs: config_1.config.scraper.frameEvaluateTimeoutMs,
|
|
1863
|
-
label: "dump-dom-before-step: evaluate",
|
|
1864
|
-
});
|
|
1865
|
-
if (typeof html === "string" && html.length > 0) {
|
|
1866
|
-
const dumpPath = (0, node_path_1.join)(runDir.root, `dom-dump-step-${i + 1}.html`);
|
|
1867
|
-
(0, node_fs_1.writeFileSync)(dumpPath, html);
|
|
1868
|
-
logger.info(`${(0, flow_runner_1.formatStepPrefix)(i, () => plan.length)}: wrote DOM dump (${html.length} bytes) to ${dumpPath}`);
|
|
1831
|
+
let lastOrigin = originOf(page.url());
|
|
1832
|
+
let exitedViaTrailingGrace = false;
|
|
1833
|
+
for (let i = 0; i < plan.length; i++) {
|
|
1834
|
+
const step = plan[i];
|
|
1835
|
+
// Liveness gate: a closed/crashed Stagehand session makes `page.url()`
|
|
1836
|
+
// throw synchronously. Every downstream observe/act probe treats that
|
|
1837
|
+
// throw as "no candidates" and, for an optional step, silently skips
|
|
1838
|
+
// it — so without this check the loop runs to `plan.length` and the
|
|
1839
|
+
// post-loop isFlowTruncated count-check sees a full completedSteps
|
|
1840
|
+
// array and reports "not truncated" even though the session died
|
|
1841
|
+
// partway through. Mirrors the same guard in runHealingFlow.
|
|
1842
|
+
const readLiveUrl = () => {
|
|
1843
|
+
try {
|
|
1844
|
+
return page.url();
|
|
1869
1845
|
}
|
|
1870
|
-
|
|
1871
|
-
|
|
1846
|
+
catch (err) {
|
|
1847
|
+
throw new errors_2.SessionTimeoutError(`${(0, flow_runner_1.formatStepPrefix)(i, () => plan.length)} session appears closed/dead (page.url() threw: ${(0, errors_1.toErrorMessage)(err)}) — aborting after ${i} of ${plan.length} steps completed`);
|
|
1872
1848
|
}
|
|
1849
|
+
};
|
|
1850
|
+
readLiveUrl();
|
|
1851
|
+
currentPhase =
|
|
1852
|
+
step.instruction
|
|
1853
|
+
.replace(/[^a-z0-9]+/gi, "-")
|
|
1854
|
+
.toLowerCase()
|
|
1855
|
+
.replace(/^-|-$/g, "")
|
|
1856
|
+
.slice(0, 24) || `step-${i}`;
|
|
1857
|
+
logger.info(`${(0, flow_runner_1.formatStepPrefix)(i, () => plan.length)} [${currentPhase}]${step.optional ? " (optional)" : ""}: ${step.instruction}`);
|
|
1858
|
+
await snapshotAndPersistCookieJar(page, jarCounter, "pre-step", currentPhase, i);
|
|
1859
|
+
// Re-gate on SPA hydration when the origin changed since the last step —
|
|
1860
|
+
// the wizard SPA (e.g. apply.<ats>.com after the Apply click) boots on
|
|
1861
|
+
// a new origin the initial-goto readiness gate never covered, so wait for
|
|
1862
|
+
// its body to render before probing rather than skipping a shell page.
|
|
1863
|
+
const currentOrigin = originOf(readLiveUrl());
|
|
1864
|
+
if (currentOrigin !== "" && currentOrigin !== lastOrigin) {
|
|
1865
|
+
logger.info(`origin changed ${lastOrigin || "(none)"} → ${currentOrigin}; re-gating on SPA hydration`);
|
|
1866
|
+
await (0, flow_runner_1.waitForSpaReady)(page, logger);
|
|
1867
|
+
lastOrigin = currentOrigin;
|
|
1873
1868
|
}
|
|
1874
|
-
|
|
1875
|
-
|
|
1869
|
+
// Debug: dump the full DOM right before this step's cascade runs. Lets
|
|
1870
|
+
// a triager see the page state exactly as the cascade sees it, without
|
|
1871
|
+
// re-running. One-shot per recon run via --dump-dom-before-step.
|
|
1872
|
+
if (dumpDomBeforeStep !== null && i + 1 === dumpDomBeforeStep) {
|
|
1873
|
+
try {
|
|
1874
|
+
const html = await (0, watchdog_1.withWatchdog)(() => page.evaluate("document.documentElement ? document.documentElement.outerHTML : ''"), {
|
|
1875
|
+
timeoutMs: config_1.config.scraper.frameEvaluateTimeoutMs,
|
|
1876
|
+
label: "dump-dom-before-step: evaluate",
|
|
1877
|
+
});
|
|
1878
|
+
if (typeof html === "string" && html.length > 0) {
|
|
1879
|
+
const dumpPath = (0, node_path_1.join)(runDir.root, `dom-dump-step-${i + 1}.html`);
|
|
1880
|
+
(0, node_fs_1.writeFileSync)(dumpPath, html);
|
|
1881
|
+
logger.info(`${(0, flow_runner_1.formatStepPrefix)(i, () => plan.length)}: wrote DOM dump (${html.length} bytes) to ${dumpPath}`);
|
|
1882
|
+
}
|
|
1883
|
+
else {
|
|
1884
|
+
logger.warn(`${(0, flow_runner_1.formatStepPrefix)(i, () => plan.length)}: DOM dump returned empty content; skipping write`);
|
|
1885
|
+
}
|
|
1886
|
+
}
|
|
1887
|
+
catch (err) {
|
|
1888
|
+
logger.warn(`${(0, flow_runner_1.formatStepPrefix)(i, () => plan.length)}: DOM dump failed: ${(0, errors_1.toErrorMessage)(err)}`);
|
|
1889
|
+
}
|
|
1876
1890
|
}
|
|
1877
|
-
|
|
1878
|
-
|
|
1879
|
-
|
|
1880
|
-
|
|
1881
|
-
|
|
1882
|
-
|
|
1883
|
-
|
|
1884
|
-
|
|
1885
|
-
|
|
1886
|
-
|
|
1887
|
-
|
|
1888
|
-
|
|
1889
|
-
|
|
1890
|
-
|
|
1891
|
-
|
|
1892
|
-
|
|
1893
|
-
|
|
1894
|
-
|
|
1895
|
-
|
|
1896
|
-
|
|
1897
|
-
|
|
1898
|
-
|
|
1899
|
-
|
|
1900
|
-
|
|
1901
|
-
|
|
1902
|
-
|
|
1903
|
-
|
|
1904
|
-
|
|
1905
|
-
|
|
1906
|
-
|
|
1907
|
-
|
|
1908
|
-
|
|
1909
|
-
|
|
1910
|
-
|
|
1891
|
+
// Baseline for wizard-restart detection: capture the highest capture
|
|
1892
|
+
// INDEX before the step so findWizardRestartSignal scans only URLs that
|
|
1893
|
+
// landed during THIS step's processing (eviction-proof disk scan).
|
|
1894
|
+
const preCaptureIdxBeforeStep = (0, flow_runner_1.latestCaptureIndex)(recentCaptures);
|
|
1895
|
+
// Resolved fresh per step (not cached across the run) so a cross-origin
|
|
1896
|
+
// iframe that attaches mid-flow (e.g. after an "Apply" click reveals a
|
|
1897
|
+
// wizard embedded later in the DOM) is picked up as soon as it's
|
|
1898
|
+
// reachable, and so a replan-appended step — spliced into `plan` after
|
|
1899
|
+
// this loop already started — resolves the same frame instead of
|
|
1900
|
+
// silently reverting to the main frame. `resolveFrameTarget` falls back
|
|
1901
|
+
// to the main-frame target when `frameSelector` is null/unresolvable,
|
|
1902
|
+
// so this is a no-op for every flow that doesn't declare one.
|
|
1903
|
+
const frameTarget = await (0, frame_target_1.resolveFrameTarget)(page, frameSelector);
|
|
1904
|
+
// A child frame CDP has just attached to may still be sitting on
|
|
1905
|
+
// about:blank (the moment right after Target.setAutoAttach fires and
|
|
1906
|
+
// before the OOPIF's own navigation lands). waitForSpaReady above only
|
|
1907
|
+
// re-gates on a TOP-document origin change, which never fires for a
|
|
1908
|
+
// same-origin-top site whose wizard lives entirely inside an iframe
|
|
1909
|
+
// (e.g. the top-window site stays on careers.example.org throughout), so without
|
|
1910
|
+
// this the cascade would probe an unnavigated frame and see 0 candidates.
|
|
1911
|
+
// No-ops (zero delay) when frameTarget.frame is null.
|
|
1912
|
+
await (0, frame_target_1.waitForChildFrameReady)(frameTarget);
|
|
1913
|
+
try {
|
|
1914
|
+
const stepPromise = (0, flow_runner_1.executeStepWithHealing)({
|
|
1915
|
+
stagehand,
|
|
1916
|
+
page,
|
|
1917
|
+
frameTarget,
|
|
1918
|
+
step: step.instruction,
|
|
1919
|
+
optional: step.optional,
|
|
1920
|
+
upload: step.upload,
|
|
1921
|
+
submitStep: step.submitStep === true,
|
|
1922
|
+
flowHasSubmitSemantics: (0, flow_runner_1.flowHasSubmitSemantics)({
|
|
1923
|
+
steps: plan.map((s) => ({ submitStep: s.submitStep === true })),
|
|
1924
|
+
submitEndpointPattern,
|
|
1925
|
+
requireSubmitEndpointMatch,
|
|
1926
|
+
}),
|
|
1927
|
+
stepIndex: i,
|
|
1928
|
+
totalSteps: () => plan.length,
|
|
1929
|
+
phase: currentPhase,
|
|
1930
|
+
signalCounter,
|
|
1931
|
+
recentCaptures,
|
|
1932
|
+
recentCaptureMeta,
|
|
1933
|
+
anthropic,
|
|
1934
|
+
rephraseModel,
|
|
1935
|
+
logger,
|
|
1936
|
+
uploadFixture,
|
|
1937
|
+
isFinalStep: i === plan.length - 1,
|
|
1911
1938
|
submitEndpointPattern,
|
|
1939
|
+
submittedStateSelectors,
|
|
1912
1940
|
requireSubmitEndpointMatch,
|
|
1913
|
-
|
|
1914
|
-
|
|
1915
|
-
|
|
1916
|
-
|
|
1917
|
-
|
|
1918
|
-
|
|
1919
|
-
|
|
1920
|
-
|
|
1921
|
-
|
|
1922
|
-
|
|
1923
|
-
|
|
1924
|
-
|
|
1925
|
-
|
|
1926
|
-
|
|
1927
|
-
|
|
1928
|
-
|
|
1929
|
-
|
|
1930
|
-
|
|
1931
|
-
|
|
1932
|
-
|
|
1933
|
-
|
|
1934
|
-
|
|
1935
|
-
|
|
1936
|
-
|
|
1937
|
-
|
|
1938
|
-
|
|
1939
|
-
|
|
1940
|
-
|
|
1941
|
-
|
|
1942
|
-
|
|
1943
|
-
|
|
1944
|
-
|
|
1945
|
-
|
|
1946
|
-
|
|
1947
|
-
|
|
1948
|
-
|
|
1949
|
-
|
|
1950
|
-
|
|
1951
|
-
|
|
1952
|
-
|
|
1953
|
-
|
|
1954
|
-
|
|
1955
|
-
|
|
1956
|
-
|
|
1957
|
-
|
|
1958
|
-
|
|
1959
|
-
|
|
1960
|
-
|
|
1961
|
-
|
|
1962
|
-
|
|
1963
|
-
lastTrajectory.targetId !== undefined &&
|
|
1964
|
-
lastTrajectory.targetId !== "") {
|
|
1965
|
-
plan[i] = { ...plan[i], targetId: lastTrajectory.targetId };
|
|
1966
|
-
}
|
|
1967
|
-
// Wizard-restart detection: if a configured restart-signal URL (e.g.
|
|
1968
|
-
// a wizard ATS's `init-apply?...&application_canceled=true`) landed during
|
|
1969
|
-
// this step, the multi-page wizard reset to page 1. Remaining steps now
|
|
1970
|
-
// target a reset page and replanning against the restarted wizard is
|
|
1971
|
-
// futile — abort with a diagnostic instead of silently cycling.
|
|
1972
|
-
const restartUrl = findWizardRestartSignal({
|
|
1973
|
-
preIdx: preCaptureIdxBeforeStep,
|
|
1974
|
-
restartSignalUrlPatterns,
|
|
1975
|
-
});
|
|
1976
|
-
if (restartUrl !== null) {
|
|
1977
|
-
throw new errors_2.StepVerificationError(`${(0, flow_runner_1.formatStepPrefix)(i, () => plan.length)} (${step.instruction.slice(0, 60)}) triggered a wizard restart (${restartUrl.slice(0, 120)}) — the application reset to the first page; aborting`, "wizard-regression");
|
|
1978
|
-
}
|
|
1979
|
-
if (stepOutcome === "skipped") {
|
|
1980
|
-
const pageStagnant = signalCounter.n === lastSuccessNetworkCount && readLiveUrl() === lastSuccessUrl;
|
|
1981
|
-
if (pageStagnant) {
|
|
1982
|
-
consecutiveStaleSkips++;
|
|
1941
|
+
advanceTransitionBodyPattern,
|
|
1942
|
+
successUrlFragments,
|
|
1943
|
+
successPageTitleHints,
|
|
1944
|
+
ownBackendHostnames,
|
|
1945
|
+
knownErrorClassPrefixes,
|
|
1946
|
+
wizardExitButtonLabels,
|
|
1947
|
+
getSuppressedAisdkElementIdErrorCount: session.getSuppressedAisdkElementIdErrorCount,
|
|
1948
|
+
trajectory,
|
|
1949
|
+
captureFn,
|
|
1950
|
+
onStepFailure: dumpStepFailure,
|
|
1951
|
+
});
|
|
1952
|
+
// Races the step's own promise against the session's teardown death
|
|
1953
|
+
// signal (bugfix-003's raceAgainstTeardown): when Stagehand's CDP
|
|
1954
|
+
// transport is reaped mid-step, the in-flight promise above never
|
|
1955
|
+
// settles because the underlying CDP request is orphaned, and with
|
|
1956
|
+
// keepAlive:true nothing else keeps the event loop alive — the
|
|
1957
|
+
// process would otherwise exit 0 before this step ever resolves.
|
|
1958
|
+
// No-op (behaves like `await stepPromise`) on providers where
|
|
1959
|
+
// session.deathSignal is undefined.
|
|
1960
|
+
const stepOutcome = await (session.deathSignal
|
|
1961
|
+
? (0, session_teardown_1.raceAgainstTeardown)(stepPromise, session.deathSignal)
|
|
1962
|
+
: stepPromise);
|
|
1963
|
+
// Second liveness gate: executeStepWithHealing can resolve normally even
|
|
1964
|
+
// when the session died mid-step, if the death happened after its last
|
|
1965
|
+
// probe. Re-check here so a swallowed mid-step death is still caught at
|
|
1966
|
+
// the loop boundary rather than let the loop run to completion.
|
|
1967
|
+
readLiveUrl();
|
|
1968
|
+
await snapshotAndPersistCookieJar(page, jarCounter, "post-step", currentPhase, i);
|
|
1969
|
+
// Stamp the step's stable DOM identity from the fast primitive that just
|
|
1970
|
+
// resolved it (pushed onto trajectory), so persistReplannedFlow can dedup
|
|
1971
|
+
// reworded re-discoveries of this field across runs on `targetId`. Only a
|
|
1972
|
+
// non-empty id is load-bearing; an empty string (element had no id) is
|
|
1973
|
+
// left off so the step falls back to structural dedup.
|
|
1974
|
+
const lastTrajectory = trajectory[trajectory.length - 1];
|
|
1975
|
+
if (lastTrajectory?.stepIndex === i &&
|
|
1976
|
+
lastTrajectory.targetId !== undefined &&
|
|
1977
|
+
lastTrajectory.targetId !== "") {
|
|
1978
|
+
plan[i] = { ...plan[i], targetId: lastTrajectory.targetId };
|
|
1979
|
+
}
|
|
1980
|
+
// Wizard-restart detection: if a configured restart-signal URL (e.g.
|
|
1981
|
+
// a wizard ATS's `init-apply?...&application_canceled=true`) landed during
|
|
1982
|
+
// this step, the multi-page wizard reset to page 1. Remaining steps now
|
|
1983
|
+
// target a reset page and replanning against the restarted wizard is
|
|
1984
|
+
// futile — abort with a diagnostic instead of silently cycling.
|
|
1985
|
+
const restartUrl = findWizardRestartSignal({
|
|
1986
|
+
preIdx: preCaptureIdxBeforeStep,
|
|
1987
|
+
restartSignalUrlPatterns,
|
|
1988
|
+
});
|
|
1989
|
+
if (restartUrl !== null) {
|
|
1990
|
+
throw new errors_2.StepVerificationError(`${(0, flow_runner_1.formatStepPrefix)(i, () => plan.length)} (${step.instruction.slice(0, 60)}) triggered a wizard restart (${restartUrl.slice(0, 120)}) — the application reset to the first page; aborting`, "wizard-regression");
|
|
1983
1991
|
}
|
|
1984
|
-
if (
|
|
1985
|
-
|
|
1992
|
+
if (stepOutcome === "skipped") {
|
|
1993
|
+
const pageStagnant = signalCounter.n === lastSuccessNetworkCount && readLiveUrl() === lastSuccessUrl;
|
|
1994
|
+
if (pageStagnant) {
|
|
1995
|
+
consecutiveStaleSkips++;
|
|
1996
|
+
}
|
|
1997
|
+
if (consecutiveStaleSkips >= STUCK_SKIP_THRESHOLD) {
|
|
1998
|
+
logger.warn(`stuck detection: ${consecutiveStaleSkips} consecutive optional steps skipped with no page advancement (url=${lastSuccessUrl}, networkCount=${lastSuccessNetworkCount}) — treating as probe-absent failure to trigger replan`);
|
|
1999
|
+
consecutiveStaleSkips = 0;
|
|
2000
|
+
throw new errors_2.StepVerificationError(`${(0, flow_runner_1.formatStepPrefix)(i, () => plan.length)} (${step.instruction.slice(0, 60)}) stuck: ${STUCK_SKIP_THRESHOLD}+ consecutive optional skips with stagnant page`, "probe-absent");
|
|
2001
|
+
}
|
|
2002
|
+
}
|
|
2003
|
+
else {
|
|
1986
2004
|
consecutiveStaleSkips = 0;
|
|
1987
|
-
|
|
2005
|
+
lastSuccessNetworkCount = signalCounter.n;
|
|
2006
|
+
lastSuccessUrl = readLiveUrl();
|
|
1988
2007
|
}
|
|
2008
|
+
completedSteps.push(step.instruction);
|
|
1989
2009
|
}
|
|
1990
|
-
|
|
1991
|
-
|
|
1992
|
-
|
|
1993
|
-
|
|
1994
|
-
|
|
1995
|
-
|
|
1996
|
-
|
|
1997
|
-
|
|
1998
|
-
|
|
1999
|
-
|
|
2000
|
-
|
|
2001
|
-
|
|
2002
|
-
|
|
2003
|
-
|
|
2004
|
-
|
|
2005
|
-
|
|
2006
|
-
|
|
2007
|
-
|
|
2008
|
-
|
|
2009
|
-
|
|
2010
|
-
|
|
2011
|
-
|
|
2012
|
-
//
|
|
2013
|
-
//
|
|
2014
|
-
|
|
2015
|
-
|
|
2016
|
-
|
|
2017
|
-
|
|
2018
|
-
|
|
2019
|
-
|
|
2020
|
-
|
|
2021
|
-
|
|
2022
|
-
|
|
2023
|
-
|
|
2024
|
-
|
|
2025
|
-
|
|
2026
|
-
|
|
2027
|
-
|
|
2028
|
-
|
|
2029
|
-
|
|
2030
|
-
|
|
2031
|
-
|
|
2032
|
-
|
|
2033
|
-
|
|
2034
|
-
|
|
2035
|
-
|
|
2036
|
-
|
|
2037
|
-
|
|
2038
|
-
|
|
2039
|
-
|
|
2040
|
-
|
|
2041
|
-
|
|
2042
|
-
|
|
2043
|
-
|
|
2044
|
-
|
|
2045
|
-
|
|
2046
|
-
|
|
2010
|
+
catch (err) {
|
|
2011
|
+
if (!(err instanceof errors_2.StepVerificationError))
|
|
2012
|
+
throw err;
|
|
2013
|
+
// Unrecoverable backend-error short-circuit. When the cascade
|
|
2014
|
+
// detected a same-window 5xx on the submit endpoint, no amount of
|
|
2015
|
+
// replan or rephrase can heal a server crash. Propagate the error
|
|
2016
|
+
// out of the flow loop so main()'s outer try/catch reports the
|
|
2017
|
+
// diagnostic — bypasses the trailing-grace and replan paths
|
|
2018
|
+
// entirely. (The cascade-exhausted and probe-absent kinds fall
|
|
2019
|
+
// through to the existing dispatcher below.)
|
|
2020
|
+
if (err.kind === "backend-error-unrecoverable") {
|
|
2021
|
+
logger.error(`backend error unrecoverable: ${err.message}; aborting run`);
|
|
2022
|
+
throw err;
|
|
2023
|
+
}
|
|
2024
|
+
if (err.kind === "wizard-regression") {
|
|
2025
|
+
// The wizard restarted; replanning against a reset page cannot recover
|
|
2026
|
+
// the lost progress. Bypass the replan dispatcher and abort.
|
|
2027
|
+
logger.error(`wizard regression: ${err.message}; aborting run`);
|
|
2028
|
+
throw err;
|
|
2029
|
+
}
|
|
2030
|
+
// Tier 1 — trailing-optional-step grace: when an OPTIONAL step at
|
|
2031
|
+
// trailing position fails verification AND a recent non-GET capture
|
|
2032
|
+
// returned 2xx, treat as a benign no-op exit. The flow's "real work"
|
|
2033
|
+
// already completed server-side (recent successful POST proves it);
|
|
2034
|
+
// the trailing step is a redundant tail that the cascade can't make
|
|
2035
|
+
// meaningful progress on (e.g. resume re-upload after the workflow
|
|
2036
|
+
// already ended, final Continue when the Submit button is in
|
|
2037
|
+
// "Saving..." state). Uses only flow-position metadata + capture HTTP
|
|
2038
|
+
// metadata — no content matching, no open-set patterns.
|
|
2039
|
+
//
|
|
2040
|
+
// When the flow declares submitEndpointPattern, the "recent 2xx" must
|
|
2041
|
+
// match it. Without this gate, the heuristic latches onto every
|
|
2042
|
+
// non-GET 2xx — including pre-submit interruption_check POSTs and
|
|
2043
|
+
// third-party analytics tracking pixels (Google Analytics, DoubleClick,
|
|
2044
|
+
// googletagmanager) that fire on form interaction events. Those look
|
|
2045
|
+
// exactly like real submissions by HTTP signal but don't represent the
|
|
2046
|
+
// actual application landing. Verified by reading captures from
|
|
2047
|
+
// /tmp/recon/graphql/ during a sweep: some tenants hit this exact
|
|
2048
|
+
// failure — only interruption_check + GA tracking POSTs fired, no
|
|
2049
|
+
// integrated_apply, yet trailing-grace declared success.
|
|
2050
|
+
if (step.optional && i >= plan.length - flow_runner_1.TRAILING_GRACE_WINDOW) {
|
|
2051
|
+
// Trailing-grace check: did the submit actually land somewhere in
|
|
2052
|
+
// the recent capture history? Ask the same Haiku judge — it has
|
|
2053
|
+
// multi-signal reasoning to distinguish real submit POSTs from
|
|
2054
|
+
// analytics/tracking 2xx that look submission-shaped.
|
|
2055
|
+
const pageTitle = await (0, watchdog_1.withWatchdog)(() => page.title(), {
|
|
2056
|
+
timeoutMs: config_1.config.scraper.frameEvaluateTimeoutMs,
|
|
2057
|
+
label: "trailing-grace: page title",
|
|
2058
|
+
}).catch(() => "");
|
|
2059
|
+
const trailingGraceVerdict = await (0, verify_submit_1.verifySubmitWithLLM)({
|
|
2060
|
+
client: anthropic,
|
|
2061
|
+
input: {
|
|
2062
|
+
// A dead session throws synchronously here too — fall back to ""
|
|
2063
|
+
// rather than let the trailing-grace check itself crash the run.
|
|
2064
|
+
pageUrl: (() => {
|
|
2065
|
+
try {
|
|
2066
|
+
return page.url();
|
|
2067
|
+
}
|
|
2068
|
+
catch {
|
|
2069
|
+
return "";
|
|
2070
|
+
}
|
|
2071
|
+
})(),
|
|
2072
|
+
pageTitle,
|
|
2073
|
+
unfocusedObserve: [],
|
|
2074
|
+
networkCaptures: recentCaptureMeta,
|
|
2075
|
+
invalidMarkerCount: 0,
|
|
2076
|
+
ownBackendHostnames,
|
|
2077
|
+
successUrlFragments,
|
|
2078
|
+
successPageTitleHints,
|
|
2079
|
+
submittedStateSelectors,
|
|
2080
|
+
},
|
|
2081
|
+
captureFn,
|
|
2082
|
+
});
|
|
2083
|
+
if (trailingGraceVerdict?.verified) {
|
|
2084
|
+
logger.info(`${(0, flow_runner_1.formatStepPrefix)(i, () => plan.length)} optional + trailing position; judge verified recent submit (${trailingGraceVerdict.rationale}) — treating verification failure as benign no-op; recon complete`);
|
|
2085
|
+
exitedViaTrailingGrace = true;
|
|
2086
|
+
break;
|
|
2087
|
+
}
|
|
2088
|
+
}
|
|
2089
|
+
if (!anthropic)
|
|
2090
|
+
throw err;
|
|
2091
|
+
// Cause-based replan budget. Probe replans are cheap (one observe +
|
|
2092
|
+
// one LLM call to detect "wrong page"), cascade replans are expensive
|
|
2093
|
+
// (four attempts × backoff + observe + LLM rephrase before we know
|
|
2094
|
+
// the step is unrecoverable). Separate budgets so cheap recoveries
|
|
2095
|
+
// don't eat into the budget reserved for expensive ones.
|
|
2096
|
+
const isProbe = err.kind === "probe-absent";
|
|
2097
|
+
const budget = isProbe ? resolvedProbeBudget : resolvedCascadeBudget;
|
|
2098
|
+
const usedSoFar = isProbe ? probeReplansUsed : cascadeReplansUsed;
|
|
2099
|
+
if (usedSoFar >= budget) {
|
|
2100
|
+
const kindsSummary = callsNdjsonPath !== null
|
|
2101
|
+
? summarizeReplanFailureKinds({
|
|
2102
|
+
callsNdjsonPath,
|
|
2103
|
+
callType: call_types_1.CALL_TYPE_RECON_REPLAN,
|
|
2104
|
+
tailCount: budget * 2,
|
|
2105
|
+
})
|
|
2106
|
+
: "";
|
|
2107
|
+
const kindsSuffix = kindsSummary ? ` — ${kindsSummary}` : "";
|
|
2108
|
+
logger.error(`${(0, flow_runner_1.formatStepPrefix)(i, () => plan.length)} ${err.kind} replan budget exhausted (${usedSoFar}/${budget}); aborting${kindsSuffix}`);
|
|
2109
|
+
throw err;
|
|
2110
|
+
}
|
|
2111
|
+
const replanIndex = replanEvents.length + 1;
|
|
2112
|
+
const originalRemaining = plan.slice(i + 1);
|
|
2113
|
+
const dumpMatch = err.message.match(/see (\/[^\s]+)$/);
|
|
2114
|
+
const dumpPath = dumpMatch ? dumpMatch[1] : "";
|
|
2115
|
+
logger.warn(`${(0, flow_runner_1.formatStepPrefix)(i, () => plan.length)} terminally failed (${err.kind}); attempting global replan #${replanIndex} (${isProbe ? "probe" : "cascade"} budget ${usedSoFar + 1}/${budget})`);
|
|
2116
|
+
const rawNewSteps = await replanRemainingFlow({
|
|
2047
2117
|
client: anthropic,
|
|
2048
|
-
|
|
2049
|
-
|
|
2050
|
-
|
|
2051
|
-
|
|
2052
|
-
|
|
2053
|
-
|
|
2054
|
-
|
|
2055
|
-
|
|
2056
|
-
return "";
|
|
2057
|
-
}
|
|
2058
|
-
})(),
|
|
2059
|
-
pageTitle,
|
|
2060
|
-
unfocusedObserve: [],
|
|
2061
|
-
networkCaptures: recentCaptureMeta,
|
|
2062
|
-
invalidMarkerCount: 0,
|
|
2063
|
-
ownBackendHostnames,
|
|
2064
|
-
successUrlFragments,
|
|
2065
|
-
successPageTitleHints,
|
|
2066
|
-
submittedStateSelectors,
|
|
2067
|
-
},
|
|
2118
|
+
originalFlow: flow.map((s) => s.instruction),
|
|
2119
|
+
completedSteps,
|
|
2120
|
+
failedStep: step.instruction,
|
|
2121
|
+
remainingSteps: originalRemaining.map((s) => s.instruction),
|
|
2122
|
+
failureDumpPath: dumpPath,
|
|
2123
|
+
page,
|
|
2124
|
+
stagehand,
|
|
2125
|
+
frameSelector,
|
|
2068
2126
|
captureFn,
|
|
2127
|
+
recentCaptures,
|
|
2128
|
+
ownBackendHostnames,
|
|
2129
|
+
trajectory,
|
|
2130
|
+
priorReplans: replanEvents,
|
|
2069
2131
|
});
|
|
2070
|
-
if (
|
|
2071
|
-
logger.
|
|
2072
|
-
|
|
2073
|
-
break;
|
|
2132
|
+
if (!rawNewSteps) {
|
|
2133
|
+
logger.error(`replan #${replanIndex} returned outcome=impossible or unparseable output; aborting`);
|
|
2134
|
+
throw err;
|
|
2074
2135
|
}
|
|
2136
|
+
// Resume-from-failure: drop any replan bridge step that re-runs an
|
|
2137
|
+
// already-completed step (see filterCompletedFromReplan). Keeps the
|
|
2138
|
+
// failed step's re-emission. originalRemaining is re-appended below.
|
|
2139
|
+
const newSteps = filterCompletedFromReplan(rawNewSteps, completedSteps, step.instruction);
|
|
2140
|
+
const droppedCompleted = rawNewSteps.length - newSteps.length;
|
|
2141
|
+
if (droppedCompleted > 0) {
|
|
2142
|
+
logger.info(`replan #${replanIndex}: dropped ${droppedCompleted} bridge step(s) that re-ran already-completed steps`);
|
|
2143
|
+
}
|
|
2144
|
+
if (newSteps.length === 0) {
|
|
2145
|
+
logger.error(`replan #${replanIndex} produced only already-completed steps (nothing new to bridge); aborting`);
|
|
2146
|
+
throw err;
|
|
2147
|
+
}
|
|
2148
|
+
// Immediate no-progress guard: if the replan's only bridge is a
|
|
2149
|
+
// re-emission of the step that just failed, resuming re-runs the whole
|
|
2150
|
+
// cascade on the identical click that just exhausted it. Abort now
|
|
2151
|
+
// instead of waiting REPLAN_CYCLE_THRESHOLD repeats for the cycle
|
|
2152
|
+
// detector — that many dead cascades cost minutes of wall-clock.
|
|
2153
|
+
if (isReplanReproposingFailedStep(newSteps, step.instruction)) {
|
|
2154
|
+
const noProgressMessage = `replan #${replanIndex} re-proposed only the just-failed step ("${step.instruction.slice(0, 60)}") with no new bridge; resuming would re-fail identically — aborting`;
|
|
2155
|
+
logger.error(noProgressMessage);
|
|
2156
|
+
throw new errors_2.StepVerificationError(noProgressMessage, "replan-cycle-detected");
|
|
2157
|
+
}
|
|
2158
|
+
// Auth-boundary guard: a Sign-In/Log-In bridge proposed after an
|
|
2159
|
+
// account-creation step already completed desyncs the page from the
|
|
2160
|
+
// re-appended original tail (see isReplanRegressingAcrossAuthBoundary).
|
|
2161
|
+
if (isReplanRegressingAcrossAuthBoundary(newSteps, completedSteps)) {
|
|
2162
|
+
const authBoundaryMessage = `replan #${replanIndex} proposed a Sign-In/Log-In step after an account-creation step already completed; resuming through Sign-In would strand the original remaining tail on an incompatible page — aborting`;
|
|
2163
|
+
logger.error(authBoundaryMessage);
|
|
2164
|
+
throw new errors_2.StepVerificationError(authBoundaryMessage, "replan-cycle-detected");
|
|
2165
|
+
}
|
|
2166
|
+
const currentPageState = await (0, flow_runner_1.snapshotPage)((0, frame_target_1.mainFrameTarget)(page), signalCounter).catch(() => ({
|
|
2167
|
+
// page.url() throws synchronously on a dead session — a raw call
|
|
2168
|
+
// here would turn this fallback itself into an unhandled throw.
|
|
2169
|
+
url: (() => {
|
|
2170
|
+
try {
|
|
2171
|
+
return page.url();
|
|
2172
|
+
}
|
|
2173
|
+
catch {
|
|
2174
|
+
return "";
|
|
2175
|
+
}
|
|
2176
|
+
})(),
|
|
2177
|
+
bodyHtmlLength: 0,
|
|
2178
|
+
}));
|
|
2179
|
+
if (isReplanCycle(replanEvents, newSteps, {
|
|
2180
|
+
url: currentPageState.url,
|
|
2181
|
+
htmlLength: currentPageState.bodyHtmlLength,
|
|
2182
|
+
})) {
|
|
2183
|
+
const cycleMessage = `replan cycle detected: identical proposal × ${REPLAN_CYCLE_THRESHOLD} under static page state; aborting`;
|
|
2184
|
+
logger.error(cycleMessage);
|
|
2185
|
+
throw new errors_2.StepVerificationError(cycleMessage, "replan-cycle-detected");
|
|
2186
|
+
}
|
|
2187
|
+
if (isProbe) {
|
|
2188
|
+
probeReplansUsed++;
|
|
2189
|
+
}
|
|
2190
|
+
else {
|
|
2191
|
+
cascadeReplansUsed++;
|
|
2192
|
+
}
|
|
2193
|
+
// err.kind narrowed to the two replan-bearing variants here: the
|
|
2194
|
+
// backend-error-unrecoverable dispatcher above throws out, and the
|
|
2195
|
+
// cycle-detected variant is only constructed at the throw site just
|
|
2196
|
+
// above this push — never caught back here.
|
|
2197
|
+
replanEvents.push({
|
|
2198
|
+
replanIndex,
|
|
2199
|
+
cause: err.kind,
|
|
2200
|
+
indexAtFailure: i,
|
|
2201
|
+
failedInstruction: step.instruction,
|
|
2202
|
+
replanSteps: newSteps,
|
|
2203
|
+
timestamp: (0, date_fns_1.formatISO)(new Date()),
|
|
2204
|
+
pageState: {
|
|
2205
|
+
url: currentPageState.url,
|
|
2206
|
+
htmlLength: currentPageState.bodyHtmlLength,
|
|
2207
|
+
},
|
|
2208
|
+
});
|
|
2209
|
+
const replanPath = dumpReplanRecord({
|
|
2210
|
+
stepIndex: i,
|
|
2211
|
+
phase: currentPhase,
|
|
2212
|
+
replanIndex,
|
|
2213
|
+
completedSteps,
|
|
2214
|
+
originalRemaining: originalRemaining.map((s) => s.instruction),
|
|
2215
|
+
newRemaining: newSteps.map((s) => s.instruction),
|
|
2216
|
+
});
|
|
2217
|
+
logger.info(`replan #${replanIndex} produced ${newSteps.length} new step(s); resuming (record: ${replanPath})`);
|
|
2218
|
+
for (const [j, s] of newSteps.entries()) {
|
|
2219
|
+
logger.info(` replanned step ${j + 1}${s.optional ? " (optional)" : ""}: ${s.instruction}`);
|
|
2220
|
+
}
|
|
2221
|
+
// Prepend recovery steps before the original remaining tail — the
|
|
2222
|
+
// replanner emits bridge steps from the failure point back to where
|
|
2223
|
+
// the original flow can resume. Idempotent fills/clicks on already-
|
|
2224
|
+
// satisfied form fields cost a few seconds each but keep the rest of
|
|
2225
|
+
// the original intent (page-0 Continue, page-1 sections, resume
|
|
2226
|
+
// upload, final submit) intact instead of replacing them with the
|
|
2227
|
+
// replanner's necessarily-truncated tail (capped at REPLAN_MAX_STEPS).
|
|
2228
|
+
// Tag replan-discovered steps with origin so persistReplannedFlow
|
|
2229
|
+
// can force them optional on write-back. originalRemaining keeps its
|
|
2230
|
+
// origin: "original" — that's what protects the canonical final
|
|
2231
|
+
// submit from being silently demoted to optional across replans.
|
|
2232
|
+
const taggedNewSteps = filterReplanDuplicatingNextAuthored(newSteps.map((s) => ({ ...s, origin: "replan" })), originalRemaining);
|
|
2233
|
+
plan.splice(i, plan.length - i, ...taggedNewSteps, ...originalRemaining);
|
|
2234
|
+
i--;
|
|
2075
2235
|
}
|
|
2076
|
-
|
|
2077
|
-
|
|
2078
|
-
|
|
2079
|
-
|
|
2080
|
-
|
|
2081
|
-
|
|
2082
|
-
|
|
2083
|
-
|
|
2084
|
-
|
|
2085
|
-
|
|
2086
|
-
|
|
2087
|
-
|
|
2088
|
-
|
|
2089
|
-
|
|
2090
|
-
|
|
2091
|
-
|
|
2092
|
-
|
|
2093
|
-
|
|
2094
|
-
|
|
2095
|
-
|
|
2096
|
-
|
|
2097
|
-
|
|
2098
|
-
|
|
2099
|
-
|
|
2100
|
-
|
|
2101
|
-
|
|
2102
|
-
|
|
2103
|
-
|
|
2104
|
-
|
|
2105
|
-
|
|
2106
|
-
|
|
2107
|
-
|
|
2108
|
-
|
|
2109
|
-
|
|
2110
|
-
|
|
2111
|
-
|
|
2112
|
-
|
|
2113
|
-
|
|
2114
|
-
|
|
2236
|
+
}
|
|
2237
|
+
if (isFlowTruncated({
|
|
2238
|
+
completedStepCount: completedSteps.length,
|
|
2239
|
+
planLength: plan.length,
|
|
2240
|
+
exitedViaTrailingGrace,
|
|
2241
|
+
})) {
|
|
2242
|
+
logger.error(`flow truncated: only ${completedSteps.length}/${plan.length} steps completed and no legitimate early-exit occurred — refusing to report success`);
|
|
2243
|
+
process.exit(1);
|
|
2244
|
+
}
|
|
2245
|
+
// Stagehand can tear its own CDP transport down mid-flow while the step
|
|
2246
|
+
// loop above keeps reporting steps as "completed" — completedSteps.length
|
|
2247
|
+
// can still equal plan.length, so isFlowTruncated alone never fires.
|
|
2248
|
+
// This must stay a separate, unconditional check rather than folding into
|
|
2249
|
+
// isFlowTruncated's boolean. Traffic is active right up to the teardown
|
|
2250
|
+
// (not idle), so this is recoverable by retrying on a fresh session —
|
|
2251
|
+
// throw instead of exiting so the caller can retry it.
|
|
2252
|
+
const cdpTransportClosedError = session.getCdpTransportClosedError?.();
|
|
2253
|
+
if (cdpTransportClosedError) {
|
|
2254
|
+
throw new errors_2.CdpTransportClosedError(`Stagehand tore down the CDP transport mid-flow: ${cdpTransportClosedError.message}`);
|
|
2255
|
+
}
|
|
2256
|
+
stopCapture();
|
|
2257
|
+
// Authoritative submission record for recon-generate: which captured POSTs
|
|
2258
|
+
// are the real submission, keyed on the flow's declared submit patterns.
|
|
2259
|
+
// Runs whenever a pattern is declared (independent of the audit gate below),
|
|
2260
|
+
// since generate consumes it even when requireSubmitEndpointMatch is off.
|
|
2261
|
+
writeSubmitManifest({
|
|
2262
|
+
runRoot: runDir.root,
|
|
2263
|
+
capturesDir: runDir.graphqlDir,
|
|
2264
|
+
submitEndpointPattern,
|
|
2265
|
+
submitBodyPattern,
|
|
2266
|
+
logger,
|
|
2267
|
+
});
|
|
2268
|
+
// End-of-run audit: when the flow declared submitEndpointPattern AND
|
|
2269
|
+
// opted into requireSubmitEndpointMatch=true, scan ALL captures from
|
|
2270
|
+
// this run for a pattern-matching 200 before declaring success. If no
|
|
2271
|
+
// match, the run "succeeded" by the verifier's lights but the actual
|
|
2272
|
+
// submission didn't land. Exit non-zero so the caller (test harness,
|
|
2273
|
+
// CI, or production runner) can distinguish silent-pass from real
|
|
2274
|
+
// success. This closes the loop the silent-pass bug exposed on 2026-
|
|
2275
|
+
// 06-09: per-step verifier accepted DOM-fallback as proof; run-level
|
|
2276
|
+
// audit catches that the network proof never actually arrived.
|
|
2277
|
+
if (requireSubmitEndpointMatch && ownBackendHostnames.length > 0) {
|
|
2278
|
+
const { auditFailed, rejectionReason } = auditFinalSubmitMatch({
|
|
2115
2279
|
ownBackendHostnames,
|
|
2116
|
-
|
|
2117
|
-
|
|
2280
|
+
capturesDir: runDir.graphqlDir,
|
|
2281
|
+
logger,
|
|
2118
2282
|
});
|
|
2119
|
-
if (
|
|
2120
|
-
|
|
2121
|
-
|
|
2122
|
-
|
|
2123
|
-
|
|
2124
|
-
|
|
2125
|
-
|
|
2126
|
-
|
|
2127
|
-
const droppedCompleted = rawNewSteps.length - newSteps.length;
|
|
2128
|
-
if (droppedCompleted > 0) {
|
|
2129
|
-
logger.info(`replan #${replanIndex}: dropped ${droppedCompleted} bridge step(s) that re-ran already-completed steps`);
|
|
2130
|
-
}
|
|
2131
|
-
if (newSteps.length === 0) {
|
|
2132
|
-
logger.error(`replan #${replanIndex} produced only already-completed steps (nothing new to bridge); aborting`);
|
|
2133
|
-
throw err;
|
|
2134
|
-
}
|
|
2135
|
-
// Immediate no-progress guard: if the replan's only bridge is a
|
|
2136
|
-
// re-emission of the step that just failed, resuming re-runs the whole
|
|
2137
|
-
// cascade on the identical click that just exhausted it. Abort now
|
|
2138
|
-
// instead of waiting REPLAN_CYCLE_THRESHOLD repeats for the cycle
|
|
2139
|
-
// detector — that many dead cascades cost minutes of wall-clock.
|
|
2140
|
-
if (isReplanReproposingFailedStep(newSteps, step.instruction)) {
|
|
2141
|
-
const noProgressMessage = `replan #${replanIndex} re-proposed only the just-failed step ("${step.instruction.slice(0, 60)}") with no new bridge; resuming would re-fail identically — aborting`;
|
|
2142
|
-
logger.error(noProgressMessage);
|
|
2143
|
-
throw new errors_2.StepVerificationError(noProgressMessage, "replan-cycle-detected");
|
|
2144
|
-
}
|
|
2145
|
-
// Auth-boundary guard: a Sign-In/Log-In bridge proposed after an
|
|
2146
|
-
// account-creation step already completed desyncs the page from the
|
|
2147
|
-
// re-appended original tail (see isReplanRegressingAcrossAuthBoundary).
|
|
2148
|
-
if (isReplanRegressingAcrossAuthBoundary(newSteps, completedSteps)) {
|
|
2149
|
-
const authBoundaryMessage = `replan #${replanIndex} proposed a Sign-In/Log-In step after an account-creation step already completed; resuming through Sign-In would strand the original remaining tail on an incompatible page — aborting`;
|
|
2150
|
-
logger.error(authBoundaryMessage);
|
|
2151
|
-
throw new errors_2.StepVerificationError(authBoundaryMessage, "replan-cycle-detected");
|
|
2283
|
+
if (auditFailed) {
|
|
2284
|
+
const reasonSuffix = rejectionReason
|
|
2285
|
+
? ` — server REJECTED submission with rejection envelope (reason: "${rejectionReason}"); HTTP layer succeeded but application was not accepted`
|
|
2286
|
+
: ` — no captured 2xx had hostname in ${JSON.stringify(ownBackendHostnames)} — submission did not land despite verifier success`;
|
|
2287
|
+
logger.error(`end-of-run audit FAILED${reasonSuffix}`);
|
|
2288
|
+
// Exit non-zero so the runner counts this as a real failure rather
|
|
2289
|
+
// than rolling silent-pass forward as success.
|
|
2290
|
+
process.exit(1);
|
|
2152
2291
|
}
|
|
2153
|
-
|
|
2154
|
-
|
|
2155
|
-
|
|
2156
|
-
|
|
2157
|
-
|
|
2158
|
-
|
|
2159
|
-
|
|
2160
|
-
|
|
2161
|
-
|
|
2162
|
-
|
|
2163
|
-
|
|
2164
|
-
|
|
2165
|
-
|
|
2166
|
-
|
|
2167
|
-
|
|
2168
|
-
|
|
2169
|
-
|
|
2170
|
-
const cycleMessage = `replan cycle detected: identical proposal × ${REPLAN_CYCLE_THRESHOLD} under static page state; aborting`;
|
|
2171
|
-
logger.error(cycleMessage);
|
|
2172
|
-
throw new errors_2.StepVerificationError(cycleMessage, "replan-cycle-detected");
|
|
2292
|
+
logger.info(`end-of-run audit PASSED: at least one captured 2xx matched submitEndpointPattern with clean response body`);
|
|
2293
|
+
}
|
|
2294
|
+
await snapshotAndPersistCookieJar(page, jarCounter, "run-complete", currentPhase, plan.length - 1);
|
|
2295
|
+
logger.info(`recon complete — ${counter.n} captures written to ${runDir.root}`);
|
|
2296
|
+
}
|
|
2297
|
+
finally {
|
|
2298
|
+
// Replay-the-discovered-path: if any replan fired and the user provided
|
|
2299
|
+
// a flow file, write the improved plan back so the next run starts
|
|
2300
|
+
// where this one ended up. Runs INSIDE finally so the cascade's
|
|
2301
|
+
// discoveries survive cascade-exhausted exits too — the persistence
|
|
2302
|
+
// mechanism is the way recon self-heals the flow across runs, so it
|
|
2303
|
+
// needs to fire on failure as much as on success. Skipped on
|
|
2304
|
+
// --no-save-replan (diagnostic dry-runs) and when --flow was used
|
|
2305
|
+
// inline (no file to write back to).
|
|
2306
|
+
if (replanEvents.length > 0) {
|
|
2307
|
+
if (!saveReplan) {
|
|
2308
|
+
logger.info(`run done with ${replanEvents.length} replan event(s); --no-save-replan, leaving flow.json unchanged`);
|
|
2173
2309
|
}
|
|
2174
|
-
if (
|
|
2175
|
-
|
|
2310
|
+
else if (!flowFile) {
|
|
2311
|
+
logger.info(`run done with ${replanEvents.length} replan event(s); --flow used (no file to write back to)`);
|
|
2176
2312
|
}
|
|
2177
2313
|
else {
|
|
2178
|
-
|
|
2179
|
-
|
|
2180
|
-
|
|
2181
|
-
|
|
2182
|
-
|
|
2183
|
-
|
|
2184
|
-
|
|
2185
|
-
|
|
2186
|
-
|
|
2187
|
-
|
|
2188
|
-
|
|
2189
|
-
|
|
2190
|
-
|
|
2191
|
-
|
|
2192
|
-
|
|
2193
|
-
|
|
2194
|
-
}
|
|
2195
|
-
|
|
2196
|
-
|
|
2197
|
-
|
|
2198
|
-
|
|
2199
|
-
|
|
2200
|
-
completedSteps,
|
|
2201
|
-
originalRemaining: originalRemaining.map((s) => s.instruction),
|
|
2202
|
-
newRemaining: newSteps.map((s) => s.instruction),
|
|
2203
|
-
});
|
|
2204
|
-
logger.info(`replan #${replanIndex} produced ${newSteps.length} new step(s); resuming (record: ${replanPath})`);
|
|
2205
|
-
for (const [j, s] of newSteps.entries()) {
|
|
2206
|
-
logger.info(` replanned step ${j + 1}${s.optional ? " (optional)" : ""}: ${s.instruction}`);
|
|
2314
|
+
logger.info(`run done; writing flow.json with ${replanEvents.length} replan event(s)`);
|
|
2315
|
+
try {
|
|
2316
|
+
persistReplannedFlow({
|
|
2317
|
+
flowFile,
|
|
2318
|
+
finalPlan: plan,
|
|
2319
|
+
replanEvents,
|
|
2320
|
+
logger,
|
|
2321
|
+
originalShape,
|
|
2322
|
+
submitEndpointPattern,
|
|
2323
|
+
submittedStateSelectors,
|
|
2324
|
+
requireSubmitEndpointMatch,
|
|
2325
|
+
successUrlFragments,
|
|
2326
|
+
successPageTitleHints,
|
|
2327
|
+
ownBackendHostnames,
|
|
2328
|
+
knownErrorClassPrefixes,
|
|
2329
|
+
});
|
|
2330
|
+
}
|
|
2331
|
+
catch (err) {
|
|
2332
|
+
// Persistence is best-effort in the finally block — a write
|
|
2333
|
+
// failure here must not eat the original cascade error.
|
|
2334
|
+
logger.error(`persistReplannedFlow threw in finally: ${(0, errors_1.toErrorMessage)(err)}`);
|
|
2335
|
+
}
|
|
2207
2336
|
}
|
|
2208
|
-
// Prepend recovery steps before the original remaining tail — the
|
|
2209
|
-
// replanner emits bridge steps from the failure point back to where
|
|
2210
|
-
// the original flow can resume. Idempotent fills/clicks on already-
|
|
2211
|
-
// satisfied form fields cost a few seconds each but keep the rest of
|
|
2212
|
-
// the original intent (page-0 Continue, page-1 sections, resume
|
|
2213
|
-
// upload, final submit) intact instead of replacing them with the
|
|
2214
|
-
// replanner's necessarily-truncated tail (capped at REPLAN_MAX_STEPS).
|
|
2215
|
-
// Tag replan-discovered steps with origin so persistReplannedFlow
|
|
2216
|
-
// can force them optional on write-back. originalRemaining keeps its
|
|
2217
|
-
// origin: "original" — that's what protects the canonical final
|
|
2218
|
-
// submit from being silently demoted to optional across replans.
|
|
2219
|
-
const taggedNewSteps = filterReplanDuplicatingNextAuthored(newSteps.map((s) => ({ ...s, origin: "replan" })), originalRemaining);
|
|
2220
|
-
plan.splice(i, plan.length - i, ...taggedNewSteps, ...originalRemaining);
|
|
2221
|
-
i--;
|
|
2222
|
-
}
|
|
2223
|
-
}
|
|
2224
|
-
if (isFlowTruncated({
|
|
2225
|
-
completedStepCount: completedSteps.length,
|
|
2226
|
-
planLength: plan.length,
|
|
2227
|
-
exitedViaTrailingGrace,
|
|
2228
|
-
})) {
|
|
2229
|
-
logger.error(`flow truncated: only ${completedSteps.length}/${plan.length} steps completed and no legitimate early-exit occurred — refusing to report success`);
|
|
2230
|
-
process.exit(1);
|
|
2231
|
-
}
|
|
2232
|
-
// Stagehand can tear its own CDP transport down mid-flow while the step
|
|
2233
|
-
// loop above keeps reporting steps as "completed" — completedSteps.length
|
|
2234
|
-
// can still equal plan.length, so isFlowTruncated alone never fires.
|
|
2235
|
-
// This must stay a separate, unconditional check rather than folding into
|
|
2236
|
-
// isFlowTruncated's boolean.
|
|
2237
|
-
const cdpTransportClosedError = session.getCdpTransportClosedError?.();
|
|
2238
|
-
if (cdpTransportClosedError) {
|
|
2239
|
-
logger.error(`Stagehand tore down the CDP transport mid-flow: ${cdpTransportClosedError.message} — refusing to report success`);
|
|
2240
|
-
process.exit(1);
|
|
2241
|
-
}
|
|
2242
|
-
stopCapture();
|
|
2243
|
-
// Authoritative submission record for recon-generate: which captured POSTs
|
|
2244
|
-
// are the real submission, keyed on the flow's declared submit patterns.
|
|
2245
|
-
// Runs whenever a pattern is declared (independent of the audit gate below),
|
|
2246
|
-
// since generate consumes it even when requireSubmitEndpointMatch is off.
|
|
2247
|
-
writeSubmitManifest({
|
|
2248
|
-
runRoot: runDir.root,
|
|
2249
|
-
capturesDir: runDir.graphqlDir,
|
|
2250
|
-
submitEndpointPattern,
|
|
2251
|
-
submitBodyPattern,
|
|
2252
|
-
logger,
|
|
2253
|
-
});
|
|
2254
|
-
// End-of-run audit: when the flow declared submitEndpointPattern AND
|
|
2255
|
-
// opted into requireSubmitEndpointMatch=true, scan ALL captures from
|
|
2256
|
-
// this run for a pattern-matching 200 before declaring success. If no
|
|
2257
|
-
// match, the run "succeeded" by the verifier's lights but the actual
|
|
2258
|
-
// submission didn't land. Exit non-zero so the caller (test harness,
|
|
2259
|
-
// CI, or production runner) can distinguish silent-pass from real
|
|
2260
|
-
// success. This closes the loop the silent-pass bug exposed on 2026-
|
|
2261
|
-
// 06-09: per-step verifier accepted DOM-fallback as proof; run-level
|
|
2262
|
-
// audit catches that the network proof never actually arrived.
|
|
2263
|
-
if (requireSubmitEndpointMatch && ownBackendHostnames.length > 0) {
|
|
2264
|
-
const { auditFailed, rejectionReason } = auditFinalSubmitMatch({
|
|
2265
|
-
ownBackendHostnames,
|
|
2266
|
-
capturesDir: runDir.graphqlDir,
|
|
2267
|
-
logger,
|
|
2268
|
-
});
|
|
2269
|
-
if (auditFailed) {
|
|
2270
|
-
const reasonSuffix = rejectionReason
|
|
2271
|
-
? ` — server REJECTED submission with rejection envelope (reason: "${rejectionReason}"); HTTP layer succeeded but application was not accepted`
|
|
2272
|
-
: ` — no captured 2xx had hostname in ${JSON.stringify(ownBackendHostnames)} — submission did not land despite verifier success`;
|
|
2273
|
-
logger.error(`end-of-run audit FAILED${reasonSuffix}`);
|
|
2274
|
-
// Exit non-zero so the runner counts this as a real failure rather
|
|
2275
|
-
// than rolling silent-pass forward as success.
|
|
2276
|
-
process.exit(1);
|
|
2277
2337
|
}
|
|
2278
|
-
|
|
2338
|
+
await session.close();
|
|
2279
2339
|
}
|
|
2280
|
-
await snapshotAndPersistCookieJar(page, jarCounter, "run-complete", currentPhase, plan.length - 1);
|
|
2281
|
-
logger.info(`recon complete — ${counter.n} captures written to ${runDir.root}`);
|
|
2282
2340
|
}
|
|
2283
|
-
|
|
2284
|
-
|
|
2285
|
-
|
|
2286
|
-
|
|
2287
|
-
|
|
2288
|
-
|
|
2289
|
-
|
|
2290
|
-
|
|
2291
|
-
|
|
2292
|
-
|
|
2293
|
-
|
|
2294
|
-
|
|
2295
|
-
|
|
2296
|
-
|
|
2297
|
-
|
|
2341
|
+
// Only a CDP-transport teardown gets retried on a fresh session — every
|
|
2342
|
+
// other failure mode (isFlowTruncated's exit(1), StepVerificationError,
|
|
2343
|
+
// SessionTimeoutError, etc.) must keep rejecting/exiting after exactly one
|
|
2344
|
+
// attempt, exactly as before this whole-flow retry was added. `withScraperRetry`
|
|
2345
|
+
// (mandated over a hand-rolled loop) can't distinguish that on its own —
|
|
2346
|
+
// its classifyScraperError policy retries SessionTimeoutError too — so
|
|
2347
|
+
// non-CdpTransportClosedError failures are caught here and stashed instead
|
|
2348
|
+
// of thrown, making the retried task look "successful" to p-retry, then
|
|
2349
|
+
// re-thrown unchanged (same instance, same type) once the retry call
|
|
2350
|
+
// settles. This is plain error propagation, not a second retry loop.
|
|
2351
|
+
let nonTransportError;
|
|
2352
|
+
try {
|
|
2353
|
+
await (0, retry_1.withScraperRetry)(async () => {
|
|
2354
|
+
try {
|
|
2355
|
+
await runFlowAttempt();
|
|
2298
2356
|
}
|
|
2299
|
-
|
|
2300
|
-
|
|
2301
|
-
|
|
2302
|
-
|
|
2303
|
-
flowFile,
|
|
2304
|
-
finalPlan: plan,
|
|
2305
|
-
replanEvents,
|
|
2306
|
-
logger,
|
|
2307
|
-
originalShape,
|
|
2308
|
-
submitEndpointPattern,
|
|
2309
|
-
submittedStateSelectors,
|
|
2310
|
-
requireSubmitEndpointMatch,
|
|
2311
|
-
successUrlFragments,
|
|
2312
|
-
successPageTitleHints,
|
|
2313
|
-
ownBackendHostnames,
|
|
2314
|
-
knownErrorClassPrefixes,
|
|
2315
|
-
});
|
|
2316
|
-
}
|
|
2317
|
-
catch (err) {
|
|
2318
|
-
// Persistence is best-effort in the finally block — a write
|
|
2319
|
-
// failure here must not eat the original cascade error.
|
|
2320
|
-
logger.error(`persistReplannedFlow threw in finally: ${(0, errors_1.toErrorMessage)(err)}`);
|
|
2321
|
-
}
|
|
2357
|
+
catch (err) {
|
|
2358
|
+
if (err instanceof errors_2.CdpTransportClosedError)
|
|
2359
|
+
throw err;
|
|
2360
|
+
nonTransportError = err;
|
|
2322
2361
|
}
|
|
2362
|
+
}, { maxAttempts: config_1.config.scraper.maxTransportRetries });
|
|
2363
|
+
}
|
|
2364
|
+
catch (err) {
|
|
2365
|
+
if (err instanceof errors_2.CdpTransportClosedError) {
|
|
2366
|
+
logger.error(`Stagehand tore down the CDP transport mid-flow on every attempt (${config_1.config.scraper.maxTransportRetries} of ${config_1.config.scraper.maxTransportRetries}): ${err.message} — refusing to report success`);
|
|
2367
|
+
process.exit(1);
|
|
2323
2368
|
}
|
|
2324
|
-
|
|
2369
|
+
throw err;
|
|
2325
2370
|
}
|
|
2371
|
+
if (nonTransportError !== undefined)
|
|
2372
|
+
throw nonTransportError;
|
|
2326
2373
|
}
|
|
2327
2374
|
if (process.argv[1] !== undefined &&
|
|
2328
2375
|
(process.argv[1].endsWith("recon-browser.ts") || process.argv[1].endsWith("recon-browser.js"))) {
|