imprint-mcp 0.2.0 → 0.3.0

package/src/imprint/backend-ladder.ts

@@ -1,28 +1,63 @@
 /**
- * Walk a list of backends in order, escalating on FORBIDDEN and satisfiable
- * STATE_MISSING; other errors return immediately. fetch-bootstrap is a gated
- * API-replay adapter, not a default DOM fallback rung: auto only reaches it
- * for workflows that declare bootstrap/captures or STATE_MISSING says browser
- * bootstrap can satisfy the missing state.
+ * Walk a list of backends in order, escalating on FORBIDDEN, NETWORK (tarpit),
+ * and satisfiable STATE_MISSING; other errors return immediately.
+ *
+ * Rung tiers:
+ *  - `fetch`           — plain HTTP API replay.
+ *  - `fetch-bootstrap` — the API ANTI-BOT path: a one-time cdp-browser mint of a
+ *    validated Akamai session jar (real Chrome used ONLY to bootstrap, then
+ *    closed), then PLAIN-fetch replay of every request with that jar. The jar is
+ *    cached (~90 min) so one bootstrap serves many searches. Auto mode always
+ *    splices this right after `fetch`; it only RUNS when `fetch` escalates, so a
+ *    healthy plain-API site never pays for it.
+ *  - `stealth-fetch`   — Playwright stealth bootstrap + native fetch (token tier).
+ *  - `playbook`        — DOM-walk LAST RESORT (needs a compiled playbook.yaml).
  */
 
-import { existsSync } from 'node:fs';
-import { resolve as pathResolve } from 'node:path';
+import { existsSync, readFileSync } from 'node:fs';
+import { dirname, resolve as pathResolve } from 'node:path';
 import type { Page } from 'playwright';
+import {
+  type CdpBrowserFetch,
+  type CdpBrowserFetchOptions,
+  type MintedJar,
+  createCdpBrowserFetch,
+} from './cdp-browser-fetch.ts';
+import {
+  clearJar,
+  loadJar,
+  newestRecording,
+  saveJar,
+  seedJarFromRecording,
+} from './cdp-jar-cache.ts';
+import { proxyUrl } from './chromium.ts';
 import { RuntimeCookieJar } from './cookie-jar.ts';
 import { createLog } from './log.ts';
 import { runPlaybook } from './playbook-runner.ts';
-import { type CredentialStore, loadCredentialStore, substituteString } from './runtime.ts';
-import { type StealthFetch, createStealthFetch } from './stealth-fetch.ts';
+import {
+  type CredentialStore,
+  executeWorkflow,
+  loadCredentialStore,
+  substituteString,
+} from './runtime.ts';
+import {
+  type BootstrapArgs,
+  type StealthFetch,
+  type TokenCache,
+  bootstrapStealthToken,
+  createStealthFetch,
+} from './stealth-fetch.ts';
+import { clearCachedToken, loadCachedToken, saveCachedToken } from './stealth-token-cache.ts';
 import type { ResolvedTool } from './tool-loader.ts';
-import type {
-  BootstrapCapture,
-  ConcreteBackend,
-  ReplayBackend,
-  StateCapability,
-  StateMissingItem,
-  ToolResult,
-  Workflow,
+import {
+  type BootstrapCapture,
+  type ConcreteBackend,
+  type ReplayBackend,
+  type StateCapability,
+  type StateMissingItem,
+  type ToolResult,
+  type Workflow,
+  WorkflowSchema,
 } from './types.ts';
 
 interface LadderResult {
@@ -41,6 +76,131 @@ const log = createLog('backend');
 
 const DEFAULT_LADDER: ConcreteBackend[] = ['fetch', 'stealth-fetch', 'playbook'];
 
+/** Process-scoped memo of the backend that last succeeded for a site on the
+ *  compile/test path (`runWorkflowWithLadder`). Lets the param-coverage suite
+ *  skip doomed rungs after the first success. Never persisted; never consulted
+ *  by production replay. Exported reset for test isolation. */
+const compileWinningBackend = new Map<string, ConcreteBackend>();
+export function __resetCompileWinningBackendForTest(): void {
+  compileWinningBackend.clear();
+}
+
+let probeTimeoutMsForTest: number | null = null;
+export function __setProbeTimeoutMsForTest(ms: number | null): void {
+  probeTimeoutMsForTest = ms;
+}
+
+/** Backend preference for the compile parallel-probe winner, LOWER = preferred.
+ *  `fetch` first (cheapest, no browser). Among the browser-backed rungs prefer
+ *  `cdp-replay` over `stealth-fetch`: cdp-replay's cold start is a one-time cost
+ *  (the pool keeps Chrome warm so later calls are ~2-5s) and it is the more
+ *  anti-bot-robust path (real Chrome re-validating its sensor between calls), so
+ *  it shouldn't lose the probe just because stealth's FIRST call clocked faster. */
+const BACKEND_PROBE_RANK: Record<string, number> = {
+  fetch: 0,
+  'cdp-replay': 1,
+  'stealth-fetch': 2,
+};
+
+/** Pick the parallel-probe winner among backends that returned real data: prefer
+ *  by `BACKEND_PROBE_RANK` (fetch < cdp-replay < stealth-fetch), with first-call
+ *  duration only as a tiebreak — so when both browser backends succeed, the
+ *  warm-poolable cdp-replay wins instead of stealth's faster cold call. Pure +
+ *  exported for unit testing. */
+export function pickProbeWinner<T extends { backend: ConcreteBackend; durationMs: number }>(
+  winners: T[],
+): T | undefined {
+  return [...winners].sort((a, b) => {
+    const ra = BACKEND_PROBE_RANK[a.backend] ?? 9;
+    const rb = BACKEND_PROBE_RANK[b.backend] ?? 9;
+    return ra !== rb ? ra - rb : a.durationMs - b.durationMs;
+  })[0];
+}
+
+/** Process-global CDP pool for the compile/test path (`runWorkflowWithLadder`).
+ *  cdp-replay stores its live Chrome here on success so subsequent calls within
+ *  the same `bun test` process reuse it (~2-5s vs ~33s cold start) — the same
+ *  mechanism as the runtime pool in mcp-server.ts. An idle timer (re)armed after
+ *  every call closes each browser shortly after the LAST call, so the host
+ *  process drains and exits cleanly (no leak, no hang) without a per-call drain.
+ *  Per-process: concurrent compile lanes are separate `bun test` processes, so
+ *  this is never shared across lanes; never consulted by production replay. */
+const compileCdpPool = new Map<string, CdpBrowserFetch>();
+const compileCdpIdleTimers = new Map<string, ReturnType<typeof setTimeout>>();
+const COMPILE_CDP_IDLE_MS = 15_000;
+
+/** Cancel pending idle-closes — called when a new call is about to reuse the pool. */
+function clearCompileCdpIdle(): void {
+  for (const t of compileCdpIdleTimers.values()) clearTimeout(t);
+  compileCdpIdleTimers.clear();
+}
+
+/** (Re)arm an idle-close timer for every pooled browser. If no further call
+ *  reuses the pool within COMPILE_CDP_IDLE_MS, the browser is closed + evicted so
+ *  the event loop drains and the process exits. The timer is intentionally NOT
+ *  unref'd: closing the browser is what lets the process exit, so the teardown
+ *  must be guaranteed to fire. */
+function armCompileCdpIdleClose(): void {
+  clearCompileCdpIdle();
+  for (const [site, cf] of compileCdpPool) {
+    const timer = setTimeout(() => {
+      compileCdpPool.delete(site);
+      compileCdpIdleTimers.delete(site);
+      // Close releases the websocket + Chrome child handles so the event loop
+      // drains and the host process exits (mirrors mcp-server's idle close).
+      void cf.close().catch(() => {});
+    }, COMPILE_CDP_IDLE_MS);
+    compileCdpIdleTimers.set(site, timer);
+  }
+}
+
+/** Test isolation: cancel idle timers + drop pooled browsers (best-effort close). */
+export function __resetCompileCdpPoolForTest(): void {
+  clearCompileCdpIdle();
+  for (const cf of compileCdpPool.values()) void cf.close().catch(() => {});
+  compileCdpPool.clear();
+}
+
+/** Freshness window for the file-backed compile-time stealth token. Matches
+ *  stealth-fetch's in-process `maxTokenAgeSeconds` default so a reused token is
+ *  not immediately considered stale by `createStealthFetch`. */
+const STEALTH_TOKEN_MAX_AGE_SECONDS = 600;
+
+/** Min spacing (ms) between LIVE requests to one origin on the compile/test path,
+ *  to stay under the transient anti-bot rate-flag (observed: ~2 rapid state-
+ *  changing requests OK, ~3-4 trips it; recovers). The param-coverage suite fires
+ *  one search per parameter — without pacing that burst flags the IP and TARPITS
+ *  every later request (exactly what made v13's `.act` tools fail compile, and
+ *  what flagged the IP during manual testing). Read per-call so tests can set
+ *  IMPRINT_COMPILE_ACT_SPACING_MS=0. Process-scoped; production replay untouched. */
+function compileActSpacingMs(): number {
+  const v = Number(process.env.IMPRINT_COMPILE_ACT_SPACING_MS ?? 25_000);
+  return Number.isFinite(v) && v > 0 ? v : 0;
+}
+const compileLastRequestAt = new Map<string, number>();
+function sleepMs(ms: number): Promise<void> {
+  return new Promise((r) => setTimeout(r, ms));
+}
+/** Await the per-origin min spacing before a compile-path live request. The
+ *  first call to an origin never waits (last=0); subsequent ones within the
+ *  window are delayed so the suite paces itself under the rate-flag. */
+async function paceCompileRequest(origin: string): Promise<void> {
+  const spacing = compileActSpacingMs();
+  if (spacing <= 0) return;
+  const last = compileLastRequestAt.get(origin) ?? 0;
+  const waitMs = last + spacing - Date.now();
+  if (waitMs > 0) {
+    log(
+      `compile pacing: waiting ${Math.round(waitMs / 1000)}s before next live request to ${origin}`,
+    );
+    await sleepMs(waitMs);
+  }
+  compileLastRequestAt.set(origin, Date.now());
+}
+export function __resetCompilePacingForTest(): void {
+  compileLastRequestAt.clear();
+}
+
 /** Expand a replayBackend choice into a concrete ladder. 'auto' prefers
  *  the probed order (if any), else the default. Explicit choice → single rung. */
 export function resolveLadder(
@@ -62,12 +222,46 @@ export async function runWithLadder(
   params: Record<string, string | number | boolean>,
   assetRoot: string,
   stealthCache: Map<string, StealthFetch>,
+  options?: {
+    skipBootstrapSplice?: boolean;
+    /** Per-site CDP browser pool so cdp-replay reuses a live Chrome across
+     *  calls (~2-5s) instead of launching a fresh one each time (~33s). */
+    cdpPool?: Map<string, CdpBrowserFetch>;
+    /** Per-session memo of the backend that last served each tool. Once set, the
+     *  next call starts at that backend instead of re-walking the doomed early
+     *  rungs — the runtime analog of the compile path's `compileWinningBackend`.
+     *  The mcp-server owns one map and ties its lifetime to `cdpPool` (a memoized
+     *  cdp-replay is only fast while its Chrome is pooled). */
+    winnerCache?: Map<string, ConcreteBackend>;
+  },
 ): Promise<LadderResult> {
   if (ladder.length === 0) {
     throw new Error('runWithLadder: empty ladder');
   }
 
-  const effectiveLadder = effectiveAutoLadder(ladder, tool.workflow);
+  const baseLadder = options?.skipBootstrapSplice
+    ? ladder
+    : effectiveAutoLadder(ladder, tool.workflow);
+
+  // Runtime winner memo. Once a backend has served this tool in THIS session,
+  // start there next time instead of re-walking the doomed early rungs (southwest
+  // re-paid an ~80s fetch-bootstrap before cdp-replay on every call). The memo
+  // reorders the POST-splice ladder — cdp-replay only exists after
+  // effectiveAutoLadder splices it in, so reordering the raw `ladder` could never
+  // memoize it. Wrap-around keeps every other rung as fallback, so a now-stale
+  // winner still escalates correctly.
+  const memoKey = `${tool.site}:${tool.workflow.toolName}`;
+  let effectiveLadder = baseLadder;
+  const memoWinner = options?.winnerCache?.get(memoKey);
+  if (memoWinner) {
+    const idx = baseLadder.indexOf(memoWinner);
+    if (idx > 0) {
+      effectiveLadder = [...baseLadder.slice(idx), ...baseLadder.slice(0, idx)];
+      log(
+        `runtime memo: ${memoKey} → start at ${memoWinner}; ladder: ${effectiveLadder.join(' → ')}`,
+      );
+    }
+  }
   const attempts: LadderResult['attempts'] = [];
   let lastResult: ToolResult | null = null;
   let skipUntilBackend: ConcreteBackend | null = null;
@@ -76,6 +270,9 @@ export async function runWithLadder(
     if (skipUntilBackend && backend !== skipUntilBackend) continue;
     if (skipUntilBackend === backend) skipUntilBackend = null;
 
+    // The playbook rung is the DOM-walk LAST RESORT (needs a playbook.yaml). The
+    // anti-bot API path is the fetch-bootstrap rung above (cdp-browser jar mint
+    // then PLAIN-fetch replay) — NOT this rung. Skip when no playbook.yaml.
     if (backend === 'playbook' && !existsSync(playbookPath(assetRoot, tool.site, tool.dir))) {
       attempts.push({
         backend,
@@ -83,7 +280,7 @@ export async function runWithLadder(
         detail: 'no playbook.yaml',
         durationMs: 0,
       });
-      log(`${backend}: skipped (prerequisite missing)`);
+      log(`${backend}: skipped (no playbook.yaml)`);
       continue;
     }
 
@@ -92,24 +289,50 @@ export async function runWithLadder(
     let result: ToolResult;
     try {
       switch (backend) {
-        case 'fetch':
-          result = await tool.toolFn(params);
+        case 'fetch': {
+          // Egress the plain `fetch` rung through IMPRINT_PROXY when set, so even
+          // the first rung (and GET-only tools) use the residential proxy IP.
+          const proxyFetch = makeProxyFetch();
+          result = await tool.toolFn(params, proxyFetch ? { fetchImpl: proxyFetch } : undefined);
           break;
+        }
         case 'fetch-bootstrap':
           result = await runFetchBootstrap(tool, params);
           break;
+        case 'cdp-replay':
+          result = await runCdpReplay(tool, params, options?.cdpPool);
+          break;
         case 'stealth-fetch': {
           const sf = ensureStealthFetch(tool, stealthCache);
-          result = await tool.toolFn(params, { fetchImpl: sf.fetchImpl });
+          // When the workflow declares a bootstrap block, mint its declared
+          // session-token state (CSRF cookies etc.) from the SAME stealth
+          // session that provides the transport cookies. Without this, a
+          // workflow escalating here from fetch-bootstrap loses the
+          // ${state.X} its requests need — the gap that made bootstrap-block
+          // tools on anti-bot sites unverifiable.
+          const initialState = tool.workflow.bootstrap
+            ? await stealthBootstrapState(sf, tool.workflow.bootstrap)
+            : undefined;
+          result = await tool.toolFn(params, { fetchImpl: sf.fetchImpl, initialState });
           break;
         }
-        case 'playbook':
+        case 'playbook': {
+          // DOM-walk last resort (the anti-bot API path is fetch-bootstrap, above).
+          // Apply workflow.json's declared parameter defaults — runPlaybook
+          // validates and throws on absent values regardless of declared defaults.
+          const paramsWithDefaults: typeof params = { ...params };
+          for (const p of tool.workflow.parameters) {
+            if (!(p.name in paramsWithDefaults) && p.default !== undefined) {
+              paramsWithDefaults[p.name] = p.default;
+            }
+          }
           result = await runPlaybook({
             playbook: playbookPath(assetRoot, tool.site, tool.dir),
-            params,
+            params: paramsWithDefaults,
             site: tool.site,
           });
           break;
+        }
       }
     } catch (err) {
       const msg = err instanceof Error ? err.message : String(err);
@@ -121,6 +344,7 @@ export async function runWithLadder(
     if (result.ok) {
       attempts.push({ backend, outcome: 'ok', detail: `succeeded in ${durationMs}ms`, durationMs });
       log(`${backend}: OK in ${durationMs}ms`);
+      options?.winnerCache?.set(memoKey, backend);
       return { result, usedBackend: backend, attempts };
     }
 
@@ -150,8 +374,48 @@ export async function runWithLadder(
       }
     }
 
-    // Non-FORBIDDEN errors don't escalate — different backend can't fix
-    // AUTH_EXPIRED, NETWORK, RATE_LIMITED.
+    // NETWORK escalates: a long timeout is usually anti-bot tarpitting
+    // (Akamai/Cloudflare/PerimeterX hang the connection rather than 403),
+    // and a different transport (stealth-fetch's minted token cookies, or
+    // playbook's full stealth browser) can fix it. Real DNS/connectivity
+    // failures die in milliseconds at every rung, so the cost ceiling is
+    // bounded by the per-rung timeout × ladder length.
+    if (result.error === 'NETWORK') {
+      attempts.push({
+        backend,
+        outcome: 'escalate',
+        detail: `${result.error}: ${result.message.slice(0, 120)}`,
+        durationMs,
+      });
+      log(`${backend}: NETWORK in ${durationMs}ms — escalating to next rung`);
+      continue;
+    }
+
+    // BAD_RESPONSE (e.g. HTTP 400) is backend-specific on anti-bot sites, so it
+    // escalates rather than stopping. A cdp-replay in-page POST can be rejected
+    // because it lacks the live Akamai sensor headers the endpoint demands, while
+    // stealth-fetch — which MINTS those sensor headers during its bootstrap —
+    // returns 200 for the byte-identical request. Validated on southwest's
+    // low-fare-calendar (cdp-replay 400, stealth-fetch 200). Stopping at the first
+    // 400 stranded the working rung; escalate so a higher-trust backend gets a
+    // shot, and the winner memo then locks onto whatever passed. A genuinely
+    // malformed request 400s at every rung and the last 400 is still returned
+    // below — cost is bounded by the ladder length.
+    if (result.error === 'BAD_RESPONSE') {
+      attempts.push({
+        backend,
+        outcome: 'escalate',
+        detail: `${result.error}: ${result.message.slice(0, 120)}`,
+        durationMs,
+      });
+      log(
+        `${backend}: BAD_RESPONSE in ${durationMs}ms — escalating (a higher-trust rung may pass)`,
+      );
+      continue;
+    }
+
+    // AUTH_EXPIRED needs a re-login; RATE_LIMITED needs backoff. Neither
+    // is fixed by switching transport.
     attempts.push({
       backend,
       outcome: 'failed',
@@ -174,33 +438,96 @@ export async function runWithLadder(
       attempts,
     };
   }
+  const lastBackend = effectiveLadder[effectiveLadder.length - 1] ?? 'fetch';
+  // Be accurate about ladder size: the parallel probe calls this with SINGLE-rung
+  // ladders, so "every backend escalated" was misleading (it described one rung,
+  // e.g. fetch-only, as if the whole ladder gave up — and fooled the integration
+  // classifier). Only say "all rungs" when there really was more than one.
   log(
-    `every backend escalated; returning last error from ${effectiveLadder[effectiveLadder.length - 1]}`,
+    effectiveLadder.length === 1
+      ? `${lastBackend}: exhausted (no fallback rung in this ladder); returning its error`
+      : `ladder exhausted: all ${effectiveLadder.length} rungs escalated (${effectiveLadder.join(' → ')}); returning last error from ${lastBackend}`,
   );
   return {
     result: lastResult,
-    usedBackend: effectiveLadder[effectiveLadder.length - 1] ?? 'fetch',
+    usedBackend: lastBackend,
     attempts,
   };
 }
 
-function effectiveAutoLadder(ladder: ConcreteBackend[], workflow: Workflow): ConcreteBackend[] {
-  if (ladder.length <= 1 || ladder.includes('fetch-bootstrap')) return ladder;
-  if (!workflowNeedsBootstrap(workflow)) return ladder;
-  const fetchIdx = ladder.indexOf('fetch');
-  if (fetchIdx === -1) return ladder;
+export function effectiveAutoLadder(
+  ladder: ConcreteBackend[],
+  workflow: Workflow,
+): ConcreteBackend[] {
+  if (ladder.length <= 1) return ladder;
   const next = [...ladder];
-  next.splice(fetchIdx + 1, 0, 'fetch-bootstrap');
+  // Splice fetch-bootstrap right after `fetch`. It is the plain-fetch API
+  // anti-bot path: a one-time cdp-browser jar mint, then PLAIN-fetch replay. It
+  // only RUNS when `fetch` escalates (FORBIDDEN/NETWORK/satisfiable
+  // STATE_MISSING), so a healthy plain-API site never pays for it. (Gating it on
+  // workflowNeedsBootstrap previously excluded inline-token workflows like
+  // costco — so we always splice now.)
+  if (!next.includes('fetch-bootstrap')) {
+    const fetchIdx = next.indexOf('fetch');
+    if (fetchIdx !== -1) {
+      next.splice(fetchIdx + 1, 0, 'fetch-bootstrap');
+    } else if (!next.includes('cdp-replay')) {
+      // `fetch` was probed-out (e.g. Akamai 403) and `cdp-replay` is not
+      // explicitly in the ladder. Splice fetch-bootstrap before stealth-fetch
+      // so the jar-based path gets a shot. When cdp-replay IS explicit, the
+      // probe already determined it's the right rung and fetch-bootstrap was
+      // exhausted — don't re-add a doomed 60s+ rung before it.
+      const sfIdx = next.indexOf('stealth-fetch');
+      if (sfIdx !== -1) next.splice(sfIdx, 0, 'fetch-bootstrap');
+    }
+  }
+  // Splice cdp-replay right after fetch-bootstrap. It runs the API requests IN a
+  // live trusted Chrome so a protected POST's self-invalidated _abck is
+  // re-validated by the page's bmak sensor between calls — the only path that
+  // SUSTAINS multiple sensitive .act POSTs (plain-fetch replay dies after ~1-2
+  // because it cannot re-post sensor data). Expensive (a real Chrome launch), so
+  // it only RUNS when fetch-bootstrap also escalates; a single-.act tool wins at
+  // fetch-bootstrap and never pays for it.
+  if (!next.includes('cdp-replay')) {
+    const fbIdx = next.indexOf('fetch-bootstrap');
+    if (fbIdx !== -1) next.splice(fbIdx + 1, 0, 'cdp-replay');
+  }
+  // For a MULTI-step state-changing anti-bot workflow, plain-fetch rungs are not
+  // just doomed — their tarpitted .act attempts BURN the per-IP rate budget
+  // before cdp-replay even runs, which can flag the IP and make cdp-replay tarpit
+  // too. Front-load cdp-replay for these so the live browser handles every
+  // protected POST from a clean slate.
+  if (prefersCdpReplayFirst(workflow)) {
+    const i = next.indexOf('cdp-replay');
+    if (i > 0) {
+      next.splice(i, 1);
+      next.unshift('cdp-replay');
+    }
+  }
   return next;
 }
 
-function workflowNeedsBootstrap(workflow: Workflow): boolean {
-  if (workflow.bootstrap) return true;
-  return workflow.requests.some((r) =>
-    (r.captures ?? []).some(
-      (c) => c.capability === 'browser_bootstrap' || c.capability === 'stealth_bootstrap',
-    ),
+/** A multi-step, state-changing, anti-bot workflow: ≥2 mutating requests AND an
+ *  anti-bot signal (a bootstrap block, or requests that depend on captured
+ *  `${state.X}` tokens). Plain-fetch replay can't sustain its sequence of
+ *  protected POSTs (each self-invalidates `_abck`); only the live-browser
+ *  cdp-replay rung can — and it should run FIRST so the doomed fetch /
+ *  fetch-bootstrap attempts don't pre-burn the per-IP .act budget. A plain
+ *  multi-POST REST API (no bootstrap, no `${state.X}`) is NOT matched, so it
+ *  keeps the cheap fetch-first order. */
+export function prefersCdpReplayFirst(workflow: Workflow): boolean {
+  const mutating = workflow.requests.filter((r) => {
+    const m = (r.method ?? 'GET').toUpperCase();
+    return r.effect === 'unsafe' || m === 'POST' || m === 'PUT' || m === 'PATCH' || m === 'DELETE';
+  });
+  if (mutating.length < 2) return false;
+  const hasStateRefs = workflow.requests.some(
+    (r) =>
+      /\$\{state\./.test(r.url ?? '') ||
+      /\$\{state\./.test(r.body ?? '') ||
+      Object.values(r.headers ?? {}).some((v) => /\$\{state\./.test(v)),
   );
+  return Boolean(workflow.bootstrap) || hasStateRefs;
 }
 
 function nextStateMissingBackend(
@@ -226,7 +553,12 @@ function capabilitySatisfiedBy(backend: ConcreteBackend, capability: StateCapabi
   if (backend === 'fetch-bootstrap') {
     return capability === 'browser_bootstrap' || capability === 'stealth_bootstrap';
   }
-  if (backend === 'stealth-fetch') return false;
+  if (backend === 'cdp-replay') {
+    return capability === 'browser_bootstrap' || capability === 'stealth_bootstrap';
+  }
+  if (backend === 'stealth-fetch') {
+    return capability === 'browser_bootstrap' || capability === 'stealth_bootstrap';
+  }
   if (backend === 'playbook') {
     return (
       capability === 'ordinary_http' ||
@@ -237,26 +569,154 @@ function capabilitySatisfiedBy(backend: ConcreteBackend, capability: StateCapabi
   return false;
 }
 
+/** Get a validated Akamai jar for this site: reuse the cached one (<=90 min,
+ *  _abck~0~) or mint a fresh one via cdp-browser (ONE real-Chrome launch — the
+ *  only mechanism that earns Akamai's trust; Playwright tarpits and never
+ *  validates _abck). The browser is closed before returning; the jar replays
+ *  via plain fetch. Returns null if Chrome can't launch (caller escalates). */
+/** Test seam: stub the cdp-browser jar mint so unit tests don't launch real
+ *  Chrome. Production leaves this null and uses the real cdp-browser path. */
+let cdpJarMinterForTest:
+  | ((baseUrl: string, bootstrapUrl: string | undefined) => Promise<MintedJar | null>)
+  | null = null;
+export function __setCdpJarMinterForTest(
+  fn: ((baseUrl: string, bootstrapUrl: string | undefined) => Promise<MintedJar | null>) | null,
+): void {
+  cdpJarMinterForTest = fn;
+}
+
+/** Test seam: stub the cdp-browser factory used by the cdp-replay rung so unit
+ *  tests don't launch real Chrome. Production leaves this null. */
+let cdpBrowserFetchFactoryForTest: ((opts: CdpBrowserFetchOptions) => CdpBrowserFetch) | null =
+  null;
+export function __setCdpBrowserFetchFactoryForTest(
+  fn: ((opts: CdpBrowserFetchOptions) => CdpBrowserFetch) | null,
+): void {
+  cdpBrowserFetchFactoryForTest = fn;
+}
+
+async function getOrMintCdpJar(
+  baseUrl: string,
+  bootstrapUrl: string | undefined,
+  siteDir: string,
+  forceFresh: boolean,
+): Promise<MintedJar | null> {
+  if (cdpJarMinterForTest) return cdpJarMinterForTest(baseUrl, bootstrapUrl);
+  if (!forceFresh) {
+    let cached = loadJar(siteDir);
+    // A recording NEWER than the cached jar supersedes it — e.g. the user
+    // re-recorded on a new IP, so the cached (old-IP) jar would tarpit. Drop the
+    // stale cache and re-seed from the fresh recording below.
+    const rec = newestRecording(siteDir);
+    if (cached && rec && rec.mtimeMs > cached.bootstrapEpoch) cached = null;
+    // No (usable) cached jar? Prefer seeding from the user's most recent
+    // RECORDING — a real-browser session whose `_abck` is HIGH-TRUST (sustains
+    // many sequential .act), strictly better than a synthetic cdp-browser mint
+    // (low-trust → tarpitted even on a fresh IP). "The recording IS the
+    // executable." Reuse the `rec` stat above so we don't re-glob.
+    if (!cached && seedJarFromRecording(siteDir, rec, bootstrapUrl)) cached = loadJar(siteDir);
+    if (cached) {
+      const provenance =
+        cached.source === 'recording'
+          ? 'recording-seeded'
+          : cached.source === 'mint'
+            ? 'cdp-minted'
+            : // pre-`source` cache: html-emptiness was the old (now-unreliable) tell
+              cached.html
+              ? 'cdp-minted'
+              : 'recording-seeded';
+      log(
+        `reusing ${provenance} jar (age ${Math.round((Date.now() - cached.bootstrapEpoch) / 1000)}s, _abck~${cached.abckFlag}~, html=${cached.html.length}b)`,
+      );
+      return cached;
+    }
+  }
+  let cf: CdpBrowserFetch | undefined;
+  try {
+    cf = createCdpBrowserFetch({ baseUrl, bootstrapUrl });
+    const jar = await cf.mintJar();
+    if (jar.abckFlag !== '0') {
+      log(`cdp jar minted with _abck~${jar.abckFlag}~ (not validated) — replay may be rejected`);
+    }
+    saveJar(siteDir, jar);
+    return jar;
+  } catch (err) {
+    log(`cdp jar mint failed: ${err instanceof Error ? err.message : String(err)}`);
+    return null;
+  } finally {
+    await cf?.close(); // browser dead; the jar outlives it
+  }
+}
+
+/** Replay transport for the bootstrap-then-fetch path: PLAIN fetch that presents
+ *  the jar's exact UA (Akamai drops the jar on a UA mismatch). Cookies are
+ *  attached by executeWorkflow's RuntimeCookieJar from bootstrappedCredentials,
+ *  so this only forces the UA. Egresses through IMPRINT_PROXY when set, so the
+ *  replay's IP matches the (proxied) browser that minted the jar — else Akamai
+ *  drops the jar on the IP mismatch. */
+function makeJarUaFetch(ua: string): typeof fetch {
+  const proxy = proxyUrl();
+  return (async (input: string | URL | Request, init?: RequestInit): Promise<Response> => {
+    const headers = new Headers(init?.headers ?? {});
+    if (ua) headers.set('user-agent', ua);
+    return globalThis.fetch(
+      input as Parameters<typeof fetch>[0],
+      {
+        ...init,
+        headers,
+        ...(proxy ? { proxy } : {}),
+      } as RequestInit,
+    );
+  }) as typeof fetch;
+}
+
+/** Plain proxied fetch for the `fetch` rung so even the first (no-jar) rung
+ *  egresses through IMPRINT_PROXY — keeps the egress IP uniform across rungs and
+ *  lets GET-only tools (e.g. location lookups) succeed from the residential
+ *  proxy. No-op (returns global fetch) when no proxy is configured. */
+function makeProxyFetch(): typeof fetch | undefined {
+  const proxy = proxyUrl();
+  if (!proxy) return undefined;
+  return (async (input: string | URL | Request, init?: RequestInit): Promise<Response> =>
+    globalThis.fetch(
+      input as Parameters<typeof fetch>[0],
+      {
+        ...init,
+        proxy,
+      } as RequestInit,
+    )) as typeof fetch;
+}
+
+/** A replay error that means the JAR is bad (clear it + re-mint), as opposed to a
+ *  transient IP rate-flag (NETWORK/RATE_LIMITED — a fresh jar won't help; back off). */
+function jarLikelyStale(result: ToolResult): boolean {
+  return !result.ok && (result.error === 'FORBIDDEN' || result.error === 'AUTH_EXPIRED');
+}
+
+/**
+ * fetch-bootstrap rung — the API anti-bot path. Mint a validated session jar via
+ * cdp-browser (real Chrome, used ONLY to bootstrap), CLOSE the browser, then
+ * replay every workflow request via PLAIN fetch with that jar. Works with or
+ * without a workflow.bootstrap block: cookie/html_regex bootstrap captures are
+ * satisfied from the minted jar + page HTML, and a workflow that captures its
+ * tokens inline (e.g. csrf via a request text_regex) just needs the jar's
+ * anti-bot cookies. Self-heals: a stale jar (403/AUTH) is cleared and re-minted
+ * once; an IP rate-flag (NETWORK) is returned for the ladder to handle (a fresh
+ * jar can't beat a transient rate tarpit).
+ */
 async function runFetchBootstrap(
   tool: ResolvedTool,
   params: Record<string, string | number | boolean>,
 ): Promise<ToolResult> {
-  if (!tool.workflow.bootstrap) {
+  let baseUrl: string;
+  try {
+    baseUrl = pickBaseUrl(tool);
+  } catch {
     return {
       ok: false,
       error: 'STATE_MISSING',
-      message: 'fetch-bootstrap requires workflow.bootstrap metadata.',
-      missing: [
-        {
-          name: 'workflow.bootstrap',
-          source: 'workflow',
-          capability: 'browser_bootstrap',
-          required: true,
-          failure: 'producer_unavailable',
-          message: 'workflow.bootstrap is missing',
-        },
-      ],
-      remediation: 'Regenerate or edit workflow.json with bootstrap metadata.',
+      message: 'fetch-bootstrap needs at least one request URL to bootstrap from.',
+      remediation: 'Regenerate workflow.json — it has no requests.',
     };
   }
 
@@ -266,88 +726,178 @@ async function runFetchBootstrap(
     values: {},
     storage: [],
   };
-  const bootstrapUrl = substituteString(tool.workflow.bootstrap.url, params, credentials, []);
-  const initialState: Record<string, unknown> = {};
-  const { chromium } = await import('playwright');
-  let browser: Awaited<ReturnType<typeof chromium.launch>> | undefined;
-  try {
-    browser = await chromium.launch({ headless: true });
-    const context = await browser.newContext();
-    if (credentials.cookies.length > 0) {
-      await context.addCookies(
-        credentials.cookies.map((c) => ({
+  const bootstrapUrl = tool.workflow.bootstrap
+    ? substituteString(tool.workflow.bootstrap.url, params, credentials, [])
+    : undefined;
+  const siteDir = pathResolve(tool.dir, '..');
+
+  for (let attempt = 0; attempt < 2; attempt++) {
+    const jar = await getOrMintCdpJar(baseUrl, bootstrapUrl, siteDir, attempt > 0);
+    if (!jar) {
+      // Couldn't even launch the bootstrap browser → let the ladder escalate.
+      const stateMissing = bootstrapFailureStateMissingResult(
+        tool.workflow,
+        'fetch-bootstrap could not launch the bootstrap browser to mint a session jar.',
+      );
+      if (stateMissing) return stateMissing;
+      return {
+        ok: false,
+        error: 'NETWORK',
+        message: 'fetch-bootstrap could not mint a session jar (browser launch failed).',
+      };
+    }
+
+    // Fast-fail an UNVALIDATED jar. A cdp-minted jar without `_abck~0~`/`bm_sv`
+    // (validated:false) is rejected by Akamai on plain-fetch replay, and a second
+    // mint just produces another unvalidated jar — so don't pay two doomed
+    // ~40s mint+replay cycles (the ~80s that made southwest's every call slow).
+    // Escalate straight to cdp-replay, which fetches INSIDE the live page (the
+    // bmak sensor re-validates `_abck` between calls) and is the only path that
+    // works once the recording is too old to seed a high-trust jar. A
+    // recording-seeded or cached jar is validated:true by construction, so the
+    // cheap plain-fetch path is untouched; `=== false` (not falsy) leaves jars
+    // without the field — older caches / test stubs — on the original path.
+    if (jar.validated === false) {
+      log(
+        'fetch-bootstrap: minted jar unvalidated (no _abck~0~/bm_sv) — plain-fetch replay doomed; escalating to cdp-replay',
+      );
+      return {
+        ok: false,
+        error: 'FORBIDDEN',
+        message: 'fetch-bootstrap: cdp-minted jar did not validate; cdp-replay (in-page) required.',
+      };
+    }
+
+    // Build credentials carrying the minted jar's cookies (executeWorkflow's
+    // RuntimeCookieJar scopes them per-request); fetchImpl only forces the UA.
+    const bootstrappedCredentials: CredentialStore = {
+      ...credentials,
+      cookies: [
+        ...credentials.cookies,
+        ...jar.cookies.map((c) => ({
           name: c.name,
           value: c.value,
-          domain: c.hostOnly ? undefined : c.domain,
-          url: c.hostOnly ? cookieUrlFor(c, bootstrapUrl) : undefined,
+          domain: c.domain,
           path: c.path,
           expires: c.expires,
           httpOnly: c.httpOnly,
           secure: c.secure,
-          sameSite: sameSiteForPlaywright(c.sameSite),
+          sameSite: c.sameSite,
+          hostOnly: !c.domain.startsWith('.'),
         })),
-      );
-    }
-    if ((credentials.storage ?? []).length > 0) {
-      await context.addInitScript((records) => {
-        const browserGlobal = globalThis as unknown as {
-          location: { origin: string };
-          localStorage: { setItem(key: string, value: string): void };
-        };
-        for (const record of records as Array<{
-          origin: string;
-          kind: 'localStorage' | 'sessionStorage';
-          key: string;
-          value: string;
-        }>) {
-          if (record.kind !== 'localStorage') continue;
-          if (browserGlobal.location.origin !== record.origin) continue;
-          browserGlobal.localStorage.setItem(record.key, record.value);
-        }
-      }, credentials.storage ?? []);
-    }
-    const page = await context.newPage();
-    await page.route('**/*', async (route) => {
-      const type = route.request().resourceType();
-      if (['image', 'media', 'font'].includes(type)) return route.abort();
-      return route.continue();
-    });
-    await page.goto(bootstrapUrl, {
-      waitUntil: tool.workflow.bootstrap.waitUntil ?? 'domcontentloaded',
-      timeout: tool.workflow.bootstrap.timeoutMs ?? 30_000,
+      ],
+    };
+
+    // Satisfy any declared bootstrap captures from the minted jar (cookie) +
+    // page HTML (html_regex). response_header/dom captures aren't available from
+    // a closed browser — required ones of those fail loud below.
+    const captureResult = jarBootstrapCaptureState(
+      tool.workflow.bootstrap,
+      jar,
+      bootstrappedCredentials,
+      bootstrapUrl ?? baseUrl,
+    );
+    if (!captureResult.ok) return captureResult.result;
+
+    const result = await tool.toolFn(params, {
+      credentials: bootstrappedCredentials,
+      initialState: captureResult.state,
+      fetchImpl: makeJarUaFetch(jar.ua),
     });
-    if (tool.workflow.bootstrap.waitMs) await page.waitForTimeout(tool.workflow.bootstrap.waitMs);
 
-    const html = await page.content();
-    for (const capture of tool.workflow.bootstrap.captures ?? []) {
-      let value: unknown;
-      try {
-        value = await evaluateBootstrapCapture(capture, page, html);
-      } catch (err) {
-        if (capture.required === false) continue;
-        return bootstrapCaptureMissingResult(
-          capture,
-          `Bootstrap capture "${capture.name}" (${capture.source}) failed: ${err instanceof Error ? err.message : String(err)}`,
-          'producer_ran_value_absent',
-        );
-      }
-      if (value !== undefined && value !== null && value !== '') {
-        initialState[capture.name] = value;
-      } else if (capture.required !== false && capture.source !== 'cookie') {
-        return bootstrapCaptureMissingResult(
-          capture,
-          `Required bootstrap capture "${capture.name}" (${capture.source}) did not produce a value.`,
-          'producer_ran_value_absent',
-        );
-      }
+    if (result.ok) return result;
+    if (attempt === 0 && jarLikelyStale(result)) {
+      log('fetch-bootstrap replay was rejected (403/auth) — clearing jar and re-minting once');
+      clearJar(siteDir);
+      continue;
+    }
+    return result;
+  }
+
+  return {
+    ok: false,
+    error: 'NETWORK',
+    message: 'fetch-bootstrap exhausted its bootstrap retries.',
+  };
+}
+
+/**
+ * cdp-replay rung — run the workflow's requests INSIDE a live trusted Chrome
+ * page (cdp-browser-fetch's in-page `fetchImpl`) instead of replaying a harvested
+ * jar via plain fetch. The decisive difference: a same-origin protected POST
+ * executes in the real page, so when its `_abck` self-invalidates the page's
+ * Akamai bmak sensor auto-re-validates it before the next call. This is the only
+ * path that SUSTAINS a SEQUENCE of sensitive `.act` POSTs (a multi-step
+ * search→agency→details flow); plain-fetch replay (fetch-bootstrap) dies after
+ * ~1-2 because it cannot re-post sensor data. Expensive (a real Chrome launch
+ * held open for the whole workflow), so it sits after fetch-bootstrap in the
+ * ladder — single-.act tools never reach it.
+ *
+ * Bootstrap state (csrf / csp-nonce) is resolved exactly as fetch-bootstrap does
+ * (via jarBootstrapCaptureState over the live page HTML + cookies harvested by
+ * mintJar) — only the transport differs.
+ */
+async function runCdpReplay(
+  tool: ResolvedTool,
+  params: Record<string, string | number | boolean>,
+  cdpPool?: Map<string, CdpBrowserFetch>,
+): Promise<ToolResult> {
+  let baseUrl: string;
+  try {
+    baseUrl = pickBaseUrl(tool);
+  } catch {
+    return {
+      ok: false,
+      error: 'STATE_MISSING',
+      message: 'cdp-replay needs at least one request URL to bootstrap from.',
+      remediation: 'Regenerate workflow.json — it has no requests.',
+    };
+  }
+
+  const credentials = (await loadCredentialStore(tool.site)) ?? {
+    site: tool.site,
+    cookies: [],
+    values: {},
+    storage: [],
+  };
+  const bootstrapUrl = tool.workflow.bootstrap
+    ? substituteString(tool.workflow.bootstrap.url, params, credentials, [])
+    : undefined;
+
+  const siteDir = pathResolve(tool.dir, '..');
+  const poolKey = tool.site;
+  const pooled = cdpPool?.get(poolKey);
+  const ownsSession = !pooled;
+
+  let cf: CdpBrowserFetch;
+  if (pooled) {
+    log('cdp-replay: reusing pooled Chrome session');
+    cf = pooled;
+  } else {
+    let seedCookies: MintedJar['cookies'] | undefined;
+    try {
+      const rec = newestRecording(siteDir);
+      let cached = loadJar(siteDir);
+      if (cached && rec && rec.mtimeMs > cached.bootstrapEpoch) cached = null;
+      if (!cached && seedJarFromRecording(siteDir, rec, bootstrapUrl)) cached = loadJar(siteDir);
+      if (cached?.cookies.length) seedCookies = cached.cookies;
+    } catch {
+      // best-effort
     }
+    cf = (cdpBrowserFetchFactoryForTest ?? createCdpBrowserFetch)({
+      baseUrl,
+      bootstrapUrl,
+      seedCookies,
+    });
+  }
 
-    const cookies = await context.cookies();
+  try {
+    const jar = await cf.mintJar();
     const bootstrappedCredentials: CredentialStore = {
       ...credentials,
       cookies: [
         ...credentials.cookies,
-        ...cookies.map((c) => ({
+        ...jar.cookies.map((c) => ({
           name: c.name,
           value: c.value,
           domain: c.domain,
@@ -360,47 +910,121 @@ async function runFetchBootstrap(
         })),
       ],
     };
-    const jar = new RuntimeCookieJar(bootstrappedCredentials.cookies);
-    for (const capture of tool.workflow.bootstrap.captures ?? []) {
-      if (capture.source !== 'cookie') continue;
-      const lookup = jar.lookup(capture.cookie, capture.url ?? bootstrapUrl, {
+    const captureResult = jarBootstrapCaptureState(
+      tool.workflow.bootstrap,
+      jar,
+      bootstrappedCredentials,
+      bootstrapUrl ?? baseUrl,
+    );
+    if (!captureResult.ok) {
+      if (ownsSession) await cf.close();
+      return captureResult.result;
+    }
+
+    const result = await tool.toolFn(params, {
+      credentials: bootstrappedCredentials,
+      initialState: captureResult.state,
+      fetchImpl: cf.fetchImpl,
+    });
+
+    if (result.ok) {
+      if (cdpPool && ownsSession) cdpPool.set(poolKey, cf);
+      try {
+        const postJar = await cf.mintJar();
+        saveJar(siteDir, postJar);
+      } catch {
+        // best-effort
+      }
+    } else {
+      if (ownsSession) {
+        await cf.close();
+      } else if (cdpPool) {
+        cdpPool.delete(poolKey);
+        log('cdp-replay: evicted degraded session from pool');
+        await cf.close();
+      }
+    }
+
+    return result;
+  } catch (err) {
+    // Session is dead — evict from pool so the next call creates a fresh one.
+    if (cdpPool) {
+      cdpPool.delete(poolKey);
+      log('cdp-replay: evicted dead session from pool');
+    }
+    if (ownsSession) await cf.close();
+    const msg = err instanceof Error ? err.message : String(err);
+    return { ok: false, error: 'NETWORK', message: `cdp-replay failed: ${msg}` };
+  }
+}
+
+/** Resolve workflow.bootstrap captures from a minted jar (cookie source) + the
+ *  bootstrap page HTML (html_regex source). Returns the initial ${state.X} map,
+ *  or a STATE_MISSING result if a required capture can't be satisfied. */
+function jarBootstrapCaptureState(
+  bootstrap: ResolvedTool['workflow']['bootstrap'],
+  jar: MintedJar,
+  credentials: CredentialStore,
+  bootstrapUrl: string,
+): { ok: true; state: Record<string, unknown> } | { ok: false; result: ToolResult } {
+  const state: Record<string, unknown> = {};
+  const captures = bootstrap?.captures ?? [];
+  if (captures.length === 0) return { ok: true, state };
+  const cookieJar = new RuntimeCookieJar(credentials.cookies);
+  for (const capture of captures) {
+    if (capture.source === 'cookie') {
+      const lookup = cookieJar.lookup(capture.cookie, capture.url ?? bootstrapUrl, {
         url: capture.url,
         domain: capture.domain,
         path: capture.path,
         sameSite: capture.sameSite,
         allowHttpOnlyProjection: capture.allowHttpOnlyProjection,
       });
-      if (lookup.ok) initialState[capture.name] = lookup.cookie.value;
+      if (lookup.ok) state[capture.name] = lookup.cookie.value;
       else if (capture.required !== false) {
-        return bootstrapCaptureMissingResult(
-          capture,
-          lookup.reason === 'ambiguous'
-            ? `Bootstrap cookie capture "${capture.name}" is ambiguous; add url/domain/path constraints.`
-            : lookup.reason === 'httponly'
-              ? `Bootstrap cookie capture "${capture.name}" targets HttpOnly cookie "${capture.cookie}" without allowHttpOnlyProjection.`
+        return {
+          ok: false,
+          result: bootstrapCaptureMissingResult(
+            capture,
+            lookup.reason === 'ambiguous'
+              ? `Bootstrap cookie capture "${capture.name}" is ambiguous; add url/domain/path constraints.`
               : `Bootstrap cookie capture "${capture.name}" did not find cookie "${capture.cookie}".`,
-          lookup.reason === 'ambiguous' ? 'ambiguous_cookie' : 'producer_ran_value_absent',
-        );
+            lookup.reason === 'ambiguous' ? 'ambiguous_cookie' : 'producer_ran_value_absent',
+          ),
+        };
       }
+    } else if (capture.source === 'html_regex') {
+      let value: string | undefined;
+      try {
+        const m = new RegExp(capture.pattern).exec(jar.html);
+        value = m?.[capture.group ?? 1] ?? m?.[0];
+      } catch {
+        value = undefined;
+      }
+      if (value) state[capture.name] = value;
+      else if (capture.required !== false) {
+        return {
+          ok: false,
+          result: bootstrapCaptureMissingResult(
+            capture,
+            `Required bootstrap capture "${capture.name}" (html_regex) did not match the bootstrap page.`,
+            'producer_ran_value_absent',
+          ),
+        };
+      }
+    } else if (capture.required !== false) {
+      // response_header / dom_* can't be resolved from a closed browser jar.
+      return {
+        ok: false,
+        result: bootstrapCaptureMissingResult(
+          capture,
+          `Bootstrap capture "${capture.name}" (${capture.source}) is not supported by the fetch-bootstrap jar path; use cookie or html_regex.`,
+          'producer_ran_value_absent',
+        ),
+      };
     }
-    return await tool.toolFn(params, {
-      credentials: bootstrappedCredentials,
-      initialState,
-    });
-  } catch (err) {
-    const stateMissing = bootstrapFailureStateMissingResult(
-      tool.workflow,
-      `fetch-bootstrap could not produce required bootstrap state: ${err instanceof Error ? err.message : String(err)}`,
-    );
-    if (stateMissing) return stateMissing;
-    return {
-      ok: false,
-      error: 'NETWORK',
-      message: `fetch-bootstrap failed: ${err instanceof Error ? err.message : String(err)}`,
-    };
-  } finally {
-    await browser?.close().catch(() => {});
   }
+  return { ok: true, state };
 }
 
 function bootstrapFailureStateMissingResult(
@@ -463,12 +1087,33 @@ function remediationForBootstrapCapabilities(capabilities: StateCapability[]): s
     : 'Run through fetch-bootstrap, or update workflow.bootstrap so Imprint can mint browser state before API replay.';
 }
 
-async function evaluateBootstrapCapture(
+// Exported for tests so the per-source logic (regex, DOM, storage, header)
+// can be unit-asserted without launching real Chromium. Internal callers
+// use it the same way; the export is just a visibility relaxation.
+export async function evaluateBootstrapCapture(
   capture: BootstrapCapture,
   page: Page,
   html: string,
+  responseHeaders: Record<string, string>,
 ): Promise<unknown> {
   switch (capture.source) {
+    case 'response_header': {
+      const raw = responseHeaders[capture.header.toLowerCase()];
+      if (raw === undefined) return undefined;
+      // Playwright's `allHeaders()` joins multi-valued headers with ", ".
+      // Most uses (CSRF, single-valued anti-replay tokens) want the whole
+      // string; mode 'first'/'last' splits when the value actually carries
+      // a comma-list. Keep the default conservative: return raw.
+      if (capture.mode === 'first' || capture.mode === 'last') {
+        const parts = raw
+          .split(',')
+          .map((p) => p.trim())
+          .filter(Boolean);
+        if (parts.length === 0) return undefined;
+        return capture.mode === 'first' ? parts[0] : parts[parts.length - 1];
+      }
+      return raw;
+    }
     case 'html_regex': {
       const match = html.match(new RegExp(capture.pattern));
       return match?.[capture.group ?? 1];
@@ -514,55 +1159,440 @@ async function evaluateBootstrapCapture(
   }
 }
 
-function sameSiteForPlaywright(
-  sameSite: string | undefined,
-): 'Strict' | 'Lax' | 'None' | undefined {
-  if (!sameSite) return undefined;
-  const lower = sameSite.toLowerCase();
-  if (lower === 'strict') return 'Strict';
-  if (lower === 'lax') return 'Lax';
-  if (lower === 'none') return 'None';
-  return undefined;
-}
-
-function cookieUrlFor(cookie: { domain: string; secure?: boolean }, fallback: string): string {
-  try {
-    const u = new URL(fallback);
-    u.hostname = cookie.domain.replace(/^\./, '');
-    u.protocol = cookie.secure ? 'https:' : u.protocol;
-    return u.toString();
-  } catch {
-    return `${cookie.secure ? 'https' : 'http'}://${cookie.domain.replace(/^\./, '')}/`;
+/** Per-site stealth fetcher; bootstrap pays its ~12s once per process. */
+/** Mint `${state.X}` values from the stealth bootstrap session for a workflow
+ *  that declares a bootstrap block. Satisfies `cookie`, `html_regex`, and
+ *  `response_header` captures from the cookies / HTML / response headers the
+ *  stealth navigation minted — all one consistent session as the transport
+ *  cookies, so a token the later API POST checks against the session resolves.
+ *  `dom_*` / storage sources need a live page and are left for the
+ *  fetch-bootstrap rung (the compile prompt steers replay-safe session tokens
+ *  to cookie/html_regex, which this covers). */
+async function stealthBootstrapState(
+  sf: StealthFetch,
+  bootstrap: NonNullable<ResolvedTool['workflow']['bootstrap']>,
+): Promise<Record<string, unknown>> {
+  const state: Record<string, unknown> = {};
+  const captures = bootstrap.captures ?? [];
+  const supported = captures.filter(
+    (c) => c.source === 'cookie' || c.source === 'html_regex' || c.source === 'response_header',
+  );
+  if (supported.length === 0) return state;
+  const tokens = await sf.ensureBootstrapped();
+  for (const cap of supported) {
+    if (cap.source === 'cookie') {
+      const hit = tokens.cookies.find((c) => c.name === cap.cookie);
+      if (hit) state[cap.name] = hit.value;
+    } else if (cap.source === 'html_regex') {
+      const html = tokens.bootstrapHtml ?? '';
+      try {
+        const m = html.match(new RegExp(cap.pattern));
+        const v = m?.[cap.group ?? 1];
+        if (v !== undefined) state[cap.name] = v;
+      } catch {
+        // invalid regex — leave unset; substitution will surface STATE_MISSING
+      }
+    } else if (cap.source === 'response_header') {
+      const v = tokens.bootstrapResponseHeaders?.[cap.header.toLowerCase()];
+      if (v !== undefined && v !== '') state[cap.name] = v;
+    }
   }
+  return state;
 }
 
-/** Per-site stealth fetcher; bootstrap pays its ~12s once per process. */
 function ensureStealthFetch(tool: ResolvedTool, cache: Map<string, StealthFetch>): StealthFetch {
   const cached = cache.get(tool.site);
   if (cached) return cached;
-  const sf = createStealthFetch({ baseUrl: pickBaseUrl(tool) });
+  const sf = createStealthFetch({
+    baseUrl: pickBaseUrl(tool),
+    // When the workflow declares a bootstrap page, navigate IT during the
+    // stealth bootstrap so the session-token cookies it sets (CSRF etc.) are
+    // minted in the same session as the anti-bot cookies. Otherwise the
+    // stealth rung can't satisfy a `${state.X}` the workflow bootstrap was
+    // supposed to provide, and escalation from fetch-bootstrap dead-ends.
+    bootstrapUrl: tool.workflow.bootstrap?.url,
+  });
   cache.set(tool.site, sf);
   return sf;
 }
 
-/** First request URL's origin — Akamai binds sensor tokens to that
- *  origin, and the origin is always literal (substitutions only appear
- *  after the domain in well-formed workflows). */
-function pickBaseUrl(tool: ResolvedTool): string {
-  const firstRequest = tool.workflow.requests[0];
-  if (!firstRequest) {
+/** Pick the URL to navigate when bootstrapping an anti-bot session.
+ *  Akamai binds sensor tokens to the origin+path the browser navigated
+ *  to, so we need an HTML page — not a JSON API endpoint.
+ *
+ *  Heuristic: skip leading requests whose path looks like a raw data
+ *  endpoint (.json, .xml, /api/, /version) — those return JSON/XML
+ *  without rendering an HTML page, so the anti-bot sensor JS never
+ *  fires and the _abck cookie stays unvalidated. Fall back to
+ *  requests[0] if every request looks like an API call. */
+export function pickBaseUrl(tool: ResolvedTool): string {
+  const requests = tool.workflow.requests;
+  if (!requests.length) {
     throw new Error(
       `Workflow ${tool.workflow.toolName} has no requests — stealth-fetch needs at least one request URL.\n→ re-record the session; recording probably stopped before any XHR fired.`,
     );
   }
-  const m = firstRequest.url.match(/^(https?:\/\/[^/]+)/);
-  if (m?.[1]) return m[1];
-  throw new Error(
-    `Could not derive bootstrap origin from URL: ${firstRequest.url}\n→ check workflow.json — the first request URL must start with https://<domain>.`,
-  );
+
+  // Prefer the first request whose Referer is an HTML page — the Referer
+  // is the page the user was on when the API call fired, so it's the
+  // correct bootstrap target. Referer is set by the browser and always
+  // points to a real navigable page.
+  for (const req of requests) {
+    const referer = req.headers?.Referer ?? req.headers?.referer;
+    if (referer) {
+      try {
+        const u = new URL(referer);
+        return `${u.origin}${u.pathname}`;
+      } catch {
+        // malformed referer — skip
+      }
+    }
+  }
+
+  // Fallback: use the origin of the first request. API paths
+  // (/api/...) aren't navigable HTML pages — the anti-bot sensor only
+  // fires on a real page load — so the bare origin (homepage) is the
+  // safest bootstrap target. The homepage loads the full SPA shell
+  // with Akamai/Cloudflare/DataDome sensor scripts, minting a valid
+  // _abck cookie that covers all paths under that origin.
+  const first = requests[0];
+  if (!first) {
+    throw new Error(
+      `Workflow ${tool.workflow.toolName} has no requests — unreachable after length check above.`,
+    );
+  }
+  try {
+    const u = new URL(first.url);
+    return u.origin;
+  } catch {
+    throw new Error(
+      `Could not parse bootstrap URL: ${first.url}\n→ check workflow.json — the first request URL must be absolute (https://...).`,
+    );
+  }
 }
 
 function playbookPath(assetRoot: string, site: string, toolDir?: string): string {
   if (toolDir) return pathResolve(toolDir, 'playbook.yaml');
   return pathResolve(assetRoot, site, 'playbook.yaml');
 }
+
+/**
+ * Compile-time integration-test convenience: dispatch a request through
+ * `runWithLadder` using only a `workflow.json` path. Avoids requiring an
+ * emitted `index.ts` (which doesn't exist when integration.test.ts runs
+ * during compile, before `imprint emit`).
+ *
+ * **Ladder is intentionally fixed to `['fetch', 'stealth-fetch']`** —
+ * the playbook rung is excluded because `playbook.yaml` is compiled in
+ * a separate later step (`imprint compile-playbook`), so at integration-
+ * test time there is no playbook to fall back to. Even if a stale
+ * playbook from a prior compile exists on disk, exercising it here would
+ * conflate two independent verification surfaces and pull a slow
+ * Playwright bootstrap into every test run.
+ *
+ * Credentials are loaded by `executeWorkflow` from the credential store
+ * for the workflow's `site` by default; pass `credentials` explicitly to
+ * override (e.g., when a test wants to assert behavior under a known
+ * credential state).
+ *
+ * The test "passes" as long as ANY backend in the ladder returns ok —
+ * fetch OR stealth-fetch. Tools whose fetch path will be blocked at
+ * runtime are still verified end-to-end via stealth-fetch.
+ */
+export async function runWorkflowWithLadder(opts: {
+  workflowPath: string;
+  params: Record<string, string | number | boolean>;
+  /** Optional credential override; otherwise loaded from the credential
+   *  store by executeWorkflow. */
+  credentials?: CredentialStore;
+}): Promise<LadderResult> {
+  if (!existsSync(opts.workflowPath)) {
+    throw new Error(`runWorkflowWithLadder: workflow.json not found at ${opts.workflowPath}`);
+  }
+  const workflow = WorkflowSchema.parse(JSON.parse(readFileSync(opts.workflowPath, 'utf8')));
+  const toolDir = dirname(opts.workflowPath);
+  // assetRoot only matters for playbook-rung path resolution, which this
+  // ladder skips. Use a conventional value for completeness.
+  const assetRoot = pathResolve(toolDir, '..', '..');
+
+  const tool: ResolvedTool = {
+    site: workflow.site ?? '',
+    dir: toolDir,
+    workflow,
+    toolFn: async (params, fnOpts) => {
+      // Thread ALL execution opts the rungs pass — fetchImpl (stealth), and
+      // crucially initialState + credentials minted by fetch-bootstrap's
+      // Chrome navigation. The production generated tool fn (tool-loader path)
+      // forwards these to executeWorkflow; this test/probe-path toolFn must do
+      // the same, otherwise a bootstrap-block tool's csrf/session state is
+      // silently dropped here and the integration test fails a workflow that
+      // actually works in production — a false waiver.
+      const o = fnOpts as
+        | {
+            fetchImpl?: typeof fetch;
+            initialState?: Record<string, unknown>;
+            credentials?: CredentialStore;
+          }
+        | undefined;
+      return executeWorkflow({
+        workflow,
+        params: params as Record<string, string | number | boolean>,
+        credentials: o?.credentials ?? opts.credentials,
+        workflowPath: opts.workflowPath,
+        fetchImpl: o?.fetchImpl,
+        initialState: o?.initialState,
+      });
+    },
+  };
+
+  const ladder: ConcreteBackend[] = ['fetch', 'fetch-bootstrap', 'cdp-replay', 'stealth-fetch'];
+
+  const memoKey = `${tool.site}::${workflow.toolName}`;
+  const memoWinner = compileWinningBackend.get(memoKey);
+
+  // Share one stealth token across this site's compile-time test processes.
+  const stealthCache = new Map<string, StealthFetch>();
+  try {
+    const siteDir = pathResolve(toolDir, '..');
+    const baseUrl = pickBaseUrl(tool);
+    let fileCacheConsumed = false;
+    const cachingBootstrap = async (args: BootstrapArgs): Promise<TokenCache> => {
+      if (!fileCacheConsumed) {
+        const cached = loadCachedToken(siteDir, STEALTH_TOKEN_MAX_AGE_SECONDS);
+        if (cached) {
+          fileCacheConsumed = true;
+          log(`reusing cached stealth token for ${tool.site || siteDir}`);
+          return cached;
+        }
+      }
+      clearCachedToken(siteDir);
+      const token = await bootstrapStealthToken(args);
+      saveCachedToken(siteDir, token);
+      fileCacheConsumed = true;
+      return token;
+    };
+    stealthCache.set(
+      tool.site,
+      createStealthFetch(
+        { baseUrl, bootstrapUrl: tool.workflow.bootstrap?.url },
+        { bootstrap: cachingBootstrap },
+      ),
+    );
+  } catch {
+    // No usable base URL → leave the cache empty; runWithLadder/ensureStealthFetch
+    // will lazily bootstrap (same behavior as before this optimization).
+  }
+
+  // Reuse the process-global compile CDP pool so cdp-replay stays warm (~2-5s)
+  // across this `bun test` process's calls; cancel any pending idle-close now
+  // that we're about to use it again. The pool is torn down by an idle timer
+  // (armed in `finally`) shortly after the LAST call — see compileCdpPool.
+  const cdpPool = compileCdpPool;
+  clearCompileCdpIdle();
+
+  try {
+    try {
+      await paceCompileRequest(new URL(pickBaseUrl(tool)).origin);
+    } catch {
+      // no parseable base URL → nothing to pace
+    }
+
+    // ── First call: parallel probe (45s deadline) ───────────────────────────
+    // Race non-overlapping backends so a tarpitted rung doesn't block a
+    // faster one. fetch-bootstrap is excluded: it launches Chrome to the
+    // same origin as cdp-replay, and two simultaneous Chromes trip Akamai's
+    // concurrent-session detection. cdp-replay is strictly better when both
+    // need Chrome; if fetch wins, fetch-bootstrap is unnecessary anyway.
+    //
+    // Uses Promise.allSettled (NOT Promise.any) deliberately: a fast OK from
+    // a lower rung (e.g. fetch returning a cached/stale 200) may not be the
+    // best result — we need all backends to settle so we can pick the
+    // fastest *correct* one. The tradeoff is wall-clock: the probe blocks
+    // until the slowest backend resolves (or hits the deadline). cdp-replay
+    // is slow on its first cold start (~33s) but subsequent calls reuse the
+    // CDP pool and complete in ~2-5s — so the first probe pays the cost but
+    // all later calls benefit from having discovered the right rung.
+    //
+    // The compile agent's integration tests MUST use a timeout >= 60s (the
+    // compile-agent.md prompt recommends this) so the test process survives
+    // the full probe duration. A 30s test timeout kills the probe before
+    // cdp-replay can finish its cold start.
+    //
+    // Each bun-test subprocess is a fresh process (memo empty), so the
+    // compile agent's iteration loop re-probes after every workflow change —
+    // no premature lock-in.
+    if (!memoWinner) {
+      const PROBE_TIMEOUT_MS = probeTimeoutMsForTest ?? 45_000;
+      const probeBackends: ConcreteBackend[] = ['fetch', 'cdp-replay', 'stealth-fetch'];
+
+      const settled = await Promise.allSettled(
+        probeBackends.map(async (b) => {
+          const t0 = Date.now();
+          // Keep a handle to the real backend run (the race's non-timeout arm) so a
+          // backend that LOSES the deadline race — still launching Chrome in the
+          // background — gets settled and its pooled browser drained, not leaked,
+          // once the probe returns.
+          const inner = runWithLadder([b], tool, opts.params, assetRoot, stealthCache, {
+            skipBootstrapSplice: true,
+            cdpPool,
+          });
+          // A backend that finishes AFTER the probe returned (it lost the race but
+          // is still cold-starting Chrome) pools its browser late — arm the idle
+          // close so it's torn down rather than left lingering.
+          void inner.finally(() => armCompileCdpIdleClose());
+          const r = await Promise.race([
+            inner,
+            sleepMs(PROBE_TIMEOUT_MS).then(
+              () =>
+                ({
+                  result: { ok: false, error: 'NETWORK', message: 'probe deadline exceeded' },
+                  usedBackend: b,
+                  attempts: [],
+                }) as LadderResult,
+            ),
+          ]);
+          return { backend: b, result: r, durationMs: Date.now() - t0 };
+        }),
+      );
+
+      const digest = settled.map((s, i) => {
+        const b = probeBackends[i];
+        if (s.status === 'rejected')
+          return `${b}: ${s.reason instanceof Error ? s.reason.message : String(s.reason)}`.slice(
+            0,
+            120,
+          );
+        const { result: lr, durationMs } = s.value;
+        return lr.result.ok
+          ? `${b}: OK in ${durationMs}ms`
+          : `${b}: ${lr.result.error} — ${lr.result.message.slice(0, 200)} (${durationMs}ms)`;
+      });
+
+      type ProbeEntry = { backend: ConcreteBackend; result: LadderResult; durationMs: number };
+      const winners = settled
+        .filter(
+          (s): s is PromiseFulfilledResult<ProbeEntry> =>
+            s.status === 'fulfilled' && s.value.result.result.ok,
+        )
+        .map((s) => s.value);
+
+      const best = pickProbeWinner(winners);
+      if (best) {
+        compileWinningBackend.set(memoKey, best.backend);
+        log(
+          `parallel probe: winner=${best.backend} (${best.durationMs}ms)\n  ${digest.join('\n  ')}`,
+        );
+        return best.result;
+      }
+
+      log(`parallel probe: all backends failed\n  ${digest.join('\n  ')}`);
+      return {
+        result: {
+          ok: false as const,
+          error: 'NETWORK' as const,
+          message: `All backends failed during parallel probe: ${digest.join('; ')}`,
+        },
+        usedBackend: ladder[ladder.length - 1] ?? 'fetch',
+        attempts: [],
+      };
+    }
+
+    // ── Memo hit: start at the memoized winner, keep all later rungs ─────
+    // Previous logic sliced earlier rungs away (`ladder.slice(idx)`), which
+    // dropped cdp-replay as a fallback when stealth-fetch (the last rung)
+    // was the winner. Now: reorder the ladder to start at the winner and
+    // wrap around so every rung remains reachable. The winner is tried first
+    // (the optimization), but if it fails the remaining rungs catch it.
+    const idx = ladder.indexOf(memoWinner);
+    const memoLadder = idx > 0 ? [...ladder.slice(idx), ...ladder.slice(0, idx)] : ladder;
+    log(
+      `compile memo: ${memoKey} previously succeeded via ${memoWinner}; ladder: ${memoLadder.join(' → ')}`,
+    );
+    const result = await runWithLadder(memoLadder, tool, opts.params, assetRoot, stealthCache, {
+      skipBootstrapSplice: true,
+      cdpPool,
+    });
+    if (result.result.ok) {
+      compileWinningBackend.set(memoKey, result.usedBackend);
+    } else {
+      compileWinningBackend.delete(memoKey);
+    }
+    return result;
+  } finally {
+    // Keep the pool warm for the next call in this process; arm an idle-close so
+    // it's torn down shortly after the LAST call — that lets a raw `bun probe.ts`
+    // exit cleanly (no 30-min hang) and never leaks a browser.
+    armCompileCdpIdleClose();
+  }
+}
+
+export interface RenderedRequest {
+  method: string;
+  /** Final, fully-substituted + transform-applied request URL. */
+  url: string;
+  /** Outgoing headers (lower/mixed case as the runtime set them). */
+  headers: Record<string, string>;
+  /** Outgoing body, or null for body-less requests. */
+  body: string | null;
+}
+
+/**
+ * Render a workflow's outgoing requests OFFLINE — no network, no browser. Runs
+ * the real `executeWorkflow` (so `${param}`/`${state}` substitution, captures,
+ * and any `requestTransformModule` all execute) but with a `fetchImpl` that
+ * returns the matching RECORDED response for each request and CAPTURES the final
+ * outgoing request before returning it.
+ *
+ * Purpose: verify a parameter actually reaches its field by diffing renders
+ * across param overrides — WITHOUT firing a live `.act` per parameter (the burst
+ * that flags anti-bot IPs and made costco's tools fail compile). The live suite
+ * then needs only ONE baseline call to prove the workflow produces real data; the
+ * per-parameter "does X reach field F" check becomes a deterministic offline diff.
+ *
+ * `recordedResponseFor(method, url)` supplies the recorded response so captures
+ * (csrf via text_regex, etc.) resolve and the transform builds the real body;
+ * return undefined to fall back to an empty `200`.
+ */
+export async function renderWorkflowRequests(opts: {
+  workflow: Workflow;
+  params: Record<string, string | number | boolean>;
+  workflowPath?: string;
+  credentials?: CredentialStore;
+  recordedResponseFor?: (
+    method: string,
+    url: string,
+  ) => { status: number; body: string; headers?: Record<string, string> } | undefined;
+}): Promise<{ requests: RenderedRequest[]; result: ToolResult }> {
+  const captured: RenderedRequest[] = [];
+  const fetchImpl: typeof fetch = (async (
+    input: string | URL | Request,
+    init?: RequestInit,
+  ): Promise<Response> => {
+    const url =
+      typeof input === 'string' ? input : input instanceof URL ? input.toString() : input.url;
+    const method = (init?.method ?? 'GET').toUpperCase();
+    const headers: Record<string, string> = {};
+    if (init?.headers) {
+      const h = new Headers(init.headers as Record<string, string>);
+      h.forEach((v, k) => {
+        headers[k] = v;
+      });
+    }
+    const body = typeof init?.body === 'string' ? init.body : init?.body ? String(init.body) : null;
+    captured.push({ method, url, headers, body });
+    const rec = opts.recordedResponseFor?.(method, url);
+    return new Response(rec?.body ?? '{}', {
+      status: rec?.status ?? 200,
+      headers: new Headers(rec?.headers ?? {}),
+    });
+  }) as typeof fetch;
+
+  const result = await executeWorkflow({
+    workflow: opts.workflow,
+    params: opts.params,
+    credentials: opts.credentials,
+    workflowPath: opts.workflowPath,
+    fetchImpl,
+  });
+  return { requests: captured, result };
+}