@detiq/crawler 0.1.3

package/src/screen-reader.ts

@@ -0,0 +1,89 @@
+import type { Page } from 'playwright';
+
+/**
+ * Reads a page as a structured outline (cheap + reliable), the way
+ * Playwright MCP / the accessibility tree works — NOT by sending pixels to a
+ * vision model. Vision is a fallback for image-only pages (added later).
+ *
+ * Returns the interactive elements found, which the Screen Reader Agent then
+ * enriches with MEANING and saves to the Zeta Core.
+ */
+export interface RawElement {
+  role: string;
+  label: string;
+  selectorHint: string;
+  expectedData?: string;
+}
+
+export async function readScreen(page: Page): Promise<RawElement[]> {
+  return page.evaluate(() => {
+    const out: { role: string; label: string; selectorHint: string; expectedData?: string }[] = [];
+    const seen = new Set<string>();
+
+    // Noise patterns to skip — cookie banners, footer junk, marketing copy buttons, version strings
+    const noiseLabel =
+      /^(cookie|consent|privacy policy|terms of use|copyright|©|\bv\d+\.\d+|accept|decline|got it|dismiss|close|skip to|powered by|copy|copied|\s*$)/i;
+
+    // Select only truly interactive elements (not generic [role] containers)
+    const nodes = document.querySelectorAll(
+      'input:not([type=hidden]), select, textarea, button, a[href]'
+    );
+
+    nodes.forEach((el) => {
+      const htmlEl = el as HTMLElement;
+
+      // Skip invisible or zero-size elements
+      const style = window.getComputedStyle(htmlEl);
+      if (style.display === 'none' || style.visibility === 'hidden' || style.opacity === '0') return;
+      const rect = el.getBoundingClientRect();
+      if (rect.width === 0 && rect.height === 0) return;
+
+      // Derive role
+      const tag = el.tagName.toLowerCase();
+      const role =
+        el.getAttribute('role') ||
+        (tag === 'input' ? ((el as HTMLInputElement).type || 'text') : tag);
+
+      // Derive the best human-readable label (prefer aria > placeholder > inner text)
+      const raw = (
+        el.getAttribute('aria-label') ||
+        el.getAttribute('placeholder') ||
+        (el.getAttribute('title') ?? '') ||
+        htmlEl.innerText?.replace(/\s+/g, ' ').trim() ||
+        el.getAttribute('name') ||
+        ''
+      ).substring(0, 80);
+
+      // Skip empty, noise patterns, or very long labels (likely container text)
+      if (!raw || noiseLabel.test(raw) || raw.length > 70) return;
+
+      // Deduplicate by role+normalised label
+      const key = `${role}:${raw.toLowerCase()}`;
+      if (seen.has(key)) return;
+      seen.add(key);
+
+      let expectedData: string | undefined;
+      if (tag === 'input') {
+        const input = el as HTMLInputElement;
+        if (input.type === 'email' || /email/i.test(raw)) expectedData = 'valid_email';
+        else if (input.type === 'password' || /password/i.test(raw)) expectedData = 'password';
+        else if (input.type === 'tel' || /phone|mobile/i.test(raw)) expectedData = 'phone';
+        else if (input.type === 'date' || /\bdate\b/i.test(raw)) expectedData = 'date';
+        else if (/\b(code|otp|token)\b/i.test(raw)) expectedData = 'one_time_code';
+        else if (/\b(search|query)\b/i.test(raw)) expectedData = 'search_term';
+      }
+
+      out.push({ role, label: raw, selectorHint: tag, expectedData });
+    });
+
+    // Prioritise: form fields > buttons > links; within each group keep order
+    const priority = (e: typeof out[0]) => {
+      if (['email', 'password', 'text', 'tel', 'date', 'number', 'select', 'textarea'].includes(e.role)) return 0;
+      if (e.role === 'button' || e.role === 'submit') return 1;
+      return 2;
+    };
+    out.sort((a, b) => priority(a) - priority(b));
+
+    return out.slice(0, 25); // cap at 25 — enough context, not overwhelming
+  });
+}

package/src/screenshot-masker.ts

@@ -0,0 +1,98 @@
+/**
+ * Screenshot masker — blur or black-box PII regions in captured screenshots.
+ *
+ * Strategy: inject a CSS overlay into the page before screenshot capture that
+ * covers detected PII-likely elements (input[type=email], input[type=password],
+ * elements with PII-containing text) with a blur or solid black mask.
+ *
+ * This is a CSS-based approach (no image processing library needed):
+ * 1. Identify likely-PII DOM elements via selectors
+ * 2. Add an overlaid <div> with blur/black filter at each element's bounding box
+ * 3. Take screenshot
+ * 4. Remove overlays
+ *
+ * Falls back gracefully if elements are not found or overlay injection fails.
+ */
+
+import type { Page } from 'playwright';
+
+export interface MaskingOpts {
+  /** Additional CSS selectors to mask (in addition to defaults) */
+  extraSelectors?: string[];
+  /** Masking style: 'blur' or 'blackout' */
+  style?: 'blur' | 'blackout';
+  /** Blur strength in px (default 8) */
+  blurPx?: number;
+}
+
+// Default PII-sensitive selectors
+const DEFAULT_PII_SELECTORS = [
+  'input[type="email"]',
+  'input[type="password"]',
+  'input[type="tel"]',
+  'input[autocomplete="cc-number"]',
+  'input[autocomplete="cc-csc"]',
+  'input[autocomplete="ssn"]',
+  '[data-pii="true"]',
+  '[data-sensitive="true"]',
+];
+
+export async function maskScreenshot(
+  page: Page,
+  opts: MaskingOpts = {},
+): Promise<{ maskedCount: number; screenshot: Buffer }> {
+  const { extraSelectors = [], style = 'blur', blurPx = 8 } = opts;
+  const selectors = [...DEFAULT_PII_SELECTORS, ...extraSelectors];
+
+  let maskedCount = 0;
+  const overlayIds: string[] = [];
+
+  // Inject CSS overlays
+  try {
+    const result = await page.evaluate(({ selectors, style, blurPx }) => {
+      const ids: string[] = [];
+      for (const sel of selectors) {
+        try {
+          const elements = Array.from(document.querySelectorAll(sel));
+          for (const el of elements) {
+            const rect = el.getBoundingClientRect();
+            if (rect.width === 0 || rect.height === 0) continue;
+            const overlay = document.createElement('div');
+            const id = `__zeta_mask_${Math.random().toString(36).slice(2)}`;
+            overlay.id = id;
+            overlay.style.cssText = [
+              'position: fixed',
+              `top: ${rect.top + window.scrollY}px`,
+              `left: ${rect.left + window.scrollX}px`,
+              `width: ${rect.width}px`,
+              `height: ${rect.height}px`,
+              'z-index: 2147483647',
+              style === 'blur'
+                ? `backdrop-filter: blur(${blurPx}px); -webkit-backdrop-filter: blur(${blurPx}px); background: rgba(0,0,0,0.1);`
+                : 'background: #000000;',
+              'pointer-events: none',
+            ].join('; ');
+            document.body.appendChild(overlay);
+            ids.push(id);
+          }
+        } catch { /* selector error */ }
+      }
+      return ids;
+    }, { selectors, style, blurPx });
+
+    overlayIds.push(...result);
+    maskedCount = result.length;
+  } catch { /* non-fatal */ }
+
+  // Take screenshot with overlays active
+  const screenshot = await page.screenshot({ type: 'png', fullPage: false });
+
+  // Remove overlays
+  if (overlayIds.length > 0) {
+    await page.evaluate((ids) => {
+      for (const id of ids) document.getElementById(id)?.remove();
+    }, overlayIds).catch(() => {});
+  }
+
+  return { maskedCount, screenshot };
+}

package/src/seo-checker.ts

@@ -0,0 +1,134 @@
+/**
+ * SEO indexability checker — determines if a page is crawlable and indexable.
+ *
+ * Checks:
+ * 1. X-Robots-Tag response header (noindex, nofollow, noarchive)
+ * 2. <meta name="robots"> tag (noindex, nofollow)
+ * 3. Canonical URL (self-canonical vs cross-canonical)
+ * 4. rel="nofollow" on outbound links
+ * 5. hreflang tags (multilingual)
+ * 6. Open Graph completeness (og:title, og:description, og:image)
+ * 7. Structured data presence (JSON-LD)
+ * 8. Meta description
+ * 9. Title tag length (< 60 chars recommended)
+ * 10. URL structure (query params, fragments, excessive depth)
+ */
+
+import type { Page } from 'playwright';
+
+export interface SeoIssue {
+  rule: string;
+  severity: 'critical' | 'warning' | 'info';
+  message: string;
+  value?: string;
+}
+
+export interface SeoCheckResult {
+  url: string;
+  isIndexable: boolean;
+  canonicalUrl?: string;
+  isSelfCanonical?: boolean;
+  hreflang: Record<string, string>; // lang → url
+  issues: SeoIssue[];
+  score: number; // 0-100
+  checkedAt: string;
+}
+
+export async function checkSeo(page: Page, url: string, responseHeaders: Record<string, string> = {}): Promise<SeoCheckResult> {
+  const issues: SeoIssue[] = [];
+  const checkedAt = new Date().toISOString();
+  let isIndexable = true;
+
+  // 1. X-Robots-Tag
+  const robotsHeader = responseHeaders['x-robots-tag'] ?? '';
+  if (/noindex/i.test(robotsHeader)) {
+    isIndexable = false;
+    issues.push({ rule: 'x-robots-noindex', severity: 'critical', message: 'X-Robots-Tag: noindex found in response header', value: robotsHeader });
+  }
+
+  // 2-9. Page-level checks
+  const pageData = await page.evaluate(() => {
+    const meta = (name: string) => document.querySelector(`meta[name="${name}"]`)?.getAttribute('content') ?? '';
+    const og = (prop: string) => document.querySelector(`meta[property="og:${prop}"]`)?.getAttribute('content') ?? '';
+    const canonical = document.querySelector('link[rel="canonical"]')?.getAttribute('href') ?? '';
+    const hreflang: Record<string, string> = {};
+    document.querySelectorAll('link[rel="alternate"][hreflang]').forEach((el: any) => {
+      hreflang[el.getAttribute('hreflang')] = el.getAttribute('href') ?? '';
+    });
+    const jsonLd = Array.from(document.querySelectorAll('script[type="application/ld+json"]')).map((s: any) => s.textContent ?? '');
+    return {
+      robotsMeta: meta('robots'),
+      description: meta('description'),
+      title: document.title ?? '',
+      canonical,
+      hreflang,
+      ogTitle: og('title'),
+      ogDescription: og('description'),
+      ogImage: og('image'),
+      hasJsonLd: jsonLd.length > 0,
+      jsonLd,
+    };
+  }).catch(() => ({
+    robotsMeta: '', description: '', title: '', canonical: '', hreflang: {},
+    ogTitle: '', ogDescription: '', ogImage: '', hasJsonLd: false, jsonLd: [],
+  }));
+
+  // 2. Meta robots
+  if (/noindex/i.test(pageData.robotsMeta)) {
+    isIndexable = false;
+    issues.push({ rule: 'meta-robots-noindex', severity: 'critical', message: 'Meta robots noindex found', value: pageData.robotsMeta });
+  }
+
+  // 3. Canonical
+  let isSelfCanonical: boolean | undefined;
+  if (pageData.canonical) {
+    isSelfCanonical = pageData.canonical === url || pageData.canonical.replace(/\/$/, '') === url.replace(/\/$/, '');
+    if (!isSelfCanonical) {
+      issues.push({ rule: 'cross-canonical', severity: 'warning', message: 'Cross-canonical points elsewhere', value: pageData.canonical });
+    }
+  } else {
+    issues.push({ rule: 'missing-canonical', severity: 'warning', message: 'No canonical link tag' });
+  }
+
+  // 4. Title
+  if (!pageData.title) {
+    issues.push({ rule: 'missing-title', severity: 'critical', message: 'Page has no title tag' });
+  } else if (pageData.title.length > 60) {
+    issues.push({ rule: 'title-too-long', severity: 'warning', message: `Title ${pageData.title.length} chars (>60)`, value: pageData.title });
+  }
+
+  // 5. Meta description
+  if (!pageData.description) {
+    issues.push({ rule: 'missing-description', severity: 'warning', message: 'Missing meta description' });
+  } else if (pageData.description.length > 160) {
+    issues.push({ rule: 'description-too-long', severity: 'info', message: `Description ${pageData.description.length} chars (>160)` });
+  }
+
+  // 6. OG tags
+  if (!pageData.ogTitle) issues.push({ rule: 'missing-og-title', severity: 'info', message: 'Missing og:title' });
+  if (!pageData.ogImage) issues.push({ rule: 'missing-og-image', severity: 'info', message: 'Missing og:image' });
+
+  // 7. Structured data
+  if (!pageData.hasJsonLd) issues.push({ rule: 'no-structured-data', severity: 'info', message: 'No JSON-LD structured data found' });
+
+  // 8. URL depth
+  try {
+    const depth = new URL(url).pathname.split('/').filter(Boolean).length;
+    if (depth > 5) issues.push({ rule: 'deep-url', severity: 'info', message: `URL depth ${depth} (>5 levels)`, value: url });
+  } catch {}
+
+  const criticals = issues.filter(i => i.severity === 'critical').length;
+  const warnings = issues.filter(i => i.severity === 'warning').length;
+  const score = Math.max(0, 100 - criticals * 20 - warnings * 5 - issues.filter(i => i.severity === 'info').length * 2);
+
+  return {
+    url,
+    isIndexable,
+    canonicalUrl: pageData.canonical || undefined,
+    isSelfCanonical,
+    hreflang: pageData.hreflang,
+    issues,
+    score,
+    checkedAt,
+  };
+}

package/src/session-capture.ts

@@ -0,0 +1,325 @@
+import { chromium } from 'playwright';
+import { login } from './auth.js';
+import { generateTOTP } from './job-queue.js';
+
+interface CaptureSession {
+  browser: any;
+  context: any;
+  timer: ReturnType<typeof setTimeout>;
+}
+
+const activeSessions = new Map<string, CaptureSession>();
+
+/**
+ * Opens a headed (visible) browser at appUrl and returns a captureId.
+ * The user logs in manually, then calls completeSessionCapture() to harvest storageState.
+ * Sessions expire after 10 minutes if not completed.
+ *
+ * Throws 'HEADLESS_ENV' if running in Docker/Linux without a display server.
+ * The API route surfaces this as a 503 so the frontend can switch to manual JSON paste.
+ */
+export async function startSessionCapture(appUrl: string): Promise<string> {
+  const inDocker = process.env.PLAYWRIGHT_IN_DOCKER === 'true';
+  const noDisplay = process.platform === 'linux' && !process.env.DISPLAY;
+  if (inDocker || noDisplay) {
+    throw new Error('HEADLESS_ENV');
+  }
+
+  const id = Math.random().toString(36).slice(2) + Date.now().toString(36);
+  const browser = await chromium.launch({ headless: false });
+  const context = await browser.newContext();
+  const page = await context.newPage();
+  await page.goto(appUrl, { waitUntil: 'domcontentloaded' }).catch(() => {});
+  const timer = setTimeout(() => cancelSessionCapture(id).catch(() => {}), 10 * 60 * 1000);
+  activeSessions.set(id, { browser, context, timer });
+  return id;
+}
+
+/** Captures storageState from the open browser, closes it, and returns the JSON string. */
+export async function completeSessionCapture(captureId: string): Promise<string> {
+  const session = activeSessions.get(captureId);
+  if (!session) throw new Error('Capture session not found or expired.');
+  clearTimeout(session.timer);
+  const state = await session.context.storageState();
+  await session.browser.close().catch(() => {});
+  activeSessions.delete(captureId);
+  return JSON.stringify(state);
+}
+
+/** Closes the browser without capturing — called on Cancel. */
+export async function cancelSessionCapture(captureId: string): Promise<void> {
+  const session = activeSessions.get(captureId);
+  if (!session) return;
+  clearTimeout(session.timer);
+  await session.browser.close().catch(() => {});
+  activeSessions.delete(captureId);
+}
+
+/**
+ * Post-login exploration: navigates nav links, cycles tabs, advances carousels,
+ * opens accordions, scrolls — so the browser accumulates full session state before capture.
+ * All best-effort: never throws, never fails the capture.
+ */
+async function exploreAppAfterLogin(page: any): Promise<void> {
+  const sleep = (ms: number) => page.waitForTimeout(ms).catch(() => {});
+
+  const isVisible = `
+    function isVisible(el) {
+      const s = window.getComputedStyle(el);
+      if (s.display === 'none' || s.visibility === 'hidden' || parseFloat(s.opacity) < 0.1) return false;
+      const r = el.getBoundingClientRect();
+      return r.width > 4 && r.height > 4;
+    }
+  `;
+
+  try {
+    const origin = await page.evaluate(() => location.origin).catch(() => '');
+    const startUrl = await page.evaluate(() => location.href).catch(() => '');
+
+    // ── 1. Expand collapsed nav menus / hamburgers ──────────────────────────
+    await page.evaluate(new Function(`
+      ${isVisible}
+      const inBadCtx = el => {
+        let p = el.parentElement;
+        while (p) {
+          const role = p.getAttribute('role') ?? '';
+          if (['dialog','alertdialog','combobox','listbox'].includes(role)) return true;
+          p = p.parentElement;
+        }
+        return false;
+      };
+      const targets = new Set();
+      document.querySelectorAll('[aria-haspopup]:not([aria-expanded="true"])').forEach(el => {
+        if (isVisible(el) && !inBadCtx(el)) targets.add(el);
+      });
+      document.querySelectorAll('[aria-expanded="false"]').forEach(el => {
+        if ((el.tagName === 'BUTTON' || el.getAttribute('role') === 'button') && isVisible(el) && !inBadCtx(el)) targets.add(el);
+      });
+      document.querySelectorAll('[data-state="closed"],[data-headlessui-state="closed"]').forEach(el => {
+        if ((el.tagName === 'BUTTON' || el.getAttribute('role') === 'button') && isVisible(el) && !inBadCtx(el)) targets.add(el);
+      });
+      targets.forEach(el => { try { el.click(); } catch {} });
+    `) as () => void).catch(() => {});
+    await sleep(600);
+
+    // ── 2. Collect nav links from the expanded DOM ──────────────────────────
+    const navLinks: string[] = await page.evaluate(new Function(`
+      ${isVisible}
+      const links = new Set();
+      const navEls = document.querySelectorAll('nav a, [role="navigation"] a, aside a, header a');
+      navEls.forEach(a => {
+        const href = a.getAttribute('href');
+        if (!href || href.startsWith('#') || href.startsWith('mailto:') || href.startsWith('tel:')) return;
+        if (!isVisible(a)) return;
+        try {
+          const url = new URL(href, location.href);
+          if (url.origin === location.origin) links.add(url.href);
+        } catch {}
+      });
+      return [...links].slice(0, 10); // cap at 10 nav pages
+    `) as () => string[]).catch(() => []);
+
+    // Dismiss opened menus before navigating
+    await page.keyboard?.press('Escape').catch(() => {});
+    await sleep(200);
+
+    // ── 3. Visit each nav link, interact, return ────────────────────────────
+    for (const link of navLinks) {
+      try {
+        if (link === startUrl) continue;
+        await page.goto(link, { waitUntil: 'networkidle', timeout: 15_000 }).catch(() => {});
+        await sleep(400);
+        await interactWithCurrentPage(page, isVisible, sleep);
+      } catch { /* skip broken page */ }
+    }
+
+    // Return to start page
+    if (navLinks.length > 0) {
+      await page.goto(startUrl, { waitUntil: 'networkidle', timeout: 15_000 }).catch(() => {});
+      await sleep(400);
+    }
+
+    // ── 4. Final interaction pass on landing page ───────────────────────────
+    await interactWithCurrentPage(page, isVisible, sleep);
+
+    await page.waitForLoadState('networkidle', { timeout: 5_000 }).catch(() => {});
+  } catch {
+    // Exploration is best-effort
+  }
+}
+
+/** Interacts with tabs, carousels, accordions, and scrolls on the current page. */
+async function interactWithCurrentPage(page: any, isVisible: string, sleep: (ms: number) => Promise<void>): Promise<void> {
+  // Cycle through tabs
+  await page.evaluate(new Function(`
+    ${isVisible}
+    const tabs = [...document.querySelectorAll('[role="tab"]')].filter(isVisible);
+    tabs.forEach(tab => { try { tab.click(); } catch {} });
+  `) as () => void).catch(() => {});
+  await sleep(400);
+
+  // Advance carousels (next buttons)
+  await page.evaluate(new Function(`
+    ${isVisible}
+    const nextBtns = [...document.querySelectorAll(
+      'button[aria-label*="next" i], button[aria-label*="forward" i], button[aria-label*="right" i], ' +
+      '[class*="carousel" i] button, [class*="slider" i] button, [data-direction="next"]'
+    )].filter(isVisible);
+    // Click next up to 3 times per carousel
+    for (let i = 0; i < 3; i++) {
+      nextBtns.forEach(btn => { try { btn.click(); } catch {} });
+    }
+  `) as () => void).catch(() => {});
+  await sleep(400);
+
+  // Open closed accordions / disclosure panels
+  await page.evaluate(new Function(`
+    ${isVisible}
+    document.querySelectorAll(
+      '[data-state="closed"] button, [aria-expanded="false"], ' +
+      'details:not([open]) > summary, [class*="accordion" i] button'
+    ).forEach(el => {
+      if (isVisible(el)) { try { el.click(); } catch {} }
+    });
+  `) as () => void).catch(() => {});
+  await sleep(300);
+
+  // Scroll to bottom then back to top
+  await page.evaluate(() => { window.scrollTo({ top: document.body.scrollHeight, behavior: 'instant' }); }).catch(() => {});
+  await sleep(300);
+  await page.evaluate(() => { window.scrollTo({ top: 0, behavior: 'instant' }); }).catch(() => {});
+}
+
+/**
+ * Headless login + session capture — works on Docker/cloud (no display needed).
+ * Tries DOM-based login first (fast, free). If it fails and llmConfig is provided,
+ * falls back to Stagehand AI-guided login (handles modals, multi-step, marketing pages).
+ *
+ * Returns { storageState, cookies, localStorage } on success.
+ * Throws a human-readable error on failure.
+ */
+export async function validateAndCaptureSession(
+  appUrl: string,
+  creds: {
+    username: string; password: string; totpSecret?: string;
+    httpBasicUsername?: string; httpBasicPassword?: string;
+  },
+  llmConfig?: { modelName: string; apiKey: string },
+  captchaOpts?: { apiKey?: string; provider?: '2captcha' | 'capsolver' },
+): Promise<{ storageState: string; cookies: number; localStorage: number }> {
+  const browser = await chromium.launch({ headless: true });
+  try {
+    const contextOpts: any = {};
+    if (creds.httpBasicUsername && creds.httpBasicPassword) {
+      contextOpts.httpCredentials = { username: creds.httpBasicUsername, password: creds.httpBasicPassword };
+    }
+    const context = await browser.newContext(contextOpts);
+    const page = await context.newPage();
+
+    const getOneTimeCode = creds.totpSecret
+      ? async () => generateTOTP(creds.totpSecret!)
+      : undefined;
+
+    let loggedIn: boolean;
+    try {
+      loggedIn = await login(page, appUrl, {
+        username: creds.username,
+        password: creds.password,
+        getOneTimeCode,
+      });
+    } catch (loginErr: any) {
+      // login() throws when it can read a real error message from the page
+      const shotBuf = await page.screenshot({ type: 'jpeg', quality: 60, fullPage: false }).catch(() => null);
+      throw Object.assign(loginErr, {
+        failureScreenshot: shotBuf ? shotBuf.toString('base64') : null,
+      });
+    }
+
+    // Stagehand AI fallback when DOM login fails and LLM config is available
+    if (!loggedIn && llmConfig) {
+      let sh: any = null;
+      try {
+        const { Stagehand } = await import('@browserbasehq/stagehand');
+        const { loginWithStagehand } = await import('./ai-login.js');
+        sh = new Stagehand({
+          env: 'LOCAL',
+          model: { modelName: llmConfig.modelName as any, apiKey: llmConfig.apiKey },
+          serverCache: false,
+          verbose: 0,
+          localBrowserLaunchOptions: { headless: true },
+        });
+        await sh.init();
+        const shPage = await (sh as any).resolvePage();
+        await shPage.goto(appUrl, { waitUntil: 'networkidle', timeout: 30_000 }).catch(() => {});
+        const shLoggedIn = await loginWithStagehand(
+          sh,
+          { username: creds.username, password: creds.password, getOneTimeCode },
+          appUrl,
+          captchaOpts,
+        );
+        if (shLoggedIn) {
+          const shCtx = (sh as any).context ?? (sh as any).browserContext;
+          if (shCtx) {
+            const shState = await shCtx.storageState();
+            const shCookies = shState.cookies?.length ?? 0;
+            const shLocalStorage = shState.origins?.reduce((s: number, o: any) => s + (o.localStorage?.length ?? 0), 0) ?? 0;
+            if (shCookies > 0 || shLocalStorage > 0) {
+              await sh.close().catch(() => {});
+              sh = null;
+              return { storageState: JSON.stringify(shState), cookies: shCookies, localStorage: shLocalStorage };
+            }
+          }
+          loggedIn = true; // Stagehand succeeded but we'll capture from the plain browser context below
+        }
+      } catch (shErr: any) {
+        const msg = String(shErr?.message ?? shErr);
+        if (msg.includes('CAPTCHA') || shErr?.isSSOError) throw shErr;
+        // Stagehand failed silently — fall through to "credentials rejected"
+      } finally {
+        if (sh) await sh.close().catch(() => {});
+      }
+    }
+
+    if (!loggedIn) {
+      const shotBuf = await page.screenshot({ type: 'jpeg', quality: 60, fullPage: false }).catch(() => null);
+      const challenge = await page.evaluate(() => {
+        const text = (document.body?.innerText ?? '').toLowerCase();
+        const hasCloudflare = /verify you are human|checking your browser|cloudflare|cf-challenge|turnstile/.test(text) ||
+          !!document.querySelector('iframe[src*="challenges.cloudflare.com"], .cf-turnstile, [data-sitekey]');
+        const hasCaptcha = hasCloudflare || /captcha|recaptcha|hcaptcha/.test(text) ||
+          !!document.querySelector('iframe[src*="recaptcha"], iframe[src*="hcaptcha"], #captcha, .captcha, [id*="captcha"], [class*="captcha"]');
+        return { hasCloudflare, hasCaptcha };
+      }).catch(() => ({ hasCloudflare: false, hasCaptcha: false }));
+      const message = challenge.hasCloudflare
+        ? 'Cloudflare human verification blocked headless validation. This cannot be bypassed reliably in code; use Live Browser or manual session capture, configure a CAPTCHA solver, or allowlist the crawler IP.'
+        : challenge.hasCaptcha
+          ? 'A CAPTCHA blocked headless validation. Configure CAPTCHA Auto-Solve or use Live Browser/manual session capture.'
+          : 'Login failed — credentials rejected or the login flow requires a manual/SSO step. Check credentials, saved login instructions, or use Live Browser session capture.';
+      const err = Object.assign(new Error(message), {
+        failureScreenshot: shotBuf ? shotBuf.toString('base64') : null,
+        reason: challenge.hasCloudflare ? 'CLOUDFLARE_CHALLENGE' : challenge.hasCaptcha ? 'CAPTCHA_CHALLENGE' : 'LOGIN_NOT_CONFIRMED',
+      });
+      throw err;
+    }
+
+    // Wait for post-login navigation to settle before capturing session
+    await page.waitForLoadState('networkidle', { timeout: 8_000 }).catch(() => {});
+
+    // Post-login exploration: navigate tabs, expand menus, scroll to trigger lazy-loaded state
+    await exploreAppAfterLogin(page);
+
+    const state = await context.storageState();
+    const cookies = state.cookies?.length ?? 0;
+    const localStorage = state.origins?.reduce(
+      (sum, o) => sum + (o.localStorage?.length ?? 0), 0
+    ) ?? 0;
+
+    if (cookies === 0 && localStorage === 0) {
+      throw new Error('Login appeared to succeed but no session data was captured — try manual session capture instead');
+    }
+
+    return { storageState: JSON.stringify(state), cookies, localStorage };
+  } finally {
+    await browser.close().catch(() => {});
+  }
+}