@detiq/crawler 0.1.7 → 0.1.9

package/dist/ai-login.js

@@ -99,7 +99,18 @@ async function loginWithStagehand(stagehand, creds, startUrl, captchaOpts, email
     await logStep("fill_password", async () => {
       const observations = await stagehand.observe("Find the password input field");
       const selector = observations[0]?.selector || 'input[type="password"]';
-      await pg.locator(selector).fill(creds.password);
+      try {
+        await pg.locator(selector).fill(creds.password);
+      } catch {
+        await pg.evaluate((pwd) => {
+          const input = document.querySelector('input[type="password"]');
+          if (!input) throw new Error("No password field found on page");
+          const setter = Object.getOwnPropertyDescriptor(window.HTMLInputElement.prototype, "value")?.set;
+          setter?.call(input, pwd);
+          input.dispatchEvent(new Event("input", { bubbles: true }));
+          input.dispatchEvent(new Event("change", { bubbles: true }));
+        }, creds.password);
+      }
     });
     await logStep("click_submit", () => actWithMouseSim(stagehand, "Click the sign in, log in, login, or continue button to submit credentials"));
     await pg.waitForLoadState("networkidle").catch(() => pg.waitForTimeout(2e3));

package/dist/ai-login.js.map

@@ -1,7 +1,7 @@
 {
   "version": 3,
   "sources": ["../src/ai-login.ts"],
-  "sourcesContent": ["import { z } from 'zod';\nimport type { Stagehand } from '@browserbasehq/stagehand';\nimport { AppBrain } from '@detiq/app-brain';\nimport { solveCaptcha } from './captcha-solver.js';\nimport { waitForEmailContent } from './email-otp.js';\nimport { logger } from './logger.js';\nimport type { Credentials } from './auth.js';\nimport { simulateMouseMovement } from './stealth/human-behavior.js';\n\nconst CLICK_INTENT = /\\b(click|tap|press|submit|select|choose|toggle)\\b/i;\n\n/** Simulate human mouse movement before click-type actions, then call stagehand.act(). */\nasync function actWithMouseSim(stagehand: Stagehand, instruction: string): Promise<any> {\n  if (CLICK_INTENT.test(instruction)) {\n    try {\n      const pg = await (stagehand as any).resolvePage();\n      const viewport = pg.viewportSize() ?? { width: 1280, height: 720 };\n      // Move mouse toward center of page with some randomness before Stagehand clicks\n      const targetX = viewport.width * 0.3 + Math.random() * viewport.width * 0.4;\n      const targetY = viewport.height * 0.2 + Math.random() * viewport.height * 0.5;\n      await simulateMouseMovement(pg, targetX, targetY);\n    } catch { /* non-fatal \u2014 don't block the action */ }\n  }\n  return stagehand.act(instruction);\n}\n\n/**\n * Recorded login step from the Chrome extension's step recorder.\n * {username} and {password} in `value` are substituted at replay time.\n */\nexport interface LoginStep {\n  action: 'fill' | 'click' | 'wait';\n  selector: string;\n  value?: string;\n  ms?: number;\n}\n\nconst TwoFASchema = z.object({\n  has2FA: z.boolean(),\n  fieldLabel: z.string().optional(),\n});\n\nconst CAPTCHA_SELECTORS = [\n  'iframe[src*=\"recaptcha\"]',\n  'iframe[src*=\"hcaptcha\"]',\n  'iframe[src*=\"challenges.cloudflare.com\"]',\n  '.cf-turnstile',\n  '[data-sitekey]',\n  '#captcha',\n  '.captcha',\n  '[id*=\"captcha\"]',\n  '[class*=\"captcha\"]',\n];\n\n/**\n * AI-guided login using Stagehand act/extract.\n *\n * @param loginInstructions - Optional natural language description of the login flow.\n *   When provided, an initial stagehand.act() is called with these instructions to\n *   navigate the UI (e.g. \"click Sign In in the nav, fill email, click Continue\").\n *   The standard 3-step credential fill follows immediately after.\n * @param opts - Optional tenant/project/job IDs for writing AUTH_STEP CrawlEvidence.\n */\nexport async function loginWithStagehand(\n  stagehand: Stagehand,\n  creds: Credentials,\n  startUrl: string,\n  captchaOpts?: { apiKey?: string; provider?: '2captcha' | 'capsolver' },\n  emailOtpOpts?: { apiKey?: string; inboxId?: string },\n  loginInstructions?: string,\n  opts?: { tenantId?: string; projectId?: string; jobId?: string },\n): Promise<boolean> {\n  async function logStep(step: string, fn: () => Promise<any>): Promise<{ success: boolean; detail: string }> {\n    const t0 = Date.now();\n    try {\n      const r = await fn();\n      const result = { success: true, detail: r?.description ?? r?.message ?? step };\n      if (opts?.tenantId && opts?.projectId) {\n        AppBrain.saveCrawlEvidence(opts.tenantId, opts.projectId, {\n          jobId: opts?.jobId,\n          type: 'AUTH_STEP',\n          payload: { step, ...result, ms: Date.now() - t0 },\n        }).catch(() => {});\n      }\n      return result;\n    } catch (e: any) {\n      const result = { success: false, detail: e.message ?? 'failed' };\n      if (opts?.tenantId && opts?.projectId) {\n        AppBrain.saveCrawlEvidence(opts.tenantId, opts.projectId, {\n          jobId: opts?.jobId,\n          type: 'AUTH_STEP',\n          payload: { step, ...result, ms: Date.now() - t0 },\n        }).catch(() => {});\n      }\n      throw e;\n    }\n  }\n\n  try {\n    const pg = await (stagehand as any).resolvePage();\n\n    // CAPTCHA detection\n    for (const sel of CAPTCHA_SELECTORS) {\n      const count = await pg.locator(sel).count().catch(() => 0);\n      if (count > 0) {\n        logger.warn({ selector: sel }, '[crawler] CAPTCHA detected on login page');\n        if (captchaOpts?.apiKey) {\n          logger.info({ provider: captchaOpts.provider ?? '2captcha' }, '[crawler] Attempting auto-solve');\n          try {\n            const result = await solveCaptcha(pg, startUrl, captchaOpts.apiKey, captchaOpts.provider ?? '2captcha');\n            if (result) {\n              logger.info({ provider: result.provider }, '[crawler] CAPTCHA solved \u2014 proceeding with login');\n              break;\n            }\n          } catch (solveErr) {\n            logger.warn({ err: solveErr }, '[crawler] CAPTCHA solve failed \u2014 falling back to manual error');\n            throw new Error(`CAPTCHA_DETECTED: Auto-solve failed (${String(solveErr)}). Use session capture instead.`);\n          }\n        } else {\n          throw new Error(\n            'CAPTCHA_DETECTED: Login page has a CAPTCHA. Use \"Session Capture \u2192 Launch Browser\" to log in manually, then capture the session. Alternatively, disable CAPTCHA for the ZeTa crawler IP.'\n          );\n        }\n        break;\n      }\n    }\n\n    // If caller provided natural language login instructions, use them to navigate to the login form\n    const MAX_LOGIN_INSTRUCTIONS = 2000;\n    if (loginInstructions && loginInstructions.length > MAX_LOGIN_INSTRUCTIONS) {\n      throw new Error(`loginInstructions exceeds ${MAX_LOGIN_INSTRUCTIONS} character limit`);\n    }\n    if (loginInstructions) {\n      await logStep('custom_recipe', () => actWithMouseSim(stagehand, loginInstructions));\n      await pg.waitForLoadState('networkidle').catch(() => pg.waitForTimeout(1500));\n    }\n\n    await logStep('fill_username', () => actWithMouseSim(stagehand, `Type \"${creds.username}\" into the username or email input field`));\n    // Don't send password to LLM \u2014 use direct Playwright fill to avoid\n    // leaking credentials into the Stagehand/LLM provider's request logs.\n    await logStep('fill_password', async () => {\n      const observations = await stagehand.observe('Find the password input field');\n      const selector = observations[0]?.selector || 'input[type=\"password\"]';\n      await pg.locator(selector).fill(creds.password);\n    });\n    await logStep('click_submit', () => actWithMouseSim(stagehand, 'Click the sign in, log in, login, or continue button to submit credentials'));\n    await pg.waitForLoadState('networkidle').catch(() => pg.waitForTimeout(2000));\n\n    const twoFA = await stagehand.extract(\n      'Is there a two-factor authentication, OTP, or verification code input visible right now?',\n      TwoFASchema as any\n    );\n\n    if (twoFA.has2FA) {\n      if (opts?.tenantId && opts?.projectId) {\n        AppBrain.saveCrawlEvidence(opts.tenantId, opts.projectId, {\n          jobId: opts?.jobId,\n          type: 'AUTH_STEP',\n          payload: { step: '2fa_detect', success: true, detail: twoFA.fieldLabel ?? '2FA detected', ms: 0 },\n        }).catch(() => {});\n      }\n\n      let code = creds.oneTimeCode ?? await creds.getOneTimeCode?.();\n\n      if (!code && emailOtpOpts?.apiKey && emailOtpOpts?.inboxId) {\n        const since = new Date().toISOString();\n        logger.info('[crawler] 2FA detected \u2014 waiting for email OTP or magic link via MailSlurp');\n        const emailContent = await waitForEmailContent(\n          { provider: 'mailslurp', apiKey: emailOtpOpts.apiKey, inboxId: emailOtpOpts.inboxId },\n          since\n        );\n        if (emailContent?.magicLink) {\n          logger.info({ url: emailContent.magicLink }, '[crawler] Magic link received \u2014 navigating');\n          await pg.goto(emailContent.magicLink, { waitUntil: 'domcontentloaded' });\n          await pg.waitForLoadState('networkidle').catch(() => {});\n          return true; // magic link navigation = login complete\n        }\n        code = emailContent?.otp ?? undefined;\n        if (code) {\n          logger.info('[crawler] Email OTP received');\n        } else {\n          logger.warn('[crawler] Email OTP timed out after 60s');\n        }\n      }\n\n      if (!code) {\n        console.warn('[crawler] 2FA detected but no oneTimeCode provided');\n        return false;\n      }\n      await logStep('2fa_inject', () => actWithMouseSim(stagehand, `Enter \"${code}\" into the ${twoFA.fieldLabel ?? 'verification code'} field`));\n      await logStep('2fa_submit', () => actWithMouseSim(stagehand, 'Click verify or confirm to submit the verification code'));\n      await pg.waitForLoadState('networkidle').catch(() => pg.waitForTimeout(2000));\n    }\n\n    // URL-based success check\n    const currentUrl = pg.url();\n    const loginPathPattern = /\\/(login|signin|sign-in|auth|authenticate|log-in|account\\/login)(\\?|$|\\/)/i;\n    try {\n      const { pathname } = new URL(currentUrl);\n      const wasRedirected = currentUrl !== startUrl;\n      const stillOnLoginPath = loginPathPattern.test(pathname);\n      if (wasRedirected && !stillOnLoginPath) {\n        logger.info({ url: currentUrl }, 'Login confirmed via URL change');\n        return true;\n      }\n    } catch { /* invalid URL \u2014 fall through */ }\n\n    const passwordCount = await pg.locator('input[type=\"password\"]').count().catch(() => 1);\n    const bodyText = await pg.evaluate(() => (document as any).body?.innerText?.trim()?.length ?? 0).catch(() => 0);\n    if (passwordCount === 0 && bodyText > 30) {\n      logger.info('Login confirmed: password field no longer visible');\n      return true;\n    }\n\n    logger.warn({ url: currentUrl }, 'Login appears failed');\n    try {\n      const screenshotPath = `/tmp/zeta-auth-failure-${Date.now()}.jpg`;\n      await pg.screenshot({ path: screenshotPath, type: 'jpeg', quality: 70, fullPage: false });\n      logger.warn({ screenshotPath }, '[crawler] Auth failure screenshot saved');\n    } catch { /* non-fatal */ }\n    return false;\n  } catch (err) {\n    const msg = String((err as Error).message ?? err);\n    if (msg.includes('CAPTCHA_DETECTED')) throw err;\n    logger.error({ err }, 'Login error');\n    return false;\n  }\n}\n"],
-  "mappings": "AAAA,SAAS,SAAS;AAElB,SAAS,gBAAgB;AACzB,SAAS,oBAAoB;AAC7B,SAAS,2BAA2B;AACpC,SAAS,cAAc;AAEvB,SAAS,6BAA6B;AAEtC,MAAM,eAAe;AAGrB,eAAe,gBAAgB,WAAsB,aAAmC;AACtF,MAAI,aAAa,KAAK,WAAW,GAAG;AAClC,QAAI;AACF,YAAM,KAAK,MAAO,UAAkB,YAAY;AAChD,YAAM,WAAW,GAAG,aAAa,KAAK,EAAE,OAAO,MAAM,QAAQ,IAAI;AAEjE,YAAM,UAAU,SAAS,QAAQ,MAAM,KAAK,OAAO,IAAI,SAAS,QAAQ;AACxE,YAAM,UAAU,SAAS,SAAS,MAAM,KAAK,OAAO,IAAI,SAAS,SAAS;AAC1E,YAAM,sBAAsB,IAAI,SAAS,OAAO;AAAA,IAClD,QAAQ;AAAA,IAA2C;AAAA,EACrD;AACA,SAAO,UAAU,IAAI,WAAW;AAClC;AAaA,MAAM,cAAc,EAAE,OAAO;AAAA,EAC3B,QAAQ,EAAE,QAAQ;AAAA,EAClB,YAAY,EAAE,OAAO,EAAE,SAAS;AAClC,CAAC;AAED,MAAM,oBAAoB;AAAA,EACxB;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AACF;AAWA,eAAsB,mBACpB,WACA,OACA,UACA,aACA,cACA,mBACA,MACkB;AAClB,iBAAe,QAAQ,MAAc,IAAuE;AAC1G,UAAM,KAAK,KAAK,IAAI;AACpB,QAAI;AACF,YAAM,IAAI,MAAM,GAAG;AACnB,YAAM,SAAS,EAAE,SAAS,MAAM,QAAQ,GAAG,eAAe,GAAG,WAAW,KAAK;AAC7E,UAAI,MAAM,YAAY,MAAM,WAAW;AACrC,iBAAS,kBAAkB,KAAK,UAAU,KAAK,WAAW;AAAA,UACxD,OAAO,MAAM;AAAA,UACb,MAAM;AAAA,UACN,SAAS,EAAE,MAAM,GAAG,QAAQ,IAAI,KAAK,IAAI,IAAI,GAAG;AAAA,QAClD,CAAC,EAAE,MAAM,MAAM;AAAA,QAAC,CAAC;AAAA,MACnB;AACA,aAAO;AAAA,IACT,SAAS,GAAQ;AACf,YAAM,SAAS,EAAE,SAAS,OAAO,QAAQ,EAAE,WAAW,SAAS;AAC/D,UAAI,MAAM,YAAY,MAAM,WAAW;AACrC,iBAAS,kBAAkB,KAAK,UAAU,KAAK,WAAW;AAAA,UACxD,OAAO,MAAM;AAAA,UACb,MAAM;AAAA,UACN,SAAS,EAAE,MAAM,GAAG,QAAQ,IAAI,KAAK,IAAI,IAAI,GAAG;AAAA,QAClD,CAAC,EAAE,MAAM,MAAM;AAAA,QAAC,CAAC;AAAA,MACnB;AACA,YAAM;AAAA,IACR;AAAA,EACF;AAEA,MAAI;AACF,UAAM,KAAK,MAAO,UAAkB,YAAY;AAGhD,eAAW,OAAO,mBAAmB;AACnC,YAAM,QAAQ,MAAM,GAAG,QAAQ,GAAG,EAAE,MAAM,EAAE,MAAM,MAAM,CAAC;AACzD,UAAI,QAAQ,GAAG;AACb,eAAO,KAAK,EAAE,UAAU,IAAI,GAAG,0CAA0C;AACzE,YAAI,aAAa,QAAQ;AACvB,iBAAO,KAAK,EAAE,UAAU,YAAY,YAAY,WAAW,GAAG,iCAAiC;AAC/F,cAAI;AACF,kBAAM,SAAS,MAAM,aAAa,IAAI,UAAU,YAAY,QAAQ,YAAY,YAAY,UAAU;AACtG,gBAAI,QAAQ;AACV,qBAAO,KAAK,EAAE,UAAU,OAAO,SAAS,GAAG,uDAAkD;AAC7F;AAAA,YACF;AAAA,UACF,SAAS,UAAU;AACjB,mBAAO,KAAK,EAAE,KAAK,SAAS,GAAG,oEAA+D;AAC9F,kBAAM,IAAI,MAAM,wCAAwC,OAAO,QAAQ,CAAC,iCAAiC;AAAA,UAC3G;AAAA,QACF,OAAO;AACL,gBAAM,IAAI;AAAA,YACR;AAAA,UACF;AAAA,QACF;AACA;AAAA,MACF;AAAA,IACF;AAGA,UAAM,yBAAyB;AAC/B,QAAI,qBAAqB,kBAAkB,SAAS,wBAAwB;AAC1E,YAAM,IAAI,MAAM,6BAA6B,sBAAsB,kBAAkB;AAAA,IACvF;AACA,QAAI,mBAAmB;AACrB,YAAM,QAAQ,iBAAiB,MAAM,gBAAgB,WAAW,iBAAiB,CAAC;AAClF,YAAM,GAAG,iBAAiB,aAAa,EAAE,MAAM,MAAM,GAAG,eAAe,IAAI,CAAC;AAAA,IAC9E;AAEA,UAAM,QAAQ,iBAAiB,MAAM,gBAAgB,WAAW,SAAS,MAAM,QAAQ,0CAA0C,CAAC;AAGlI,UAAM,QAAQ,iBAAiB,YAAY;AACzC,YAAM,eAAe,MAAM,UAAU,QAAQ,+BAA+B;AAC5E,YAAM,WAAW,aAAa,CAAC,GAAG,YAAY;AAC9C,YAAM,GAAG,QAAQ,QAAQ,EAAE,KAAK,MAAM,QAAQ;AAAA,IAChD,CAAC;AACD,UAAM,QAAQ,gBAAgB,MAAM,gBAAgB,WAAW,4EAA4E,CAAC;AAC5I,UAAM,GAAG,iBAAiB,aAAa,EAAE,MAAM,MAAM,GAAG,eAAe,GAAI,CAAC;AAE5E,UAAM,QAAQ,MAAM,UAAU;AAAA,MAC5B;AAAA,MACA;AAAA,IACF;AAEA,QAAI,MAAM,QAAQ;AAChB,UAAI,MAAM,YAAY,MAAM,WAAW;AACrC,iBAAS,kBAAkB,KAAK,UAAU,KAAK,WAAW;AAAA,UACxD,OAAO,MAAM;AAAA,UACb,MAAM;AAAA,UACN,SAAS,EAAE,MAAM,cAAc,SAAS,MAAM,QAAQ,MAAM,cAAc,gBAAgB,IAAI,EAAE;AAAA,QAClG,CAAC,EAAE,MAAM,MAAM;AAAA,QAAC,CAAC;AAAA,MACnB;AAEA,UAAI,OAAO,MAAM,eAAe,MAAM,MAAM,iBAAiB;AAE7D,UAAI,CAAC,QAAQ,cAAc,UAAU,cAAc,SAAS;AAC1D,cAAM,SAAQ,oBAAI,KAAK,GAAE,YAAY;AACrC,eAAO,KAAK,iFAA4E;AACxF,cAAM,eAAe,MAAM;AAAA,UACzB,EAAE,UAAU,aAAa,QAAQ,aAAa,QAAQ,SAAS,aAAa,QAAQ;AAAA,UACpF;AAAA,QACF;AACA,YAAI,cAAc,WAAW;AAC3B,iBAAO,KAAK,EAAE,KAAK,aAAa,UAAU,GAAG,iDAA4C;AACzF,gBAAM,GAAG,KAAK,aAAa,WAAW,EAAE,WAAW,mBAAmB,CAAC;AACvE,gBAAM,GAAG,iBAAiB,aAAa,EAAE,MAAM,MAAM;AAAA,UAAC,CAAC;AACvD,iBAAO;AAAA,QACT;AACA,eAAO,cAAc,OAAO;AAC5B,YAAI,MAAM;AACR,iBAAO,KAAK,8BAA8B;AAAA,QAC5C,OAAO;AACL,iBAAO,KAAK,yCAAyC;AAAA,QACvD;AAAA,MACF;AAEA,UAAI,CAAC,MAAM;AACT,gBAAQ,KAAK,oDAAoD;AACjE,eAAO;AAAA,MACT;AACA,YAAM,QAAQ,cAAc,MAAM,gBAAgB,WAAW,UAAU,IAAI,cAAc,MAAM,cAAc,mBAAmB,QAAQ,CAAC;AACzI,YAAM,QAAQ,cAAc,MAAM,gBAAgB,WAAW,yDAAyD,CAAC;AACvH,YAAM,GAAG,iBAAiB,aAAa,EAAE,MAAM,MAAM,GAAG,eAAe,GAAI,CAAC;AAAA,IAC9E;AAGA,UAAM,aAAa,GAAG,IAAI;AAC1B,UAAM,mBAAmB;AACzB,QAAI;AACF,YAAM,EAAE,SAAS,IAAI,IAAI,IAAI,UAAU;AACvC,YAAM,gBAAgB,eAAe;AACrC,YAAM,mBAAmB,iBAAiB,KAAK,QAAQ;AACvD,UAAI,iBAAiB,CAAC,kBAAkB;AACtC,eAAO,KAAK,EAAE,KAAK,WAAW,GAAG,gCAAgC;AACjE,eAAO;AAAA,MACT;AAAA,IACF,QAAQ;AAAA,IAAmC;AAE3C,UAAM,gBAAgB,MAAM,GAAG,QAAQ,wBAAwB,EAAE,MAAM,EAAE,MAAM,MAAM,CAAC;AACtF,UAAM,WAAW,MAAM,GAAG,SAAS,MAAO,SAAiB,MAAM,WAAW,KAAK,GAAG,UAAU,CAAC,EAAE,MAAM,MAAM,CAAC;AAC9G,QAAI,kBAAkB,KAAK,WAAW,IAAI;AACxC,aAAO,KAAK,mDAAmD;AAC/D,aAAO;AAAA,IACT;AAEA,WAAO,KAAK,EAAE,KAAK,WAAW,GAAG,sBAAsB;AACvD,QAAI;AACF,YAAM,iBAAiB,0BAA0B,KAAK,IAAI,CAAC;AAC3D,YAAM,GAAG,WAAW,EAAE,MAAM,gBAAgB,MAAM,QAAQ,SAAS,IAAI,UAAU,MAAM,CAAC;AACxF,aAAO,KAAK,EAAE,eAAe,GAAG,yCAAyC;AAAA,IAC3E,QAAQ;AAAA,IAAkB;AAC1B,WAAO;AAAA,EACT,SAAS,KAAK;AACZ,UAAM,MAAM,OAAQ,IAAc,WAAW,GAAG;AAChD,QAAI,IAAI,SAAS,kBAAkB,EAAG,OAAM;AAC5C,WAAO,MAAM,EAAE,IAAI,GAAG,aAAa;AACnC,WAAO;AAAA,EACT;AACF;",
+  "sourcesContent": ["import { z } from 'zod';\nimport type { Stagehand } from '@browserbasehq/stagehand';\nimport { AppBrain } from '@detiq/app-brain';\nimport { solveCaptcha } from './captcha-solver.js';\nimport { waitForEmailContent } from './email-otp.js';\nimport { logger } from './logger.js';\nimport type { Credentials } from './auth.js';\nimport { simulateMouseMovement } from './stealth/human-behavior.js';\n\nconst CLICK_INTENT = /\\b(click|tap|press|submit|select|choose|toggle)\\b/i;\n\n/** Simulate human mouse movement before click-type actions, then call stagehand.act(). */\nasync function actWithMouseSim(stagehand: Stagehand, instruction: string): Promise<any> {\n  if (CLICK_INTENT.test(instruction)) {\n    try {\n      const pg = await (stagehand as any).resolvePage();\n      const viewport = pg.viewportSize() ?? { width: 1280, height: 720 };\n      // Move mouse toward center of page with some randomness before Stagehand clicks\n      const targetX = viewport.width * 0.3 + Math.random() * viewport.width * 0.4;\n      const targetY = viewport.height * 0.2 + Math.random() * viewport.height * 0.5;\n      await simulateMouseMovement(pg, targetX, targetY);\n    } catch { /* non-fatal \u2014 don't block the action */ }\n  }\n  return stagehand.act(instruction);\n}\n\n/**\n * Recorded login step from the Chrome extension's step recorder.\n * {username} and {password} in `value` are substituted at replay time.\n */\nexport interface LoginStep {\n  action: 'fill' | 'click' | 'wait';\n  selector: string;\n  value?: string;\n  ms?: number;\n}\n\nconst TwoFASchema = z.object({\n  has2FA: z.boolean(),\n  fieldLabel: z.string().optional(),\n});\n\nconst CAPTCHA_SELECTORS = [\n  'iframe[src*=\"recaptcha\"]',\n  'iframe[src*=\"hcaptcha\"]',\n  'iframe[src*=\"challenges.cloudflare.com\"]',\n  '.cf-turnstile',\n  '[data-sitekey]',\n  '#captcha',\n  '.captcha',\n  '[id*=\"captcha\"]',\n  '[class*=\"captcha\"]',\n];\n\n/**\n * AI-guided login using Stagehand act/extract.\n *\n * @param loginInstructions - Optional natural language description of the login flow.\n *   When provided, an initial stagehand.act() is called with these instructions to\n *   navigate the UI (e.g. \"click Sign In in the nav, fill email, click Continue\").\n *   The standard 3-step credential fill follows immediately after.\n * @param opts - Optional tenant/project/job IDs for writing AUTH_STEP CrawlEvidence.\n */\nexport async function loginWithStagehand(\n  stagehand: Stagehand,\n  creds: Credentials,\n  startUrl: string,\n  captchaOpts?: { apiKey?: string; provider?: '2captcha' | 'capsolver' },\n  emailOtpOpts?: { apiKey?: string; inboxId?: string },\n  loginInstructions?: string,\n  opts?: { tenantId?: string; projectId?: string; jobId?: string },\n): Promise<boolean> {\n  async function logStep(step: string, fn: () => Promise<any>): Promise<{ success: boolean; detail: string }> {\n    const t0 = Date.now();\n    try {\n      const r = await fn();\n      const result = { success: true, detail: r?.description ?? r?.message ?? step };\n      if (opts?.tenantId && opts?.projectId) {\n        AppBrain.saveCrawlEvidence(opts.tenantId, opts.projectId, {\n          jobId: opts?.jobId,\n          type: 'AUTH_STEP',\n          payload: { step, ...result, ms: Date.now() - t0 },\n        }).catch(() => {});\n      }\n      return result;\n    } catch (e: any) {\n      const result = { success: false, detail: e.message ?? 'failed' };\n      if (opts?.tenantId && opts?.projectId) {\n        AppBrain.saveCrawlEvidence(opts.tenantId, opts.projectId, {\n          jobId: opts?.jobId,\n          type: 'AUTH_STEP',\n          payload: { step, ...result, ms: Date.now() - t0 },\n        }).catch(() => {});\n      }\n      throw e;\n    }\n  }\n\n  try {\n    const pg = await (stagehand as any).resolvePage();\n\n    // CAPTCHA detection\n    for (const sel of CAPTCHA_SELECTORS) {\n      const count = await pg.locator(sel).count().catch(() => 0);\n      if (count > 0) {\n        logger.warn({ selector: sel }, '[crawler] CAPTCHA detected on login page');\n        if (captchaOpts?.apiKey) {\n          logger.info({ provider: captchaOpts.provider ?? '2captcha' }, '[crawler] Attempting auto-solve');\n          try {\n            const result = await solveCaptcha(pg, startUrl, captchaOpts.apiKey, captchaOpts.provider ?? '2captcha');\n            if (result) {\n              logger.info({ provider: result.provider }, '[crawler] CAPTCHA solved \u2014 proceeding with login');\n              break;\n            }\n          } catch (solveErr) {\n            logger.warn({ err: solveErr }, '[crawler] CAPTCHA solve failed \u2014 falling back to manual error');\n            throw new Error(`CAPTCHA_DETECTED: Auto-solve failed (${String(solveErr)}). Use session capture instead.`);\n          }\n        } else {\n          throw new Error(\n            'CAPTCHA_DETECTED: Login page has a CAPTCHA. Use \"Session Capture \u2192 Launch Browser\" to log in manually, then capture the session. Alternatively, disable CAPTCHA for the ZeTa crawler IP.'\n          );\n        }\n        break;\n      }\n    }\n\n    // If caller provided natural language login instructions, use them to navigate to the login form\n    const MAX_LOGIN_INSTRUCTIONS = 2000;\n    if (loginInstructions && loginInstructions.length > MAX_LOGIN_INSTRUCTIONS) {\n      throw new Error(`loginInstructions exceeds ${MAX_LOGIN_INSTRUCTIONS} character limit`);\n    }\n    if (loginInstructions) {\n      await logStep('custom_recipe', () => actWithMouseSim(stagehand, loginInstructions));\n      await pg.waitForLoadState('networkidle').catch(() => pg.waitForTimeout(1500));\n    }\n\n    await logStep('fill_username', () => actWithMouseSim(stagehand, `Type \"${creds.username}\" into the username or email input field`));\n    // Don't send password to LLM \u2014 use direct Playwright fill to avoid\n    // leaking credentials into the Stagehand/LLM provider's request logs.\n    await logStep('fill_password', async () => {\n      const observations = await stagehand.observe('Find the password input field');\n      const selector = observations[0]?.selector || 'input[type=\"password\"]';\n      try {\n        await pg.locator(selector).fill(creds.password);\n      } catch {\n        // Stagehand locator failed (e.g. form re-rendered, proxy can't locate element).\n        // Fall back to direct DOM fill via evaluate so the password never touches the LLM.\n        await pg.evaluate((pwd: string) => {\n          const input = document.querySelector('input[type=\"password\"]') as HTMLInputElement | null;\n          if (!input) throw new Error('No password field found on page');\n          const setter = Object.getOwnPropertyDescriptor(window.HTMLInputElement.prototype, 'value')?.set;\n          setter?.call(input, pwd);\n          input.dispatchEvent(new Event('input', { bubbles: true }));\n          input.dispatchEvent(new Event('change', { bubbles: true }));\n        }, creds.password);\n      }\n    });\n    await logStep('click_submit', () => actWithMouseSim(stagehand, 'Click the sign in, log in, login, or continue button to submit credentials'));\n    await pg.waitForLoadState('networkidle').catch(() => pg.waitForTimeout(2000));\n\n    const twoFA = await stagehand.extract(\n      'Is there a two-factor authentication, OTP, or verification code input visible right now?',\n      TwoFASchema as any\n    );\n\n    if (twoFA.has2FA) {\n      if (opts?.tenantId && opts?.projectId) {\n        AppBrain.saveCrawlEvidence(opts.tenantId, opts.projectId, {\n          jobId: opts?.jobId,\n          type: 'AUTH_STEP',\n          payload: { step: '2fa_detect', success: true, detail: twoFA.fieldLabel ?? '2FA detected', ms: 0 },\n        }).catch(() => {});\n      }\n\n      let code = creds.oneTimeCode ?? await creds.getOneTimeCode?.();\n\n      if (!code && emailOtpOpts?.apiKey && emailOtpOpts?.inboxId) {\n        const since = new Date().toISOString();\n        logger.info('[crawler] 2FA detected \u2014 waiting for email OTP or magic link via MailSlurp');\n        const emailContent = await waitForEmailContent(\n          { provider: 'mailslurp', apiKey: emailOtpOpts.apiKey, inboxId: emailOtpOpts.inboxId },\n          since\n        );\n        if (emailContent?.magicLink) {\n          logger.info({ url: emailContent.magicLink }, '[crawler] Magic link received \u2014 navigating');\n          await pg.goto(emailContent.magicLink, { waitUntil: 'domcontentloaded' });\n          await pg.waitForLoadState('networkidle').catch(() => {});\n          return true; // magic link navigation = login complete\n        }\n        code = emailContent?.otp ?? undefined;\n        if (code) {\n          logger.info('[crawler] Email OTP received');\n        } else {\n          logger.warn('[crawler] Email OTP timed out after 60s');\n        }\n      }\n\n      if (!code) {\n        console.warn('[crawler] 2FA detected but no oneTimeCode provided');\n        return false;\n      }\n      await logStep('2fa_inject', () => actWithMouseSim(stagehand, `Enter \"${code}\" into the ${twoFA.fieldLabel ?? 'verification code'} field`));\n      await logStep('2fa_submit', () => actWithMouseSim(stagehand, 'Click verify or confirm to submit the verification code'));\n      await pg.waitForLoadState('networkidle').catch(() => pg.waitForTimeout(2000));\n    }\n\n    // URL-based success check\n    const currentUrl = pg.url();\n    const loginPathPattern = /\\/(login|signin|sign-in|auth|authenticate|log-in|account\\/login)(\\?|$|\\/)/i;\n    try {\n      const { pathname } = new URL(currentUrl);\n      const wasRedirected = currentUrl !== startUrl;\n      const stillOnLoginPath = loginPathPattern.test(pathname);\n      if (wasRedirected && !stillOnLoginPath) {\n        logger.info({ url: currentUrl }, 'Login confirmed via URL change');\n        return true;\n      }\n    } catch { /* invalid URL \u2014 fall through */ }\n\n    const passwordCount = await pg.locator('input[type=\"password\"]').count().catch(() => 1);\n    const bodyText = await pg.evaluate(() => (document as any).body?.innerText?.trim()?.length ?? 0).catch(() => 0);\n    if (passwordCount === 0 && bodyText > 30) {\n      logger.info('Login confirmed: password field no longer visible');\n      return true;\n    }\n\n    logger.warn({ url: currentUrl }, 'Login appears failed');\n    try {\n      const screenshotPath = `/tmp/zeta-auth-failure-${Date.now()}.jpg`;\n      await pg.screenshot({ path: screenshotPath, type: 'jpeg', quality: 70, fullPage: false });\n      logger.warn({ screenshotPath }, '[crawler] Auth failure screenshot saved');\n    } catch { /* non-fatal */ }\n    return false;\n  } catch (err) {\n    const msg = String((err as Error).message ?? err);\n    if (msg.includes('CAPTCHA_DETECTED')) throw err;\n    logger.error({ err }, 'Login error');\n    return false;\n  }\n}\n"],
+  "mappings": "AAAA,SAAS,SAAS;AAElB,SAAS,gBAAgB;AACzB,SAAS,oBAAoB;AAC7B,SAAS,2BAA2B;AACpC,SAAS,cAAc;AAEvB,SAAS,6BAA6B;AAEtC,MAAM,eAAe;AAGrB,eAAe,gBAAgB,WAAsB,aAAmC;AACtF,MAAI,aAAa,KAAK,WAAW,GAAG;AAClC,QAAI;AACF,YAAM,KAAK,MAAO,UAAkB,YAAY;AAChD,YAAM,WAAW,GAAG,aAAa,KAAK,EAAE,OAAO,MAAM,QAAQ,IAAI;AAEjE,YAAM,UAAU,SAAS,QAAQ,MAAM,KAAK,OAAO,IAAI,SAAS,QAAQ;AACxE,YAAM,UAAU,SAAS,SAAS,MAAM,KAAK,OAAO,IAAI,SAAS,SAAS;AAC1E,YAAM,sBAAsB,IAAI,SAAS,OAAO;AAAA,IAClD,QAAQ;AAAA,IAA2C;AAAA,EACrD;AACA,SAAO,UAAU,IAAI,WAAW;AAClC;AAaA,MAAM,cAAc,EAAE,OAAO;AAAA,EAC3B,QAAQ,EAAE,QAAQ;AAAA,EAClB,YAAY,EAAE,OAAO,EAAE,SAAS;AAClC,CAAC;AAED,MAAM,oBAAoB;AAAA,EACxB;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AACF;AAWA,eAAsB,mBACpB,WACA,OACA,UACA,aACA,cACA,mBACA,MACkB;AAClB,iBAAe,QAAQ,MAAc,IAAuE;AAC1G,UAAM,KAAK,KAAK,IAAI;AACpB,QAAI;AACF,YAAM,IAAI,MAAM,GAAG;AACnB,YAAM,SAAS,EAAE,SAAS,MAAM,QAAQ,GAAG,eAAe,GAAG,WAAW,KAAK;AAC7E,UAAI,MAAM,YAAY,MAAM,WAAW;AACrC,iBAAS,kBAAkB,KAAK,UAAU,KAAK,WAAW;AAAA,UACxD,OAAO,MAAM;AAAA,UACb,MAAM;AAAA,UACN,SAAS,EAAE,MAAM,GAAG,QAAQ,IAAI,KAAK,IAAI,IAAI,GAAG;AAAA,QAClD,CAAC,EAAE,MAAM,MAAM;AAAA,QAAC,CAAC;AAAA,MACnB;AACA,aAAO;AAAA,IACT,SAAS,GAAQ;AACf,YAAM,SAAS,EAAE,SAAS,OAAO,QAAQ,EAAE,WAAW,SAAS;AAC/D,UAAI,MAAM,YAAY,MAAM,WAAW;AACrC,iBAAS,kBAAkB,KAAK,UAAU,KAAK,WAAW;AAAA,UACxD,OAAO,MAAM;AAAA,UACb,MAAM;AAAA,UACN,SAAS,EAAE,MAAM,GAAG,QAAQ,IAAI,KAAK,IAAI,IAAI,GAAG;AAAA,QAClD,CAAC,EAAE,MAAM,MAAM;AAAA,QAAC,CAAC;AAAA,MACnB;AACA,YAAM;AAAA,IACR;AAAA,EACF;AAEA,MAAI;AACF,UAAM,KAAK,MAAO,UAAkB,YAAY;AAGhD,eAAW,OAAO,mBAAmB;AACnC,YAAM,QAAQ,MAAM,GAAG,QAAQ,GAAG,EAAE,MAAM,EAAE,MAAM,MAAM,CAAC;AACzD,UAAI,QAAQ,GAAG;AACb,eAAO,KAAK,EAAE,UAAU,IAAI,GAAG,0CAA0C;AACzE,YAAI,aAAa,QAAQ;AACvB,iBAAO,KAAK,EAAE,UAAU,YAAY,YAAY,WAAW,GAAG,iCAAiC;AAC/F,cAAI;AACF,kBAAM,SAAS,MAAM,aAAa,IAAI,UAAU,YAAY,QAAQ,YAAY,YAAY,UAAU;AACtG,gBAAI,QAAQ;AACV,qBAAO,KAAK,EAAE,UAAU,OAAO,SAAS,GAAG,uDAAkD;AAC7F;AAAA,YACF;AAAA,UACF,SAAS,UAAU;AACjB,mBAAO,KAAK,EAAE,KAAK,SAAS,GAAG,oEAA+D;AAC9F,kBAAM,IAAI,MAAM,wCAAwC,OAAO,QAAQ,CAAC,iCAAiC;AAAA,UAC3G;AAAA,QACF,OAAO;AACL,gBAAM,IAAI;AAAA,YACR;AAAA,UACF;AAAA,QACF;AACA;AAAA,MACF;AAAA,IACF;AAGA,UAAM,yBAAyB;AAC/B,QAAI,qBAAqB,kBAAkB,SAAS,wBAAwB;AAC1E,YAAM,IAAI,MAAM,6BAA6B,sBAAsB,kBAAkB;AAAA,IACvF;AACA,QAAI,mBAAmB;AACrB,YAAM,QAAQ,iBAAiB,MAAM,gBAAgB,WAAW,iBAAiB,CAAC;AAClF,YAAM,GAAG,iBAAiB,aAAa,EAAE,MAAM,MAAM,GAAG,eAAe,IAAI,CAAC;AAAA,IAC9E;AAEA,UAAM,QAAQ,iBAAiB,MAAM,gBAAgB,WAAW,SAAS,MAAM,QAAQ,0CAA0C,CAAC;AAGlI,UAAM,QAAQ,iBAAiB,YAAY;AACzC,YAAM,eAAe,MAAM,UAAU,QAAQ,+BAA+B;AAC5E,YAAM,WAAW,aAAa,CAAC,GAAG,YAAY;AAC9C,UAAI;AACF,cAAM,GAAG,QAAQ,QAAQ,EAAE,KAAK,MAAM,QAAQ;AAAA,MAChD,QAAQ;AAGN,cAAM,GAAG,SAAS,CAAC,QAAgB;AACjC,gBAAM,QAAQ,SAAS,cAAc,wBAAwB;AAC7D,cAAI,CAAC,MAAO,OAAM,IAAI,MAAM,iCAAiC;AAC7D,gBAAM,SAAS,OAAO,yBAAyB,OAAO,iBAAiB,WAAW,OAAO,GAAG;AAC5F,kBAAQ,KAAK,OAAO,GAAG;AACvB,gBAAM,cAAc,IAAI,MAAM,SAAS,EAAE,SAAS,KAAK,CAAC,CAAC;AACzD,gBAAM,cAAc,IAAI,MAAM,UAAU,EAAE,SAAS,KAAK,CAAC,CAAC;AAAA,QAC5D,GAAG,MAAM,QAAQ;AAAA,MACnB;AAAA,IACF,CAAC;AACD,UAAM,QAAQ,gBAAgB,MAAM,gBAAgB,WAAW,4EAA4E,CAAC;AAC5I,UAAM,GAAG,iBAAiB,aAAa,EAAE,MAAM,MAAM,GAAG,eAAe,GAAI,CAAC;AAE5E,UAAM,QAAQ,MAAM,UAAU;AAAA,MAC5B;AAAA,MACA;AAAA,IACF;AAEA,QAAI,MAAM,QAAQ;AAChB,UAAI,MAAM,YAAY,MAAM,WAAW;AACrC,iBAAS,kBAAkB,KAAK,UAAU,KAAK,WAAW;AAAA,UACxD,OAAO,MAAM;AAAA,UACb,MAAM;AAAA,UACN,SAAS,EAAE,MAAM,cAAc,SAAS,MAAM,QAAQ,MAAM,cAAc,gBAAgB,IAAI,EAAE;AAAA,QAClG,CAAC,EAAE,MAAM,MAAM;AAAA,QAAC,CAAC;AAAA,MACnB;AAEA,UAAI,OAAO,MAAM,eAAe,MAAM,MAAM,iBAAiB;AAE7D,UAAI,CAAC,QAAQ,cAAc,UAAU,cAAc,SAAS;AAC1D,cAAM,SAAQ,oBAAI,KAAK,GAAE,YAAY;AACrC,eAAO,KAAK,iFAA4E;AACxF,cAAM,eAAe,MAAM;AAAA,UACzB,EAAE,UAAU,aAAa,QAAQ,aAAa,QAAQ,SAAS,aAAa,QAAQ;AAAA,UACpF;AAAA,QACF;AACA,YAAI,cAAc,WAAW;AAC3B,iBAAO,KAAK,EAAE,KAAK,aAAa,UAAU,GAAG,iDAA4C;AACzF,gBAAM,GAAG,KAAK,aAAa,WAAW,EAAE,WAAW,mBAAmB,CAAC;AACvE,gBAAM,GAAG,iBAAiB,aAAa,EAAE,MAAM,MAAM;AAAA,UAAC,CAAC;AACvD,iBAAO;AAAA,QACT;AACA,eAAO,cAAc,OAAO;AAC5B,YAAI,MAAM;AACR,iBAAO,KAAK,8BAA8B;AAAA,QAC5C,OAAO;AACL,iBAAO,KAAK,yCAAyC;AAAA,QACvD;AAAA,MACF;AAEA,UAAI,CAAC,MAAM;AACT,gBAAQ,KAAK,oDAAoD;AACjE,eAAO;AAAA,MACT;AACA,YAAM,QAAQ,cAAc,MAAM,gBAAgB,WAAW,UAAU,IAAI,cAAc,MAAM,cAAc,mBAAmB,QAAQ,CAAC;AACzI,YAAM,QAAQ,cAAc,MAAM,gBAAgB,WAAW,yDAAyD,CAAC;AACvH,YAAM,GAAG,iBAAiB,aAAa,EAAE,MAAM,MAAM,GAAG,eAAe,GAAI,CAAC;AAAA,IAC9E;AAGA,UAAM,aAAa,GAAG,IAAI;AAC1B,UAAM,mBAAmB;AACzB,QAAI;AACF,YAAM,EAAE,SAAS,IAAI,IAAI,IAAI,UAAU;AACvC,YAAM,gBAAgB,eAAe;AACrC,YAAM,mBAAmB,iBAAiB,KAAK,QAAQ;AACvD,UAAI,iBAAiB,CAAC,kBAAkB;AACtC,eAAO,KAAK,EAAE,KAAK,WAAW,GAAG,gCAAgC;AACjE,eAAO;AAAA,MACT;AAAA,IACF,QAAQ;AAAA,IAAmC;AAE3C,UAAM,gBAAgB,MAAM,GAAG,QAAQ,wBAAwB,EAAE,MAAM,EAAE,MAAM,MAAM,CAAC;AACtF,UAAM,WAAW,MAAM,GAAG,SAAS,MAAO,SAAiB,MAAM,WAAW,KAAK,GAAG,UAAU,CAAC,EAAE,MAAM,MAAM,CAAC;AAC9G,QAAI,kBAAkB,KAAK,WAAW,IAAI;AACxC,aAAO,KAAK,mDAAmD;AAC/D,aAAO;AAAA,IACT;AAEA,WAAO,KAAK,EAAE,KAAK,WAAW,GAAG,sBAAsB;AACvD,QAAI;AACF,YAAM,iBAAiB,0BAA0B,KAAK,IAAI,CAAC;AAC3D,YAAM,GAAG,WAAW,EAAE,MAAM,gBAAgB,MAAM,QAAQ,SAAS,IAAI,UAAU,MAAM,CAAC;AACxF,aAAO,KAAK,EAAE,eAAe,GAAG,yCAAyC;AAAA,IAC3E,QAAQ;AAAA,IAAkB;AAC1B,WAAO;AAAA,EACT,SAAS,KAAK;AACZ,UAAM,MAAM,OAAQ,IAAc,WAAW,GAAG;AAChD,QAAI,IAAI,SAAS,kBAAkB,EAAG,OAAM;AAC5C,WAAO,MAAM,EAAE,IAAI,GAAG,aAAa;AACnC,WAAO;AAAA,EACT;AACF;",
   "names": []
 }

package/dist/crawler.js

@@ -314,6 +314,15 @@ const TwoFASchema = z.object({
 });
 async function crawlProject(opts, onProgress) {
   if (opts.screenshotDir) await fs.mkdir(opts.screenshotDir, { recursive: true });
+  const uploadToCloud = async (localPath, key) => {
+    if (!opts.storageConfig) return void 0;
+    try {
+      const { uploadScreenshotToTenantStorage } = await import("@detiq/app-brain");
+      return await uploadScreenshotToTenantStorage(opts.storageConfig, localPath, key);
+    } catch {
+      return void 0;
+    }
+  };
   const isDocker = !!process.env.PLAYWRIGHT_IN_DOCKER || process.env.NODE_ENV === "production";
   const toStagehandModel = (model, provider) => model.includes("/") ? model : `${provider}/${model}`;
   let stagehandModel = toStagehandModel(config.llmModel, "openai");
@@ -601,6 +610,15 @@ async function crawlProject(opts, onProgress) {
       localStorage.setItem("token", token);
     }, opts.authToken);
   }
+  const crawlerSecret = process.env.CRAWLER_INTERNAL_SECRET;
+  if (crawlerSecret) {
+    const loginPattern = `${new URL(opts.appUrl).origin}**/auth/login*`;
+    await page.route(loginPattern, (route) => {
+      route.continue({
+        headers: { ...route.request().headers(), "x-zeta-crawler": crawlerSecret }
+      });
+    });
+  }
   await page.addInitScript(() => {
     try {
       window.__zetaCLS = 0;
@@ -736,7 +754,8 @@ async function crawlProject(opts, onProgress) {
           }, state.origins);
         }
         await onProgress?.(12, "Restoring session");
-        await rawPage.reload({ waitUntil: "domcontentloaded" });
+        await rawPage.goto(opts.appUrl, { waitUntil: "domcontentloaded" }).catch(() => {
+        });
         await waitForStableDOM(page);
         if (page.url() === opts.appUrl || page.url() === opts.appUrl.replace(/\/$/, "")) {
           await waitForURLChange(page, opts.appUrl).catch(() => {
@@ -1178,7 +1197,12 @@ async function crawlProject(opts, onProgress) {
     let aiRateLimitWarned = false;
     let totalAttemptsThisSegment = 0;
     const maxAttempts = Math.max(segmentSize * 4, 200);
+    let pendingSessionRestore = null;
     while (queue.length > 0 && visited.size < maxScreens && visited.size < segmentSize) {
+      if (pendingSessionRestore) {
+        await pendingSessionRestore();
+        pendingSessionRestore = null;
+      }
       if (++totalAttemptsThisSegment > maxAttempts) {
         logger.warn({ maxAttempts, visited: visited.size, queued: queue.length }, "[crawler] attempt cap reached \u2014 stopping to prevent URL explosion");
         break;
@@ -1250,6 +1274,45 @@ async function crawlProject(opts, onProgress) {
             }
           };
           rawPage.on("response", onRedirectResponse);
+          const AUTH_PATH_REGEX = /\/(login|register|signup|sign-in|sign-up|forgot-password|reset-password)([-/]|$)/i;
+          const isAuthRoute = AUTH_PATH_REGEX.test(new URL(url).pathname);
+          const browserCtx = playwrightBrowser?.contexts()[0];
+          if (isAuthRoute && browserCtx) {
+            const currentState = await browserCtx.storageState();
+            const hasSession = (currentState.cookies?.length ?? 0) > 0 || currentState.origins?.some((o) => (o.localStorage?.length ?? 0) > 0);
+            if (hasSession) {
+              pendingSessionRestore = async () => {
+                const ctx = playwrightBrowser?.contexts()[0];
+                if (ctx) {
+                  await ctx.addCookies(currentState.cookies);
+                  await rawPage.evaluate((origins) => {
+                    for (const entry of origins ?? []) {
+                      if (window.location.origin === entry.origin) {
+                        for (const item of entry.localStorage) {
+                          try {
+                            localStorage.setItem(item.name, item.value);
+                          } catch {
+                          }
+                        }
+                      }
+                    }
+                  }, currentState.origins).catch(() => {
+                  });
+                  logger.info("[crawler] restored session state after public auth route");
+                }
+              };
+              await browserCtx.clearCookies();
+              await rawPage.evaluate(() => {
+                try {
+                  localStorage.clear();
+                  sessionStorage.clear();
+                } catch {
+                }
+              }).catch(() => {
+              });
+              logger.info({ url }, "[crawler] temporarily cleared session for public auth route");
+            }
+          }
           await page.goto(url, { waitUntil: "domcontentloaded", timeout: 3e4 }).catch((navErr) => {
             logger.warn({ url, err: navErr?.message ?? String(navErr) }, "[crawler] navigation failed \u2014 skipping");
           });
@@ -1732,8 +1795,14 @@ async function crawlProject(opts, onProgress) {
           }
         })()}`);
         const [, elements, nextLinks, atElements] = await Promise.all([
-          page.screenshot({ path: shotPath, fullPage: true }).then(() => AppBrain.saveScreenshot(opts.tenantId, opts.projectId, screen.id, shotPath, screen.version, "DESKTOP")),
-          extractElements(stagehand, page, elementLLM ?? void 0).catch(() => []),
+          page.screenshot({ path: shotPath, fullPage: true }).then(async () => {
+            const cloudUrl = await uploadToCloud(shotPath, `${opts.tenantId}/${opts.projectId}/screenshots/${path.basename(shotPath)}`);
+            return AppBrain.saveScreenshot(opts.tenantId, opts.projectId, screen.id, shotPath, screen.version, "DESKTOP", cloudUrl);
+          }).catch((err) => console.error("[crawler] page.screenshot failed:", err)),
+          extractElements(stagehand, page, elementLLM ?? void 0).catch((err) => {
+            console.error("[crawler] extractElements failed:", err);
+            return [];
+          }),
           // Always run AI link discovery (even when cache exists) — cache is merged in, not used as a replacement.
           // Skipping AI when cache exists was a bug: stale cache from broken crawls permanently blocked discovery.
           discoverLinks(stagehand, origin, page, linkLLM ?? void 0, aiSkipCount > 0).then((freshLinks) => {
@@ -2155,7 +2224,7 @@ ${snippet}`,
                 fcp: rawPerf?.fcp,
                 cls: rawPerf?.cls,
                 ttfb: rawPerf?.ttfb,
-                viewport: "DESKTOP",
+                viewport: opts.viewport ?? "DESKTOP",
                 ...lhScores ? {
                   lighthousePerf: lhScores.perf,
                   lighthouseA11y: lhScores.a11y,
@@ -2287,6 +2356,10 @@ ${snippet}`,
         }
       }
     }
+    if (pendingSessionRestore) {
+      await pendingSessionRestore();
+      pendingSessionRestore = null;
+    }
     try {
       const { runUrlPatternCompletion } = await import("./url-pattern-completer.js");
       const llmCfgForPatterns = await getTaskAgentLLMConfig(opts.tenantId, "crawl").catch(() => null);
@@ -2555,8 +2628,7 @@ ${snippet}`,
         logger.info({ url: sessionCheckUrl }, "[P5] Session alive \u2014 proceeding with viewport pass");
       }
     } catch (err) {
-      logger.warn({ err: String(err) }, "[P5] Session health check failed \u2014 skipping viewport pass to avoid bad screenshots");
-      sessionAliveForViewports = false;
+      logger.warn({ err: String(err) }, "[P5] Session health check navigation failed \u2014 proceeding with viewport pass anyway");
     }
     const extraProfiles = (opts.deviceProfiles ?? []).filter((p) => p !== "DESKTOP");
     if (sessionAliveForViewports && extraProfiles.length > 0 && discovered.length > 0) {
@@ -2607,7 +2679,8 @@ ${snippet}`,
                 mobilePage.screenshot({ path: shotPath, fullPage: true }),
                 domFallbackElements(mobilePage)
               ]);
-              await AppBrain.saveScreenshot(opts.tenantId, opts.projectId, disc.screenId, shotPath, disc.version, profile);
+              const cloudUrl = await uploadToCloud(shotPath, `${opts.tenantId}/${opts.projectId}/screenshots/${path.basename(shotPath)}`);
+              await AppBrain.saveScreenshot(opts.tenantId, opts.projectId, disc.screenId, shotPath, disc.version, profile, cloudUrl);
               AppBrain.autoCreateVisualBaseline(opts.tenantId, opts.projectId, disc.screenId, profile, shotPath, gitInfo.branch, gitInfo.commit || void 0).catch(() => {
               });
               for (const el of vpElements) {
@@ -3569,7 +3642,15 @@ async function captureAuthDiscoveryScreen(page, opts, requestedUrl, source, ordi
   });
   const screen = await AppBrain.saveScreen(opts.tenantId, opts.projectId, name, screenUrl, opts.jobId, source === "entry" ? "ENTRY_POINT" : "SEED_URL").catch(() => null);
   if (!screen) return { captured: false, finalUrl, flowType };
-  await AppBrain.saveScreenshot(opts.tenantId, opts.projectId, screen.id, shotPath, screen.version, "DESKTOP").catch(() => {
+  let authCloudUrl;
+  if (opts.storageConfig) {
+    try {
+      const { uploadScreenshotToTenantStorage } = await import("@detiq/app-brain");
+      authCloudUrl = await uploadScreenshotToTenantStorage(opts.storageConfig, shotPath, `${opts.tenantId}/${opts.projectId}/screenshots/${path.basename(shotPath)}`);
+    } catch {
+    }
+  }
+  await AppBrain.saveScreenshot(opts.tenantId, opts.projectId, screen.id, shotPath, screen.version, "DESKTOP", authCloudUrl).catch(() => {
   });
   await AppBrain.saveCrawlEvidence(opts.tenantId, opts.projectId, {
     screenId: screen.id,