@detiq/crawler 0.2.14 → 1.0.1

package/dist/session-capture.d.ts

@@ -32,4 +32,5 @@ export declare function validateAndCaptureSession(appUrl: string, creds: {
     storageState: string;
     cookies: number;
     localStorage: number;
+    finalUrl: string;
 }>;

package/dist/session-capture.js

@@ -243,6 +243,7 @@ async function validateAndCaptureSession(appUrl, creds, llmConfig, captchaOpts)
     }
     await page.waitForLoadState("networkidle", { timeout: 8e3 }).catch(() => {
     });
+    const finalUrl = page.url();
     await exploreAppAfterLogin(page);
     const state = await context.storageState();
     const cookies = state.cookies?.length ?? 0;
@@ -253,7 +254,7 @@ async function validateAndCaptureSession(appUrl, creds, llmConfig, captchaOpts)
     if (cookies === 0 && localStorage === 0) {
       throw new Error("Login appeared to succeed but no session data was captured \u2014 try manual session capture instead");
     }
-    return { storageState: JSON.stringify(state), cookies, localStorage };
+    return { storageState: JSON.stringify(state), cookies, localStorage, finalUrl };
   } finally {
     await browser.close().catch(() => {
     });

package/dist/session-capture.js.map

@@ -1,7 +1,7 @@
 {
   "version": 3,
   "sources": ["../src/session-capture.ts"],
-  "sourcesContent": ["import { chromium } from 'playwright';\nimport { randomBytes } from 'node:crypto';\nimport { login } from './auth.js';\nimport { generateTOTP } from './job-queue.js';\n\ninterface CaptureSession {\n  browser: any;\n  context: any;\n  timer: ReturnType<typeof setTimeout>;\n}\n\nconst activeSessions = new Map<string, CaptureSession>();\n\n/**\n * Opens a headed (visible) browser at appUrl and returns a captureId.\n * The user logs in manually, then calls completeSessionCapture() to harvest storageState.\n * Sessions expire after 10 minutes if not completed.\n *\n * Throws 'HEADLESS_ENV' if running in Docker/Linux without a display server.\n * The API route surfaces this as a 503 so the frontend can switch to manual JSON paste.\n */\nexport async function startSessionCapture(appUrl: string): Promise<string> {\n  const inDocker = process.env.PLAYWRIGHT_IN_DOCKER === 'true';\n  const noDisplay = process.platform === 'linux' && !process.env.DISPLAY;\n  if (inDocker || noDisplay) {\n    throw new Error('HEADLESS_ENV');\n  }\n\n  const id = randomBytes(16).toString('hex');\n  const browser = await chromium.launch({ headless: false });\n  const context = await browser.newContext();\n  const page = await context.newPage();\n  await page.goto(appUrl, { waitUntil: 'domcontentloaded' }).catch(() => {});\n  const timer = setTimeout(() => cancelSessionCapture(id).catch(() => {}), 10 * 60 * 1000);\n  activeSessions.set(id, { browser, context, timer });\n  return id;\n}\n\n/** Captures storageState from the open browser, closes it, and returns the JSON string. */\nexport async function completeSessionCapture(captureId: string): Promise<string> {\n  const session = activeSessions.get(captureId);\n  if (!session) throw new Error('Capture session not found or expired.');\n  clearTimeout(session.timer);\n  const state = await session.context.storageState();\n  await session.browser.close().catch(() => {});\n  activeSessions.delete(captureId);\n  return JSON.stringify(state);\n}\n\n/** Closes the browser without capturing \u2014 called on Cancel. */\nexport async function cancelSessionCapture(captureId: string): Promise<void> {\n  const session = activeSessions.get(captureId);\n  if (!session) return;\n  clearTimeout(session.timer);\n  await session.browser.close().catch(() => {});\n  activeSessions.delete(captureId);\n}\n\n/**\n * Post-login exploration: navigates nav links, cycles tabs, advances carousels,\n * opens accordions, scrolls \u2014 so the browser accumulates full session state before capture.\n * All best-effort: never throws, never fails the capture.\n */\nasync function exploreAppAfterLogin(page: any): Promise<void> {\n  const sleep = (ms: number) => page.waitForTimeout(ms).catch(() => {});\n\n  const isVisible = `\n    function isVisible(el) {\n      const s = window.getComputedStyle(el);\n      if (s.display === 'none' || s.visibility === 'hidden' || parseFloat(s.opacity) < 0.1) return false;\n      const r = el.getBoundingClientRect();\n      return r.width > 4 && r.height > 4;\n    }\n  `;\n\n  try {\n    const origin = await page.evaluate(() => location.origin).catch(() => '');\n    const startUrl = await page.evaluate(() => location.href).catch(() => '');\n\n    // \u2500\u2500 1. Expand collapsed nav menus / hamburgers \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n    await page.evaluate(new Function(`\n      ${isVisible}\n      const inBadCtx = el => {\n        let p = el.parentElement;\n        while (p) {\n          const role = p.getAttribute('role') ?? '';\n          if (['dialog','alertdialog','combobox','listbox'].includes(role)) return true;\n          p = p.parentElement;\n        }\n        return false;\n      };\n      const targets = new Set();\n      document.querySelectorAll('[aria-haspopup]:not([aria-expanded=\"true\"])').forEach(el => {\n        if (isVisible(el) && !inBadCtx(el)) targets.add(el);\n      });\n      document.querySelectorAll('[aria-expanded=\"false\"]').forEach(el => {\n        if ((el.tagName === 'BUTTON' || el.getAttribute('role') === 'button') && isVisible(el) && !inBadCtx(el)) targets.add(el);\n      });\n      document.querySelectorAll('[data-state=\"closed\"],[data-headlessui-state=\"closed\"]').forEach(el => {\n        if ((el.tagName === 'BUTTON' || el.getAttribute('role') === 'button') && isVisible(el) && !inBadCtx(el)) targets.add(el);\n      });\n      targets.forEach(el => { try { el.click(); } catch {} });\n    `) as () => void).catch(() => {});\n    await sleep(600);\n\n    // \u2500\u2500 2. Collect nav links from the expanded DOM \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n    const navLinks: string[] = await page.evaluate(new Function(`\n      ${isVisible}\n      const links = new Set();\n      const navEls = document.querySelectorAll('nav a, [role=\"navigation\"] a, aside a, header a');\n      navEls.forEach(a => {\n        const href = a.getAttribute('href');\n        if (!href || href.startsWith('#') || href.startsWith('mailto:') || href.startsWith('tel:')) return;\n        if (!isVisible(a)) return;\n        try {\n          const url = new URL(href, location.href);\n          if (url.origin === location.origin) links.add(url.href);\n        } catch {}\n      });\n      return [...links].slice(0, 10); // cap at 10 nav pages\n    `) as () => string[]).catch(() => []);\n\n    // Dismiss opened menus before navigating\n    await page.keyboard?.press('Escape').catch(() => {});\n    await sleep(200);\n\n    // \u2500\u2500 3. Visit each nav link, interact, return \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n    for (const link of navLinks) {\n      try {\n        if (link === startUrl) continue;\n        await page.goto(link, { waitUntil: 'networkidle', timeout: 15_000 }).catch(() => {});\n        await sleep(400);\n        await interactWithCurrentPage(page, isVisible, sleep);\n      } catch { /* skip broken page */ }\n    }\n\n    // Return to start page\n    if (navLinks.length > 0) {\n      await page.goto(startUrl, { waitUntil: 'networkidle', timeout: 15_000 }).catch(() => {});\n      await sleep(400);\n    }\n\n    // \u2500\u2500 4. Final interaction pass on landing page \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n    await interactWithCurrentPage(page, isVisible, sleep);\n\n    await page.waitForLoadState('networkidle', { timeout: 5_000 }).catch(() => {});\n  } catch {\n    // Exploration is best-effort\n  }\n}\n\n/** Interacts with tabs, carousels, accordions, and scrolls on the current page. */\nasync function interactWithCurrentPage(page: any, isVisible: string, sleep: (ms: number) => Promise<void>): Promise<void> {\n  // Cycle through tabs\n  await page.evaluate(new Function(`\n    ${isVisible}\n    const tabs = [...document.querySelectorAll('[role=\"tab\"]')].filter(isVisible);\n    tabs.forEach(tab => { try { tab.click(); } catch {} });\n  `) as () => void).catch(() => {});\n  await sleep(400);\n\n  // Advance carousels (next buttons)\n  await page.evaluate(new Function(`\n    ${isVisible}\n    const nextBtns = [...document.querySelectorAll(\n      'button[aria-label*=\"next\" i], button[aria-label*=\"forward\" i], button[aria-label*=\"right\" i], ' +\n      '[class*=\"carousel\" i] button, [class*=\"slider\" i] button, [data-direction=\"next\"]'\n    )].filter(isVisible);\n    // Click next up to 3 times per carousel\n    for (let i = 0; i < 3; i++) {\n      nextBtns.forEach(btn => { try { btn.click(); } catch {} });\n    }\n  `) as () => void).catch(() => {});\n  await sleep(400);\n\n  // Open closed accordions / disclosure panels\n  await page.evaluate(new Function(`\n    ${isVisible}\n    document.querySelectorAll(\n      '[data-state=\"closed\"] button, [aria-expanded=\"false\"], ' +\n      'details:not([open]) > summary, [class*=\"accordion\" i] button'\n    ).forEach(el => {\n      if (isVisible(el)) { try { el.click(); } catch {} }\n    });\n  `) as () => void).catch(() => {});\n  await sleep(300);\n\n  // Scroll to bottom then back to top\n  await page.evaluate(() => { window.scrollTo({ top: document.body.scrollHeight, behavior: 'instant' }); }).catch(() => {});\n  await sleep(300);\n  await page.evaluate(() => { window.scrollTo({ top: 0, behavior: 'instant' }); }).catch(() => {});\n}\n\n/**\n * Headless login + session capture \u2014 works on Docker/cloud (no display needed).\n * Tries DOM-based login first (fast, free). If it fails and llmConfig is provided,\n * falls back to Stagehand AI-guided login (handles modals, multi-step, marketing pages).\n *\n * Returns { storageState, cookies, localStorage } on success.\n * Throws a human-readable error on failure.\n */\nexport async function validateAndCaptureSession(\n  appUrl: string,\n  creds: {\n    username: string; password: string; totpSecret?: string;\n    httpBasicUsername?: string; httpBasicPassword?: string;\n  },\n  llmConfig?: { modelName: string; apiKey: string },\n  captchaOpts?: { apiKey?: string; provider?: '2captcha' | 'capsolver' },\n): Promise<{ storageState: string; cookies: number; localStorage: number }> {\n  const browser = await chromium.launch({ headless: true });\n  try {\n    const contextOpts: any = {};\n    if (creds.httpBasicUsername && creds.httpBasicPassword) {\n      contextOpts.httpCredentials = { username: creds.httpBasicUsername, password: creds.httpBasicPassword };\n    }\n    const context = await browser.newContext(contextOpts);\n    const page = await context.newPage();\n\n    const getOneTimeCode = creds.totpSecret\n      ? async () => generateTOTP(creds.totpSecret!)\n      : undefined;\n\n    let loggedIn: boolean;\n    try {\n      loggedIn = await login(page, appUrl, {\n        username: creds.username,\n        password: creds.password,\n        getOneTimeCode,\n      });\n    } catch (loginErr: any) {\n      // login() throws when it can read a real error message from the page\n      const shotBuf = await page.screenshot({ type: 'jpeg', quality: 60, fullPage: false }).catch(() => null);\n      throw Object.assign(loginErr, {\n        failureScreenshot: shotBuf ? shotBuf.toString('base64') : null,\n      });\n    }\n\n    // Stagehand AI fallback when DOM login fails and LLM config is available\n    if (!loggedIn && llmConfig) {\n      let sh: any = null;\n      try {\n        const { Stagehand } = await import('@browserbasehq/stagehand');\n        const { loginWithStagehand } = await import('./ai-login.js');\n        sh = new Stagehand({\n          env: 'LOCAL',\n          model: { modelName: llmConfig.modelName as any, apiKey: llmConfig.apiKey },\n          serverCache: false,\n          verbose: 0,\n          localBrowserLaunchOptions: { headless: true },\n        });\n        await sh.init();\n        const shPage = await (sh as any).resolvePage();\n        await shPage.goto(appUrl, { waitUntil: 'networkidle', timeout: 30_000 }).catch(() => {});\n        const shLoggedIn = await loginWithStagehand(\n          sh,\n          { username: creds.username, password: creds.password, getOneTimeCode },\n          appUrl,\n          captchaOpts,\n        );\n        if (shLoggedIn) {\n          const shCtx = (sh as any).context ?? (sh as any).browserContext;\n          if (shCtx) {\n            const shState = await shCtx.storageState();\n            const shCookies = shState.cookies?.length ?? 0;\n            const shLocalStorage = shState.origins?.reduce((s: number, o: any) => s + (o.localStorage?.length ?? 0), 0) ?? 0;\n            if (shCookies > 0 || shLocalStorage > 0) {\n              await sh.close().catch(() => {});\n              sh = null;\n              return { storageState: JSON.stringify(shState), cookies: shCookies, localStorage: shLocalStorage };\n            }\n          }\n          loggedIn = true; // Stagehand succeeded but we'll capture from the plain browser context below\n        }\n      } catch (shErr: any) {\n        const msg = String(shErr?.message ?? shErr);\n        if (msg.includes('CAPTCHA') || shErr?.isSSOError) throw shErr;\n        // Stagehand failed silently \u2014 fall through to \"credentials rejected\"\n      } finally {\n        if (sh) await sh.close().catch(() => {});\n      }\n    }\n\n    if (!loggedIn) {\n      const shotBuf = await page.screenshot({ type: 'jpeg', quality: 60, fullPage: false }).catch(() => null);\n      const challenge = await page.evaluate(() => {\n        const text = (document.body?.innerText ?? '').toLowerCase();\n        const hasCloudflare = /verify you are human|checking your browser|cloudflare|cf-challenge|turnstile/.test(text) ||\n          !!document.querySelector('iframe[src*=\"challenges.cloudflare.com\"], .cf-turnstile, [data-sitekey]');\n        const hasCaptcha = hasCloudflare || /captcha|recaptcha|hcaptcha/.test(text) ||\n          !!document.querySelector('iframe[src*=\"recaptcha\"], iframe[src*=\"hcaptcha\"], #captcha, .captcha, [id*=\"captcha\"], [class*=\"captcha\"]');\n        return { hasCloudflare, hasCaptcha };\n      }).catch(() => ({ hasCloudflare: false, hasCaptcha: false }));\n      const message = challenge.hasCloudflare\n        ? 'Cloudflare human verification blocked headless validation. This cannot be bypassed reliably in code; use Live Browser or manual session capture, configure a CAPTCHA solver, or allowlist the crawler IP.'\n        : challenge.hasCaptcha\n          ? 'A CAPTCHA blocked headless validation. Configure CAPTCHA Auto-Solve or use Live Browser/manual session capture.'\n          : 'Login failed \u2014 credentials rejected or the login flow requires a manual/SSO step. Check credentials, saved login instructions, or use Live Browser session capture.';\n      const err = Object.assign(new Error(message), {\n        failureScreenshot: shotBuf ? shotBuf.toString('base64') : null,\n        reason: challenge.hasCloudflare ? 'CLOUDFLARE_CHALLENGE' : challenge.hasCaptcha ? 'CAPTCHA_CHALLENGE' : 'LOGIN_NOT_CONFIRMED',\n      });\n      throw err;\n    }\n\n    // Wait for post-login navigation to settle before capturing session\n    await page.waitForLoadState('networkidle', { timeout: 8_000 }).catch(() => {});\n\n    // Post-login exploration: navigate tabs, expand menus, scroll to trigger lazy-loaded state\n    await exploreAppAfterLogin(page);\n\n    const state = await context.storageState();\n    const cookies = state.cookies?.length ?? 0;\n    const localStorage = state.origins?.reduce(\n      (sum, o) => sum + (o.localStorage?.length ?? 0), 0\n    ) ?? 0;\n\n    if (cookies === 0 && localStorage === 0) {\n      throw new Error('Login appeared to succeed but no session data was captured \u2014 try manual session capture instead');\n    }\n\n    return { storageState: JSON.stringify(state), cookies, localStorage };\n  } finally {\n    await browser.close().catch(() => {});\n  }\n}\n"],
-  "mappings": "AAAA,SAAS,gBAAgB;AACzB,SAAS,mBAAmB;AAC5B,SAAS,aAAa;AACtB,SAAS,oBAAoB;AAQ7B,MAAM,iBAAiB,oBAAI,IAA4B;AAUvD,eAAsB,oBAAoB,QAAiC;AACzE,QAAM,WAAW,QAAQ,IAAI,yBAAyB;AACtD,QAAM,YAAY,QAAQ,aAAa,WAAW,CAAC,QAAQ,IAAI;AAC/D,MAAI,YAAY,WAAW;AACzB,UAAM,IAAI,MAAM,cAAc;AAAA,EAChC;AAEA,QAAM,KAAK,YAAY,EAAE,EAAE,SAAS,KAAK;AACzC,QAAM,UAAU,MAAM,SAAS,OAAO,EAAE,UAAU,MAAM,CAAC;AACzD,QAAM,UAAU,MAAM,QAAQ,WAAW;AACzC,QAAM,OAAO,MAAM,QAAQ,QAAQ;AACnC,QAAM,KAAK,KAAK,QAAQ,EAAE,WAAW,mBAAmB,CAAC,EAAE,MAAM,MAAM;AAAA,EAAC,CAAC;AACzE,QAAM,QAAQ,WAAW,MAAM,qBAAqB,EAAE,EAAE,MAAM,MAAM;AAAA,EAAC,CAAC,GAAG,KAAK,KAAK,GAAI;AACvF,iBAAe,IAAI,IAAI,EAAE,SAAS,SAAS,MAAM,CAAC;AAClD,SAAO;AACT;AAGA,eAAsB,uBAAuB,WAAoC;AAC/E,QAAM,UAAU,eAAe,IAAI,SAAS;AAC5C,MAAI,CAAC,QAAS,OAAM,IAAI,MAAM,uCAAuC;AACrE,eAAa,QAAQ,KAAK;AAC1B,QAAM,QAAQ,MAAM,QAAQ,QAAQ,aAAa;AACjD,QAAM,QAAQ,QAAQ,MAAM,EAAE,MAAM,MAAM;AAAA,EAAC,CAAC;AAC5C,iBAAe,OAAO,SAAS;AAC/B,SAAO,KAAK,UAAU,KAAK;AAC7B;AAGA,eAAsB,qBAAqB,WAAkC;AAC3E,QAAM,UAAU,eAAe,IAAI,SAAS;AAC5C,MAAI,CAAC,QAAS;AACd,eAAa,QAAQ,KAAK;AAC1B,QAAM,QAAQ,QAAQ,MAAM,EAAE,MAAM,MAAM;AAAA,EAAC,CAAC;AAC5C,iBAAe,OAAO,SAAS;AACjC;AAOA,eAAe,qBAAqB,MAA0B;AAC5D,QAAM,QAAQ,CAAC,OAAe,KAAK,eAAe,EAAE,EAAE,MAAM,MAAM;AAAA,EAAC,CAAC;AAEpE,QAAM,YAAY;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AASlB,MAAI;AACF,UAAM,SAAS,MAAM,KAAK,SAAS,MAAM,SAAS,MAAM,EAAE,MAAM,MAAM,EAAE;AACxE,UAAM,WAAW,MAAM,KAAK,SAAS,MAAM,SAAS,IAAI,EAAE,MAAM,MAAM,EAAE;AAGxE,UAAM,KAAK,SAAS,IAAI,SAAS;AAAA,QAC7B,SAAS;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,KAqBZ,CAAe,EAAE,MAAM,MAAM;AAAA,IAAC,CAAC;AAChC,UAAM,MAAM,GAAG;AAGf,UAAM,WAAqB,MAAM,KAAK,SAAS,IAAI,SAAS;AAAA,QACxD,SAAS;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,KAaZ,CAAmB,EAAE,MAAM,MAAM,CAAC,CAAC;AAGpC,UAAM,KAAK,UAAU,MAAM,QAAQ,EAAE,MAAM,MAAM;AAAA,IAAC,CAAC;AACnD,UAAM,MAAM,GAAG;AAGf,eAAW,QAAQ,UAAU;AAC3B,UAAI;AACF,YAAI,SAAS,SAAU;AACvB,cAAM,KAAK,KAAK,MAAM,EAAE,WAAW,eAAe,SAAS,KAAO,CAAC,EAAE,MAAM,MAAM;AAAA,QAAC,CAAC;AACnF,cAAM,MAAM,GAAG;AACf,cAAM,wBAAwB,MAAM,WAAW,KAAK;AAAA,MACtD,QAAQ;AAAA,MAAyB;AAAA,IACnC;AAGA,QAAI,SAAS,SAAS,GAAG;AACvB,YAAM,KAAK,KAAK,UAAU,EAAE,WAAW,eAAe,SAAS,KAAO,CAAC,EAAE,MAAM,MAAM;AAAA,MAAC,CAAC;AACvF,YAAM,MAAM,GAAG;AAAA,IACjB;AAGA,UAAM,wBAAwB,MAAM,WAAW,KAAK;AAEpD,UAAM,KAAK,iBAAiB,eAAe,EAAE,SAAS,IAAM,CAAC,EAAE,MAAM,MAAM;AAAA,IAAC,CAAC;AAAA,EAC/E,QAAQ;AAAA,EAER;AACF;AAGA,eAAe,wBAAwB,MAAW,WAAmB,OAAqD;AAExH,QAAM,KAAK,SAAS,IAAI,SAAS;AAAA,MAC7B,SAAS;AAAA;AAAA;AAAA,GAGZ,CAAe,EAAE,MAAM,MAAM;AAAA,EAAC,CAAC;AAChC,QAAM,MAAM,GAAG;AAGf,QAAM,KAAK,SAAS,IAAI,SAAS;AAAA,MAC7B,SAAS;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,GASZ,CAAe,EAAE,MAAM,MAAM;AAAA,EAAC,CAAC;AAChC,QAAM,MAAM,GAAG;AAGf,QAAM,KAAK,SAAS,IAAI,SAAS;AAAA,MAC7B,SAAS;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,GAOZ,CAAe,EAAE,MAAM,MAAM;AAAA,EAAC,CAAC;AAChC,QAAM,MAAM,GAAG;AAGf,QAAM,KAAK,SAAS,MAAM;AAAE,WAAO,SAAS,EAAE,KAAK,SAAS,KAAK,cAAc,UAAU,UAAU,CAAC;AAAA,EAAG,CAAC,EAAE,MAAM,MAAM;AAAA,EAAC,CAAC;AACxH,QAAM,MAAM,GAAG;AACf,QAAM,KAAK,SAAS,MAAM;AAAE,WAAO,SAAS,EAAE,KAAK,GAAG,UAAU,UAAU,CAAC;AAAA,EAAG,CAAC,EAAE,MAAM,MAAM;AAAA,EAAC,CAAC;AACjG;AAUA,eAAsB,0BACpB,QACA,OAIA,WACA,aAC0E;AAC1E,QAAM,UAAU,MAAM,SAAS,OAAO,EAAE,UAAU,KAAK,CAAC;AACxD,MAAI;AACF,UAAM,cAAmB,CAAC;AAC1B,QAAI,MAAM,qBAAqB,MAAM,mBAAmB;AACtD,kBAAY,kBAAkB,EAAE,UAAU,MAAM,mBAAmB,UAAU,MAAM,kBAAkB;AAAA,IACvG;AACA,UAAM,UAAU,MAAM,QAAQ,WAAW,WAAW;AACpD,UAAM,OAAO,MAAM,QAAQ,QAAQ;AAEnC,UAAM,iBAAiB,MAAM,aACzB,YAAY,aAAa,MAAM,UAAW,IAC1C;AAEJ,QAAI;AACJ,QAAI;AACF,iBAAW,MAAM,MAAM,MAAM,QAAQ;AAAA,QACnC,UAAU,MAAM;AAAA,QAChB,UAAU,MAAM;AAAA,QAChB;AAAA,MACF,CAAC;AAAA,IACH,SAAS,UAAe;AAEtB,YAAM,UAAU,MAAM,KAAK,WAAW,EAAE,MAAM,QAAQ,SAAS,IAAI,UAAU,MAAM,CAAC,EAAE,MAAM,MAAM,IAAI;AACtG,YAAM,OAAO,OAAO,UAAU;AAAA,QAC5B,mBAAmB,UAAU,QAAQ,SAAS,QAAQ,IAAI;AAAA,MAC5D,CAAC;AAAA,IACH;AAGA,QAAI,CAAC,YAAY,WAAW;AAC1B,UAAI,KAAU;AACd,UAAI;AACF,cAAM,EAAE,UAAU,IAAI,MAAM,OAAO,0BAA0B;AAC7D,cAAM,EAAE,mBAAmB,IAAI,MAAM,OAAO,eAAe;AAC3D,aAAK,IAAI,UAAU;AAAA,UACjB,KAAK;AAAA,UACL,OAAO,EAAE,WAAW,UAAU,WAAkB,QAAQ,UAAU,OAAO;AAAA,UACzE,aAAa;AAAA,UACb,SAAS;AAAA,UACT,2BAA2B,EAAE,UAAU,KAAK;AAAA,QAC9C,CAAC;AACD,cAAM,GAAG,KAAK;AACd,cAAM,SAAS,MAAO,GAAW,YAAY;AAC7C,cAAM,OAAO,KAAK,QAAQ,EAAE,WAAW,eAAe,SAAS,IAAO,CAAC,EAAE,MAAM,MAAM;AAAA,QAAC,CAAC;AACvF,cAAM,aAAa,MAAM;AAAA,UACvB;AAAA,UACA,EAAE,UAAU,MAAM,UAAU,UAAU,MAAM,UAAU,eAAe;AAAA,UACrE;AAAA,UACA;AAAA,QACF;AACA,YAAI,YAAY;AACd,gBAAM,QAAS,GAAW,WAAY,GAAW;AACjD,cAAI,OAAO;AACT,kBAAM,UAAU,MAAM,MAAM,aAAa;AACzC,kBAAM,YAAY,QAAQ,SAAS,UAAU;AAC7C,kBAAM,iBAAiB,QAAQ,SAAS,OAAO,CAAC,GAAW,MAAW,KAAK,EAAE,cAAc,UAAU,IAAI,CAAC,KAAK;AAC/G,gBAAI,YAAY,KAAK,iBAAiB,GAAG;AACvC,oBAAM,GAAG,MAAM,EAAE,MAAM,MAAM;AAAA,cAAC,CAAC;AAC/B,mBAAK;AACL,qBAAO,EAAE,cAAc,KAAK,UAAU,OAAO,GAAG,SAAS,WAAW,cAAc,eAAe;AAAA,YACnG;AAAA,UACF;AACA,qBAAW;AAAA,QACb;AAAA,MACF,SAAS,OAAY;AACnB,cAAM,MAAM,OAAO,OAAO,WAAW,KAAK;AAC1C,YAAI,IAAI,SAAS,SAAS,KAAK,OAAO,WAAY,OAAM;AAAA,MAE1D,UAAE;AACA,YAAI,GAAI,OAAM,GAAG,MAAM,EAAE,MAAM,MAAM;AAAA,QAAC,CAAC;AAAA,MACzC;AAAA,IACF;AAEA,QAAI,CAAC,UAAU;AACb,YAAM,UAAU,MAAM,KAAK,WAAW,EAAE,MAAM,QAAQ,SAAS,IAAI,UAAU,MAAM,CAAC,EAAE,MAAM,MAAM,IAAI;AACtG,YAAM,YAAY,MAAM,KAAK,SAAS,MAAM;AAC1C,cAAM,QAAQ,SAAS,MAAM,aAAa,IAAI,YAAY;AAC1D,cAAM,gBAAgB,+EAA+E,KAAK,IAAI,KAC5G,CAAC,CAAC,SAAS,cAAc,yEAAyE;AACpG,cAAM,aAAa,iBAAiB,6BAA6B,KAAK,IAAI,KACxE,CAAC,CAAC,SAAS,cAAc,4GAA4G;AACvI,eAAO,EAAE,eAAe,WAAW;AAAA,MACrC,CAAC,EAAE,MAAM,OAAO,EAAE,eAAe,OAAO,YAAY,MAAM,EAAE;AAC5D,YAAM,UAAU,UAAU,gBACtB,8MACA,UAAU,aACR,oHACA;AACN,YAAM,MAAM,OAAO,OAAO,IAAI,MAAM,OAAO,GAAG;AAAA,QAC5C,mBAAmB,UAAU,QAAQ,SAAS,QAAQ,IAAI;AAAA,QAC1D,QAAQ,UAAU,gBAAgB,yBAAyB,UAAU,aAAa,sBAAsB;AAAA,MAC1G,CAAC;AACD,YAAM;AAAA,IACR;AAGA,UAAM,KAAK,iBAAiB,eAAe,EAAE,SAAS,IAAM,CAAC,EAAE,MAAM,MAAM;AAAA,IAAC,CAAC;AAG7E,UAAM,qBAAqB,IAAI;AAE/B,UAAM,QAAQ,MAAM,QAAQ,aAAa;AACzC,UAAM,UAAU,MAAM,SAAS,UAAU;AACzC,UAAM,eAAe,MAAM,SAAS;AAAA,MAClC,CAAC,KAAK,MAAM,OAAO,EAAE,cAAc,UAAU;AAAA,MAAI;AAAA,IACnD,KAAK;AAEL,QAAI,YAAY,KAAK,iBAAiB,GAAG;AACvC,YAAM,IAAI,MAAM,sGAAiG;AAAA,IACnH;AAEA,WAAO,EAAE,cAAc,KAAK,UAAU,KAAK,GAAG,SAAS,aAAa;AAAA,EACtE,UAAE;AACA,UAAM,QAAQ,MAAM,EAAE,MAAM,MAAM;AAAA,IAAC,CAAC;AAAA,EACtC;AACF;",
+  "sourcesContent": ["import { chromium } from 'playwright';\nimport { randomBytes } from 'node:crypto';\nimport { login } from './auth.js';\nimport { generateTOTP } from './job-queue.js';\n\ninterface CaptureSession {\n  browser: any;\n  context: any;\n  timer: ReturnType<typeof setTimeout>;\n}\n\nconst activeSessions = new Map<string, CaptureSession>();\n\n/**\n * Opens a headed (visible) browser at appUrl and returns a captureId.\n * The user logs in manually, then calls completeSessionCapture() to harvest storageState.\n * Sessions expire after 10 minutes if not completed.\n *\n * Throws 'HEADLESS_ENV' if running in Docker/Linux without a display server.\n * The API route surfaces this as a 503 so the frontend can switch to manual JSON paste.\n */\nexport async function startSessionCapture(appUrl: string): Promise<string> {\n  const inDocker = process.env.PLAYWRIGHT_IN_DOCKER === 'true';\n  const noDisplay = process.platform === 'linux' && !process.env.DISPLAY;\n  if (inDocker || noDisplay) {\n    throw new Error('HEADLESS_ENV');\n  }\n\n  const id = randomBytes(16).toString('hex');\n  const browser = await chromium.launch({ headless: false });\n  const context = await browser.newContext();\n  const page = await context.newPage();\n  await page.goto(appUrl, { waitUntil: 'domcontentloaded' }).catch(() => {});\n  const timer = setTimeout(() => cancelSessionCapture(id).catch(() => {}), 10 * 60 * 1000);\n  activeSessions.set(id, { browser, context, timer });\n  return id;\n}\n\n/** Captures storageState from the open browser, closes it, and returns the JSON string. */\nexport async function completeSessionCapture(captureId: string): Promise<string> {\n  const session = activeSessions.get(captureId);\n  if (!session) throw new Error('Capture session not found or expired.');\n  clearTimeout(session.timer);\n  const state = await session.context.storageState();\n  await session.browser.close().catch(() => {});\n  activeSessions.delete(captureId);\n  return JSON.stringify(state);\n}\n\n/** Closes the browser without capturing \u2014 called on Cancel. */\nexport async function cancelSessionCapture(captureId: string): Promise<void> {\n  const session = activeSessions.get(captureId);\n  if (!session) return;\n  clearTimeout(session.timer);\n  await session.browser.close().catch(() => {});\n  activeSessions.delete(captureId);\n}\n\n/**\n * Post-login exploration: navigates nav links, cycles tabs, advances carousels,\n * opens accordions, scrolls \u2014 so the browser accumulates full session state before capture.\n * All best-effort: never throws, never fails the capture.\n */\nasync function exploreAppAfterLogin(page: any): Promise<void> {\n  const sleep = (ms: number) => page.waitForTimeout(ms).catch(() => {});\n\n  const isVisible = `\n    function isVisible(el) {\n      const s = window.getComputedStyle(el);\n      if (s.display === 'none' || s.visibility === 'hidden' || parseFloat(s.opacity) < 0.1) return false;\n      const r = el.getBoundingClientRect();\n      return r.width > 4 && r.height > 4;\n    }\n  `;\n\n  try {\n    const origin = await page.evaluate(() => location.origin).catch(() => '');\n    const startUrl = await page.evaluate(() => location.href).catch(() => '');\n\n    // \u2500\u2500 1. Expand collapsed nav menus / hamburgers \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n    await page.evaluate(new Function(`\n      ${isVisible}\n      const inBadCtx = el => {\n        let p = el.parentElement;\n        while (p) {\n          const role = p.getAttribute('role') ?? '';\n          if (['dialog','alertdialog','combobox','listbox'].includes(role)) return true;\n          p = p.parentElement;\n        }\n        return false;\n      };\n      const targets = new Set();\n      document.querySelectorAll('[aria-haspopup]:not([aria-expanded=\"true\"])').forEach(el => {\n        if (isVisible(el) && !inBadCtx(el)) targets.add(el);\n      });\n      document.querySelectorAll('[aria-expanded=\"false\"]').forEach(el => {\n        if ((el.tagName === 'BUTTON' || el.getAttribute('role') === 'button') && isVisible(el) && !inBadCtx(el)) targets.add(el);\n      });\n      document.querySelectorAll('[data-state=\"closed\"],[data-headlessui-state=\"closed\"]').forEach(el => {\n        if ((el.tagName === 'BUTTON' || el.getAttribute('role') === 'button') && isVisible(el) && !inBadCtx(el)) targets.add(el);\n      });\n      targets.forEach(el => { try { el.click(); } catch {} });\n    `) as () => void).catch(() => {});\n    await sleep(600);\n\n    // \u2500\u2500 2. Collect nav links from the expanded DOM \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n    const navLinks: string[] = await page.evaluate(new Function(`\n      ${isVisible}\n      const links = new Set();\n      const navEls = document.querySelectorAll('nav a, [role=\"navigation\"] a, aside a, header a');\n      navEls.forEach(a => {\n        const href = a.getAttribute('href');\n        if (!href || href.startsWith('#') || href.startsWith('mailto:') || href.startsWith('tel:')) return;\n        if (!isVisible(a)) return;\n        try {\n          const url = new URL(href, location.href);\n          if (url.origin === location.origin) links.add(url.href);\n        } catch {}\n      });\n      return [...links].slice(0, 10); // cap at 10 nav pages\n    `) as () => string[]).catch(() => []);\n\n    // Dismiss opened menus before navigating\n    await page.keyboard?.press('Escape').catch(() => {});\n    await sleep(200);\n\n    // \u2500\u2500 3. Visit each nav link, interact, return \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n    for (const link of navLinks) {\n      try {\n        if (link === startUrl) continue;\n        await page.goto(link, { waitUntil: 'networkidle', timeout: 15_000 }).catch(() => {});\n        await sleep(400);\n        await interactWithCurrentPage(page, isVisible, sleep);\n      } catch { /* skip broken page */ }\n    }\n\n    // Return to start page\n    if (navLinks.length > 0) {\n      await page.goto(startUrl, { waitUntil: 'networkidle', timeout: 15_000 }).catch(() => {});\n      await sleep(400);\n    }\n\n    // \u2500\u2500 4. Final interaction pass on landing page \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n    await interactWithCurrentPage(page, isVisible, sleep);\n\n    await page.waitForLoadState('networkidle', { timeout: 5_000 }).catch(() => {});\n  } catch {\n    // Exploration is best-effort\n  }\n}\n\n/** Interacts with tabs, carousels, accordions, and scrolls on the current page. */\nasync function interactWithCurrentPage(page: any, isVisible: string, sleep: (ms: number) => Promise<void>): Promise<void> {\n  // Cycle through tabs\n  await page.evaluate(new Function(`\n    ${isVisible}\n    const tabs = [...document.querySelectorAll('[role=\"tab\"]')].filter(isVisible);\n    tabs.forEach(tab => { try { tab.click(); } catch {} });\n  `) as () => void).catch(() => {});\n  await sleep(400);\n\n  // Advance carousels (next buttons)\n  await page.evaluate(new Function(`\n    ${isVisible}\n    const nextBtns = [...document.querySelectorAll(\n      'button[aria-label*=\"next\" i], button[aria-label*=\"forward\" i], button[aria-label*=\"right\" i], ' +\n      '[class*=\"carousel\" i] button, [class*=\"slider\" i] button, [data-direction=\"next\"]'\n    )].filter(isVisible);\n    // Click next up to 3 times per carousel\n    for (let i = 0; i < 3; i++) {\n      nextBtns.forEach(btn => { try { btn.click(); } catch {} });\n    }\n  `) as () => void).catch(() => {});\n  await sleep(400);\n\n  // Open closed accordions / disclosure panels\n  await page.evaluate(new Function(`\n    ${isVisible}\n    document.querySelectorAll(\n      '[data-state=\"closed\"] button, [aria-expanded=\"false\"], ' +\n      'details:not([open]) > summary, [class*=\"accordion\" i] button'\n    ).forEach(el => {\n      if (isVisible(el)) { try { el.click(); } catch {} }\n    });\n  `) as () => void).catch(() => {});\n  await sleep(300);\n\n  // Scroll to bottom then back to top\n  await page.evaluate(() => { window.scrollTo({ top: document.body.scrollHeight, behavior: 'instant' }); }).catch(() => {});\n  await sleep(300);\n  await page.evaluate(() => { window.scrollTo({ top: 0, behavior: 'instant' }); }).catch(() => {});\n}\n\n/**\n * Headless login + session capture \u2014 works on Docker/cloud (no display needed).\n * Tries DOM-based login first (fast, free). If it fails and llmConfig is provided,\n * falls back to Stagehand AI-guided login (handles modals, multi-step, marketing pages).\n *\n * Returns { storageState, cookies, localStorage } on success.\n * Throws a human-readable error on failure.\n */\nexport async function validateAndCaptureSession(\n  appUrl: string,\n  creds: {\n    username: string; password: string; totpSecret?: string;\n    httpBasicUsername?: string; httpBasicPassword?: string;\n  },\n  llmConfig?: { modelName: string; apiKey: string },\n  captchaOpts?: { apiKey?: string; provider?: '2captcha' | 'capsolver' },\n): Promise<{ storageState: string; cookies: number; localStorage: number; finalUrl?: string }> {\n  const browser = await chromium.launch({ headless: true });\n  try {\n    const contextOpts: any = {};\n    if (creds.httpBasicUsername && creds.httpBasicPassword) {\n      contextOpts.httpCredentials = { username: creds.httpBasicUsername, password: creds.httpBasicPassword };\n    }\n    const context = await browser.newContext(contextOpts);\n    const page = await context.newPage();\n\n    const getOneTimeCode = creds.totpSecret\n      ? async () => generateTOTP(creds.totpSecret!)\n      : undefined;\n\n    let loggedIn: boolean;\n    try {\n      loggedIn = await login(page, appUrl, {\n        username: creds.username,\n        password: creds.password,\n        getOneTimeCode,\n      });\n    } catch (loginErr: any) {\n      // login() throws when it can read a real error message from the page\n      const shotBuf = await page.screenshot({ type: 'jpeg', quality: 60, fullPage: false }).catch(() => null);\n      throw Object.assign(loginErr, {\n        failureScreenshot: shotBuf ? shotBuf.toString('base64') : null,\n      });\n    }\n\n    // Stagehand AI fallback when DOM login fails and LLM config is available\n    if (!loggedIn && llmConfig) {\n      let sh: any = null;\n      try {\n        const { Stagehand } = await import('@browserbasehq/stagehand');\n        const { loginWithStagehand } = await import('./ai-login.js');\n        sh = new Stagehand({\n          env: 'LOCAL',\n          model: { modelName: llmConfig.modelName as any, apiKey: llmConfig.apiKey },\n          serverCache: false,\n          verbose: 0,\n          localBrowserLaunchOptions: { headless: true },\n        });\n        await sh.init();\n        const shPage = await (sh as any).resolvePage();\n        await shPage.goto(appUrl, { waitUntil: 'networkidle', timeout: 30_000 }).catch(() => {});\n        const shLoggedIn = await loginWithStagehand(\n          sh,\n          { username: creds.username, password: creds.password, getOneTimeCode },\n          appUrl,\n          captchaOpts,\n        );\n        if (shLoggedIn) {\n          const shCtx = (sh as any).context ?? (sh as any).browserContext;\n          if (shCtx) {\n            const shState = await shCtx.storageState();\n            const shCookies = shState.cookies?.length ?? 0;\n            const shLocalStorage = shState.origins?.reduce((s: number, o: any) => s + (o.localStorage?.length ?? 0), 0) ?? 0;\n            if (shCookies > 0 || shLocalStorage > 0) {\n              await sh.close().catch(() => {});\n              sh = null;\n              return { storageState: JSON.stringify(shState), cookies: shCookies, localStorage: shLocalStorage };\n            }\n          }\n          loggedIn = true; // Stagehand succeeded but we'll capture from the plain browser context below\n        }\n      } catch (shErr: any) {\n        const msg = String(shErr?.message ?? shErr);\n        if (msg.includes('CAPTCHA') || shErr?.isSSOError) throw shErr;\n        // Stagehand failed silently \u2014 fall through to \"credentials rejected\"\n      } finally {\n        if (sh) await sh.close().catch(() => {});\n      }\n    }\n\n    if (!loggedIn) {\n      const shotBuf = await page.screenshot({ type: 'jpeg', quality: 60, fullPage: false }).catch(() => null);\n      const challenge = await page.evaluate(() => {\n        const text = (document.body?.innerText ?? '').toLowerCase();\n        const hasCloudflare = /verify you are human|checking your browser|cloudflare|cf-challenge|turnstile/.test(text) ||\n          !!document.querySelector('iframe[src*=\"challenges.cloudflare.com\"], .cf-turnstile, [data-sitekey]');\n        const hasCaptcha = hasCloudflare || /captcha|recaptcha|hcaptcha/.test(text) ||\n          !!document.querySelector('iframe[src*=\"recaptcha\"], iframe[src*=\"hcaptcha\"], #captcha, .captcha, [id*=\"captcha\"], [class*=\"captcha\"]');\n        return { hasCloudflare, hasCaptcha };\n      }).catch(() => ({ hasCloudflare: false, hasCaptcha: false }));\n      const message = challenge.hasCloudflare\n        ? 'Cloudflare human verification blocked headless validation. This cannot be bypassed reliably in code; use Live Browser or manual session capture, configure a CAPTCHA solver, or allowlist the crawler IP.'\n        : challenge.hasCaptcha\n          ? 'A CAPTCHA blocked headless validation. Configure CAPTCHA Auto-Solve or use Live Browser/manual session capture.'\n          : 'Login failed \u2014 credentials rejected or the login flow requires a manual/SSO step. Check credentials, saved login instructions, or use Live Browser session capture.';\n      const err = Object.assign(new Error(message), {\n        failureScreenshot: shotBuf ? shotBuf.toString('base64') : null,\n        reason: challenge.hasCloudflare ? 'CLOUDFLARE_CHALLENGE' : challenge.hasCaptcha ? 'CAPTCHA_CHALLENGE' : 'LOGIN_NOT_CONFIRMED',\n      });\n      throw err;\n    }\n\n    // Wait for post-login navigation to settle before capturing session\n    await page.waitForLoadState('networkidle', { timeout: 8_000 }).catch(() => {});\n\n    // Capture post-login landing URL before exploration navigates away\n    const finalUrl = page.url();\n\n    // Post-login exploration: navigate tabs, expand menus, scroll to trigger lazy-loaded state\n    await exploreAppAfterLogin(page);\n\n    const state = await context.storageState();\n    const cookies = state.cookies?.length ?? 0;\n    const localStorage = state.origins?.reduce(\n      (sum, o) => sum + (o.localStorage?.length ?? 0), 0\n    ) ?? 0;\n\n    if (cookies === 0 && localStorage === 0) {\n      throw new Error('Login appeared to succeed but no session data was captured \u2014 try manual session capture instead');\n    }\n\n    return { storageState: JSON.stringify(state), cookies, localStorage, finalUrl };\n  } finally {\n    await browser.close().catch(() => {});\n  }\n}\n"],
+  "mappings": "AAAA,SAAS,gBAAgB;AACzB,SAAS,mBAAmB;AAC5B,SAAS,aAAa;AACtB,SAAS,oBAAoB;AAQ7B,MAAM,iBAAiB,oBAAI,IAA4B;AAUvD,eAAsB,oBAAoB,QAAiC;AACzE,QAAM,WAAW,QAAQ,IAAI,yBAAyB;AACtD,QAAM,YAAY,QAAQ,aAAa,WAAW,CAAC,QAAQ,IAAI;AAC/D,MAAI,YAAY,WAAW;AACzB,UAAM,IAAI,MAAM,cAAc;AAAA,EAChC;AAEA,QAAM,KAAK,YAAY,EAAE,EAAE,SAAS,KAAK;AACzC,QAAM,UAAU,MAAM,SAAS,OAAO,EAAE,UAAU,MAAM,CAAC;AACzD,QAAM,UAAU,MAAM,QAAQ,WAAW;AACzC,QAAM,OAAO,MAAM,QAAQ,QAAQ;AACnC,QAAM,KAAK,KAAK,QAAQ,EAAE,WAAW,mBAAmB,CAAC,EAAE,MAAM,MAAM;AAAA,EAAC,CAAC;AACzE,QAAM,QAAQ,WAAW,MAAM,qBAAqB,EAAE,EAAE,MAAM,MAAM;AAAA,EAAC,CAAC,GAAG,KAAK,KAAK,GAAI;AACvF,iBAAe,IAAI,IAAI,EAAE,SAAS,SAAS,MAAM,CAAC;AAClD,SAAO;AACT;AAGA,eAAsB,uBAAuB,WAAoC;AAC/E,QAAM,UAAU,eAAe,IAAI,SAAS;AAC5C,MAAI,CAAC,QAAS,OAAM,IAAI,MAAM,uCAAuC;AACrE,eAAa,QAAQ,KAAK;AAC1B,QAAM,QAAQ,MAAM,QAAQ,QAAQ,aAAa;AACjD,QAAM,QAAQ,QAAQ,MAAM,EAAE,MAAM,MAAM;AAAA,EAAC,CAAC;AAC5C,iBAAe,OAAO,SAAS;AAC/B,SAAO,KAAK,UAAU,KAAK;AAC7B;AAGA,eAAsB,qBAAqB,WAAkC;AAC3E,QAAM,UAAU,eAAe,IAAI,SAAS;AAC5C,MAAI,CAAC,QAAS;AACd,eAAa,QAAQ,KAAK;AAC1B,QAAM,QAAQ,QAAQ,MAAM,EAAE,MAAM,MAAM;AAAA,EAAC,CAAC;AAC5C,iBAAe,OAAO,SAAS;AACjC;AAOA,eAAe,qBAAqB,MAA0B;AAC5D,QAAM,QAAQ,CAAC,OAAe,KAAK,eAAe,EAAE,EAAE,MAAM,MAAM;AAAA,EAAC,CAAC;AAEpE,QAAM,YAAY;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AASlB,MAAI;AACF,UAAM,SAAS,MAAM,KAAK,SAAS,MAAM,SAAS,MAAM,EAAE,MAAM,MAAM,EAAE;AACxE,UAAM,WAAW,MAAM,KAAK,SAAS,MAAM,SAAS,IAAI,EAAE,MAAM,MAAM,EAAE;AAGxE,UAAM,KAAK,SAAS,IAAI,SAAS;AAAA,QAC7B,SAAS;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,KAqBZ,CAAe,EAAE,MAAM,MAAM;AAAA,IAAC,CAAC;AAChC,UAAM,MAAM,GAAG;AAGf,UAAM,WAAqB,MAAM,KAAK,SAAS,IAAI,SAAS;AAAA,QACxD,SAAS;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,KAaZ,CAAmB,EAAE,MAAM,MAAM,CAAC,CAAC;AAGpC,UAAM,KAAK,UAAU,MAAM,QAAQ,EAAE,MAAM,MAAM;AAAA,IAAC,CAAC;AACnD,UAAM,MAAM,GAAG;AAGf,eAAW,QAAQ,UAAU;AAC3B,UAAI;AACF,YAAI,SAAS,SAAU;AACvB,cAAM,KAAK,KAAK,MAAM,EAAE,WAAW,eAAe,SAAS,KAAO,CAAC,EAAE,MAAM,MAAM;AAAA,QAAC,CAAC;AACnF,cAAM,MAAM,GAAG;AACf,cAAM,wBAAwB,MAAM,WAAW,KAAK;AAAA,MACtD,QAAQ;AAAA,MAAyB;AAAA,IACnC;AAGA,QAAI,SAAS,SAAS,GAAG;AACvB,YAAM,KAAK,KAAK,UAAU,EAAE,WAAW,eAAe,SAAS,KAAO,CAAC,EAAE,MAAM,MAAM;AAAA,MAAC,CAAC;AACvF,YAAM,MAAM,GAAG;AAAA,IACjB;AAGA,UAAM,wBAAwB,MAAM,WAAW,KAAK;AAEpD,UAAM,KAAK,iBAAiB,eAAe,EAAE,SAAS,IAAM,CAAC,EAAE,MAAM,MAAM;AAAA,IAAC,CAAC;AAAA,EAC/E,QAAQ;AAAA,EAER;AACF;AAGA,eAAe,wBAAwB,MAAW,WAAmB,OAAqD;AAExH,QAAM,KAAK,SAAS,IAAI,SAAS;AAAA,MAC7B,SAAS;AAAA;AAAA;AAAA,GAGZ,CAAe,EAAE,MAAM,MAAM;AAAA,EAAC,CAAC;AAChC,QAAM,MAAM,GAAG;AAGf,QAAM,KAAK,SAAS,IAAI,SAAS;AAAA,MAC7B,SAAS;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,GASZ,CAAe,EAAE,MAAM,MAAM;AAAA,EAAC,CAAC;AAChC,QAAM,MAAM,GAAG;AAGf,QAAM,KAAK,SAAS,IAAI,SAAS;AAAA,MAC7B,SAAS;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,GAOZ,CAAe,EAAE,MAAM,MAAM;AAAA,EAAC,CAAC;AAChC,QAAM,MAAM,GAAG;AAGf,QAAM,KAAK,SAAS,MAAM;AAAE,WAAO,SAAS,EAAE,KAAK,SAAS,KAAK,cAAc,UAAU,UAAU,CAAC;AAAA,EAAG,CAAC,EAAE,MAAM,MAAM;AAAA,EAAC,CAAC;AACxH,QAAM,MAAM,GAAG;AACf,QAAM,KAAK,SAAS,MAAM;AAAE,WAAO,SAAS,EAAE,KAAK,GAAG,UAAU,UAAU,CAAC;AAAA,EAAG,CAAC,EAAE,MAAM,MAAM;AAAA,EAAC,CAAC;AACjG;AAUA,eAAsB,0BACpB,QACA,OAIA,WACA,aAC6F;AAC7F,QAAM,UAAU,MAAM,SAAS,OAAO,EAAE,UAAU,KAAK,CAAC;AACxD,MAAI;AACF,UAAM,cAAmB,CAAC;AAC1B,QAAI,MAAM,qBAAqB,MAAM,mBAAmB;AACtD,kBAAY,kBAAkB,EAAE,UAAU,MAAM,mBAAmB,UAAU,MAAM,kBAAkB;AAAA,IACvG;AACA,UAAM,UAAU,MAAM,QAAQ,WAAW,WAAW;AACpD,UAAM,OAAO,MAAM,QAAQ,QAAQ;AAEnC,UAAM,iBAAiB,MAAM,aACzB,YAAY,aAAa,MAAM,UAAW,IAC1C;AAEJ,QAAI;AACJ,QAAI;AACF,iBAAW,MAAM,MAAM,MAAM,QAAQ;AAAA,QACnC,UAAU,MAAM;AAAA,QAChB,UAAU,MAAM;AAAA,QAChB;AAAA,MACF,CAAC;AAAA,IACH,SAAS,UAAe;AAEtB,YAAM,UAAU,MAAM,KAAK,WAAW,EAAE,MAAM,QAAQ,SAAS,IAAI,UAAU,MAAM,CAAC,EAAE,MAAM,MAAM,IAAI;AACtG,YAAM,OAAO,OAAO,UAAU;AAAA,QAC5B,mBAAmB,UAAU,QAAQ,SAAS,QAAQ,IAAI;AAAA,MAC5D,CAAC;AAAA,IACH;AAGA,QAAI,CAAC,YAAY,WAAW;AAC1B,UAAI,KAAU;AACd,UAAI;AACF,cAAM,EAAE,UAAU,IAAI,MAAM,OAAO,0BAA0B;AAC7D,cAAM,EAAE,mBAAmB,IAAI,MAAM,OAAO,eAAe;AAC3D,aAAK,IAAI,UAAU;AAAA,UACjB,KAAK;AAAA,UACL,OAAO,EAAE,WAAW,UAAU,WAAkB,QAAQ,UAAU,OAAO;AAAA,UACzE,aAAa;AAAA,UACb,SAAS;AAAA,UACT,2BAA2B,EAAE,UAAU,KAAK;AAAA,QAC9C,CAAC;AACD,cAAM,GAAG,KAAK;AACd,cAAM,SAAS,MAAO,GAAW,YAAY;AAC7C,cAAM,OAAO,KAAK,QAAQ,EAAE,WAAW,eAAe,SAAS,IAAO,CAAC,EAAE,MAAM,MAAM;AAAA,QAAC,CAAC;AACvF,cAAM,aAAa,MAAM;AAAA,UACvB;AAAA,UACA,EAAE,UAAU,MAAM,UAAU,UAAU,MAAM,UAAU,eAAe;AAAA,UACrE;AAAA,UACA;AAAA,QACF;AACA,YAAI,YAAY;AACd,gBAAM,QAAS,GAAW,WAAY,GAAW;AACjD,cAAI,OAAO;AACT,kBAAM,UAAU,MAAM,MAAM,aAAa;AACzC,kBAAM,YAAY,QAAQ,SAAS,UAAU;AAC7C,kBAAM,iBAAiB,QAAQ,SAAS,OAAO,CAAC,GAAW,MAAW,KAAK,EAAE,cAAc,UAAU,IAAI,CAAC,KAAK;AAC/G,gBAAI,YAAY,KAAK,iBAAiB,GAAG;AACvC,oBAAM,GAAG,MAAM,EAAE,MAAM,MAAM;AAAA,cAAC,CAAC;AAC/B,mBAAK;AACL,qBAAO,EAAE,cAAc,KAAK,UAAU,OAAO,GAAG,SAAS,WAAW,cAAc,eAAe;AAAA,YACnG;AAAA,UACF;AACA,qBAAW;AAAA,QACb;AAAA,MACF,SAAS,OAAY;AACnB,cAAM,MAAM,OAAO,OAAO,WAAW,KAAK;AAC1C,YAAI,IAAI,SAAS,SAAS,KAAK,OAAO,WAAY,OAAM;AAAA,MAE1D,UAAE;AACA,YAAI,GAAI,OAAM,GAAG,MAAM,EAAE,MAAM,MAAM;AAAA,QAAC,CAAC;AAAA,MACzC;AAAA,IACF;AAEA,QAAI,CAAC,UAAU;AACb,YAAM,UAAU,MAAM,KAAK,WAAW,EAAE,MAAM,QAAQ,SAAS,IAAI,UAAU,MAAM,CAAC,EAAE,MAAM,MAAM,IAAI;AACtG,YAAM,YAAY,MAAM,KAAK,SAAS,MAAM;AAC1C,cAAM,QAAQ,SAAS,MAAM,aAAa,IAAI,YAAY;AAC1D,cAAM,gBAAgB,+EAA+E,KAAK,IAAI,KAC5G,CAAC,CAAC,SAAS,cAAc,yEAAyE;AACpG,cAAM,aAAa,iBAAiB,6BAA6B,KAAK,IAAI,KACxE,CAAC,CAAC,SAAS,cAAc,4GAA4G;AACvI,eAAO,EAAE,eAAe,WAAW;AAAA,MACrC,CAAC,EAAE,MAAM,OAAO,EAAE,eAAe,OAAO,YAAY,MAAM,EAAE;AAC5D,YAAM,UAAU,UAAU,gBACtB,8MACA,UAAU,aACR,oHACA;AACN,YAAM,MAAM,OAAO,OAAO,IAAI,MAAM,OAAO,GAAG;AAAA,QAC5C,mBAAmB,UAAU,QAAQ,SAAS,QAAQ,IAAI;AAAA,QAC1D,QAAQ,UAAU,gBAAgB,yBAAyB,UAAU,aAAa,sBAAsB;AAAA,MAC1G,CAAC;AACD,YAAM;AAAA,IACR;AAGA,UAAM,KAAK,iBAAiB,eAAe,EAAE,SAAS,IAAM,CAAC,EAAE,MAAM,MAAM;AAAA,IAAC,CAAC;AAG7E,UAAM,WAAW,KAAK,IAAI;AAG1B,UAAM,qBAAqB,IAAI;AAE/B,UAAM,QAAQ,MAAM,QAAQ,aAAa;AACzC,UAAM,UAAU,MAAM,SAAS,UAAU;AACzC,UAAM,eAAe,MAAM,SAAS;AAAA,MAClC,CAAC,KAAK,MAAM,OAAO,EAAE,cAAc,UAAU;AAAA,MAAI;AAAA,IACnD,KAAK;AAEL,QAAI,YAAY,KAAK,iBAAiB,GAAG;AACvC,YAAM,IAAI,MAAM,sGAAiG;AAAA,IACnH;AAEA,WAAO,EAAE,cAAc,KAAK,UAAU,KAAK,GAAG,SAAS,cAAc,SAAS;AAAA,EAChF,UAAE;AACA,UAAM,QAAQ,MAAM,EAAE,MAAM,MAAM;AAAA,IAAC,CAAC;AAAA,EACtC;AACF;",
   "names": []
 }

package/dist/tls-fingerprint.js

@@ -46,7 +46,7 @@ async function probeTlsFingerprint(host, port = 443) {
         protocol: socket.getProtocol() ?? "unknown",
         cipher: socket.getCipher()?.name ?? "unknown",
         authorized: socket.authorized,
-        subjectCN: cert?.subject?.CN ?? null
+        subjectCN: cert?.subject?.CN?.toString() ?? null
       });
       socket.destroy();
     });

package/dist/tls-fingerprint.js.map

@@ -1,7 +1,7 @@
 {
   "version": 3,
   "sources": ["../src/tls-fingerprint.ts"],
-  "sourcesContent": ["/**\n * TLS fingerprint spoofing \u2014 Chrome JA3 cipher suite ordering.\n *\n * Node.js TLS doesn't expose JA3 hash control directly, but ordering\n * cipher suites to match Chrome's TLS ClientHello reduces fingerprint\n * detectability vs the default OpenSSL order.\n *\n * For full JA3 bypass, pair with the browser-stealth engine (Camoufox).\n */\n\nimport { Agent as HttpsAgent } from 'node:https';\nimport { connect as tlsConnect } from 'node:tls';\n\n// Chrome 120+ TLS 1.3 + 1.2 cipher suite order\nexport const CHROME_CIPHERS = [\n  'TLS_AES_128_GCM_SHA256',\n  'TLS_AES_256_GCM_SHA384',\n  'TLS_CHACHA20_POLY1305_SHA256',\n  'ECDHE-ECDSA-AES128-GCM-SHA256',\n  'ECDHE-RSA-AES128-GCM-SHA256',\n  'ECDHE-ECDSA-AES256-GCM-SHA384',\n  'ECDHE-RSA-AES256-GCM-SHA384',\n  'ECDHE-ECDSA-CHACHA20-POLY1305',\n  'ECDHE-RSA-CHACHA20-POLY1305',\n  'ECDHE-RSA-AES128-SHA',\n  'ECDHE-RSA-AES256-SHA',\n  'AES128-GCM-SHA256',\n  'AES256-GCM-SHA384',\n  'AES128-SHA',\n  'AES256-SHA',\n].join(':');\n\n// Chrome's ECDH curve preference order\nexport const CHROME_CURVES = 'X25519:P-256:P-384';\n\nexport interface TlsFingerprintOptions {\n  rejectUnauthorized?: boolean;\n  keepAlive?: boolean;\n  keepAliveMsecs?: number;\n  maxSockets?: number;\n}\n\n/**\n * Returns an HTTPS agent configured with Chrome-like TLS settings.\n * Drop-in replacement for the default agent in fetch/got/axios.\n */\nexport function createChromeTlsAgent(opts: TlsFingerprintOptions = {}): HttpsAgent {\n  return new HttpsAgent({\n    ciphers: CHROME_CIPHERS,\n    ecdhCurve: CHROME_CURVES,\n    minVersion: 'TLSv1.2',\n    maxVersion: 'TLSv1.3',\n    rejectUnauthorized: opts.rejectUnauthorized ?? true,\n    keepAlive: opts.keepAlive ?? true,\n    keepAliveMsecs: opts.keepAliveMsecs ?? 10_000,\n    maxSockets: opts.maxSockets ?? 50,\n  });\n}\n\n/**\n * Probe a host and return its JA3-relevant TLS parameters.\n * Useful for verifying that the fingerprint changed vs default.\n */\nexport async function probeTlsFingerprint(host: string, port = 443): Promise<{\n  protocol: string;\n  cipher: string;\n  authorized: boolean;\n  subjectCN: string | null;\n}> {\n  return new Promise((resolve, reject) => {\n    const socket = tlsConnect({\n      host,\n      port,\n      servername: host,\n      ciphers: CHROME_CIPHERS,\n      ecdhCurve: CHROME_CURVES,\n      minVersion: 'TLSv1.2',\n      rejectUnauthorized: false,\n    }, () => {\n      const cert = socket.getPeerCertificate();\n      resolve({\n        protocol: socket.getProtocol() ?? 'unknown',\n        cipher: socket.getCipher()?.name ?? 'unknown',\n        authorized: socket.authorized,\n        subjectCN: cert?.subject?.CN ?? null,\n      });\n      socket.destroy();\n    });\n    socket.on('error', reject);\n    setTimeout(() => { socket.destroy(); reject(new Error('TLS probe timeout')); }, 5_000);\n  });\n}\n\n/**\n * Node.js fetch dispatcher options with Chrome TLS agent.\n * Pass as the `dispatcher` option to undici or as `agent` to node-fetch.\n */\nexport const chromeTlsAgent = createChromeTlsAgent();\n\nexport const CHROME_TLS_HEADERS = {\n  'Accept-Encoding': 'gzip, deflate, br, zstd',\n  'Accept-Language': 'en-US,en;q=0.9',\n  'sec-ch-ua': '\"Google Chrome\";v=\"120\", \"Chromium\";v=\"120\", \"Not-A.Brand\";v=\"99\"',\n  'sec-ch-ua-mobile': '?0',\n  'sec-ch-ua-platform': '\"Windows\"',\n  'Sec-Fetch-Dest': 'document',\n  'Sec-Fetch-Mode': 'navigate',\n  'Sec-Fetch-Site': 'none',\n  'Sec-Fetch-User': '?1',\n  'Upgrade-Insecure-Requests': '1',\n};\n"],
-  "mappings": "AAUA,SAAS,SAAS,kBAAkB;AACpC,SAAS,WAAW,kBAAkB;AAG/B,MAAM,iBAAiB;AAAA,EAC5B;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AACF,EAAE,KAAK,GAAG;AAGH,MAAM,gBAAgB;AAatB,SAAS,qBAAqB,OAA8B,CAAC,GAAe;AACjF,SAAO,IAAI,WAAW;AAAA,IACpB,SAAS;AAAA,IACT,WAAW;AAAA,IACX,YAAY;AAAA,IACZ,YAAY;AAAA,IACZ,oBAAoB,KAAK,sBAAsB;AAAA,IAC/C,WAAW,KAAK,aAAa;AAAA,IAC7B,gBAAgB,KAAK,kBAAkB;AAAA,IACvC,YAAY,KAAK,cAAc;AAAA,EACjC,CAAC;AACH;AAMA,eAAsB,oBAAoB,MAAc,OAAO,KAK5D;AACD,SAAO,IAAI,QAAQ,CAAC,SAAS,WAAW;AACtC,UAAM,SAAS,WAAW;AAAA,MACxB;AAAA,MACA;AAAA,MACA,YAAY;AAAA,MACZ,SAAS;AAAA,MACT,WAAW;AAAA,MACX,YAAY;AAAA,MACZ,oBAAoB;AAAA,IACtB,GAAG,MAAM;AACP,YAAM,OAAO,OAAO,mBAAmB;AACvC,cAAQ;AAAA,QACN,UAAU,OAAO,YAAY,KAAK;AAAA,QAClC,QAAQ,OAAO,UAAU,GAAG,QAAQ;AAAA,QACpC,YAAY,OAAO;AAAA,QACnB,WAAW,MAAM,SAAS,MAAM;AAAA,MAClC,CAAC;AACD,aAAO,QAAQ;AAAA,IACjB,CAAC;AACD,WAAO,GAAG,SAAS,MAAM;AACzB,eAAW,MAAM;AAAE,aAAO,QAAQ;AAAG,aAAO,IAAI,MAAM,mBAAmB,CAAC;AAAA,IAAG,GAAG,GAAK;AAAA,EACvF,CAAC;AACH;AAMO,MAAM,iBAAiB,qBAAqB;AAE5C,MAAM,qBAAqB;AAAA,EAChC,mBAAmB;AAAA,EACnB,mBAAmB;AAAA,EACnB,aAAa;AAAA,EACb,oBAAoB;AAAA,EACpB,sBAAsB;AAAA,EACtB,kBAAkB;AAAA,EAClB,kBAAkB;AAAA,EAClB,kBAAkB;AAAA,EAClB,kBAAkB;AAAA,EAClB,6BAA6B;AAC/B;",
+  "sourcesContent": ["/**\n * TLS fingerprint spoofing \u2014 Chrome JA3 cipher suite ordering.\n *\n * Node.js TLS doesn't expose JA3 hash control directly, but ordering\n * cipher suites to match Chrome's TLS ClientHello reduces fingerprint\n * detectability vs the default OpenSSL order.\n *\n * For full JA3 bypass, pair with the browser-stealth engine (Camoufox).\n */\n\nimport { Agent as HttpsAgent } from 'node:https';\nimport { connect as tlsConnect } from 'node:tls';\n\n// Chrome 120+ TLS 1.3 + 1.2 cipher suite order\nexport const CHROME_CIPHERS = [\n  'TLS_AES_128_GCM_SHA256',\n  'TLS_AES_256_GCM_SHA384',\n  'TLS_CHACHA20_POLY1305_SHA256',\n  'ECDHE-ECDSA-AES128-GCM-SHA256',\n  'ECDHE-RSA-AES128-GCM-SHA256',\n  'ECDHE-ECDSA-AES256-GCM-SHA384',\n  'ECDHE-RSA-AES256-GCM-SHA384',\n  'ECDHE-ECDSA-CHACHA20-POLY1305',\n  'ECDHE-RSA-CHACHA20-POLY1305',\n  'ECDHE-RSA-AES128-SHA',\n  'ECDHE-RSA-AES256-SHA',\n  'AES128-GCM-SHA256',\n  'AES256-GCM-SHA384',\n  'AES128-SHA',\n  'AES256-SHA',\n].join(':');\n\n// Chrome's ECDH curve preference order\nexport const CHROME_CURVES = 'X25519:P-256:P-384';\n\nexport interface TlsFingerprintOptions {\n  rejectUnauthorized?: boolean;\n  keepAlive?: boolean;\n  keepAliveMsecs?: number;\n  maxSockets?: number;\n}\n\n/**\n * Returns an HTTPS agent configured with Chrome-like TLS settings.\n * Drop-in replacement for the default agent in fetch/got/axios.\n */\nexport function createChromeTlsAgent(opts: TlsFingerprintOptions = {}): HttpsAgent {\n  return new HttpsAgent({\n    ciphers: CHROME_CIPHERS,\n    ecdhCurve: CHROME_CURVES,\n    minVersion: 'TLSv1.2',\n    maxVersion: 'TLSv1.3',\n    rejectUnauthorized: opts.rejectUnauthorized ?? true,\n    keepAlive: opts.keepAlive ?? true,\n    keepAliveMsecs: opts.keepAliveMsecs ?? 10_000,\n    maxSockets: opts.maxSockets ?? 50,\n  });\n}\n\n/**\n * Probe a host and return its JA3-relevant TLS parameters.\n * Useful for verifying that the fingerprint changed vs default.\n */\nexport async function probeTlsFingerprint(host: string, port = 443): Promise<{\n  protocol: string;\n  cipher: string;\n  authorized: boolean;\n  subjectCN: string | null;\n}> {\n  return new Promise((resolve, reject) => {\n    const socket = tlsConnect({\n      host,\n      port,\n      servername: host,\n      ciphers: CHROME_CIPHERS,\n      ecdhCurve: CHROME_CURVES,\n      minVersion: 'TLSv1.2',\n      rejectUnauthorized: false,\n    }, () => {\n      const cert = socket.getPeerCertificate();\n      resolve({\n        protocol: socket.getProtocol() ?? 'unknown',\n        cipher: socket.getCipher()?.name ?? 'unknown',\n        authorized: socket.authorized,\n        subjectCN: cert?.subject?.CN?.toString() ?? null,\n      });\n      socket.destroy();\n    });\n    socket.on('error', reject);\n    setTimeout(() => { socket.destroy(); reject(new Error('TLS probe timeout')); }, 5_000);\n  });\n}\n\n/**\n * Node.js fetch dispatcher options with Chrome TLS agent.\n * Pass as the `dispatcher` option to undici or as `agent` to node-fetch.\n */\nexport const chromeTlsAgent = createChromeTlsAgent();\n\nexport const CHROME_TLS_HEADERS = {\n  'Accept-Encoding': 'gzip, deflate, br, zstd',\n  'Accept-Language': 'en-US,en;q=0.9',\n  'sec-ch-ua': '\"Google Chrome\";v=\"120\", \"Chromium\";v=\"120\", \"Not-A.Brand\";v=\"99\"',\n  'sec-ch-ua-mobile': '?0',\n  'sec-ch-ua-platform': '\"Windows\"',\n  'Sec-Fetch-Dest': 'document',\n  'Sec-Fetch-Mode': 'navigate',\n  'Sec-Fetch-Site': 'none',\n  'Sec-Fetch-User': '?1',\n  'Upgrade-Insecure-Requests': '1',\n};\n"],
+  "mappings": "AAUA,SAAS,SAAS,kBAAkB;AACpC,SAAS,WAAW,kBAAkB;AAG/B,MAAM,iBAAiB;AAAA,EAC5B;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AACF,EAAE,KAAK,GAAG;AAGH,MAAM,gBAAgB;AAatB,SAAS,qBAAqB,OAA8B,CAAC,GAAe;AACjF,SAAO,IAAI,WAAW;AAAA,IACpB,SAAS;AAAA,IACT,WAAW;AAAA,IACX,YAAY;AAAA,IACZ,YAAY;AAAA,IACZ,oBAAoB,KAAK,sBAAsB;AAAA,IAC/C,WAAW,KAAK,aAAa;AAAA,IAC7B,gBAAgB,KAAK,kBAAkB;AAAA,IACvC,YAAY,KAAK,cAAc;AAAA,EACjC,CAAC;AACH;AAMA,eAAsB,oBAAoB,MAAc,OAAO,KAK5D;AACD,SAAO,IAAI,QAAQ,CAAC,SAAS,WAAW;AACtC,UAAM,SAAS,WAAW;AAAA,MACxB;AAAA,MACA;AAAA,MACA,YAAY;AAAA,MACZ,SAAS;AAAA,MACT,WAAW;AAAA,MACX,YAAY;AAAA,MACZ,oBAAoB;AAAA,IACtB,GAAG,MAAM;AACP,YAAM,OAAO,OAAO,mBAAmB;AACvC,cAAQ;AAAA,QACN,UAAU,OAAO,YAAY,KAAK;AAAA,QAClC,QAAQ,OAAO,UAAU,GAAG,QAAQ;AAAA,QACpC,YAAY,OAAO;AAAA,QACnB,WAAW,MAAM,SAAS,IAAI,SAAS,KAAK;AAAA,MAC9C,CAAC;AACD,aAAO,QAAQ;AAAA,IACjB,CAAC;AACD,WAAO,GAAG,SAAS,MAAM;AACzB,eAAW,MAAM;AAAE,aAAO,QAAQ;AAAG,aAAO,IAAI,MAAM,mBAAmB,CAAC;AAAA,IAAG,GAAG,GAAK;AAAA,EACvF,CAAC;AACH;AAMO,MAAM,iBAAiB,qBAAqB;AAE5C,MAAM,qBAAqB;AAAA,EAChC,mBAAmB;AAAA,EACnB,mBAAmB;AAAA,EACnB,aAAa;AAAA,EACb,oBAAoB;AAAA,EACpB,sBAAsB;AAAA,EACtB,kBAAkB;AAAA,EAClB,kBAAkB;AAAA,EAClB,kBAAkB;AAAA,EAClB,kBAAkB;AAAA,EAClB,6BAA6B;AAC/B;",
   "names": []
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@detiq/crawler",
-  "version": "0.2.14",
+  "version": "1.0.1",
   "private": false,
   "publishConfig": {
     "access": "public",
@@ -31,6 +31,7 @@
     "mailparser": "^3.9.12",
     "pino": "^10.3.1",
     "pino-pretty": "^13.1.3",
+    "pixelmatch": "^6.0.0",
     "playwright": "^1.55.0",
     "playwright-extra": "^4.3.6",
     "puppeteer-extra-plugin-stealth": "^2.11.2",

package/src/crawler.ts

@@ -74,6 +74,16 @@ import { generateOpenApiSpec, specToYaml } from './openapi-generator.js';
 import { WebhookNotifier } from './webhook-notifier.js';
 import { generatePlaywrightTest } from './playwright-recorder.js';
 
+// Module-level cache for per-crawl LLM config — avoids redundant DB round-trips inside per-page loops.
+// Keyed by tenantId so multi-tenant workers don't share configs across tenants.
+const _crawlLLMCache = new Map<string, any>();
+async function getCachedCrawlLLM(tenantId: string): Promise<any> {
+  if (_crawlLLMCache.has(tenantId)) return _crawlLLMCache.get(tenantId);
+  const cfg = await getTaskAgentLLMConfig(tenantId, 'crawl').catch(() => null);
+  _crawlLLMCache.set(tenantId, cfg);
+  return cfg;
+}
+
 function getGitInfo(): { branch: string; commit: string } {
   try {
     const branch = execSync('git rev-parse --abbrev-ref HEAD', { encoding: 'utf8', timeout: 3000 }).trim();
@@ -1707,12 +1717,12 @@ export async function crawlProject(opts: {
           continue;
         }
         // Full page-ready sequence:
-        // 1. networkidle — waits for XHR/fetch after initial load (data grids, dashboards)
+        // 1. load — waits for initial page load (faster than networkidle; XHR handled by waitForStableDOM)
         // 2. DOM stability — waits for React/Vue hydration to finish mutating the DOM
         // 3. Modal capture — screenshot app dialogs BEFORE dismissal (they are testable UI states)
         // 4. Overlay dismissal — removes cookie banners/modals before element extraction
         // 5. Lazy-load scroll — triggers IntersectionObserver for off-screen images/components
-        await page.waitForLoadState('networkidle', { timeout: 8_000 }).catch(() => { });
+        await page.waitForLoadState('load', { timeout: 8_000 }).catch(() => { });
         await waitForStableDOM(page);
 
         if (!opts.discoveryMode) {
@@ -1806,7 +1816,7 @@ export async function crawlProject(opts: {
               throw new Error('[AUTH_FAILURE] Re-authentication failed. Session is unrecoverable. Re-capture the session in project settings and retry the crawl.');
             }
           } else {
-            logger.warn('[crawler] Session expired mid-crawl, no credentials to recover — stopping BFS');
+            logger.warn('[crawler] Session expired mid-crawl, no credentials to recover — skipping URL');
             AppBrain.saveCrawlEvidence(opts.tenantId, opts.projectId, {
               jobId: opts.jobId,
               type: 'AUTH_FAILURE',
@@ -2203,8 +2213,8 @@ export async function crawlProject(opts: {
         const linkRecipe = await AppBrain.getActionRecipe(
           opts.tenantId, opts.projectId, new URL(url).hostname, `link-discovery:${canonUrl}`
         ).catch(() => null);
-        const cachedLinks: string[] | null = (linkRecipe && linkRecipe.successRate >= 0.7)
-          ? ((linkRecipe.steps as any[])
+        const cachedLinks: string[] | null = (linkRecipe && (linkRecipe as any).successRate >= 0.7)
+          ? (((linkRecipe as any).steps as any[])
             .map((s: any) => s.href)
             .filter((href: string | undefined): href is string => !!href && isUsefulNavigationUrl(href, origin)))
           : null;
@@ -2326,7 +2336,7 @@ export async function crawlProject(opts: {
               // Generate a one-sentence AI summary from the markdown — fire-and-forget
               const summaryPromise = (async () => {
                 try {
-                  const llmCfg = await getTaskAgentLLMConfig(opts.tenantId, 'crawl').catch(() => undefined);
+                  const llmCfg = await getCachedCrawlLLM(opts.tenantId);
                   const snippet = extracted.markdown.slice(0, 1200);
                   const { text, usage } = await askTracked(
                     'You are a QA analyst. Given page content, write ONE crisp sentence describing what this screen does or shows. Be specific. No preamble.',
@@ -2486,21 +2496,48 @@ export async function crawlProject(opts: {
                   }
                 }
 
-                const { changeType, recommendation, changes: allChanges, bugReportDraft, aiSummary } = verdict;
+                let { changeType, recommendation, changes: allChanges, bugReportDraft, aiSummary } = verdict;
+
+                // R7: Apply per-project visual diff threshold — suppress diffs below the threshold
+                if (changeType !== 'NONE') {
+                  try {
+                    const project = await AppBrain.getProject(opts.tenantId, opts.projectId);
+                    const threshold = (project as any)?.visualDiffThreshold ?? null;
+                    if (threshold !== null && pHashDistance < threshold) {
+                      changeType = 'NONE';
+                    }
+                  } catch { /* non-fatal — use verdict as-is */ }
+                }
+
                 if (changeType !== 'NONE') {
                   let newScreenshotCloudUrl: string | undefined;
+                  let diffImageCloudUrl: string | undefined;
                   try {
                     const { getTenantStorageConfig, uploadScreenshotToTenantStorage, buildArtifactKey } = await import('@detiq/app-brain');
                     const storageCfg = await getTenantStorageConfig(opts.tenantId).catch(() => null);
                     if (storageCfg) {
                       const key = buildArtifactKey(opts.tenantId, opts.projectId, 'visual-diffs', `${screen.id}-v${screen.version}.png`, storageCfg.prefix);
                       newScreenshotCloudUrl = await uploadScreenshotToTenantStorage(storageCfg, shotPath, key);
+
+                      // R5: Generate and upload pixelmatch diff image (changed pixels in red)
+                      try {
+                        const { generatePixelmatchDiff } = await import('./phash.js');
+                        const diffBuffer = await generatePixelmatchDiff(baseline.storagePath, shotPath, baseline.ignoredRegions ?? []);
+                        const diffTmpPath = `${shotPath}.diff.png`;
+                        await fs.writeFile(diffTmpPath, diffBuffer);
+                        const diffKey = buildArtifactKey(opts.tenantId, opts.projectId, 'visual-diffs', `${screen.id}-v${screen.version}-diff.png`, storageCfg.prefix);
+                        diffImageCloudUrl = await uploadScreenshotToTenantStorage(storageCfg, diffTmpPath, diffKey);
+                        fs.unlink(diffTmpPath).catch(() => {});
+                      } catch (diffErr) {
+                        logger.warn({ screenId: screen.id, err: String(diffErr) }, '[crawler] pixelmatch diff generation failed, skipping');
+                      }
                     }
                   } catch { /* non-fatal — falls back to local path */ }
                   await AppBrain.saveVisualDiff(opts.tenantId, {
                     baselineId: baseline.id,
                     newScreenshotPath: shotPath,
                     newScreenshotCloudUrl,
+                    diffImageCloudUrl,
                     diffScore: pHashDistance,
                     changeType,
                     changes: allChanges,
@@ -2522,18 +2559,22 @@ export async function crawlProject(opts: {
         }
 
         const linkMappings = await extractLinkMappings(page, origin);
-        for (const el of elements) {
-          const saved = await AppBrain.saveElement(opts.tenantId, opts.projectId, screen.id, el.meaning, el.role, el.expectedData, el.notes, undefined, true, el.boundingRect, el.ariaState, el.parentLandmark);
-          // Populate ElementFingerprint — fire-and-forget, failures are non-fatal
-          const label = el.meaning;
-          const role = el.role ?? 'unknown';
-          const selectorHint = atByRole.get(role) ?? role;
-          const selectors = [
-            { type: 'role-label', value: `${role}:${label}`, confidence: 0.9 },
-            { type: 'aria-label', value: label, confidence: 0.85 },
-            { type: 'tag-hint', value: selectorHint, confidence: 0.6 },
-          ];
-          AppBrain.saveElementFingerprint(opts.tenantId, opts.projectId, saved.id, selectors).catch(console.warn);
+        // Batch element saves: 4 concurrent DB writes instead of serial round-trips
+        for (let _ei = 0; _ei < elements.length; _ei += 4) {
+          const _batch = elements.slice(_ei, _ei + 4);
+          await Promise.all(_batch.map(async (el: any) => {
+            const saved = await AppBrain.saveElement(opts.tenantId, opts.projectId, screen.id, el.meaning, el.role, el.expectedData, el.notes, undefined, true, el.boundingRect, el.ariaState, el.parentLandmark);
+            // Populate ElementFingerprint — fire-and-forget, failures are non-fatal
+            const label = el.meaning;
+            const role = el.role ?? 'unknown';
+            const selectorHint = atByRole.get(role) ?? role;
+            const selectors = [
+              { type: 'role-label', value: `${role}:${label}`, confidence: 0.9 },
+              { type: 'aria-label', value: label, confidence: 0.85 },
+              { type: 'tag-hint', value: selectorHint, confidence: 0.6 },
+            ];
+            AppBrain.saveElementFingerprint(opts.tenantId, opts.projectId, (saved as any).id, selectors).catch(console.warn);
+          }));
         }
         for (const lm of linkMappings) {
           const safety = assessActionSafety(lm.label, domainProfile);
@@ -2876,7 +2917,7 @@ export async function crawlProject(opts: {
     // Saves URL_CANDIDATES evidence so the ActivityDrawer shows what was found.
     try {
       const { runUrlPatternCompletion } = await import('./url-pattern-completer.js');
-      const llmCfgForPatterns = await getTaskAgentLLMConfig(opts.tenantId, 'crawl').catch(() => null);
+      const llmCfgForPatterns = await getCachedCrawlLLM(opts.tenantId);
       const urlPatternCandidates = await runUrlPatternCompletion(
         Array.from(visited.keys()),
         opts.appUrl,
@@ -2944,7 +2985,7 @@ export async function crawlProject(opts: {
     if (visited.size >= 3) {
       try {
         await onProgress?.(80, 'Running gap analysis…');
-        const llmCfgForGap = await getTaskAgentLLMConfig(opts.tenantId, 'crawl').catch(() => null);
+        const llmCfgForGap = await getCachedCrawlLLM(opts.tenantId);
         const discoveredForGap = Array.from(visited.entries()).map(([url, screenId]) => {
           const d = discovered.find(dd => dd.screenId === screenId);
           return { url, name: d ? (new URL(url).pathname.split('/').filter(Boolean).pop() ?? 'screen') : undefined, elements: d?.elements };
@@ -4168,7 +4209,6 @@ async function waitForScreenshotReady(page: any): Promise<void> {
       ]);
     }).catch(() => { }));
     await withHardCap(waitForStableDOM(page, 2_000, 500));
-    await withHardCap(page.waitForTimeout(250).catch(() => { }));
   } catch { /* readiness is best-effort; never fail the crawl for a continuously mutating page */ }
 }
 

package/src/job-queue.ts

@@ -32,7 +32,7 @@ async function notifyAdmins(tenantId: string, data: {
         where: { tenantId, role: { in: ['OWNER', 'ADMIN'] } },
         select: { userId: true },
       })
-    );
+    ) as any[];
     await withTenant(tenantId, (tx) =>
       tx.notification.createMany({
         data: members.map((m: { userId: string }) => ({
@@ -189,17 +189,17 @@ const runningControllers = new Map<string, AbortController>();
 // double-processed. See startJobWorkerLoop() below.
 
 const WORKER_ID = `${os.hostname()}:${process.pid}`;
-const STALE_JOB_TIMEOUT_MINUTES = Number(process.env.JOB_STALE_TIMEOUT_MINUTES ?? 30);
-const WORKER_POLL_INTERVAL_MS = Number(process.env.JOB_WORKER_POLL_INTERVAL_MS ?? 15_000);
-const WORKER_CONCURRENCY = Number(process.env.JOB_WORKER_CONCURRENCY ?? 3);
-const JOB_HEARTBEAT_INTERVAL_MS = Number(process.env.JOB_HEARTBEAT_INTERVAL_MS ?? 15_000);
+const STALE_JOB_TIMEOUT_MINUTES = Number(process.env.JOB_STALE_TIMEOUT_MINUTES) || 30;
+const WORKER_POLL_INTERVAL_MS = Number(process.env.JOB_WORKER_POLL_INTERVAL_MS) || 15_000;
+const WORKER_CONCURRENCY = Number(process.env.JOB_WORKER_CONCURRENCY) || 3;
+const JOB_HEARTBEAT_INTERVAL_MS = Number(process.env.JOB_HEARTBEAT_INTERVAL_MS) || 15_000;
 // How many headless browsers a single RUN_TEST_SUITE job launches at once —
 // distinct from WORKER_CONCURRENCY (how many *jobs* the poll loop claims at
 // once). Bounded by default to avoid exhausting a container's memory; the
 // primary scale-out path for running more tests at once is still horizontal
 // (more worker containers / SQS), not raising this past what one container
 // can safely hold — see startJobWorkerLoop's doc comment.
-const TEST_EXEC_CONCURRENCY = Number(process.env.TEST_EXEC_CONCURRENCY ?? 3);
+const TEST_EXEC_CONCURRENCY = Number(process.env.TEST_EXEC_CONCURRENCY) || 3;
 
 type ClaimedJob = { id: string; tenantId: string; projectId: string; type: string; config: any };
 
@@ -254,7 +254,7 @@ export async function requeueStaleRunningJobs(): Promise<void> {
     RETURNING id
   `;
   if (rows.length > 0) {
-    logger.warn({ jobIds: rows.map((r) => r.id), staleTimeoutMinutes: STALE_JOB_TIMEOUT_MINUTES }, '[job-worker] requeued stale RUNNING job(s)');
+    logger.warn({ jobIds: rows.map((r: any) => r.id), staleTimeoutMinutes: STALE_JOB_TIMEOUT_MINUTES }, '[job-worker] requeued stale RUNNING job(s)');
   }
 }
 
@@ -567,14 +567,15 @@ export interface ProcessJobOpts {
 export async function processJob(jobId: string, config: any, opts?: ProcessJobOpts) {
   const { tenantId, projectId } = config;
 
-  // Mark as RUNNING.
+  // Mark as RUNNING — skip if already CANCELLED (race: user cancelled before worker picked it up).
   const jobStartedAt = new Date();
-  await withTenant(tenantId, (tx) =>
-    tx.platformJob.update({
-      where: { id: jobId },
+  const claimed = await withTenant(tenantId, (tx) =>
+    tx.platformJob.updateMany({
+      where: { id: jobId, status: { not: 'CANCELLED' } },
       data: { status: 'RUNNING', startedAt: jobStartedAt, heartbeatAt: jobStartedAt, workerId: WORKER_ID },
     })
   );
+  if (claimed.count === 0) return;
 
   const controller = new AbortController();
   runningControllers.set(jobId, controller);
@@ -1469,7 +1470,7 @@ async function processGenerateSkillFilesJob(jobId: string, config: any) {
         select: { result: true },
       })
     );
-    const lastResult = (lastCrawl?.result ?? {}) as any;
+    const lastResult = ((lastCrawl as any)?.result ?? {}) as any;
     await updateJobProgress(tenantId, jobId, 25, `Generating ${screens.length} app intelligence skill files…`);
     await generateTaxonomySkillFiles(tenantId, projectId, screens, lastResult.apiEndpoints, lastResult.performanceMetrics);
     await updateJobProgress(tenantId, jobId, 85, 'Refreshing testing intelligence skill files…');
@@ -1702,7 +1703,9 @@ async function processDiscoverJob(jobId: string, config: any, opts?: ProcessJobO
         authToken: crawlCfg.authToken,
         authTokenHeader: crawlCfg.authTokenHeader,
         authTokenPrefix: crawlCfg.authTokenPrefix,
-        startUrls: sitemapUrls.length > 0 ? sitemapUrls : undefined,
+        startUrls: crawlCfg.postLoginUrl
+          ? [crawlCfg.postLoginUrl, ...(sitemapUrls.length > 0 ? sitemapUrls : [])]
+          : (sitemapUrls.length > 0 ? sitemapUrls : undefined),
         screenshotDir,
         maxScreens: maxDiscoverUrls,
         proxyUrl,
@@ -1820,7 +1823,7 @@ async function processMonitorJob(jobId: string, config: any, opts?: ProcessJobOp
     const projectScreens: Array<{ url: string | null; elementHash: string | null }> =
       await withTenant(tenantId, (tx) =>
         tx.screen.findMany({ where: { projectId }, select: { url: true, elementHash: true } })
-      ).catch(() => []);
+      ).catch(() => []) as any;
     const screenHashMap: Record<string, string> = {};
     for (const s of projectScreens) {
       if (s.url && s.elementHash) screenHashMap[s.url] = s.elementHash;
@@ -2155,7 +2158,7 @@ async function processRepoScanJob(jobId: string, config: any) {
 
     const conn = await withTenant(tenantId, (tx) =>
       tx.gitHubRepoConnection.findFirst({ where: { id: repoConnectionId, tenantId } })
-    );
+    ) as any;
     if (!conn) throw new Error(`RepoConnection ${repoConnectionId} not found`);
 
     // Decrypt token
@@ -2288,7 +2291,7 @@ async function processRepoScanJob(jobId: string, config: any) {
     const { buildCoverageMap } = await import('@detiq/agents');
     const testCases = await withTenant(tenantId, (tx) =>
       tx.testCase.findMany({ where: { projectId, tenantId, status: { not: 'RETIRED' } }, select: { id: true, title: true, ref: true } })
-    );
+    ) as any[];
 
     const mappings = await buildCoverageMap(testCases, allExtracted, tenantId);
 
@@ -2296,7 +2299,7 @@ async function processRepoScanJob(jobId: string, config: any) {
     for (const m of mappings) {
       const existing = await withTenant(tenantId, (tx) =>
         tx.coverageMapping.findUnique({ where: { testCaseId: m.testCaseId } })
-      );
+      ) as any;
       if (existing?.manualOverride) continue;
 
       await withTenant(tenantId, (tx) =>
@@ -2425,8 +2428,8 @@ async function processBatchGenerateAlignedJob(jobId: string, config: any) {
           where: { tenantId, projectId, status: { not: 'RETIRED' } },
           select: { id: true },
         })
-      );
-      caseIds = cases.map((c) => c.id);
+      ) as any[];
+      caseIds = cases.map((c: any) => c.id);
     }
 
     await updateJobProgress(tenantId, jobId, 15, `Generating scripts for ${caseIds.length} test cases`);
@@ -2674,7 +2677,7 @@ async function processNewPagesPipelineJob(jobId: string, config: any) {
         select: { id: true, url: true, name: true },
         take: 100,
       })
-    );
+    ) as any[];
 
     if (newScreens.length === 0) {
       await updateJobProgress(tenantId, jobId, 100, 'No new screens to process');

package/src/phash.ts

@@ -57,6 +57,39 @@ function hammingDistance(a: bigint, b: bigint): number {
   return count;
 }
 
+/**
+ * Generates a pixelmatch diff image highlighting changed pixels in red.
+ * Applies the same ignoredRegions mask as screenshotDiffPercent so masked
+ * zones don't show as "changed" in the visual output.
+ * Returns a PNG Buffer suitable for upload to S3.
+ */
+export async function generatePixelmatchDiff(
+  pathA: string,
+  pathB: string,
+  ignoredRegions: Array<{ x: number; y: number; width: number; height: number }> = [],
+): Promise<Buffer> {
+  const metaA = await sharp(pathA).metadata();
+  const W = metaA.width ?? 1200;
+  const H = metaA.height ?? 800;
+
+  const toRawRGBA = async (path: string): Promise<Buffer> => {
+    const base = ignoredRegions.length > 0 ? sharp(await applyMask(path, ignoredRegions)) : sharp(path);
+    const { data } = await base.resize(W, H, { fit: 'fill' }).ensureAlpha().raw().toBuffer({ resolveWithObject: true });
+    return data;
+  };
+
+  const [rawA, rawB] = await Promise.all([toRawRGBA(pathA), toRawRGBA(pathB)]);
+  const { default: pixelmatch } = await import('pixelmatch');
+  const diffData = Buffer.alloc(W * H * 4);
+  pixelmatch(rawA, rawB, diffData, W, H, {
+    threshold: 0.1,
+    alpha: 0.3,
+    diffColor: [255, 0, 0],
+    aaColor: [255, 255, 0],
+  });
+  return sharp(diffData, { raw: { width: W, height: H, channels: 4 } }).png().toBuffer();
+}
+
 export async function screenshotDiffPercent(
   pathA: string,
   pathB: string,

package/src/session-capture.ts

@@ -207,7 +207,7 @@ export async function validateAndCaptureSession(
   },
   llmConfig?: { modelName: string; apiKey: string },
   captchaOpts?: { apiKey?: string; provider?: '2captcha' | 'capsolver' },
-): Promise<{ storageState: string; cookies: number; localStorage: number }> {
+): Promise<{ storageState: string; cookies: number; localStorage: number; finalUrl?: string }> {
   const browser = await chromium.launch({ headless: true });
   try {
     const contextOpts: any = {};
@@ -306,6 +306,9 @@ export async function validateAndCaptureSession(
     // Wait for post-login navigation to settle before capturing session
     await page.waitForLoadState('networkidle', { timeout: 8_000 }).catch(() => {});
 
+    // Capture post-login landing URL before exploration navigates away
+    const finalUrl = page.url();
+
     // Post-login exploration: navigate tabs, expand menus, scroll to trigger lazy-loaded state
     await exploreAppAfterLogin(page);
 
@@ -319,7 +322,7 @@ export async function validateAndCaptureSession(
       throw new Error('Login appeared to succeed but no session data was captured — try manual session capture instead');
     }
 
-    return { storageState: JSON.stringify(state), cookies, localStorage };
+    return { storageState: JSON.stringify(state), cookies, localStorage, finalUrl };
   } finally {
     await browser.close().catch(() => {});
   }

package/src/stealth/human-behavior.ts

@@ -108,19 +108,12 @@ export async function infiniteScrollToBottom(
 ): Promise<{ scrollCount: number; heightReached: number }> {
   const { maxScrolls = 50, scrollDelay = 400, stabilizeMs = 600 } = opts;
   let scrollCount = 0;
-  let sameHeightCount = 0;
-  let lastHeight = 0;
+  let stableScrollCount = 0;
+  let previousHeight = 0;
 
   try {
     while (scrollCount < maxScrolls) {
-      const currentHeight: number = await page.evaluate(() => document.body.scrollHeight).catch(() => 0);
-      if (currentHeight === lastHeight) {
-        sameHeightCount++;
-        if (sameHeightCount >= 3) break; // no new content
-      } else {
-        sameHeightCount = 0;
-        lastHeight = currentHeight;
-      }
+      previousHeight = await page.evaluate(() => document.documentElement.scrollHeight).catch(() => 0);
 
       const scrollBy = 500 + Math.floor(Math.random() * 300);
       await page.evaluate((by: number) => window.scrollBy(0, by), scrollBy).catch(() => {});
@@ -129,10 +122,19 @@ export async function infiniteScrollToBottom(
       // Wait for potential lazy-loaded content
       await page.waitForTimeout(stabilizeMs).catch(() => {});
       scrollCount++;
+
+      const newHeight: number = await page.evaluate(() => document.documentElement.scrollHeight).catch(() => 0);
+      if (newHeight - previousHeight < 10) {
+        stableScrollCount++;
+        if (stableScrollCount >= 3) break; // no meaningful new content after 3 consecutive non-growing scrolls
+      } else {
+        stableScrollCount = 0;
+      }
+      previousHeight = newHeight;
     }
   } catch { /* non-fatal */ }
 
-  return { scrollCount, heightReached: lastHeight };
+  return { scrollCount, heightReached: previousHeight };
 }
 
 export async function extractIframeContent(page: Page): Promise<Array<{ src: string; content: string; markdown: string }>> {