@detiq/crawler 1.1.7 → 1.1.9

package/dist/session-capture.js

@@ -1,47 +1,59 @@
-import { chromium } from "playwright";
-import { randomBytes } from "node:crypto";
-import { login } from "./auth.js";
-import { generateTOTP } from "./job-queue.js";
-const activeSessions = /* @__PURE__ */ new Map();
-async function startSessionCapture(appUrl) {
-  const inDocker = process.env.PLAYWRIGHT_IN_DOCKER === "true";
-  const noDisplay = process.platform === "linux" && !process.env.DISPLAY;
-  if (inDocker || noDisplay) {
-    throw new Error("HEADLESS_ENV");
-  }
-  const id = randomBytes(16).toString("hex");
-  const browser = await chromium.launch({ headless: false });
-  const context = await browser.newContext();
-  const page = await context.newPage();
-  await page.goto(appUrl, { waitUntil: "domcontentloaded" }).catch(() => {
-  });
-  const timer = setTimeout(() => cancelSessionCapture(id).catch(() => {
-  }), 10 * 60 * 1e3);
-  activeSessions.set(id, { browser, context, timer });
-  return id;
+import { chromium } from 'playwright';
+import { randomBytes } from 'node:crypto';
+import { login } from './auth.js';
+import { generateTOTP } from './job-queue.js';
+const activeSessions = new Map();
+/**
+ * Opens a headed (visible) browser at appUrl and returns a captureId.
+ * The user logs in manually, then calls completeSessionCapture() to harvest storageState.
+ * Sessions expire after 10 minutes if not completed.
+ *
+ * Throws 'HEADLESS_ENV' if running in Docker/Linux without a display server.
+ * The API route surfaces this as a 503 so the frontend can switch to manual JSON paste.
+ */
+export async function startSessionCapture(appUrl) {
+    const inDocker = process.env.PLAYWRIGHT_IN_DOCKER === 'true';
+    const noDisplay = process.platform === 'linux' && !process.env.DISPLAY;
+    if (inDocker || noDisplay) {
+        throw new Error('HEADLESS_ENV');
+    }
+    const id = randomBytes(16).toString('hex');
+    const browser = await chromium.launch({ headless: false });
+    const context = await browser.newContext();
+    const page = await context.newPage();
+    await page.goto(appUrl, { waitUntil: 'domcontentloaded' }).catch(() => { });
+    const timer = setTimeout(() => cancelSessionCapture(id).catch(() => { }), 10 * 60 * 1000);
+    activeSessions.set(id, { browser, context, timer });
+    return id;
 }
-async function completeSessionCapture(captureId) {
-  const session = activeSessions.get(captureId);
-  if (!session) throw new Error("Capture session not found or expired.");
-  clearTimeout(session.timer);
-  const state = await session.context.storageState();
-  await session.browser.close().catch(() => {
-  });
-  activeSessions.delete(captureId);
-  return JSON.stringify(state);
+/** Captures storageState from the open browser, closes it, and returns the JSON string. */
+export async function completeSessionCapture(captureId) {
+    const session = activeSessions.get(captureId);
+    if (!session)
+        throw new Error('Capture session not found or expired.');
+    clearTimeout(session.timer);
+    const state = await session.context.storageState();
+    await session.browser.close().catch(() => { });
+    activeSessions.delete(captureId);
+    return JSON.stringify(state);
 }
-async function cancelSessionCapture(captureId) {
-  const session = activeSessions.get(captureId);
-  if (!session) return;
-  clearTimeout(session.timer);
-  await session.browser.close().catch(() => {
-  });
-  activeSessions.delete(captureId);
+/** Closes the browser without capturing — called on Cancel. */
+export async function cancelSessionCapture(captureId) {
+    const session = activeSessions.get(captureId);
+    if (!session)
+        return;
+    clearTimeout(session.timer);
+    await session.browser.close().catch(() => { });
+    activeSessions.delete(captureId);
 }
+/**
+ * Post-login exploration: navigates nav links, cycles tabs, advances carousels,
+ * opens accordions, scrolls — so the browser accumulates full session state before capture.
+ * All best-effort: never throws, never fails the capture.
+ */
 async function exploreAppAfterLogin(page) {
-  const sleep = (ms) => page.waitForTimeout(ms).catch(() => {
-  });
-  const isVisible = `
+    const sleep = (ms) => page.waitForTimeout(ms).catch(() => { });
+    const isVisible = `
     function isVisible(el) {
       const s = window.getComputedStyle(el);
       if (s.display === 'none' || s.visibility === 'hidden' || parseFloat(s.opacity) < 0.1) return false;
@@ -49,10 +61,11 @@ async function exploreAppAfterLogin(page) {
       return r.width > 4 && r.height > 4;
     }
   `;
-  try {
-    const origin = await page.evaluate(() => location.origin).catch(() => "");
-    const startUrl = await page.evaluate(() => location.href).catch(() => "");
-    await page.evaluate(new Function(`
+    try {
+        const origin = await page.evaluate(() => location.origin).catch(() => '');
+        const startUrl = await page.evaluate(() => location.href).catch(() => '');
+        // ── 1. Expand collapsed nav menus / hamburgers ──────────────────────────
+        await page.evaluate(new Function(`
       ${isVisible}
       const inBadCtx = el => {
         let p = el.parentElement;
@@ -74,10 +87,10 @@ async function exploreAppAfterLogin(page) {
         if ((el.tagName === 'BUTTON' || el.getAttribute('role') === 'button') && isVisible(el) && !inBadCtx(el)) targets.add(el);
       });
       targets.forEach(el => { try { el.click(); } catch {} });
-    `)).catch(() => {
-    });
-    await sleep(600);
-    const navLinks = await page.evaluate(new Function(`
+    `)).catch(() => { });
+        await sleep(600);
+        // ── 2. Collect nav links from the expanded DOM ──────────────────────────
+        const navLinks = await page.evaluate(new Function(`
       ${isVisible}
       const links = new Set();
       const navEls = document.querySelectorAll('nav a, [role="navigation"] a, aside a, header a');
@@ -92,39 +105,44 @@ async function exploreAppAfterLogin(page) {
       });
       return [...links].slice(0, 10); // cap at 10 nav pages
     `)).catch(() => []);
-    await page.keyboard?.press("Escape").catch(() => {
-    });
-    await sleep(200);
-    for (const link of navLinks) {
-      try {
-        if (link === startUrl) continue;
-        await page.goto(link, { waitUntil: "networkidle", timeout: 15e3 }).catch(() => {
-        });
-        await sleep(400);
+        // Dismiss opened menus before navigating
+        await page.keyboard?.press('Escape').catch(() => { });
+        await sleep(200);
+        // ── 3. Visit each nav link, interact, return ────────────────────────────
+        for (const link of navLinks) {
+            try {
+                if (link === startUrl)
+                    continue;
+                await page.goto(link, { waitUntil: 'networkidle', timeout: 15_000 }).catch(() => { });
+                await sleep(400);
+                await interactWithCurrentPage(page, isVisible, sleep);
+            }
+            catch { /* skip broken page */ }
+        }
+        // Return to start page
+        if (navLinks.length > 0) {
+            await page.goto(startUrl, { waitUntil: 'networkidle', timeout: 15_000 }).catch(() => { });
+            await sleep(400);
+        }
+        // ── 4. Final interaction pass on landing page ───────────────────────────
         await interactWithCurrentPage(page, isVisible, sleep);
-      } catch {
-      }
+        await page.waitForLoadState('networkidle', { timeout: 5_000 }).catch(() => { });
     }
-    if (navLinks.length > 0) {
-      await page.goto(startUrl, { waitUntil: "networkidle", timeout: 15e3 }).catch(() => {
-      });
-      await sleep(400);
+    catch {
+        // Exploration is best-effort
     }
-    await interactWithCurrentPage(page, isVisible, sleep);
-    await page.waitForLoadState("networkidle", { timeout: 5e3 }).catch(() => {
-    });
-  } catch {
-  }
 }
+/** Interacts with tabs, carousels, accordions, and scrolls on the current page. */
 async function interactWithCurrentPage(page, isVisible, sleep) {
-  await page.evaluate(new Function(`
+    // Cycle through tabs
+    await page.evaluate(new Function(`
     ${isVisible}
     const tabs = [...document.querySelectorAll('[role="tab"]')].filter(isVisible);
     tabs.forEach(tab => { try { tab.click(); } catch {} });
-  `)).catch(() => {
-  });
-  await sleep(400);
-  await page.evaluate(new Function(`
+  `)).catch(() => { });
+    await sleep(400);
+    // Advance carousels (next buttons)
+    await page.evaluate(new Function(`
     ${isVisible}
     const nextBtns = [...document.querySelectorAll(
       'button[aria-label*="next" i], button[aria-label*="forward" i], button[aria-label*="right" i], ' +
@@ -134,10 +152,10 @@ async function interactWithCurrentPage(page, isVisible, sleep) {
     for (let i = 0; i < 3; i++) {
       nextBtns.forEach(btn => { try { btn.click(); } catch {} });
     }
-  `)).catch(() => {
-  });
-  await sleep(400);
-  await page.evaluate(new Function(`
+  `)).catch(() => { });
+    await sleep(400);
+    // Open closed accordions / disclosure panels
+    await page.evaluate(new Function(`
     ${isVisible}
     document.querySelectorAll(
       '[data-state="closed"] button, [aria-expanded="false"], ' +
@@ -145,125 +163,127 @@ async function interactWithCurrentPage(page, isVisible, sleep) {
     ).forEach(el => {
       if (isVisible(el)) { try { el.click(); } catch {} }
     });
-  `)).catch(() => {
-  });
-  await sleep(300);
-  await page.evaluate(() => {
-    window.scrollTo({ top: document.body.scrollHeight, behavior: "instant" });
-  }).catch(() => {
-  });
-  await sleep(300);
-  await page.evaluate(() => {
-    window.scrollTo({ top: 0, behavior: "instant" });
-  }).catch(() => {
-  });
+  `)).catch(() => { });
+    await sleep(300);
+    // Scroll to bottom then back to top
+    await page.evaluate(() => { window.scrollTo({ top: document.body.scrollHeight, behavior: 'instant' }); }).catch(() => { });
+    await sleep(300);
+    await page.evaluate(() => { window.scrollTo({ top: 0, behavior: 'instant' }); }).catch(() => { });
 }
-async function validateAndCaptureSession(appUrl, creds, llmConfig, captchaOpts) {
-  const browser = await chromium.launch({ headless: true });
-  try {
-    const contextOpts = {};
-    if (creds.httpBasicUsername && creds.httpBasicPassword) {
-      contextOpts.httpCredentials = { username: creds.httpBasicUsername, password: creds.httpBasicPassword };
-    }
-    const context = await browser.newContext(contextOpts);
-    const page = await context.newPage();
-    const getOneTimeCode = creds.totpSecret ? async () => generateTOTP(creds.totpSecret) : void 0;
-    let loggedIn;
+/**
+ * Headless login + session capture — works on Docker/cloud (no display needed).
+ * Tries DOM-based login first (fast, free). If it fails and llmConfig is provided,
+ * falls back to Stagehand AI-guided login (handles modals, multi-step, marketing pages).
+ *
+ * Returns { storageState, cookies, localStorage } on success.
+ * Throws a human-readable error on failure.
+ */
+export async function validateAndCaptureSession(appUrl, creds, llmConfig, captchaOpts) {
+    const browser = await chromium.launch({ headless: true });
     try {
-      loggedIn = await login(page, appUrl, {
-        username: creds.username,
-        password: creds.password,
-        getOneTimeCode
-      });
-    } catch (loginErr) {
-      const shotBuf = await page.screenshot({ type: "jpeg", quality: 60, fullPage: false }).catch(() => null);
-      throw Object.assign(loginErr, {
-        failureScreenshot: shotBuf ? shotBuf.toString("base64") : null
-      });
-    }
-    if (!loggedIn && llmConfig) {
-      let sh = null;
-      try {
-        const { Stagehand } = await import("@browserbasehq/stagehand");
-        const { loginWithStagehand } = await import("./ai-login.js");
-        sh = new Stagehand({
-          env: "LOCAL",
-          model: { modelName: llmConfig.modelName, apiKey: llmConfig.apiKey },
-          serverCache: false,
-          verbose: 0,
-          localBrowserLaunchOptions: { headless: true }
-        });
-        await sh.init();
-        const shPage = await sh.resolvePage();
-        await shPage.goto(appUrl, { waitUntil: "networkidle", timeout: 3e4 }).catch(() => {
-        });
-        const shLoggedIn = await loginWithStagehand(
-          sh,
-          { username: creds.username, password: creds.password, getOneTimeCode },
-          appUrl,
-          captchaOpts
-        );
-        if (shLoggedIn) {
-          const shCtx = sh.context ?? sh.browserContext;
-          if (shCtx) {
-            const shState = await shCtx.storageState();
-            const shCookies = shState.cookies?.length ?? 0;
-            const shLocalStorage = shState.origins?.reduce((s, o) => s + (o.localStorage?.length ?? 0), 0) ?? 0;
-            if (shCookies > 0 || shLocalStorage > 0) {
-              await sh.close().catch(() => {
-              });
-              sh = null;
-              return { storageState: JSON.stringify(shState), cookies: shCookies, localStorage: shLocalStorage };
+        const contextOpts = {};
+        if (creds.httpBasicUsername && creds.httpBasicPassword) {
+            contextOpts.httpCredentials = { username: creds.httpBasicUsername, password: creds.httpBasicPassword };
+        }
+        const context = await browser.newContext(contextOpts);
+        const page = await context.newPage();
+        const getOneTimeCode = creds.totpSecret
+            ? async () => generateTOTP(creds.totpSecret)
+            : undefined;
+        let loggedIn;
+        try {
+            loggedIn = await login(page, appUrl, {
+                username: creds.username,
+                password: creds.password,
+                getOneTimeCode,
+            });
+        }
+        catch (loginErr) {
+            // login() throws when it can read a real error message from the page
+            const shotBuf = await page.screenshot({ type: 'jpeg', quality: 60, fullPage: false }).catch(() => null);
+            throw Object.assign(loginErr, {
+                failureScreenshot: shotBuf ? shotBuf.toString('base64') : null,
+            });
+        }
+        // Stagehand AI fallback when DOM login fails and LLM config is available
+        if (!loggedIn && llmConfig) {
+            let sh = null;
+            try {
+                const { Stagehand } = await import('@browserbasehq/stagehand');
+                const { loginWithStagehand } = await import('./ai-login.js');
+                sh = new Stagehand({
+                    env: 'LOCAL',
+                    model: { modelName: llmConfig.modelName, apiKey: llmConfig.apiKey },
+                    serverCache: false,
+                    verbose: 0,
+                    localBrowserLaunchOptions: { headless: true },
+                });
+                await sh.init();
+                const shPage = await sh.resolvePage();
+                await shPage.goto(appUrl, { waitUntil: 'networkidle', timeout: 30_000 }).catch(() => { });
+                const shLoggedIn = await loginWithStagehand(sh, { username: creds.username, password: creds.password, getOneTimeCode }, appUrl, captchaOpts);
+                if (shLoggedIn) {
+                    const shCtx = sh.context ?? sh.browserContext;
+                    if (shCtx) {
+                        const shState = await shCtx.storageState();
+                        const shCookies = shState.cookies?.length ?? 0;
+                        const shLocalStorage = shState.origins?.reduce((s, o) => s + (o.localStorage?.length ?? 0), 0) ?? 0;
+                        if (shCookies > 0 || shLocalStorage > 0) {
+                            await sh.close().catch(() => { });
+                            sh = null;
+                            return { storageState: JSON.stringify(shState), cookies: shCookies, localStorage: shLocalStorage };
+                        }
+                    }
+                    loggedIn = true; // Stagehand succeeded but we'll capture from the plain browser context below
+                }
+            }
+            catch (shErr) {
+                const msg = String(shErr?.message ?? shErr);
+                if (msg.includes('CAPTCHA') || shErr?.isSSOError)
+                    throw shErr;
+                // Stagehand failed silently — fall through to "credentials rejected"
+            }
+            finally {
+                if (sh)
+                    await sh.close().catch(() => { });
             }
-          }
-          loggedIn = true;
         }
-      } catch (shErr) {
-        const msg = String(shErr?.message ?? shErr);
-        if (msg.includes("CAPTCHA") || shErr?.isSSOError) throw shErr;
-      } finally {
-        if (sh) await sh.close().catch(() => {
-        });
-      }
-    }
-    if (!loggedIn) {
-      const shotBuf = await page.screenshot({ type: "jpeg", quality: 60, fullPage: false }).catch(() => null);
-      const challenge = await page.evaluate(() => {
-        const text = (document.body?.innerText ?? "").toLowerCase();
-        const hasCloudflare = /verify you are human|checking your browser|cloudflare|cf-challenge|turnstile/.test(text) || !!document.querySelector('iframe[src*="challenges.cloudflare.com"], .cf-turnstile, [data-sitekey]');
-        const hasCaptcha = hasCloudflare || /captcha|recaptcha|hcaptcha/.test(text) || !!document.querySelector('iframe[src*="recaptcha"], iframe[src*="hcaptcha"], #captcha, .captcha, [id*="captcha"], [class*="captcha"]');
-        return { hasCloudflare, hasCaptcha };
-      }).catch(() => ({ hasCloudflare: false, hasCaptcha: false }));
-      const message = challenge.hasCloudflare ? "Cloudflare human verification blocked headless validation. This cannot be bypassed reliably in code; use Live Browser or manual session capture, configure a CAPTCHA solver, or allowlist the crawler IP." : challenge.hasCaptcha ? "A CAPTCHA blocked headless validation. Configure CAPTCHA Auto-Solve or use Live Browser/manual session capture." : "Login failed \u2014 credentials rejected or the login flow requires a manual/SSO step. Check credentials, saved login instructions, or use Live Browser session capture.";
-      const err = Object.assign(new Error(message), {
-        failureScreenshot: shotBuf ? shotBuf.toString("base64") : null,
-        reason: challenge.hasCloudflare ? "CLOUDFLARE_CHALLENGE" : challenge.hasCaptcha ? "CAPTCHA_CHALLENGE" : "LOGIN_NOT_CONFIRMED"
-      });
-      throw err;
+        if (!loggedIn) {
+            const shotBuf = await page.screenshot({ type: 'jpeg', quality: 60, fullPage: false }).catch(() => null);
+            const challenge = await page.evaluate(() => {
+                const text = (document.body?.innerText ?? '').toLowerCase();
+                const hasCloudflare = /verify you are human|checking your browser|cloudflare|cf-challenge|turnstile/.test(text) ||
+                    !!document.querySelector('iframe[src*="challenges.cloudflare.com"], .cf-turnstile, [data-sitekey]');
+                const hasCaptcha = hasCloudflare || /captcha|recaptcha|hcaptcha/.test(text) ||
+                    !!document.querySelector('iframe[src*="recaptcha"], iframe[src*="hcaptcha"], #captcha, .captcha, [id*="captcha"], [class*="captcha"]');
+                return { hasCloudflare, hasCaptcha };
+            }).catch(() => ({ hasCloudflare: false, hasCaptcha: false }));
+            const message = challenge.hasCloudflare
+                ? 'Cloudflare human verification blocked headless validation. This cannot be bypassed reliably in code; use Live Browser or manual session capture, configure a CAPTCHA solver, or allowlist the crawler IP.'
+                : challenge.hasCaptcha
+                    ? 'A CAPTCHA blocked headless validation. Configure CAPTCHA Auto-Solve or use Live Browser/manual session capture.'
+                    : 'Login failed — credentials rejected or the login flow requires a manual/SSO step. Check credentials, saved login instructions, or use Live Browser session capture.';
+            const err = Object.assign(new Error(message), {
+                failureScreenshot: shotBuf ? shotBuf.toString('base64') : null,
+                reason: challenge.hasCloudflare ? 'CLOUDFLARE_CHALLENGE' : challenge.hasCaptcha ? 'CAPTCHA_CHALLENGE' : 'LOGIN_NOT_CONFIRMED',
+            });
+            throw err;
+        }
+        // Wait for post-login navigation to settle before capturing session
+        await page.waitForLoadState('networkidle', { timeout: 8_000 }).catch(() => { });
+        // Capture post-login landing URL before exploration navigates away
+        const finalUrl = page.url();
+        // Post-login exploration: navigate tabs, expand menus, scroll to trigger lazy-loaded state
+        await exploreAppAfterLogin(page);
+        const state = await context.storageState();
+        const cookies = state.cookies?.length ?? 0;
+        const localStorage = state.origins?.reduce((sum, o) => sum + (o.localStorage?.length ?? 0), 0) ?? 0;
+        if (cookies === 0 && localStorage === 0) {
+            throw new Error('Login appeared to succeed but no session data was captured — try manual session capture instead');
+        }
+        return { storageState: JSON.stringify(state), cookies, localStorage, finalUrl };
     }
-    await page.waitForLoadState("networkidle", { timeout: 8e3 }).catch(() => {
-    });
-    const finalUrl = page.url();
-    await exploreAppAfterLogin(page);
-    const state = await context.storageState();
-    const cookies = state.cookies?.length ?? 0;
-    const localStorage = state.origins?.reduce(
-      (sum, o) => sum + (o.localStorage?.length ?? 0),
-      0
-    ) ?? 0;
-    if (cookies === 0 && localStorage === 0) {
-      throw new Error("Login appeared to succeed but no session data was captured \u2014 try manual session capture instead");
+    finally {
+        await browser.close().catch(() => { });
     }
-    return { storageState: JSON.stringify(state), cookies, localStorage, finalUrl };
-  } finally {
-    await browser.close().catch(() => {
-    });
-  }
 }
-export {
-  cancelSessionCapture,
-  completeSessionCapture,
-  startSessionCapture,
-  validateAndCaptureSession
-};
-//# sourceMappingURL=session-capture.js.map