@detiq/crawler 1.1.9 → 1.1.11

package/dist/session-capture.js

@@ -1,59 +1,47 @@
-import { chromium } from 'playwright';
-import { randomBytes } from 'node:crypto';
-import { login } from './auth.js';
-import { generateTOTP } from './job-queue.js';
-const activeSessions = new Map();
-/**
- * Opens a headed (visible) browser at appUrl and returns a captureId.
- * The user logs in manually, then calls completeSessionCapture() to harvest storageState.
- * Sessions expire after 10 minutes if not completed.
- *
- * Throws 'HEADLESS_ENV' if running in Docker/Linux without a display server.
- * The API route surfaces this as a 503 so the frontend can switch to manual JSON paste.
- */
-export async function startSessionCapture(appUrl) {
-    const inDocker = process.env.PLAYWRIGHT_IN_DOCKER === 'true';
-    const noDisplay = process.platform === 'linux' && !process.env.DISPLAY;
-    if (inDocker || noDisplay) {
-        throw new Error('HEADLESS_ENV');
-    }
-    const id = randomBytes(16).toString('hex');
-    const browser = await chromium.launch({ headless: false });
-    const context = await browser.newContext();
-    const page = await context.newPage();
-    await page.goto(appUrl, { waitUntil: 'domcontentloaded' }).catch(() => { });
-    const timer = setTimeout(() => cancelSessionCapture(id).catch(() => { }), 10 * 60 * 1000);
-    activeSessions.set(id, { browser, context, timer });
-    return id;
+import { chromium } from "playwright";
+import { randomBytes } from "node:crypto";
+import { login } from "./auth.js";
+import { generateTOTP } from "./job-queue.js";
+const activeSessions = /* @__PURE__ */ new Map();
+async function startSessionCapture(appUrl) {
+  const inDocker = process.env.PLAYWRIGHT_IN_DOCKER === "true";
+  const noDisplay = process.platform === "linux" && !process.env.DISPLAY;
+  if (inDocker || noDisplay) {
+    throw new Error("HEADLESS_ENV");
+  }
+  const id = randomBytes(16).toString("hex");
+  const browser = await chromium.launch({ headless: false });
+  const context = await browser.newContext();
+  const page = await context.newPage();
+  await page.goto(appUrl, { waitUntil: "domcontentloaded" }).catch(() => {
+  });
+  const timer = setTimeout(() => cancelSessionCapture(id).catch(() => {
+  }), 10 * 60 * 1e3);
+  activeSessions.set(id, { browser, context, timer });
+  return id;
 }
-/** Captures storageState from the open browser, closes it, and returns the JSON string. */
-export async function completeSessionCapture(captureId) {
-    const session = activeSessions.get(captureId);
-    if (!session)
-        throw new Error('Capture session not found or expired.');
-    clearTimeout(session.timer);
-    const state = await session.context.storageState();
-    await session.browser.close().catch(() => { });
-    activeSessions.delete(captureId);
-    return JSON.stringify(state);
+async function completeSessionCapture(captureId) {
+  const session = activeSessions.get(captureId);
+  if (!session) throw new Error("Capture session not found or expired.");
+  clearTimeout(session.timer);
+  const state = await session.context.storageState();
+  await session.browser.close().catch(() => {
+  });
+  activeSessions.delete(captureId);
+  return JSON.stringify(state);
 }
-/** Closes the browser without capturing — called on Cancel. */
-export async function cancelSessionCapture(captureId) {
-    const session = activeSessions.get(captureId);
-    if (!session)
-        return;
-    clearTimeout(session.timer);
-    await session.browser.close().catch(() => { });
-    activeSessions.delete(captureId);
+async function cancelSessionCapture(captureId) {
+  const session = activeSessions.get(captureId);
+  if (!session) return;
+  clearTimeout(session.timer);
+  await session.browser.close().catch(() => {
+  });
+  activeSessions.delete(captureId);
 }
-/**
- * Post-login exploration: navigates nav links, cycles tabs, advances carousels,
- * opens accordions, scrolls — so the browser accumulates full session state before capture.
- * All best-effort: never throws, never fails the capture.
- */
 async function exploreAppAfterLogin(page) {
-    const sleep = (ms) => page.waitForTimeout(ms).catch(() => { });
-    const isVisible = `
+  const sleep = (ms) => page.waitForTimeout(ms).catch(() => {
+  });
+  const isVisible = `
     function isVisible(el) {
       const s = window.getComputedStyle(el);
       if (s.display === 'none' || s.visibility === 'hidden' || parseFloat(s.opacity) < 0.1) return false;
@@ -61,11 +49,10 @@ async function exploreAppAfterLogin(page) {
       return r.width > 4 && r.height > 4;
     }
   `;
-    try {
-        const origin = await page.evaluate(() => location.origin).catch(() => '');
-        const startUrl = await page.evaluate(() => location.href).catch(() => '');
-        // ── 1. Expand collapsed nav menus / hamburgers ──────────────────────────
-        await page.evaluate(new Function(`
+  try {
+    const origin = await page.evaluate(() => location.origin).catch(() => "");
+    const startUrl = await page.evaluate(() => location.href).catch(() => "");
+    await page.evaluate(new Function(`
       ${isVisible}
       const inBadCtx = el => {
         let p = el.parentElement;
@@ -87,10 +74,10 @@ async function exploreAppAfterLogin(page) {
         if ((el.tagName === 'BUTTON' || el.getAttribute('role') === 'button') && isVisible(el) && !inBadCtx(el)) targets.add(el);
       });
       targets.forEach(el => { try { el.click(); } catch {} });
-    `)).catch(() => { });
-        await sleep(600);
-        // ── 2. Collect nav links from the expanded DOM ──────────────────────────
-        const navLinks = await page.evaluate(new Function(`
+    `)).catch(() => {
+    });
+    await sleep(600);
+    const navLinks = await page.evaluate(new Function(`
       ${isVisible}
       const links = new Set();
       const navEls = document.querySelectorAll('nav a, [role="navigation"] a, aside a, header a');
@@ -105,44 +92,39 @@ async function exploreAppAfterLogin(page) {
       });
       return [...links].slice(0, 10); // cap at 10 nav pages
     `)).catch(() => []);
-        // Dismiss opened menus before navigating
-        await page.keyboard?.press('Escape').catch(() => { });
-        await sleep(200);
-        // ── 3. Visit each nav link, interact, return ────────────────────────────
-        for (const link of navLinks) {
-            try {
-                if (link === startUrl)
-                    continue;
-                await page.goto(link, { waitUntil: 'networkidle', timeout: 15_000 }).catch(() => { });
-                await sleep(400);
-                await interactWithCurrentPage(page, isVisible, sleep);
-            }
-            catch { /* skip broken page */ }
-        }
-        // Return to start page
-        if (navLinks.length > 0) {
-            await page.goto(startUrl, { waitUntil: 'networkidle', timeout: 15_000 }).catch(() => { });
-            await sleep(400);
-        }
-        // ── 4. Final interaction pass on landing page ───────────────────────────
+    await page.keyboard?.press("Escape").catch(() => {
+    });
+    await sleep(200);
+    for (const link of navLinks) {
+      try {
+        if (link === startUrl) continue;
+        await page.goto(link, { waitUntil: "networkidle", timeout: 15e3 }).catch(() => {
+        });
+        await sleep(400);
         await interactWithCurrentPage(page, isVisible, sleep);
-        await page.waitForLoadState('networkidle', { timeout: 5_000 }).catch(() => { });
+      } catch {
+      }
     }
-    catch {
-        // Exploration is best-effort
+    if (navLinks.length > 0) {
+      await page.goto(startUrl, { waitUntil: "networkidle", timeout: 15e3 }).catch(() => {
+      });
+      await sleep(400);
     }
+    await interactWithCurrentPage(page, isVisible, sleep);
+    await page.waitForLoadState("networkidle", { timeout: 5e3 }).catch(() => {
+    });
+  } catch {
+  }
 }
-/** Interacts with tabs, carousels, accordions, and scrolls on the current page. */
 async function interactWithCurrentPage(page, isVisible, sleep) {
-    // Cycle through tabs
-    await page.evaluate(new Function(`
+  await page.evaluate(new Function(`
     ${isVisible}
     const tabs = [...document.querySelectorAll('[role="tab"]')].filter(isVisible);
     tabs.forEach(tab => { try { tab.click(); } catch {} });
-  `)).catch(() => { });
-    await sleep(400);
-    // Advance carousels (next buttons)
-    await page.evaluate(new Function(`
+  `)).catch(() => {
+  });
+  await sleep(400);
+  await page.evaluate(new Function(`
     ${isVisible}
     const nextBtns = [...document.querySelectorAll(
       'button[aria-label*="next" i], button[aria-label*="forward" i], button[aria-label*="right" i], ' +
@@ -152,10 +134,10 @@ async function interactWithCurrentPage(page, isVisible, sleep) {
     for (let i = 0; i < 3; i++) {
       nextBtns.forEach(btn => { try { btn.click(); } catch {} });
     }
-  `)).catch(() => { });
-    await sleep(400);
-    // Open closed accordions / disclosure panels
-    await page.evaluate(new Function(`
+  `)).catch(() => {
+  });
+  await sleep(400);
+  await page.evaluate(new Function(`
     ${isVisible}
     document.querySelectorAll(
       '[data-state="closed"] button, [aria-expanded="false"], ' +
@@ -163,127 +145,125 @@ async function interactWithCurrentPage(page, isVisible, sleep) {
     ).forEach(el => {
       if (isVisible(el)) { try { el.click(); } catch {} }
     });
-  `)).catch(() => { });
-    await sleep(300);
-    // Scroll to bottom then back to top
-    await page.evaluate(() => { window.scrollTo({ top: document.body.scrollHeight, behavior: 'instant' }); }).catch(() => { });
-    await sleep(300);
-    await page.evaluate(() => { window.scrollTo({ top: 0, behavior: 'instant' }); }).catch(() => { });
+  `)).catch(() => {
+  });
+  await sleep(300);
+  await page.evaluate(() => {
+    window.scrollTo({ top: document.body.scrollHeight, behavior: "instant" });
+  }).catch(() => {
+  });
+  await sleep(300);
+  await page.evaluate(() => {
+    window.scrollTo({ top: 0, behavior: "instant" });
+  }).catch(() => {
+  });
 }
-/**
- * Headless login + session capture — works on Docker/cloud (no display needed).
- * Tries DOM-based login first (fast, free). If it fails and llmConfig is provided,
- * falls back to Stagehand AI-guided login (handles modals, multi-step, marketing pages).
- *
- * Returns { storageState, cookies, localStorage } on success.
- * Throws a human-readable error on failure.
- */
-export async function validateAndCaptureSession(appUrl, creds, llmConfig, captchaOpts) {
-    const browser = await chromium.launch({ headless: true });
+async function validateAndCaptureSession(appUrl, creds, llmConfig, captchaOpts) {
+  const browser = await chromium.launch({ headless: true });
+  try {
+    const contextOpts = {};
+    if (creds.httpBasicUsername && creds.httpBasicPassword) {
+      contextOpts.httpCredentials = { username: creds.httpBasicUsername, password: creds.httpBasicPassword };
+    }
+    const context = await browser.newContext(contextOpts);
+    const page = await context.newPage();
+    const getOneTimeCode = creds.totpSecret ? async () => generateTOTP(creds.totpSecret) : void 0;
+    let loggedIn;
     try {
-        const contextOpts = {};
-        if (creds.httpBasicUsername && creds.httpBasicPassword) {
-            contextOpts.httpCredentials = { username: creds.httpBasicUsername, password: creds.httpBasicPassword };
-        }
-        const context = await browser.newContext(contextOpts);
-        const page = await context.newPage();
-        const getOneTimeCode = creds.totpSecret
-            ? async () => generateTOTP(creds.totpSecret)
-            : undefined;
-        let loggedIn;
-        try {
-            loggedIn = await login(page, appUrl, {
-                username: creds.username,
-                password: creds.password,
-                getOneTimeCode,
-            });
-        }
-        catch (loginErr) {
-            // login() throws when it can read a real error message from the page
-            const shotBuf = await page.screenshot({ type: 'jpeg', quality: 60, fullPage: false }).catch(() => null);
-            throw Object.assign(loginErr, {
-                failureScreenshot: shotBuf ? shotBuf.toString('base64') : null,
-            });
-        }
-        // Stagehand AI fallback when DOM login fails and LLM config is available
-        if (!loggedIn && llmConfig) {
-            let sh = null;
-            try {
-                const { Stagehand } = await import('@browserbasehq/stagehand');
-                const { loginWithStagehand } = await import('./ai-login.js');
-                sh = new Stagehand({
-                    env: 'LOCAL',
-                    model: { modelName: llmConfig.modelName, apiKey: llmConfig.apiKey },
-                    serverCache: false,
-                    verbose: 0,
-                    localBrowserLaunchOptions: { headless: true },
-                });
-                await sh.init();
-                const shPage = await sh.resolvePage();
-                await shPage.goto(appUrl, { waitUntil: 'networkidle', timeout: 30_000 }).catch(() => { });
-                const shLoggedIn = await loginWithStagehand(sh, { username: creds.username, password: creds.password, getOneTimeCode }, appUrl, captchaOpts);
-                if (shLoggedIn) {
-                    const shCtx = sh.context ?? sh.browserContext;
-                    if (shCtx) {
-                        const shState = await shCtx.storageState();
-                        const shCookies = shState.cookies?.length ?? 0;
-                        const shLocalStorage = shState.origins?.reduce((s, o) => s + (o.localStorage?.length ?? 0), 0) ?? 0;
-                        if (shCookies > 0 || shLocalStorage > 0) {
-                            await sh.close().catch(() => { });
-                            sh = null;
-                            return { storageState: JSON.stringify(shState), cookies: shCookies, localStorage: shLocalStorage };
-                        }
-                    }
-                    loggedIn = true; // Stagehand succeeded but we'll capture from the plain browser context below
-                }
-            }
-            catch (shErr) {
-                const msg = String(shErr?.message ?? shErr);
-                if (msg.includes('CAPTCHA') || shErr?.isSSOError)
-                    throw shErr;
-                // Stagehand failed silently — fall through to "credentials rejected"
-            }
-            finally {
-                if (sh)
-                    await sh.close().catch(() => { });
+      loggedIn = await login(page, appUrl, {
+        username: creds.username,
+        password: creds.password,
+        getOneTimeCode
+      });
+    } catch (loginErr) {
+      const shotBuf = await page.screenshot({ type: "jpeg", quality: 60, fullPage: false }).catch(() => null);
+      throw Object.assign(loginErr, {
+        failureScreenshot: shotBuf ? shotBuf.toString("base64") : null
+      });
+    }
+    if (!loggedIn && llmConfig) {
+      let sh = null;
+      try {
+        const { Stagehand } = await import("@browserbasehq/stagehand");
+        const { loginWithStagehand } = await import("./ai-login.js");
+        sh = new Stagehand({
+          env: "LOCAL",
+          model: { modelName: llmConfig.modelName, apiKey: llmConfig.apiKey },
+          serverCache: false,
+          verbose: 0,
+          localBrowserLaunchOptions: { headless: true }
+        });
+        await sh.init();
+        const shPage = await sh.resolvePage();
+        await shPage.goto(appUrl, { waitUntil: "networkidle", timeout: 3e4 }).catch(() => {
+        });
+        const shLoggedIn = await loginWithStagehand(
+          sh,
+          { username: creds.username, password: creds.password, getOneTimeCode },
+          appUrl,
+          captchaOpts
+        );
+        if (shLoggedIn) {
+          const shCtx = sh.context ?? sh.browserContext;
+          if (shCtx) {
+            const shState = await shCtx.storageState();
+            const shCookies = shState.cookies?.length ?? 0;
+            const shLocalStorage = shState.origins?.reduce((s, o) => s + (o.localStorage?.length ?? 0), 0) ?? 0;
+            if (shCookies > 0 || shLocalStorage > 0) {
+              await sh.close().catch(() => {
+              });
+              sh = null;
+              return { storageState: JSON.stringify(shState), cookies: shCookies, localStorage: shLocalStorage };
             }
+          }
+          loggedIn = true;
         }
-        if (!loggedIn) {
-            const shotBuf = await page.screenshot({ type: 'jpeg', quality: 60, fullPage: false }).catch(() => null);
-            const challenge = await page.evaluate(() => {
-                const text = (document.body?.innerText ?? '').toLowerCase();
-                const hasCloudflare = /verify you are human|checking your browser|cloudflare|cf-challenge|turnstile/.test(text) ||
-                    !!document.querySelector('iframe[src*="challenges.cloudflare.com"], .cf-turnstile, [data-sitekey]');
-                const hasCaptcha = hasCloudflare || /captcha|recaptcha|hcaptcha/.test(text) ||
-                    !!document.querySelector('iframe[src*="recaptcha"], iframe[src*="hcaptcha"], #captcha, .captcha, [id*="captcha"], [class*="captcha"]');
-                return { hasCloudflare, hasCaptcha };
-            }).catch(() => ({ hasCloudflare: false, hasCaptcha: false }));
-            const message = challenge.hasCloudflare
-                ? 'Cloudflare human verification blocked headless validation. This cannot be bypassed reliably in code; use Live Browser or manual session capture, configure a CAPTCHA solver, or allowlist the crawler IP.'
-                : challenge.hasCaptcha
-                    ? 'A CAPTCHA blocked headless validation. Configure CAPTCHA Auto-Solve or use Live Browser/manual session capture.'
-                    : 'Login failed — credentials rejected or the login flow requires a manual/SSO step. Check credentials, saved login instructions, or use Live Browser session capture.';
-            const err = Object.assign(new Error(message), {
-                failureScreenshot: shotBuf ? shotBuf.toString('base64') : null,
-                reason: challenge.hasCloudflare ? 'CLOUDFLARE_CHALLENGE' : challenge.hasCaptcha ? 'CAPTCHA_CHALLENGE' : 'LOGIN_NOT_CONFIRMED',
-            });
-            throw err;
-        }
-        // Wait for post-login navigation to settle before capturing session
-        await page.waitForLoadState('networkidle', { timeout: 8_000 }).catch(() => { });
-        // Capture post-login landing URL before exploration navigates away
-        const finalUrl = page.url();
-        // Post-login exploration: navigate tabs, expand menus, scroll to trigger lazy-loaded state
-        await exploreAppAfterLogin(page);
-        const state = await context.storageState();
-        const cookies = state.cookies?.length ?? 0;
-        const localStorage = state.origins?.reduce((sum, o) => sum + (o.localStorage?.length ?? 0), 0) ?? 0;
-        if (cookies === 0 && localStorage === 0) {
-            throw new Error('Login appeared to succeed but no session data was captured — try manual session capture instead');
-        }
-        return { storageState: JSON.stringify(state), cookies, localStorage, finalUrl };
+      } catch (shErr) {
+        const msg = String(shErr?.message ?? shErr);
+        if (msg.includes("CAPTCHA") || shErr?.isSSOError) throw shErr;
+      } finally {
+        if (sh) await sh.close().catch(() => {
+        });
+      }
     }
-    finally {
-        await browser.close().catch(() => { });
+    if (!loggedIn) {
+      const shotBuf = await page.screenshot({ type: "jpeg", quality: 60, fullPage: false }).catch(() => null);
+      const challenge = await page.evaluate(() => {
+        const text = (document.body?.innerText ?? "").toLowerCase();
+        const hasCloudflare = /verify you are human|checking your browser|cloudflare|cf-challenge|turnstile/.test(text) || !!document.querySelector('iframe[src*="challenges.cloudflare.com"], .cf-turnstile, [data-sitekey]');
+        const hasCaptcha = hasCloudflare || /captcha|recaptcha|hcaptcha/.test(text) || !!document.querySelector('iframe[src*="recaptcha"], iframe[src*="hcaptcha"], #captcha, .captcha, [id*="captcha"], [class*="captcha"]');
+        return { hasCloudflare, hasCaptcha };
+      }).catch(() => ({ hasCloudflare: false, hasCaptcha: false }));
+      const message = challenge.hasCloudflare ? "Cloudflare human verification blocked headless validation. This cannot be bypassed reliably in code; use Live Browser or manual session capture, configure a CAPTCHA solver, or allowlist the crawler IP." : challenge.hasCaptcha ? "A CAPTCHA blocked headless validation. Configure CAPTCHA Auto-Solve or use Live Browser/manual session capture." : "Login failed \u2014 credentials rejected or the login flow requires a manual/SSO step. Check credentials, saved login instructions, or use Live Browser session capture.";
+      const err = Object.assign(new Error(message), {
+        failureScreenshot: shotBuf ? shotBuf.toString("base64") : null,
+        reason: challenge.hasCloudflare ? "CLOUDFLARE_CHALLENGE" : challenge.hasCaptcha ? "CAPTCHA_CHALLENGE" : "LOGIN_NOT_CONFIRMED"
+      });
+      throw err;
     }
+    await page.waitForLoadState("networkidle", { timeout: 8e3 }).catch(() => {
+    });
+    const finalUrl = page.url();
+    await exploreAppAfterLogin(page);
+    const state = await context.storageState();
+    const cookies = state.cookies?.length ?? 0;
+    const localStorage = state.origins?.reduce(
+      (sum, o) => sum + (o.localStorage?.length ?? 0),
+      0
+    ) ?? 0;
+    if (cookies === 0 && localStorage === 0) {
+      throw new Error("Login appeared to succeed but no session data was captured \u2014 try manual session capture instead");
+    }
+    return { storageState: JSON.stringify(state), cookies, localStorage, finalUrl };
+  } finally {
+    await browser.close().catch(() => {
+    });
+  }
 }
+export {
+  cancelSessionCapture,
+  completeSessionCapture,
+  startSessionCapture,
+  validateAndCaptureSession
+};
+//# sourceMappingURL=session-capture.js.map