@detiq/crawler 0.1.3

package/src/ai-login.ts

@@ -0,0 +1,218 @@
+import { z } from 'zod';
+import type { Stagehand } from '@browserbasehq/stagehand';
+import { AppBrain } from '@detiq/app-brain';
+import { solveCaptcha } from './captcha-solver.js';
+import { waitForEmailContent } from './email-otp.js';
+import { logger } from './logger.js';
+import type { Credentials } from './auth.js';
+import { simulateMouseMovement } from './stealth/human-behavior.js';
+
+const CLICK_INTENT = /\b(click|tap|press|submit|select|choose|toggle)\b/i;
+
+/** Simulate human mouse movement before click-type actions, then call stagehand.act(). */
+async function actWithMouseSim(stagehand: Stagehand, instruction: string): Promise<any> {
+  if (CLICK_INTENT.test(instruction)) {
+    try {
+      const pg = await (stagehand as any).resolvePage();
+      const viewport = pg.viewportSize() ?? { width: 1280, height: 720 };
+      // Move mouse toward center of page with some randomness before Stagehand clicks
+      const targetX = viewport.width * 0.3 + Math.random() * viewport.width * 0.4;
+      const targetY = viewport.height * 0.2 + Math.random() * viewport.height * 0.5;
+      await simulateMouseMovement(pg, targetX, targetY);
+    } catch { /* non-fatal — don't block the action */ }
+  }
+  return stagehand.act(instruction);
+}
+
+/**
+ * Recorded login step from the Chrome extension's step recorder.
+ * {username} and {password} in `value` are substituted at replay time.
+ */
+export interface LoginStep {
+  action: 'fill' | 'click' | 'wait';
+  selector: string;
+  value?: string;
+  ms?: number;
+}
+
+const TwoFASchema = z.object({
+  has2FA: z.boolean(),
+  fieldLabel: z.string().optional(),
+});
+
+const CAPTCHA_SELECTORS = [
+  'iframe[src*="recaptcha"]',
+  'iframe[src*="hcaptcha"]',
+  'iframe[src*="challenges.cloudflare.com"]',
+  '.cf-turnstile',
+  '[data-sitekey]',
+  '#captcha',
+  '.captcha',
+  '[id*="captcha"]',
+  '[class*="captcha"]',
+];
+
+/**
+ * AI-guided login using Stagehand act/extract.
+ *
+ * @param loginInstructions - Optional natural language description of the login flow.
+ *   When provided, an initial stagehand.act() is called with these instructions to
+ *   navigate the UI (e.g. "click Sign In in the nav, fill email, click Continue").
+ *   The standard 3-step credential fill follows immediately after.
+ * @param opts - Optional tenant/project/job IDs for writing AUTH_STEP CrawlEvidence.
+ */
+export async function loginWithStagehand(
+  stagehand: Stagehand,
+  creds: Credentials,
+  startUrl: string,
+  captchaOpts?: { apiKey?: string; provider?: '2captcha' | 'capsolver' },
+  emailOtpOpts?: { apiKey?: string; inboxId?: string },
+  loginInstructions?: string,
+  opts?: { tenantId?: string; projectId?: string; jobId?: string },
+): Promise<boolean> {
+  async function logStep(step: string, fn: () => Promise<any>): Promise<{ success: boolean; detail: string }> {
+    const t0 = Date.now();
+    try {
+      const r = await fn();
+      const result = { success: true, detail: r?.description ?? r?.message ?? step };
+      if (opts?.tenantId && opts?.projectId) {
+        AppBrain.saveCrawlEvidence(opts.tenantId, opts.projectId, {
+          jobId: opts?.jobId,
+          type: 'AUTH_STEP',
+          payload: { step, ...result, ms: Date.now() - t0 },
+        }).catch(() => {});
+      }
+      return result;
+    } catch (e: any) {
+      const result = { success: false, detail: e.message ?? 'failed' };
+      if (opts?.tenantId && opts?.projectId) {
+        AppBrain.saveCrawlEvidence(opts.tenantId, opts.projectId, {
+          jobId: opts?.jobId,
+          type: 'AUTH_STEP',
+          payload: { step, ...result, ms: Date.now() - t0 },
+        }).catch(() => {});
+      }
+      throw e;
+    }
+  }
+
+  try {
+    const pg = await (stagehand as any).resolvePage();
+
+    // CAPTCHA detection
+    for (const sel of CAPTCHA_SELECTORS) {
+      const count = await pg.locator(sel).count().catch(() => 0);
+      if (count > 0) {
+        logger.warn({ selector: sel }, '[crawler] CAPTCHA detected on login page');
+        if (captchaOpts?.apiKey) {
+          logger.info({ provider: captchaOpts.provider ?? '2captcha' }, '[crawler] Attempting auto-solve');
+          try {
+            const result = await solveCaptcha(pg, startUrl, captchaOpts.apiKey, captchaOpts.provider ?? '2captcha');
+            if (result) {
+              logger.info({ provider: result.provider }, '[crawler] CAPTCHA solved — proceeding with login');
+              break;
+            }
+          } catch (solveErr) {
+            logger.warn({ err: solveErr }, '[crawler] CAPTCHA solve failed — falling back to manual error');
+            throw new Error(`CAPTCHA_DETECTED: Auto-solve failed (${String(solveErr)}). Use session capture instead.`);
+          }
+        } else {
+          throw new Error(
+            'CAPTCHA_DETECTED: Login page has a CAPTCHA. Use "Session Capture → Launch Browser" to log in manually, then capture the session. Alternatively, disable CAPTCHA for the ZeTa crawler IP.'
+          );
+        }
+        break;
+      }
+    }
+
+    // If caller provided natural language login instructions, use them to navigate to the login form
+    if (loginInstructions) {
+      await logStep('custom_recipe', () => actWithMouseSim(stagehand, loginInstructions));
+      await pg.waitForLoadState('networkidle').catch(() => pg.waitForTimeout(1500));
+    }
+
+    await logStep('fill_username', () => actWithMouseSim(stagehand, `Type "${creds.username}" into the username or email input field`));
+    await logStep('fill_password', () => actWithMouseSim(stagehand, `Type "${creds.password}" into the password input field`));
+    await logStep('click_submit', () => actWithMouseSim(stagehand, 'Click the sign in, log in, login, or continue button to submit credentials'));
+    await pg.waitForLoadState('networkidle').catch(() => pg.waitForTimeout(2000));
+
+    const twoFA = await stagehand.extract(
+      'Is there a two-factor authentication, OTP, or verification code input visible right now?',
+      TwoFASchema as any
+    );
+
+    if (twoFA.has2FA) {
+      if (opts?.tenantId && opts?.projectId) {
+        AppBrain.saveCrawlEvidence(opts.tenantId, opts.projectId, {
+          jobId: opts?.jobId,
+          type: 'AUTH_STEP',
+          payload: { step: '2fa_detect', success: true, detail: twoFA.fieldLabel ?? '2FA detected', ms: 0 },
+        }).catch(() => {});
+      }
+
+      let code = creds.oneTimeCode ?? await creds.getOneTimeCode?.();
+
+      if (!code && emailOtpOpts?.apiKey && emailOtpOpts?.inboxId) {
+        const since = new Date().toISOString();
+        logger.info('[crawler] 2FA detected — waiting for email OTP or magic link via MailSlurp');
+        const emailContent = await waitForEmailContent(
+          { provider: 'mailslurp', apiKey: emailOtpOpts.apiKey, inboxId: emailOtpOpts.inboxId },
+          since
+        );
+        if (emailContent?.magicLink) {
+          logger.info({ url: emailContent.magicLink }, '[crawler] Magic link received — navigating');
+          await pg.goto(emailContent.magicLink, { waitUntil: 'domcontentloaded' });
+          await pg.waitForLoadState('networkidle').catch(() => {});
+          return true; // magic link navigation = login complete
+        }
+        code = emailContent?.otp ?? undefined;
+        if (code) {
+          logger.info('[crawler] Email OTP received');
+        } else {
+          logger.warn('[crawler] Email OTP timed out after 60s');
+        }
+      }
+
+      if (!code) {
+        console.warn('[crawler] 2FA detected but no oneTimeCode provided');
+        return false;
+      }
+      await logStep('2fa_inject', () => actWithMouseSim(stagehand, `Enter "${code}" into the ${twoFA.fieldLabel ?? 'verification code'} field`));
+      await logStep('2fa_submit', () => actWithMouseSim(stagehand, 'Click verify or confirm to submit the verification code'));
+      await pg.waitForLoadState('networkidle').catch(() => pg.waitForTimeout(2000));
+    }
+
+    // URL-based success check
+    const currentUrl = pg.url();
+    const loginPathPattern = /\/(login|signin|sign-in|auth|authenticate|log-in|account\/login)(\?|$|\/)/i;
+    try {
+      const { pathname } = new URL(currentUrl);
+      const wasRedirected = currentUrl !== startUrl;
+      const stillOnLoginPath = loginPathPattern.test(pathname);
+      if (wasRedirected && !stillOnLoginPath) {
+        logger.info({ url: currentUrl }, 'Login confirmed via URL change');
+        return true;
+      }
+    } catch { /* invalid URL — fall through */ }
+
+    const passwordCount = await pg.locator('input[type="password"]').count().catch(() => 1);
+    const bodyText = await pg.evaluate(() => (document as any).body?.innerText?.trim()?.length ?? 0).catch(() => 0);
+    if (passwordCount === 0 && bodyText > 30) {
+      logger.info('Login confirmed: password field no longer visible');
+      return true;
+    }
+
+    logger.warn({ url: currentUrl }, 'Login appears failed');
+    try {
+      const screenshotPath = `/tmp/zeta-auth-failure-${Date.now()}.jpg`;
+      await pg.screenshot({ path: screenshotPath, type: 'jpeg', quality: 70, fullPage: false });
+      logger.warn({ screenshotPath }, '[crawler] Auth failure screenshot saved');
+    } catch { /* non-fatal */ }
+    return false;
+  } catch (err) {
+    const msg = String((err as Error).message ?? err);
+    if (msg.includes('CAPTCHA_DETECTED')) throw err;
+    logger.error({ err }, 'Login error');
+    return false;
+  }
+}

package/src/auth-token-rotator.ts

@@ -0,0 +1,89 @@
+/**
+ * Auth token rotator — manage a pool of authenticated sessions for multi-account crawls.
+ *
+ * Use case: crawl the same app as admin+user+guest in one crawl job to discover
+ * role-specific screens that only appear when authenticated with different privileges.
+ * Each session is a Playwright storageState JSON string.
+ *
+ * Sessions are checked out (locked) before use and returned after the page is done.
+ * Round-robin assignment with stale-session eviction after maxAgeMs.
+ */
+
+export interface SessionEntry {
+  id: string;
+  role: string;          // 'admin' | 'user' | 'guest' | custom label
+  storageState: string;  // Playwright StorageState JSON
+  createdAt: number;     // Date.now()
+  checkedOutAt: number | null;
+  requestCount: number;
+}
+
+export interface SessionPoolOpts {
+  sessions: Array<{ role: string; storageState: string }>;
+  maxAgeMs?: number;        // evict sessions older than this (default: 4h)
+  maxRequestsPerSession?: number; // rotate after N requests (default: 100)
+}
+
+export class AuthTokenRotator {
+  private pool: SessionEntry[];
+  private readonly maxAgeMs: number;
+  private readonly maxRequests: number;
+
+  constructor(opts: SessionPoolOpts) {
+    this.maxAgeMs = opts.maxAgeMs ?? 4 * 60 * 60 * 1000;
+    this.maxRequests = opts.maxRequestsPerSession ?? 100;
+    this.pool = opts.sessions.map((s, i) => ({
+      id: `session-${i}-${s.role}`,
+      role: s.role,
+      storageState: s.storageState,
+      createdAt: Date.now(),
+      checkedOutAt: null,
+      requestCount: 0,
+    }));
+  }
+
+  /** Get the next available session. Returns null if all are checked out. */
+  checkout(): SessionEntry | null {
+    const now = Date.now();
+    const available = this.pool.filter(s =>
+      !s.checkedOutAt &&
+      now - s.createdAt < this.maxAgeMs &&
+      s.requestCount < this.maxRequests,
+    );
+    if (available.length === 0) return null;
+    // Round-robin: pick lowest requestCount
+    available.sort((a, b) => a.requestCount - b.requestCount);
+    const session = available[0];
+    session.checkedOutAt = now;
+    return session;
+  }
+
+  /** Return a session to the pool after use. */
+  checkin(sessionId: string, updatedStorageState?: string): void {
+    const s = this.pool.find(e => e.id === sessionId);
+    if (!s) return;
+    s.checkedOutAt = null;
+    s.requestCount++;
+    if (updatedStorageState) s.storageState = updatedStorageState;
+  }
+
+  /** Get all available sessions (for parallel crawl setup). */
+  getAll(): ReadonlyArray<SessionEntry> {
+    return this.pool;
+  }
+
+  /** Stats for monitoring. */
+  stats(): { total: number; available: number; checkedOut: number; stale: number } {
+    const now = Date.now();
+    const stale = this.pool.filter(s => now - s.createdAt >= this.maxAgeMs || s.requestCount >= this.maxRequests).length;
+    const checkedOut = this.pool.filter(s => !!s.checkedOutAt).length;
+    return { total: this.pool.length, available: this.pool.length - checkedOut - stale, checkedOut, stale };
+  }
+
+  /** Add a new session to the pool at runtime. */
+  addSession(role: string, storageState: string): string {
+    const id = `session-${this.pool.length}-${role}`;
+    this.pool.push({ id, role, storageState, createdAt: Date.now(), checkedOutAt: null, requestCount: 0 });
+    return id;
+  }
+}