@detiq/crawler 0.1.3

package/src/auth.ts

@@ -0,0 +1,396 @@
+import type { Page } from 'playwright';
+
+export interface Credentials {
+  username: string;
+  password: string;
+  oneTimeCode?: string;
+  getOneTimeCode?: () => Promise<string>;
+}
+
+/**
+ * Recorded login step captured by the Chrome extension.
+ * {username} and {password} in `value` are substituted at replay time.
+ */
+export interface LoginStep {
+  action: 'fill' | 'click' | 'wait';
+  selector: string;
+  value?: string;
+  ms?: number;
+}
+
+// ─── Helpers ──────────────────────────────────────────────────────────────────
+
+async function firstVisible(page: Page, selector: string) {
+  const loc = page.locator(selector);
+  const count = await loc.count().catch(() => 0);
+  for (let i = 0; i < count; i += 1) {
+    const item = loc.nth(i);
+    if (await item.isVisible().catch(() => false)) return item;
+  }
+  // iframe fallback — Salesforce, Zendesk, and embedded login widgets hide forms in iframes
+  for (const frame of page.frames()) {
+    if (frame === page.mainFrame()) continue;
+    const floc = frame.locator(selector);
+    const fcount = await floc.count().catch(() => 0);
+    for (let i = 0; i < fcount; i += 1) {
+      const item = floc.nth(i);
+      if (await item.isVisible().catch(() => false)) return item;
+    }
+  }
+  return null;
+}
+
+/** Type into framework-controlled inputs — handles React, Angular, Vue, and plain HTML forms. */
+async function reactFill(field: Awaited<ReturnType<typeof firstVisible>>, value: string) {
+  if (!field) return;
+  await field.click();
+  await field.selectText().catch(() => {});
+  await field.press('Control+a').catch(() => {});
+  await field.press('Backspace').catch(() => {});
+  await field.pressSequentially(value, { delay: 30 });
+  // Fire native input + change events for Angular/Vue/custom frameworks that ignore DOM keypresses.
+  // React's synthetic listener catches pressSequentially above; this covers the rest.
+  await field.evaluate((el, v) => {
+    const input = el as HTMLInputElement;
+    const nativeInputValueSetter = Object.getOwnPropertyDescriptor(window.HTMLInputElement.prototype, 'value')?.set;
+    if (nativeInputValueSetter) nativeInputValueSetter.call(input, v);
+    input.dispatchEvent(new Event('input', { bubbles: true }));
+    input.dispatchEvent(new Event('change', { bubbles: true }));
+    input.dispatchEvent(new Event('blur', { bubbles: true }));
+  }, value).catch(() => {});
+}
+
+/**
+ * Find and click the most likely submit button.
+ * Waits up to 2 s for React's disabled={!email||!password} to clear before clicking.
+ */
+async function clickLikelySubmit(page: Page) {
+  // Prefer explicit submit selectors first; "Continue" is last resort to avoid
+  // accidentally clicking "Continue with Google/GitHub/Apple" SSO buttons.
+  let button = await firstVisible(page, [
+    'button[type="submit"]',
+    'input[type="submit"]',
+    'button:has-text("Next")',
+    'button:has-text("Sign in")',
+    'button:has-text("Log in")',
+    'button:has-text("Login")',
+    'button:has-text("Sign up")',
+  ].join(', '));
+
+  if (!button) {
+    // Find "Continue" buttons but exclude OAuth provider ones ("Continue with Google" etc.)
+    const candidates = page.locator('button:has-text("Continue")');
+    const count = await candidates.count().catch(() => 0);
+    for (let i = 0; i < count; i++) {
+      const c = candidates.nth(i);
+      if (!(await c.isVisible().catch(() => false))) continue;
+      const text = (await c.textContent().catch(() => '')) ?? '';
+      if (/continue\s+with\s+/i.test(text)) continue; // skip "Continue with Google/GitHub/Apple"
+      button = c;
+      break;
+    }
+  }
+  if (button) {
+    // Poll for enabled state — waitForFunction not available on Stagehand V3 proxy
+    const handle = await button.elementHandle().catch(() => null);
+    if (handle) {
+      const deadline = Date.now() + 2_000;
+      while (Date.now() < deadline) {
+        const disabled = await handle.evaluate((el) => (el as HTMLButtonElement).disabled).catch(() => false);
+        if (!disabled) break;
+        await new Promise(r => setTimeout(r, 100));
+      }
+    }
+    await Promise.all([
+      page.waitForLoadState('networkidle').catch(() => {}),
+      button.click({ timeout: 5_000 }).catch(() => {}), // page may navigate before click completes — ignore
+    ]);
+    return true;
+  }
+  await page.keyboard.press('Enter');
+  await page.waitForLoadState('networkidle').catch(() => {});
+  return false;
+}
+
+/** Dismiss cookie/GDPR consent banners so they don't block form fields. */
+async function dismissCookieBanners(page: Page) {
+  const acceptSel = [
+    'button:has-text("Accept")',
+    'button:has-text("Accept all")',
+    'button:has-text("Allow all")',
+    'button:has-text("I agree")',
+    'button:has-text("Agree")',
+    'button:has-text("OK")',
+    'button:has-text("Got it")',
+    'button:has-text("Close")',
+    '[aria-label*="Accept" i]',
+    '[aria-label*="Dismiss" i]',
+    '[class*="cookie"] button',
+    '[id*="cookie"] button',
+    '[class*="consent"] button',
+    '[class*="banner"] button[class*="close" i]',
+  ].join(', ');
+  const btn = await firstVisible(page, acceptSel);
+  if (btn) await btn.click().catch(() => {});
+  await page.waitForTimeout(300).catch(() => {});
+}
+
+/**
+ * Look for a Sign In / Log In nav link on a marketing homepage and navigate to the actual login page.
+ * Returns true if navigation happened.
+ */
+async function navigateToLoginPage(page: Page, appUrl: string): Promise<boolean> {
+  // First try common login URL suffixes
+  const origin = (() => { try { return new URL(appUrl).origin; } catch { return ''; } })();
+  const loginPaths = ['/login', '/sign-in', '/signin', '/auth/login', '/auth', '/account/login', '/user/login', '/app/login'];
+
+  // Look for a "Sign In" / "Login" / "Get started" link in the nav
+  const navLoginSel = [
+    'a:has-text("Sign in")',
+    'a:has-text("Sign In")',
+    'a:has-text("Log in")',
+    'a:has-text("Log In")',
+    'a:has-text("Login")',
+    'a:has-text("Get started")',
+    'a[href*="login" i]',
+    'a[href*="sign-in" i]',
+    'a[href*="signin" i]',
+    'a[href*="auth" i]',
+    'button:has-text("Sign in")',
+    'button:has-text("Log in")',
+    'button:has-text("Login")',
+  ].join(', ');
+
+  const navLink = await firstVisible(page, navLoginSel);
+  if (navLink) {
+    const href = await navLink.getAttribute('href').catch(() => null);
+    // If the link has an href, navigate directly — avoids modal-only flows
+    if (href && href.startsWith('http')) {
+      await page.goto(href, { waitUntil: 'networkidle', timeout: 15_000 }).catch(() => {});
+      return true;
+    }
+    // Click and immediately check for email field (modal opened) OR wait for navigation
+    await navLink.click();
+    const emailAppeared = await page.waitForSelector(EMAIL_SELECTOR, { timeout: 3_000 })
+      .then(() => true).catch(() => false);
+    if (!emailAppeared) {
+      // No modal — likely navigating to a new page
+      await page.waitForLoadState('networkidle', { timeout: 8_000 }).catch(() => {});
+    }
+    return true;
+  }
+
+  // Fallback: try common login paths on the same origin
+  if (origin) {
+    for (const p of loginPaths) {
+      const loginUrl = `${origin}${p}`;
+      const res = await page.goto(loginUrl, { waitUntil: 'networkidle', timeout: 10_000 }).catch(() => null);
+      if (res && res.status() < 400) {
+        // Check if this page actually has a login form
+        const hasForm = (await page.locator(
+          'input[type="email"], input[type="password"], input[name*="email" i], input[name*="user" i]'
+        ).count().catch(() => 0)) > 0;
+        if (hasForm) return true;
+      }
+    }
+    // Restore original page
+    await page.goto(appUrl, { waitUntil: 'networkidle', timeout: 15_000 }).catch(() => {});
+  }
+
+  return false;
+}
+
+/** Detect if current page redirected to a 3rd-party OAuth provider (SSO flow we can't automate). */
+function isOAuthProviderUrl(url: string): string | null {
+  const u = url.toLowerCase();
+  if (u.includes('accounts.google.com')) return 'Google';
+  if (u.includes('github.com/login')) return 'GitHub';
+  if (u.includes('login.microsoftonline.com')) return 'Microsoft';
+  if (u.includes('login.live.com')) return 'Microsoft';
+  if (u.includes('appleid.apple.com')) return 'Apple';
+  if (u.includes('okta.com')) return 'Okta';
+  if (u.includes('auth0.com')) return 'Auth0';
+  if (u.includes('onelogin.com')) return 'OneLogin';
+  if (u.includes('pingidentity.com')) return 'Ping Identity';
+  return null;
+}
+
+const EMAIL_SELECTOR = 'input[type="email"], input[name*="user" i], input[name*="email" i], input[autocomplete="username"]';
+const PASS_SELECTOR = 'input[type="password"], input[autocomplete="current-password"]';
+
+/** Replay recorded login steps from the Chrome extension. Returns true if login succeeded. */
+async function replayLoginRecipe(page: Page, steps: LoginStep[], creds: { username: string; password: string }): Promise<boolean> {
+  for (const step of steps) {
+    try {
+      if (step.action === 'wait') {
+        await page.waitForTimeout(step.ms ?? 500).catch(() => {});
+      } else if (step.action === 'click') {
+        const el = await firstVisible(page, step.selector);
+        if (!el) return false;
+        await el.click({ timeout: 5_000 }).catch(() => {});
+      } else if (step.action === 'fill') {
+        const el = await firstVisible(page, step.selector);
+        if (!el) return false;
+        const val = (step.value ?? '')
+          .replace('{username}', creds.username)
+          .replace('{password}', creds.password);
+        await reactFill(el, val);
+      }
+      await page.waitForTimeout(200).catch(() => {});
+    } catch { return false; }
+  }
+  await page.waitForLoadState('networkidle', { timeout: 5_000 }).catch(() => {});
+  const postUrl = page.url();
+  const LOGIN_PATH_RE = /\/(login|signin|sign-in|auth|log-in|sso|oauth)(\/|$|\?)/i;
+  if (!LOGIN_PATH_RE.test(new URL(postUrl).pathname)) return true; // navigated away — success
+  const passCount = await page.locator('input[type="password"]').count().catch(() => 0);
+  return passCount === 0;
+}
+
+// ─── Main login function ───────────────────────────────────────────────────────
+
+export async function login(page: Page, appUrl: string, creds: Credentials & { loginRecipe?: LoginStep[] }): Promise<boolean> {
+  await page.goto(appUrl, { waitUntil: 'networkidle', timeout: 30_000 }).catch(() => {});
+
+  // Recipe replay — try exact recorded steps first when available
+  if (creds.loginRecipe && creds.loginRecipe.length > 0) {
+    const recipeOk = await replayLoginRecipe(page, creds.loginRecipe, creds);
+    if (recipeOk) return true;
+    // Recipe failed — reload page and fall through to DOM heuristic
+    await page.goto(appUrl, { waitUntil: 'networkidle', timeout: 30_000 }).catch(() => {});
+  }
+
+  // Dismiss cookie banners that might block form interaction
+  await dismissCookieBanners(page);
+
+  let userField = await firstVisible(page, EMAIL_SELECTOR);
+
+  // No email field on the landing page — this is a marketing homepage.
+  // Try to navigate to the actual login page.
+  if (!userField) {
+    await navigateToLoginPage(page, appUrl);
+    // NOTE: do NOT call dismissCookieBanners here — it can close login modals
+    // navigateToLoginPage already waits for email field or networkidle
+    userField = await firstVisible(page, EMAIL_SELECTOR);
+    if (!userField) {
+      // Last attempt: wait up to 5s for form to appear (SPA redirect or slow modal)
+      await page.waitForSelector(EMAIL_SELECTOR, { timeout: 5_000 }).catch(() => {});
+      userField = await firstVisible(page, EMAIL_SELECTOR);
+    }
+  }
+
+  if (!userField) return false;
+
+  await reactFill(userField, creds.username);
+  await page.keyboard.press('Tab');
+
+  let passField = await firstVisible(page, PASS_SELECTOR);
+  if (!passField) {
+    // Multi-step form: email-first, then password on next screen
+    await clickLikelySubmit(page);
+    // Wait for password field to appear (up to 8s) rather than a fixed 2s sleep —
+    // slow IdP servers (Okta, Azure) take 3-5s to validate email and render the next step.
+    await page.waitForSelector(PASS_SELECTOR, { timeout: 8_000 }).catch(() => {});
+    // Check for SSO redirect after email submission
+    const ssoProvider = isOAuthProviderUrl(page.url());
+    if (ssoProvider) {
+      throw Object.assign(
+        new Error(
+          `This app uses ${ssoProvider} SSO — ZeTa cannot automate OAuth login. ` +
+          `Use the ZeTa Capture Chrome Extension instead: log in normally in Chrome, ` +
+          `then click the extension to capture your session.`
+        ),
+        { isLoginError: true, isSSOError: true, ssoProvider }
+      );
+    }
+    passField = await firstVisible(page, PASS_SELECTOR);
+  }
+
+  if (!passField) {
+    // Still no password field — check if we landed on an SSO provider
+    const ssoProvider = isOAuthProviderUrl(page.url());
+    if (ssoProvider) {
+      throw Object.assign(
+        new Error(
+          `This app uses ${ssoProvider} SSO — ZeTa cannot automate OAuth login. ` +
+          `Use the ZeTa Capture Chrome Extension instead: log in normally in Chrome, ` +
+          `then click the extension to capture your session.`
+        ),
+        { isLoginError: true, isSSOError: true, ssoProvider }
+      );
+    }
+    return false;
+  }
+
+  await reactFill(passField, creds.password);
+  // Press Enter — submits regardless of button disabled state
+  await page.waitForTimeout(150).catch(() => {});
+  await Promise.all([
+    page.waitForLoadState('networkidle').catch(() => {}),
+    page.keyboard.press('Enter'),
+  ]);
+
+  // Fallback: if Enter didn't navigate away, try Tab → click
+  const stillHasPass = await page.locator(PASS_SELECTOR).count().catch(() => 0);
+  if (stillHasPass > 0) {
+    await page.keyboard.press('Tab');
+    await clickLikelySubmit(page);
+  }
+
+  // Wait for post-submit navigation to complete — waitForLoadState resolves immediately
+  // if page is already idle, so use waitForURL to detect actual navigation away from login.
+  const LOGIN_PATH_RE_NAV = /\/(login|signin|sign-in|auth|log-in|sso|oauth)(\/|$|\?)/i;
+  if (LOGIN_PATH_RE_NAV.test(new URL(page.url()).pathname)) {
+    await page.waitForURL(
+      (url) => { try { return !LOGIN_PATH_RE_NAV.test(new URL(url).pathname); } catch { return true; } },
+      { timeout: 12_000 }
+    ).catch(() => {});
+  }
+
+  // Check for SSO redirect after password submit
+  const ssoAfterLogin = isOAuthProviderUrl(page.url());
+  if (ssoAfterLogin) {
+    throw Object.assign(
+      new Error(
+        `This app uses ${ssoAfterLogin} SSO — ZeTa cannot automate OAuth login. ` +
+        `Use the ZeTa Capture Chrome Extension instead.`
+      ),
+      { isLoginError: true, isSSOError: true, ssoProvider: ssoAfterLogin }
+    );
+  }
+
+  // Handle OTP / 2FA
+  const otpField = await firstVisible(page, [
+    'input[autocomplete="one-time-code"]',
+    'input[name*="otp" i]',
+    'input[name*="code" i]',
+    'input[aria-label*="code" i]',
+    'input[placeholder*="code" i]',
+  ].join(', '));
+  if (otpField) {
+    const code = creds.oneTimeCode ?? await creds.getOneTimeCode?.();
+    if (!code) return false;
+    await reactFill(otpField, code);
+    await clickLikelySubmit(page);
+  }
+
+  // Success check: navigated away from login page, and/or no password field remains.
+  // Use URL as primary signal — dashboards can have hidden password fields (profile forms, etc.)
+  // that would cause false negatives if we only check input[type="password"].
+  const postLoginUrl = page.url();
+  const LOGIN_PATH_RE = /\/(login|signin|sign-in|auth|log-in|sso|oauth)(\/|$|\?)/i;
+  const stillOnLoginPath = LOGIN_PATH_RE.test(new URL(postLoginUrl).pathname);
+  const stillOnLogin = await page.locator(PASS_SELECTOR).count();
+
+  // Only treat as failure if BOTH: still on a login-path URL AND password field present
+  if (stillOnLoginPath && stillOnLogin > 0) {
+    const errorLoc = page.locator('[role="alert"], .error, [class*="error"], [class*="alert"]');
+    await errorLoc.first().waitFor({ state: 'visible', timeout: 2_000 }).catch(() => {});
+    const pageError = await errorLoc.first().textContent().catch(() => null);
+    if (pageError?.trim()) {
+      throw Object.assign(new Error(pageError.trim()), { isLoginError: true });
+    }
+    return false;
+  }
+  return true;
+}

package/src/browser-extension.ts

@@ -0,0 +1,186 @@
+/**
+ * Browser extension injection support.
+ *
+ * Chrome extensions require a persistent profile directory — incompatible
+ * with headless Docker by default. This module handles:
+ *   1. Persistent profile directory per extension set
+ *   2. Extension download from .crx URL or load from local path
+ *   3. Launch args for Playwright / Stagehand
+ *
+ * Usage:
+ *   const profile = await prepareExtensionProfile(['/path/to/ext.crx']);
+ *   // pass profile.userDataDir and profile.args to browser launch
+ */
+
+import { createHash, randomBytes } from 'node:crypto';
+import { createWriteStream, existsSync, mkdirSync } from 'node:fs';
+import { writeFile, mkdir, rm } from 'node:fs/promises';
+import { join } from 'node:path';
+import { tmpdir } from 'node:os';
+import { pipeline } from 'node:stream/promises';
+import { Readable } from 'node:stream';
+
+const PROFILE_BASE = process.env.EXTENSION_PROFILE_DIR ?? join(tmpdir(), 'zeta-ext-profiles');
+
+export interface ExtensionSource {
+  /** Local .crx file path or https:// URL to download from */
+  source: string;
+  /** Friendly name for logging */
+  name?: string;
+}
+
+export interface ExtensionProfile {
+  /** Chrome user-data-dir containing Preferences + extension unpacked dirs */
+  userDataDir: string;
+  /** Args to append to chromium launch — includes --load-extension and --disable-extensions-except */
+  args: string[];
+  /** Cleanup the profile directory when done */
+  cleanup: () => Promise<void>;
+}
+
+/**
+ * Download a .crx from a URL to a temp file. Returns path.
+ */
+async function downloadCrx(url: string): Promise<string> {
+  const dest = join(tmpdir(), `zeta-ext-${randomBytes(6).toString('hex')}.crx`);
+  const res = await fetch(url);
+  if (!res.ok) throw new Error(`Failed to download extension from ${url}: ${res.status}`);
+  const writer = createWriteStream(dest);
+  await pipeline(Readable.fromWeb(res.body as any), writer);
+  return dest;
+}
+
+/**
+ * Unpack a .crx file to a directory Chrome can load as an unpacked extension.
+ * CRX3 format: 4-byte magic + 4-byte version + 4-byte header_size + proto header + zip body
+ */
+async function unpackCrx(crxPath: string, destDir: string): Promise<void> {
+  const { readFile } = await import('node:fs/promises');
+  const buf = await readFile(crxPath);
+
+  const magic = buf.readUInt32LE(0);
+  if (magic !== 0x43723234) {
+    // Not CRX — assume it's already a zip (unpacked extension)
+    await mkdir(destDir, { recursive: true });
+    await writeFile(join(destDir, '__raw.crx'), buf);
+    return;
+  }
+
+  const version = buf.readUInt32LE(4);
+  let zipStart: number;
+  if (version === 3) {
+    const headerSize = buf.readUInt32LE(8);
+    zipStart = 12 + headerSize;
+  } else if (version === 2) {
+    const pubKeyLen = buf.readUInt32LE(8);
+    const sigLen = buf.readUInt32LE(12);
+    zipStart = 16 + pubKeyLen + sigLen;
+  } else {
+    throw new Error(`Unknown CRX version ${version}`);
+  }
+
+  const zipBuf = buf.subarray(zipStart);
+  const tmpZip = join(tmpdir(), `zeta-ext-${randomBytes(4).toString('hex')}.zip`);
+  await writeFile(tmpZip, zipBuf);
+
+  await mkdir(destDir, { recursive: true });
+
+  // Extract using unzipper (available in Node without native deps)
+  // @ts-ignore – no types for unzipper
+  const unzipper = await import('unzipper').catch(() => null);
+  if (unzipper) {
+    await unzipper.Open.file(tmpZip).then((d: any) => d.extract({ path: destDir }));
+  } else {
+    // Fallback: shell unzip
+    const { execFile } = await import('node:child_process');
+    await new Promise<void>((res, rej) =>
+      execFile('unzip', ['-o', tmpZip, '-d', destDir], (err) => err ? rej(err) : res())
+    );
+  }
+  await rm(tmpZip, { force: true });
+}
+
+/**
+ * Prepare a Chrome user-data-dir with the given extensions loaded.
+ * Returns profile info + cleanup function.
+ */
+export async function prepareExtensionProfile(
+  extensions: ExtensionSource[]
+): Promise<ExtensionProfile> {
+  // Stable profile key based on extension sources
+  const key = createHash('md5')
+    .update(extensions.map(e => e.source).sort().join('|'))
+    .digest('hex')
+    .slice(0, 12);
+
+  const profileDir = join(PROFILE_BASE, key);
+  const extensionDirs: string[] = [];
+
+  mkdirSync(profileDir, { recursive: true });
+
+  for (const ext of extensions) {
+    const extHash = createHash('md5').update(ext.source).digest('hex').slice(0, 8);
+    const extDir = join(profileDir, `ext-${extHash}`);
+
+    if (!existsSync(extDir)) {
+      let crxPath = ext.source;
+
+      if (ext.source.startsWith('http://') || ext.source.startsWith('https://')) {
+        crxPath = await downloadCrx(ext.source);
+        await unpackCrx(crxPath, extDir);
+        await rm(crxPath, { force: true });
+      } else if (ext.source.endsWith('.crx')) {
+        await unpackCrx(ext.source, extDir);
+      } else {
+        // Assume already-unpacked extension directory — symlink or copy
+        extensionDirs.push(ext.source);
+        continue;
+      }
+    }
+
+    extensionDirs.push(extDir);
+  }
+
+  const userDataDir = join(profileDir, 'user-data');
+  mkdirSync(userDataDir, { recursive: true });
+
+  // Write a minimal Preferences file so Chrome doesn't show first-run UI
+  const defaultsDir = join(userDataDir, 'Default');
+  mkdirSync(defaultsDir, { recursive: true });
+  const prefsPath = join(defaultsDir, 'Preferences');
+  if (!existsSync(prefsPath)) {
+    await writeFile(prefsPath, JSON.stringify({
+      browser: { has_seen_welcome_page: true },
+      extensions: { alerts: { initialized: true } },
+    }), 'utf8');
+  }
+
+  const loadExtensionArg = extensionDirs.join(',');
+  const args: string[] = [
+    `--load-extension=${loadExtensionArg}`,
+    `--disable-extensions-except=${loadExtensionArg}`,
+    `--user-data-dir=${userDataDir}`,
+    '--no-first-run',
+    '--no-default-browser-check',
+  ];
+
+  return {
+    userDataDir,
+    args,
+    cleanup: async () => {
+      await rm(profileDir, { recursive: true, force: true });
+    },
+  };
+}
+
+/**
+ * Ad-blocker extension profile using the uBlock Origin CRX.
+ * Reduces noise from ad networks during crawls.
+ */
+export async function adBlockerProfile(): Promise<ExtensionProfile> {
+  // uBlock Origin 1.57.2 from GitHub release (MIT licensed)
+  return prepareExtensionProfile([{
+    name: 'uBlock Origin',
+    source: 'https://github.com/gorhill/uBlock/releases/download/1.57.2/uBlock0_1.57.2.chromium.zip',
+  }]);
+}