@vellumai/assistant 0.3.0

package/src/tools/browser/auth-detector.ts

@@ -0,0 +1,347 @@
+import type { Page } from './browser-manager.js';
+
+export type AuthChallengeType = 'login' | '2fa' | 'oauth_consent' | 'captcha';
+
+export interface AuthField {
+  type: 'email' | 'password' | 'code' | 'approval';
+  selector: string;
+  label: string;
+}
+
+export interface AuthChallenge {
+  type: AuthChallengeType;
+  service?: string;
+  fields: AuthField[];
+  url: string;
+}
+
+// ── URL pattern matching ─────────────────────────────────────────────
+
+interface ServicePattern {
+  pattern: RegExp;
+  service: string;
+  /** When set, the URL pathname must also match this pattern for the service to be recognised. */
+  pathPattern?: RegExp;
+}
+
+const SERVICE_PATTERNS: ServicePattern[] = [
+  // Auth-subdomain services — hostname match is sufficient
+  { pattern: /accounts\.google\.com/, service: 'Google' },
+  { pattern: /login\.microsoftonline\.com/, service: 'Microsoft' },
+  { pattern: /appleid\.apple\.com/, service: 'Apple' },
+  { pattern: /login\.salesforce\.com/, service: 'Salesforce' },
+  { pattern: /id\.atlassian\.com/, service: 'Atlassian' },
+  { pattern: /auth0\.com/, service: 'Auth0' },
+  { pattern: /okta\.com/, service: 'Okta' },
+  // General-domain services — need both hostname AND path match
+  { pattern: /github\.com/, service: 'GitHub', pathPattern: /^\/(login|session)/ },
+];
+
+const GENERIC_AUTH_PATTERNS: RegExp[] = [
+  /\/login\b/i,
+  /\/signin\b/i,
+  /\/sign-in\b/i,
+  /\/auth\b/i,
+  /\/oauth\b/i,
+  /\/sso\b/i,
+];
+
+/**
+ * Identify the service name from a URL. Returns undefined if no known
+ * service is matched.
+ */
+export function identifyService(url: string): string | undefined {
+  let parsed: URL;
+  try {
+    parsed = new URL(url);
+  } catch {
+    return undefined;
+  }
+  for (const { pattern, service, pathPattern } of SERVICE_PATTERNS) {
+    if (pattern.test(parsed.hostname)) {
+      if (pathPattern && !pathPattern.test(parsed.pathname)) continue;
+      return service;
+    }
+  }
+  return undefined;
+}
+
+/**
+ * Check whether a URL matches a known auth-related path pattern.
+ */
+export function isAuthUrl(url: string): boolean {
+  let parsedUrl: URL;
+  try {
+    parsedUrl = new URL(url);
+  } catch {
+    return false;
+  }
+  // Known service URLs are always auth-related (respecting pathPattern when present)
+  if (
+    SERVICE_PATTERNS.some(
+      ({ pattern, pathPattern }) =>
+        pattern.test(parsedUrl.hostname) && (!pathPattern || pathPattern.test(parsedUrl.pathname)),
+    )
+  )
+    return true;
+  // Generic path patterns — match against pathname only to avoid false positives
+  // from query parameters or fragments that happen to contain auth-related words.
+  return GENERIC_AUTH_PATTERNS.some((p) => p.test(parsedUrl.pathname));
+}
+
+// ── DOM-based detection ──────────────────────────────────────────────
+
+interface DomDetectionResult {
+  type: AuthChallengeType;
+  fields: AuthField[];
+}
+
+/**
+ * JavaScript expression evaluated inside the page to detect auth-related
+ * DOM elements. Returns a serialisable result or null.
+ *
+ * The expression is a self-contained IIFE so it can be passed as a string
+ * to `page.evaluate()`.
+ */
+const DOM_DETECT_EXPRESSION = `(() => {
+  const fields = [];
+
+  // ── Google-specific selectors ──────────────────────────────────
+  const googleEmail = document.querySelector('#identifierId');
+  if (googleEmail) {
+    fields.push({ type: 'email', selector: '#identifierId', label: 'Google email' });
+  }
+  const googlePw = document.querySelector('input[type="password"][name="Passwd"]');
+  if (googlePw) {
+    fields.push({ type: 'password', selector: 'input[type="password"][name="Passwd"]', label: 'Google password' });
+  }
+
+  // ── Generic password inputs ────────────────────────────────────
+  const pwInputs = document.querySelectorAll('input[type="password"]');
+  for (const el of pwInputs) {
+    const sel = 'input[type="password"]';
+    // Avoid duplicating Google password already added
+    if (el.getAttribute('name') === 'Passwd') continue;
+    const label = el.getAttribute('aria-label') || el.getAttribute('placeholder') || 'password';
+    fields.push({ type: 'password', selector: sel, label });
+  }
+
+  // ── Email / username inputs ────────────────────────────────────
+  const emailInputs = document.querySelectorAll(
+    'input[type="email"], input[name="email"], input[name="username"], input[autocomplete="username"]'
+  );
+  for (const el of emailInputs) {
+    if (el.id === 'identifierId') continue; // already handled
+    const sel = el.id ? '#' + el.id : 'input[type="email"]';
+    const label = el.getAttribute('aria-label') || el.getAttribute('placeholder') || 'email';
+    fields.push({ type: 'email', selector: sel, label });
+  }
+
+  // ── 2FA / verification code inputs ─────────────────────────────
+  const bodyText = (document.body && document.body.innerText) || '';
+  const is2FAText = /verification code|two.?factor|2fa|authenticator|one.?time/i.test(bodyText);
+  const codeInputs = document.querySelectorAll(
+    'input[autocomplete="one-time-code"], input[name*="code"], input[name*="otp"], input[name*="token"], input[id*="code"], input[id*="otp"]'
+  );
+
+  if (is2FAText || codeInputs.length > 0) {
+    for (const el of codeInputs) {
+      const sel = el.id ? '#' + el.id : (el.getAttribute('name') ? 'input[name="' + el.getAttribute('name') + '"]' : 'input[autocomplete="one-time-code"]');
+      const label = el.getAttribute('aria-label') || el.getAttribute('placeholder') || 'verification code';
+      fields.push({ type: 'code', selector: sel, label });
+    }
+    // If we detected 2FA text but no specific code input, still report
+    if (codeInputs.length === 0 && is2FAText) {
+      fields.push({ type: 'code', selector: '', label: 'verification code (text detected)' });
+    }
+  }
+
+  // ── OAuth consent buttons ──────────────────────────────────────
+  const buttons = document.querySelectorAll('button, input[type="submit"], [role="button"]');
+  const consentPatterns = /^(allow|approve|grant access|authorize|accept|consent)$/i;
+  let hasConsentButton = false;
+  for (const btn of buttons) {
+    const text = (btn.textContent || '').trim();
+    if (consentPatterns.test(text)) {
+      hasConsentButton = true;
+      const sel = btn.id ? '#' + btn.id : 'button';
+      fields.push({ type: 'approval', selector: sel, label: text });
+    }
+  }
+
+  // ── Sign in / Log in buttons (to support login detection) ──────
+  const signInPatterns = /^(sign in|log in|login|sign up|continue)$/i;
+  let hasSignInButton = false;
+  for (const btn of buttons) {
+    const text = (btn.textContent || '').trim();
+    if (signInPatterns.test(text)) {
+      hasSignInButton = true;
+      break;
+    }
+  }
+
+  // ── Classify ───────────────────────────────────────────────────
+  const hasPassword = fields.some(f => f.type === 'password');
+  const hasEmail = fields.some(f => f.type === 'email');
+  const hasCode = fields.some(f => f.type === 'code');
+  const hasApproval = fields.some(f => f.type === 'approval');
+
+  if (hasApproval) {
+    return { type: 'oauth_consent', fields };
+  }
+  if (hasCode && !hasPassword) {
+    return { type: '2fa', fields: fields.filter(f => f.type === 'code') };
+  }
+  if (hasPassword || (hasEmail && hasSignInButton)) {
+    return { type: 'login', fields: fields.filter(f => f.type === 'email' || f.type === 'password') };
+  }
+  if (is2FAText) {
+    return { type: '2fa', fields: fields.filter(f => f.type === 'code') };
+  }
+
+  return null;
+})()`;
+
+// ── CAPTCHA / Cloudflare detection ───────────────────────────────────
+
+const CAPTCHA_DETECT_EXPRESSION = `(() => {
+  // Cloudflare Turnstile / interstitial
+  const title = document.title || '';
+  if (/just a moment/i.test(title)) return true;
+
+  const bodyText = (document.body && document.body.innerText) || '';
+  if (/verify you are human|performing security verification/i.test(bodyText)) return true;
+
+  // Cloudflare-specific DOM elements
+  const cfSelectors = [
+    '#challenge-running',
+    '#challenge-stage',
+    '.cf-turnstile',
+    'iframe[src*="challenges.cloudflare.com"]',
+  ];
+  for (const sel of cfSelectors) {
+    if (document.querySelector(sel)) return true;
+  }
+
+  // reCAPTCHA — only flag visible challenges, not invisible v3 scoring widgets.
+  // The challenge iframe (api2/bframe) or a visible .g-recaptcha container indicates
+  // an interactive CAPTCHA the user must solve.
+  if (document.querySelector('iframe[src*="recaptcha/api2/bframe"]')) return true;
+  const recaptchaContainer = document.querySelector('.g-recaptcha');
+  if (recaptchaContainer) {
+    const rect = recaptchaContainer.getBoundingClientRect();
+    if (rect.width > 0 && rect.height > 0) return true;
+  }
+
+  // hCaptcha — only flag when a visible challenge container is present
+  const hcaptchaContainer = document.querySelector('#hcaptcha-container, .h-captcha');
+  if (hcaptchaContainer) {
+    const rect = hcaptchaContainer.getBoundingClientRect();
+    if (rect.width > 0 && rect.height > 0) return true;
+  }
+
+  // hCaptcha fallback — catch custom-rendered hCaptcha in arbitrary host elements
+  // by looking for any visible hCaptcha iframe (check all, not just the first)
+  const hcaptchaIframes = document.querySelectorAll('iframe[src*="hcaptcha"]');
+  for (const iframe of hcaptchaIframes) {
+    const rect = iframe.getBoundingClientRect();
+    if (rect.width > 0 && rect.height > 0) return true;
+  }
+
+  return false;
+})()`;
+
+/**
+ * Detect whether the current page presents a CAPTCHA or Cloudflare
+ * challenge that requires human interaction.
+ */
+export async function detectCaptchaChallenge(page: Page): Promise<AuthChallenge | null> {
+  try {
+    const isCaptcha = await page.evaluate(CAPTCHA_DETECT_EXPRESSION) as boolean;
+    if (isCaptcha) {
+      return {
+        type: 'captcha',
+        fields: [],
+        url: page.url(),
+      };
+    }
+    return null;
+  } catch {
+    return null;
+  }
+}
+
+/**
+ * Detect whether the current page presents an authentication challenge
+ * (login form, 2FA prompt, or OAuth consent screen).
+ *
+ * Detection uses two complementary strategies:
+ * 1. URL pattern matching against known auth providers.
+ * 2. DOM inspection for login, 2FA, and consent UI elements.
+ *
+ * Returns an {@link AuthChallenge} if a challenge is detected, or `null`
+ * if the page does not appear to be an auth page.
+ */
+export async function detectAuthChallenge(page: Page): Promise<AuthChallenge | null> {
+  try {
+    const currentUrl = page.url();
+    const service = identifyService(currentUrl);
+    const urlIsAuth = isAuthUrl(currentUrl);
+
+    // DOM-based detection
+    const domResult = (await page.evaluate(DOM_DETECT_EXPRESSION)) as DomDetectionResult | null;
+
+    if (domResult) {
+      return {
+        type: domResult.type,
+        service,
+        fields: domResult.fields,
+        url: currentUrl,
+      };
+    }
+
+    // If the URL strongly suggests an auth page but the DOM didn't
+    // yield specific fields, still report it as a generic login challenge.
+    if (urlIsAuth) {
+      return {
+        type: 'login',
+        service,
+        fields: [],
+        url: currentUrl,
+      };
+    }
+
+    return null;
+  } catch {
+    // If page.evaluate throws (e.g. page closed, navigation), treat as no challenge
+    return null;
+  }
+}
+
+/**
+ * Format an {@link AuthChallenge} into a human-readable string suitable
+ * for appending to browser_navigate output.
+ */
+export function formatAuthChallenge(challenge: AuthChallenge): string {
+  const serviceName = challenge.service ? `${challenge.service} ` : '';
+  const typeLabel =
+    challenge.type === 'login'
+      ? 'login page'
+      : challenge.type === '2fa'
+        ? '2FA verification'
+        : challenge.type === 'captcha'
+          ? 'CAPTCHA verification'
+          : 'OAuth consent screen';
+
+  const lines: string[] = [
+    `\u26a0\ufe0f Auth challenge detected: ${serviceName}${typeLabel}`,
+    `  Type: ${challenge.type}`,
+  ];
+
+  if (challenge.fields.length > 0) {
+    const fieldDescs = challenge.fields.map((f) => `${f.label} (${f.type})`).join(', ');
+    lines.push(`  Fields: ${fieldDescs}`);
+  }
+
+  return lines.join('\n');
+}

package/src/tools/browser/auto-navigate.ts

@@ -0,0 +1,270 @@
+/**
+ * CDP-based auto-navigation for any domain.
+ *
+ * Drives Chrome through a domain's pages by discovering internal links,
+ * so the NetworkRecorder captures the API surface without manual browsing.
+ */
+
+import { getLogger } from '../../util/logger.js';
+
+const log = getLogger('auto-navigate');
+
+const CDP_BASE = 'http://localhost:9222';
+const MAX_PAGES = 15;
+const PAGE_WAIT_MS = 3500;
+const SCROLL_WAIT_MS = 2000;
+
+/** Minimal CDP client — connects to one page tab. */
+class MiniCDP {
+  private ws: WebSocket | null = null;
+  private nextId = 1;
+  private callbacks = new Map<number, { resolve: (v: unknown) => void; reject: (e: Error) => void }>();
+
+  async connect(wsUrl: string): Promise<void> {
+    return new Promise((resolve, reject) => {
+      const ws = new WebSocket(wsUrl);
+      ws.onopen = () => { this.ws = ws; resolve(); };
+      ws.onerror = (e) => reject(new Error(`CDP error: ${e}`));
+      ws.onclose = () => {
+        this.ws = null;
+        for (const [, cb] of this.callbacks) {
+          cb.reject(new Error('WebSocket closed'));
+        }
+        this.callbacks.clear();
+      };
+      ws.onmessage = (event) => {
+        const msg = JSON.parse(String(event.data));
+        if (msg.id != null) {
+          const cb = this.callbacks.get(msg.id);
+          if (cb) {
+            this.callbacks.delete(msg.id);
+            if (msg.error) { cb.reject(new Error(msg.error.message)); } else { cb.resolve(msg.result); }
+          }
+        }
+      };
+    });
+  }
+
+  async send(method: string, params?: Record<string, unknown>): Promise<unknown> {
+    if (!this.ws) throw new Error('Not connected');
+    const id = this.nextId++;
+    return new Promise((resolve, reject) => {
+      this.callbacks.set(id, { resolve, reject });
+      this.ws!.send(JSON.stringify({ id, method, params }));
+    });
+  }
+
+  close() { this.ws?.close(); }
+}
+
+/**
+ * Navigate Chrome through a domain's pages to trigger API calls.
+ * Discovers internal links from the DOM and visits up to ~15 unique paths.
+ *
+ * @param domain The domain to crawl (e.g. "example.com").
+ * @param abortSignal Optional signal to stop navigation early.
+ * @returns List of visited page URLs.
+ */
+export async function autoNavigate(domain: string, abortSignal?: { aborted: boolean }): Promise<string[]> {
+  let wsUrl: string | null = null;
+  try {
+    const res = await fetch(`${CDP_BASE}/json/list`);
+    if (!res.ok) {
+      log.warn('CDP not available for auto-navigation');
+      return [];
+    }
+    const targets = (await res.json()) as Array<{ type: string; url: string; webSocketDebuggerUrl: string }>;
+    const domainTab = targets.find(t => {
+      if (t.type !== 'page') return false;
+      try {
+        const hostname = new URL(t.url).hostname;
+        return hostname === domain || hostname.endsWith('.' + domain);
+      } catch { return false; }
+    });
+    wsUrl = domainTab?.webSocketDebuggerUrl ?? targets.find(t => t.type === 'page')?.webSocketDebuggerUrl ?? null;
+  } catch (err) {
+    log.warn({ err }, 'Failed to discover Chrome tabs');
+    return [];
+  }
+
+  if (!wsUrl) {
+    log.warn('No Chrome tab found for auto-navigation');
+    return [];
+  }
+
+  const cdp = new MiniCDP();
+  try {
+    await cdp.connect(wsUrl);
+  } catch (err) {
+    log.warn({ err }, 'Failed to connect CDP for auto-navigation');
+    return [];
+  }
+
+  await cdp.send('Page.enable').catch(() => {});
+
+  const rootUrl = `https://${domain}/`;
+  const visited = new Set<string>();
+  const visitedUrls: string[] = [];
+
+  // Navigate to the domain root first
+  try {
+    await cdp.send('Page.navigate', { url: rootUrl });
+    await sleep(PAGE_WAIT_MS);
+    visited.add('/');
+    visitedUrls.push(rootUrl);
+    log.info({ url: rootUrl }, 'Visited root page');
+  } catch (err) {
+    log.warn({ err }, 'Failed to navigate to domain root');
+    cdp.close();
+    return [];
+  }
+
+  if (abortSignal?.aborted) { cdp.close(); return visitedUrls; }
+
+  // Scroll the root page to trigger lazy content
+  await scrollPage(cdp);
+  await sleep(SCROLL_WAIT_MS);
+
+  // Click common interactive elements on the root page
+  await clickInteractiveElements(cdp);
+  await sleep(SCROLL_WAIT_MS);
+
+  // Discover internal links from the current page
+  const discoveredLinks = await discoverInternalLinks(cdp, domain);
+  log.info({ count: discoveredLinks.length }, 'Discovered internal links from root');
+
+  // Visit discovered pages
+  for (const link of discoveredLinks) {
+    if (abortSignal?.aborted) break;
+    if (visited.size >= MAX_PAGES) break;
+    if (visited.has(link.key)) continue;
+
+    const url = link.url;
+    log.info({ url }, 'Auto-navigate visiting page');
+
+    try {
+      await cdp.send('Page.navigate', { url });
+      await sleep(PAGE_WAIT_MS);
+      visited.add(link.key);
+      visitedUrls.push(url);
+
+      // Scroll to trigger lazy-loaded content
+      await scrollPage(cdp);
+      await sleep(SCROLL_WAIT_MS);
+
+      // Click interactive elements to trigger more API calls
+      await clickInteractiveElements(cdp);
+      await sleep(1500);
+
+      // Discover more links from this page
+      const newLinks = await discoverInternalLinks(cdp, domain);
+      for (const nl of newLinks) {
+        if (!visited.has(nl.key) && !discoveredLinks.some(l => l.key === nl.key)) {
+          discoveredLinks.push(nl);
+        }
+      }
+
+      log.info({ url }, 'Auto-navigate page completed');
+    } catch (err) {
+      log.warn({ err, url }, 'Auto-navigate page failed');
+    }
+  }
+
+  cdp.close();
+  log.info({ visited: visitedUrls.length, total: discoveredLinks.length + 1 }, 'Auto-navigation finished');
+  return visitedUrls;
+}
+
+interface DiscoveredLink {
+  /** Full URL to navigate to (preserves subdomain). */
+  url: string;
+  /** Deduplication key: origin + pathname. */
+  key: string;
+}
+
+/** Extract internal links from the current page DOM, preserving subdomains. */
+async function discoverInternalLinks(cdp: MiniCDP, domain: string): Promise<DiscoveredLink[]> {
+  try {
+    const result = await cdp.send('Runtime.evaluate', {
+      expression: `
+        (function() {
+          const domain = ${JSON.stringify(domain)};
+          const seen = new Set();
+          const links = [];
+          for (const a of document.querySelectorAll('a[href]')) {
+            const href = a.getAttribute('href');
+            if (!href) continue;
+            try {
+              const url = new URL(href, location.origin);
+              if (url.hostname !== domain && !url.hostname.endsWith('.' + domain)) continue;
+              const path = url.pathname;
+              // Skip anchors, query-only links, file downloads, and trivial paths
+              if (path === '/' || path === '') continue;
+              if (path.match(/\\.(png|jpg|jpeg|gif|svg|css|js|woff|pdf|zip)$/i)) continue;
+              const key = url.origin + url.pathname;
+              if (!seen.has(key)) {
+                seen.add(key);
+                links.push({ url: url.origin + url.pathname, key });
+              }
+            } catch { /* skip malformed URLs */ }
+          }
+          return links;
+        })()
+      `,
+      awaitPromise: false,
+      returnByValue: true,
+    }) as { result?: { value?: DiscoveredLink[] } };
+    return result?.result?.value ?? [];
+  } catch {
+    return [];
+  }
+}
+
+/** Scroll the page to trigger lazy-loaded content. */
+async function scrollPage(cdp: MiniCDP): Promise<void> {
+  await cdp.send('Runtime.evaluate', {
+    expression: 'window.scrollBy(0, 800)',
+    awaitPromise: false,
+  }).catch(() => {});
+}
+
+/** Click common interactive elements (tabs, nav buttons) to trigger API calls. */
+async function clickInteractiveElements(cdp: MiniCDP): Promise<void> {
+  const selectors = [
+    'nav a:not([href="/"])',
+    '[role="tab"]',
+    '[role="tablist"] button',
+    'button[data-tab]',
+    '.tab, .nav-tab, .nav-link',
+  ];
+
+  for (const selector of selectors) {
+    await clickInPage(cdp, selector);
+    await sleep(800);
+  }
+}
+
+async function clickInPage(cdp: MiniCDP, selector: string): Promise<boolean> {
+  try {
+    const result = await cdp.send('Runtime.evaluate', {
+      expression: `
+        (function() {
+          const el = document.querySelector(${JSON.stringify(selector)});
+          if (!el) return false;
+          el.scrollIntoView({ block: 'center' });
+          el.click();
+          return true;
+        })()
+      `,
+      awaitPromise: false,
+      returnByValue: true,
+    }) as { result?: { value?: boolean } };
+    return result?.result?.value === true;
+  } catch {
+    return false;
+  }
+}
+
+function sleep(ms: number): Promise<void> {
+  return new Promise(r => setTimeout(r, ms));
+}