@detiq/crawler 0.1.3

package/src/url-pattern-completer.ts

@@ -0,0 +1,114 @@
+/**
+ * P1: URL Pattern LLM Completion
+ *
+ * After BFS crawl completes, runs a single LLM call to predict URLs that exist
+ * but were never linked from within the app. Validates each candidate with
+ * HTTP HEAD before emitting as crawl evidence (URL_CANDIDATES).
+ *
+ * Cost: ~1 LLM call + N HEAD requests. Takes ~5-30 seconds depending on
+ * candidate count. Does NOT launch a browser.
+ */
+
+import { logger as rootLogger } from './logger.js';
+
+const logger = rootLogger.child({ module: 'url-pattern-completer' });
+
+export interface UrlCandidate {
+  url: string;
+  confidence: number;
+  reason: string;
+  status: 'hit' | 'miss' | 'error';
+}
+
+interface LlmCfg {
+  provider: string;
+  model: string;
+  apiKey?: string;
+  baseUrl?: string;
+}
+
+/**
+ * Returns validated URL candidates (HEAD=200) predicted by LLM from
+ * the pattern of already-discovered URLs.
+ */
+export async function runUrlPatternCompletion(
+  discoveredUrls: string[],
+  appUrl: string,
+  llmCfg: LlmCfg | null | undefined,
+): Promise<UrlCandidate[]> {
+  if (!llmCfg?.apiKey || discoveredUrls.length < 2) return [];
+
+  const origin = (() => { try { return new URL(appUrl).origin; } catch { return ''; } })();
+  if (!origin) return [];
+
+  // Only send internal URLs (same origin, no assets)
+  const ASSET_EXT = /\.(png|jpg|jpeg|gif|svg|ico|webp|woff|woff2|ttf|eot|css|js|map|json|xml|txt|pdf)$/i;
+  const internalUrls = discoveredUrls
+    .filter(u => { try { return new URL(u).origin === origin && !ASSET_EXT.test(u); } catch { return false; } })
+    .slice(0, 60); // cap context size
+
+  if (internalUrls.length < 2) return [];
+
+  const prompt = `You are analyzing a web application's URL structure to find screens that exist but weren't discovered during automated crawling.
+
+Discovered URLs:
+${internalUrls.map(u => { try { return new URL(u).pathname + new URL(u).search; } catch { return u; } }).join('\n')}
+
+Based on these URL patterns, suggest up to 15 additional URLs that likely exist in this application but weren't found — typically settings subpages, admin panels, profile variants, paginated views, or feature-gated routes.
+
+Rules:
+- Only suggest URLs on the same domain
+- Use relative paths (starting with /)
+- Skip generic guesses (/login, /register, /home) unless the pattern strongly implies them
+- Focus on patterns YOU SEE in the discovered URLs (e.g. if /settings/profile exists, try /settings/security, /settings/billing)
+- Confidence 0.0–1.0: how certain you are the page exists
+
+Respond with ONLY valid JSON, no explanation:
+{"candidates":[{"path":"/settings/billing","confidence":0.85,"reason":"settings subpage pattern from /settings/profile"},{"path":"/admin/users","confidence":0.6,"reason":"admin prefix implied by /admin/dashboard"}]}`;
+
+  try {
+    const Anthropic = (await import('@anthropic-ai/sdk')).default;
+    const client = new Anthropic({ apiKey: llmCfg.apiKey });
+    const response = await client.messages.create({
+      model: llmCfg.model ?? 'claude-haiku-4-5-20251001',
+      max_tokens: 1024,
+      // @ts-ignore: cache_control supported in beta
+      system: [{ type: 'text', text: 'You are a URL pattern analyst. Always respond with valid JSON only.', cache_control: { type: 'ephemeral' } }],
+      messages: [{ role: 'user', content: prompt }],
+    });
+
+    const text = (response.content[0] as any)?.text ?? '';
+    const jsonMatch = text.match(/\{[\s\S]*\}/);
+    if (!jsonMatch) return [];
+    const parsed = JSON.parse(jsonMatch[0]);
+    const rawCandidates: Array<{ path: string; confidence: number; reason: string }> =
+      parsed.candidates ?? [];
+
+    if (!Array.isArray(rawCandidates) || rawCandidates.length === 0) return [];
+
+    // HEAD-validate each candidate
+    const results: UrlCandidate[] = [];
+    for (const c of rawCandidates.slice(0, 20)) {
+      if (!c.path || typeof c.path !== 'string') continue;
+      const fullUrl = c.path.startsWith('http') ? c.path : `${origin}${c.path.startsWith('/') ? '' : '/'}${c.path}`;
+      try {
+        const controller = new AbortController();
+        const timeout = setTimeout(() => controller.abort(), 6_000);
+        const res = await fetch(fullUrl, { method: 'HEAD', signal: controller.signal, redirect: 'follow' }).catch(() => null);
+        clearTimeout(timeout);
+        const status: UrlCandidate['status'] = res?.ok ? 'hit' : 'miss';
+        results.push({ url: fullUrl, confidence: c.confidence ?? 0.5, reason: c.reason ?? '', status });
+        logger.info({ url: fullUrl, httpStatus: res?.status ?? 'timeout', status }, '[url-completer] candidate checked');
+      } catch {
+        results.push({ url: fullUrl, confidence: c.confidence ?? 0.5, reason: c.reason ?? '', status: 'error' });
+      }
+    }
+
+    const hits = results.filter(r => r.status === 'hit');
+    logger.info({ total: results.length, hits: hits.length }, '[url-completer] pattern completion done');
+    return results;
+  } catch (err) {
+    logger.warn({ err: String(err) }, '[url-completer] LLM call failed — skipping pattern completion');
+    return [];
+  }
+}

package/src/visual-diff-verdict.test.ts

@@ -0,0 +1,80 @@
+import test from 'node:test';
+import assert from 'node:assert/strict';
+import { pHashVerdict, mergeAiVisualVerdict } from './visual-diff-verdict.js';
+
+test('pHashVerdict: IDENTICAL maps to NONE/APPROVE', () => {
+  const v = pHashVerdict(0, 'IDENTICAL', []);
+  assert.equal(v.changeType, 'NONE');
+  assert.equal(v.recommendation, 'APPROVE');
+});
+
+test('pHashVerdict: MODERATE change level maps to SIGNIFICANT changeType', () => {
+  const v = pHashVerdict(12, 'MODERATE', []);
+  assert.equal(v.changeType, 'SIGNIFICANT');
+});
+
+test('pHashVerdict: SIGNIFICANT change level maps to CRITICAL changeType (naming overlap between phash and VisualDiff scales)', () => {
+  const v = pHashVerdict(35, 'SIGNIFICANT', []);
+  assert.equal(v.changeType, 'CRITICAL');
+});
+
+test('pHashVerdict: recommendation escalates with distance regardless of changeLevel', () => {
+  assert.equal(pHashVerdict(0, 'MINOR', []).recommendation, 'APPROVE');
+  assert.equal(pHashVerdict(10, 'MINOR', []).recommendation, 'REVIEW');
+  assert.equal(pHashVerdict(25, 'MINOR', []).recommendation, 'FLAG');
+});
+
+test('mergeAiVisualVerdict: AI downgrading pHash CRITICAL to MINOR (anti-aliasing noise) produces no bug report draft', () => {
+  const pHash = pHashVerdict(35, 'SIGNIFICANT', [{ location: 'header', severity: 'Cosmetic', description: 'pixel shift' }]);
+  const merged = mergeAiVisualVerdict(
+    pHash,
+    { changeType: 'MINOR', recommendation: 'APPROVE', changes: [], diffScore: 3 },
+    'Login Screen',
+  );
+  assert.equal(merged.changeType, 'MINOR');
+  assert.equal(merged.recommendation, 'APPROVE');
+  assert.equal(merged.bugReportDraft, undefined, 'a downgraded verdict must not carry a stale bug report draft');
+});
+
+test('mergeAiVisualVerdict: AI confirming CRITICAL produces a bug report draft with the AI diff score, not the pHash one', () => {
+  const pHash = pHashVerdict(35, 'SIGNIFICANT', []);
+  const merged = mergeAiVisualVerdict(
+    pHash,
+    { changeType: 'CRITICAL', recommendation: 'FLAG', changes: [{ location: 'nav', severity: 'Functional', description: 'Logo missing' }], diffScore: 62.4 },
+    'Dashboard',
+  );
+  assert.equal(merged.changeType, 'CRITICAL');
+  assert.ok(merged.bugReportDraft);
+  assert.equal(merged.bugReportDraft!.severity, 'High');
+  assert.match(merged.bugReportDraft!.actual, /62\.4%/);
+  assert.equal(merged.changes.length, 1);
+  assert.equal(merged.changes[0].description, 'Logo missing');
+});
+
+test('mergeAiVisualVerdict: SIGNIFICANT (not just CRITICAL) also gets a bug report draft, with Medium severity', () => {
+  const pHash = pHashVerdict(20, 'MODERATE', []);
+  const merged = mergeAiVisualVerdict(
+    pHash,
+    { changeType: 'SIGNIFICANT', recommendation: 'REVIEW', changes: [], diffScore: 18 },
+    'Checkout',
+  );
+  assert.ok(merged.bugReportDraft);
+  assert.equal(merged.bugReportDraft!.severity, 'Medium');
+});
+
+test('mergeAiVisualVerdict: empty AI changes array keeps the pHash-derived changes instead of wiping them', () => {
+  const pHash = pHashVerdict(35, 'SIGNIFICANT', [{ location: 'footer', severity: 'Cosmetic', description: 'layout shift' }]);
+  const merged = mergeAiVisualVerdict(
+    pHash,
+    { changeType: 'CRITICAL', recommendation: 'FLAG', changes: [], diffScore: 40 },
+    'Settings',
+  );
+  assert.equal(merged.changes.length, 1);
+  assert.equal(merged.changes[0].description, 'layout shift');
+});
+
+test('mergeAiVisualVerdict: malformed AI result (no changeType) falls back to the pHash verdict untouched', () => {
+  const pHash = pHashVerdict(35, 'SIGNIFICANT', []);
+  const merged = mergeAiVisualVerdict(pHash, { changeType: '', recommendation: '', changes: [], diffScore: 0 }, 'Home');
+  assert.deepEqual(merged, pHash);
+});

package/src/visual-diff-verdict.ts

@@ -0,0 +1,57 @@
+/**
+ * Pure visual-diff verdict logic, split out of crawler.ts so it can be unit-tested
+ * without pulling in crawler.ts's full import graph (Playwright, Stagehand, and
+ * transitively @detiq/database, which instantiates a PrismaClient at module load and
+ * so requires a live DATABASE_URL just to import the module).
+ */
+
+export interface VisualDiffVerdict {
+  changeType: 'NONE' | 'MINOR' | 'SIGNIFICANT' | 'CRITICAL';
+  recommendation: 'APPROVE' | 'REVIEW' | 'FLAG' | 'BLOCK';
+  changes: Array<{ location: string; severity: string; description: string }>;
+  bugReportDraft?: { title: string; severity: string; steps: string[]; expected: string; actual: string };
+  aiSummary?: string;
+}
+
+/** Cheap, always-on first-pass verdict from perceptual hash distance + layout/element diffs. */
+export function pHashVerdict(
+  pHashDistance: number,
+  changeLevel: 'IDENTICAL' | 'MINOR' | 'MODERATE' | 'SIGNIFICANT',
+  changes: Array<{ location: string; severity: string; description: string }>,
+): VisualDiffVerdict {
+  const changeType: VisualDiffVerdict['changeType'] = changeLevel === 'IDENTICAL' ? 'NONE'
+    : changeLevel === 'MINOR' ? 'MINOR'
+      : changeLevel === 'MODERATE' ? 'SIGNIFICANT'
+        : 'CRITICAL';
+  const recommendation: VisualDiffVerdict['recommendation'] = pHashDistance === 0 ? 'APPROVE'
+    : pHashDistance < 5 ? 'APPROVE'
+      : pHashDistance < 20 ? 'REVIEW'
+        : 'FLAG';
+  return { changeType, recommendation, changes };
+}
+
+/** Replaces a pHash-only verdict with the AI's semantic judgment once it's available,
+ *  and drafts a bug report for anything still CRITICAL/SIGNIFICANT after AI review
+ *  (pHash flags plenty of anti-aliasing/font-rendering noise the AI correctly discards
+ *  as NONE/MINOR — only what survives AI review is worth a draft). */
+export function mergeAiVisualVerdict(
+  pHashResult: VisualDiffVerdict,
+  aiResult: { changeType: string; changes: any[]; recommendation: string; diffScore: number; summary?: string },
+  screenName: string,
+): VisualDiffVerdict {
+  if (!aiResult.changeType) return pHashResult;
+  const changeType = aiResult.changeType as VisualDiffVerdict['changeType'];
+  const recommendation = aiResult.recommendation as VisualDiffVerdict['recommendation'];
+  const changes = Array.isArray(aiResult.changes) && aiResult.changes.length > 0 ? aiResult.changes : pHashResult.changes;
+  const aiSummary = aiResult.summary ?? undefined;
+  const bugReportDraft = (changeType === 'CRITICAL' || changeType === 'SIGNIFICANT')
+    ? {
+        title: `Visual regression on "${screenName}"`,
+        severity: changeType === 'CRITICAL' ? 'High' : 'Medium',
+        steps: ['Navigate to the screen', 'Compare current view to baseline screenshot'],
+        expected: 'Screen appearance should match the stored baseline.',
+        actual: aiSummary || `Screen has changed (diff score: ${aiResult.diffScore.toFixed(1)}%)`,
+      }
+    : undefined;
+  return { changeType, recommendation, changes, bugReportDraft, aiSummary };
+}

package/src/webhook-notifier.ts

@@ -0,0 +1,109 @@
+/**
+ * Webhook notifier — fire HTTP POST callbacks on crawl lifecycle events.
+ *
+ * Supports: crawl.started, crawl.progress, crawl.completed, crawl.failed
+ * Payload format compatible with Slack, Discord, and generic REST webhooks.
+ * Retries 3x on failure with exponential backoff.
+ */
+
+export type CrawlEventType = 'crawl.started' | 'crawl.progress' | 'crawl.completed' | 'crawl.failed';
+
+export interface WebhookPayload {
+  event: CrawlEventType;
+  jobId: string;
+  projectId: string;
+  tenantId: string;
+  timestamp: string;
+  data: Record<string, unknown>;
+}
+
+export interface WebhookNotifierOpts {
+  url: string;
+  secret?: string;       // HMAC-SHA256 signing secret — adds X-ZeTa-Signature header
+  headers?: Record<string, string>;
+  timeoutMs?: number;    // default 10s
+  maxRetries?: number;   // default 3
+}
+
+async function hmacSign(secret: string, payload: string): Promise<string> {
+  const key = await crypto.subtle.importKey(
+    'raw',
+    new TextEncoder().encode(secret),
+    { name: 'HMAC', hash: 'SHA-256' },
+    false,
+    ['sign'],
+  );
+  const sig = await crypto.subtle.sign('HMAC', key, new TextEncoder().encode(payload));
+  return Array.from(new Uint8Array(sig)).map(b => b.toString(16).padStart(2, '0')).join('');
+}
+
+export class WebhookNotifier {
+  private readonly opts: Required<Omit<WebhookNotifierOpts, 'secret'>> & { secret?: string };
+
+  constructor(opts: WebhookNotifierOpts) {
+    this.opts = {
+      url: opts.url,
+      secret: opts.secret,
+      headers: opts.headers ?? {},
+      timeoutMs: opts.timeoutMs ?? 10_000,
+      maxRetries: opts.maxRetries ?? 3,
+    };
+  }
+
+  async notify(
+    event: CrawlEventType,
+    jobId: string,
+    projectId: string,
+    tenantId: string,
+    data: Record<string, unknown> = {},
+  ): Promise<void> {
+    const payload: WebhookPayload = {
+      event,
+      jobId,
+      projectId,
+      tenantId,
+      timestamp: new Date().toISOString(),
+      data,
+    };
+    const body = JSON.stringify(payload);
+    const headers: Record<string, string> = {
+      'Content-Type': 'application/json',
+      'User-Agent': 'ZeTa-Crawler/1.0',
+      ...this.opts.headers,
+    };
+    if (this.opts.secret) {
+      headers['X-ZeTa-Signature'] = `sha256=${await hmacSign(this.opts.secret, body)}`;
+    }
+
+    let lastErr: any;
+    for (let attempt = 0; attempt <= this.opts.maxRetries; attempt++) {
+      try {
+        const res = await fetch(this.opts.url, {
+          method: 'POST',
+          headers,
+          body,
+          signal: AbortSignal.timeout(this.opts.timeoutMs),
+        });
+        if (res.ok) return;
+        lastErr = new Error(`Webhook HTTP ${res.status}`);
+      } catch (e) {
+        lastErr = e;
+      }
+      if (attempt < this.opts.maxRetries) {
+        await new Promise(r => setTimeout(r, 1_000 * Math.pow(2, attempt)));
+      }
+    }
+    console.error(`[webhook] Failed to deliver ${event} after ${this.opts.maxRetries + 1} attempts:`, lastErr?.message);
+  }
+
+  /** Slack-compatible message format */
+  static slackPayload(event: CrawlEventType, data: Record<string, unknown>): Record<string, unknown> {
+    const emoji = event === 'crawl.completed' ? '✅' : event === 'crawl.failed' ? '❌' : '🔄';
+    return {
+      text: `${emoji} *ZeTa Crawl Event: ${event}*`,
+      blocks: [
+        { type: 'section', text: { type: 'mrkdwn', text: `${emoji} *${event}*\n${Object.entries(data).map(([k, v]) => `• *${k}*: ${v}`).join('\n')}` } },
+      ],
+    };
+  }
+}

package/src/websocket-detector.ts

@@ -0,0 +1,83 @@
+/**
+ * WebSocket detection — intercepts WS connections opened during page crawl.
+ * Records connection URLs, message schemas, and message counts.
+ * This lets ZeTa discover real-time API contracts (chat, notifications, presence)
+ * that would be invisible to HTTP-only crawlers.
+ */
+
+export interface WsConnection {
+  url: string;
+  framesSent: number;
+  framesReceived: number;
+  /** Sampled message bodies (first 10 per connection, truncated to 500 chars) */
+  samples: Array<{ direction: 'sent' | 'received'; payload: string; ts: string }>;
+  /** Inferred protocol: 'json-rpc' | 'socket.io' | 'graphql-ws' | 'raw' */
+  protocol: string;
+  openedAt: string;
+}
+
+function inferProtocol(sample: string): string {
+  try {
+    const obj = JSON.parse(sample);
+    if (obj?.jsonrpc) return 'json-rpc';
+    if (obj?.type === 'connection_init' || obj?.type === 'subscribe') return 'graphql-ws';
+    if (typeof obj?.sid === 'string' || sample.startsWith('0{')) return 'socket.io';
+    return 'json';
+  } catch {
+    return 'raw';
+  }
+}
+
+export function attachWebSocketDetector(cdpSession: any): {
+  getConnections(): WsConnection[];
+  detach(): void;
+} {
+  const connections = new Map<string, WsConnection>();
+
+  const onCreated = (evt: any) => {
+    const { requestId, url, timestamp } = evt ?? {};
+    if (!requestId || !url) return;
+    connections.set(requestId, {
+      url,
+      framesSent: 0,
+      framesReceived: 0,
+      samples: [],
+      protocol: 'unknown',
+      openedAt: new Date(timestamp * 1000).toISOString(),
+    });
+  };
+
+  const onFrame = (evt: any, direction: 'sent' | 'received') => {
+    const { requestId, response } = evt ?? {};
+    if (!requestId) return;
+    const conn = connections.get(requestId);
+    if (!conn) return;
+    if (direction === 'sent') conn.framesSent++;
+    else conn.framesReceived++;
+    if (conn.samples.length < 10 && response?.payloadData) {
+      const payload = String(response.payloadData).slice(0, 500);
+      conn.samples.push({ direction, payload, ts: new Date().toISOString() });
+      if (conn.protocol === 'unknown') conn.protocol = inferProtocol(payload);
+    }
+  };
+
+  const onSent = (evt: any) => onFrame(evt, 'sent');
+  const onReceived = (evt: any) => onFrame(evt, 'received');
+
+  if (cdpSession?.on) {
+    cdpSession.on('Network.webSocketCreated', onCreated);
+    cdpSession.on('Network.webSocketFrameSent', onSent);
+    cdpSession.on('Network.webSocketFrameReceived', onReceived);
+  }
+
+  return {
+    getConnections: () => [...connections.values()],
+    detach() {
+      if (cdpSession?.off) {
+        cdpSession.off('Network.webSocketCreated', onCreated);
+        cdpSession.off('Network.webSocketFrameSent', onSent);
+        cdpSession.off('Network.webSocketFrameReceived', onReceived);
+      }
+    },
+  };
+}

package/src/worker.ts

@@ -0,0 +1,48 @@
+/**
+ * Standalone worker process entrypoint for a dedicated crawler/worker container,
+ * separate from the API server (see infrastructure/ecs-task-definitions/crawler.json).
+ *
+ * index.ts is a side-effect-free library barrel imported by @detiq/api — it must NOT
+ * start any background loop itself, or importing a single helper from @detiq/crawler
+ * would silently spin up a second worker inside the API process too. This file is
+ * the actual entrypoint that keeps a dedicated worker container's process alive and
+ * consuming jobs: `node dist/worker.js`.
+ */
+import { startJobWorkerLoop, processDataRetentionPurge } from './job-queue.js';
+import { logger } from './logger.js';
+
+async function main() {
+  logger.info('[worker] starting standalone crawler worker process');
+
+  // Durable DB-backed queue — claims every non-CRAWL job type, and CRAWL too when
+  // JOB_QUEUE_BACKEND isn't 'sqs'. Safe to run in many worker replicas at once.
+  startJobWorkerLoop();
+
+  // Daily data-retention purge: runs immediately on startup then every 24 h.
+  // Only runs in the dedicated worker container (this file), not in the API
+  // process, so there is exactly one purge sweep regardless of API replica count.
+  const runPurge = () => {
+    processDataRetentionPurge().catch((err) => {
+      logger.error({ err: String(err) }, '[worker] data-retention purge failed');
+    });
+  };
+  runPurge();
+  setInterval(runPurge, 24 * 60 * 60 * 1000);
+
+  // Optional: also consume the SQS transport for horizontally-scaled CRAWL fleets.
+  if (process.env.JOB_QUEUE_BACKEND === 'sqs') {
+    const { startSqsWorker } = await import('./sqs-queue.js');
+    startSqsWorker().catch((err) => {
+      logger.error({ err: String(err) }, '[worker] SQS worker crashed');
+      process.exit(1);
+    });
+  }
+}
+
+process.on('SIGTERM', () => { logger.info('[worker] SIGTERM received, exiting'); process.exit(0); });
+process.on('SIGINT', () => { logger.info('[worker] SIGINT received, exiting'); process.exit(0); });
+
+main().catch((err) => {
+  logger.error({ err: String(err) }, '[worker] fatal startup error');
+  process.exit(1);
+});

package/tsconfig.json

@@ -0,0 +1,13 @@
+{
+  "extends": "../../tsconfig.base.json",
+  "compilerOptions": {
+    "rootDir": "src",
+    "outDir": "dist",
+    "lib": ["ES2022", "DOM"],
+    "isolatedModules": true,
+    "composite": false,
+    "noEmitOnError": false
+  },
+  "include": ["src/**/*.ts"],
+  "exclude": ["src/**/*.test.ts"]
+}

package/types/accessibility-auditor.d.ts

@@ -0,0 +1,29 @@
+/**
+ * Accessibility auditor — lightweight a11y checks using Playwright's aria snapshot
+ * and manual DOM queries. No axe-core dependency needed.
+ *
+ * Checks:
+ * 1. Images without alt text
+ * 2. Form inputs without associated labels
+ * 3. Buttons/links with no accessible name
+ * 4. Missing lang attribute on html element
+ * 5. Missing page title
+ * 6. Empty headings
+ * 7. Color contrast (limited — detects text with inline style color only)
+ * 8. Keyboard-unfocusable interactive elements (tabIndex=-1 on buttons)
+ */
+import type { Page } from 'playwright';
+export interface A11yViolation {
+    rule: string;
+    severity: 'critical' | 'serious' | 'moderate' | 'minor';
+    count: number;
+    examples: string[];
+}
+export interface A11yAuditResult {
+    url: string;
+    violations: A11yViolation[];
+    passCount: number;
+    score: number;
+    auditedAt: string;
+}
+export declare function auditAccessibility(page: Page, url: string): Promise<A11yAuditResult>;

package/types/ai-login.d.ts

@@ -0,0 +1,32 @@
+import type { Stagehand } from '@browserbasehq/stagehand';
+import type { Credentials } from './auth.js';
+/**
+ * Recorded login step from the Chrome extension's step recorder.
+ * {username} and {password} in `value` are substituted at replay time.
+ */
+export interface LoginStep {
+    action: 'fill' | 'click' | 'wait';
+    selector: string;
+    value?: string;
+    ms?: number;
+}
+/**
+ * AI-guided login using Stagehand act/extract.
+ *
+ * @param loginInstructions - Optional natural language description of the login flow.
+ *   When provided, an initial stagehand.act() is called with these instructions to
+ *   navigate the UI (e.g. "click Sign In in the nav, fill email, click Continue").
+ *   The standard 3-step credential fill follows immediately after.
+ * @param opts - Optional tenant/project/job IDs for writing AUTH_STEP CrawlEvidence.
+ */
+export declare function loginWithStagehand(stagehand: Stagehand, creds: Credentials, startUrl: string, captchaOpts?: {
+    apiKey?: string;
+    provider?: '2captcha' | 'capsolver';
+}, emailOtpOpts?: {
+    apiKey?: string;
+    inboxId?: string;
+}, loginInstructions?: string, opts?: {
+    tenantId?: string;
+    projectId?: string;
+    jobId?: string;
+}): Promise<boolean>;

package/types/auth-token-rotator.d.ts

@@ -0,0 +1,47 @@
+/**
+ * Auth token rotator — manage a pool of authenticated sessions for multi-account crawls.
+ *
+ * Use case: crawl the same app as admin+user+guest in one crawl job to discover
+ * role-specific screens that only appear when authenticated with different privileges.
+ * Each session is a Playwright storageState JSON string.
+ *
+ * Sessions are checked out (locked) before use and returned after the page is done.
+ * Round-robin assignment with stale-session eviction after maxAgeMs.
+ */
+export interface SessionEntry {
+    id: string;
+    role: string;
+    storageState: string;
+    createdAt: number;
+    checkedOutAt: number | null;
+    requestCount: number;
+}
+export interface SessionPoolOpts {
+    sessions: Array<{
+        role: string;
+        storageState: string;
+    }>;
+    maxAgeMs?: number;
+    maxRequestsPerSession?: number;
+}
+export declare class AuthTokenRotator {
+    private pool;
+    private readonly maxAgeMs;
+    private readonly maxRequests;
+    constructor(opts: SessionPoolOpts);
+    /** Get the next available session. Returns null if all are checked out. */
+    checkout(): SessionEntry | null;
+    /** Return a session to the pool after use. */
+    checkin(sessionId: string, updatedStorageState?: string): void;
+    /** Get all available sessions (for parallel crawl setup). */
+    getAll(): ReadonlyArray<SessionEntry>;
+    /** Stats for monitoring. */
+    stats(): {
+        total: number;
+        available: number;
+        checkedOut: number;
+        stale: number;
+    };
+    /** Add a new session to the pool at runtime. */
+    addSession(role: string, storageState: string): string;
+}

package/types/auth.d.ts

@@ -0,0 +1,20 @@
+import type { Page } from 'playwright';
+export interface Credentials {
+    username: string;
+    password: string;
+    oneTimeCode?: string;
+    getOneTimeCode?: () => Promise<string>;
+}
+/**
+ * Recorded login step captured by the Chrome extension.
+ * {username} and {password} in `value` are substituted at replay time.
+ */
+export interface LoginStep {
+    action: 'fill' | 'click' | 'wait';
+    selector: string;
+    value?: string;
+    ms?: number;
+}
+export declare function login(page: Page, appUrl: string, creds: Credentials & {
+    loginRecipe?: LoginStep[];
+}): Promise<boolean>;

package/types/browser-extension.d.ts

@@ -0,0 +1,37 @@
+/**
+ * Browser extension injection support.
+ *
+ * Chrome extensions require a persistent profile directory — incompatible
+ * with headless Docker by default. This module handles:
+ *   1. Persistent profile directory per extension set
+ *   2. Extension download from .crx URL or load from local path
+ *   3. Launch args for Playwright / Stagehand
+ *
+ * Usage:
+ *   const profile = await prepareExtensionProfile(['/path/to/ext.crx']);
+ *   // pass profile.userDataDir and profile.args to browser launch
+ */
+export interface ExtensionSource {
+    /** Local .crx file path or https:// URL to download from */
+    source: string;
+    /** Friendly name for logging */
+    name?: string;
+}
+export interface ExtensionProfile {
+    /** Chrome user-data-dir containing Preferences + extension unpacked dirs */
+    userDataDir: string;
+    /** Args to append to chromium launch — includes --load-extension and --disable-extensions-except */
+    args: string[];
+    /** Cleanup the profile directory when done */
+    cleanup: () => Promise<void>;
+}
+/**
+ * Prepare a Chrome user-data-dir with the given extensions loaded.
+ * Returns profile info + cleanup function.
+ */
+export declare function prepareExtensionProfile(extensions: ExtensionSource[]): Promise<ExtensionProfile>;
+/**
+ * Ad-blocker extension profile using the uBlock Origin CRX.
+ * Reduces noise from ad networks during crawls.
+ */
+export declare function adBlockerProfile(): Promise<ExtensionProfile>;