@detiq/crawler 0.1.3

package/Dockerfile

@@ -0,0 +1,49 @@
+# ZeTa-AI Crawler — includes Playwright + Chromium
+# Playwright needs a real Linux environment with system deps.
+# This service CANNOT run on Vercel serverless — use Railway/Render.
+# IMPORTANT: ECS Fargate task definition must set sharedMemorySize: 2048 (MB) for Chromium stability
+FROM mcr.microsoft.com/playwright:v1.55.0-jammy AS runner
+WORKDIR /app
+ENV NODE_ENV=production
+
+# Install Node.js 20 (Playwright base image ships with 18)
+RUN apt-get update && apt-get install -y curl && \
+    curl -fsSL https://deb.nodesource.com/setup_20.x | bash - && \
+    apt-get install -y nodejs && \
+    apt-get clean && rm -rf /var/lib/apt/lists/*
+
+# Workspace manifests (layer cache)
+COPY package.json package-lock.json ./
+COPY packages/core/package.json         packages/core/
+COPY packages/database/package.json     packages/database/
+COPY packages/app-brain/package.json    packages/app-brain/
+COPY packages/agents/package.json       packages/agents/
+COPY packages/crawler/package.json      packages/crawler/
+RUN npm ci
+
+# Source
+COPY tsconfig.base.json ./
+COPY packages/core       packages/core
+COPY packages/database   packages/database
+COPY packages/app-brain  packages/app-brain
+COPY packages/agents     packages/agents
+COPY packages/crawler    packages/crawler
+
+# Build
+RUN npx prisma generate --schema packages/database/prisma/schema.prisma && \
+    npm run -w @detiq/core build && \
+    npm run -w @detiq/database build && \
+    npm run -w @detiq/app-brain build && \
+    npm run -w @detiq/agents build && \
+    npm run -w @detiq/crawler build
+
+# Screenshot cache dir
+RUN mkdir -p /app/.screenshots-cache
+
+EXPOSE 4001
+# index.js is a side-effect-free library barrel (also imported by @detiq/api) — it holds
+# no listener/timer, so running it directly starts the process, finishes its imports,
+# and exits immediately with nothing ever consuming the job queue. worker.js is the
+# actual long-running entrypoint: it starts the durable job-poll loop (and the SQS
+# consumer when JOB_QUEUE_BACKEND=sqs) and keeps the process alive.
+CMD ["node", "packages/crawler/dist/worker.js"]

package/README.md

@@ -0,0 +1,11 @@
+# @detiq/crawler — The Explorer
+
+Crawls an app, **gets past the login** (the star of Phase 1), discovers
+screens, captures screenshots, and reads each page's elements into the App
+Brain.
+
+- `auth.ts` — login handling. **Invest here**; it's the competitive edge.
+- `screen-reader.ts` — reads pages as a cheap structured outline (accessibility
+  tree), not pixels. Vision is a later fallback.
+- `crawler.ts` — the discovery loop. Swap in an AI navigation library
+  (Stagehand / Browser Use) to explore more intelligently.

package/dist/accessibility-auditor.d.ts

@@ -0,0 +1,29 @@
+/**
+ * Accessibility auditor — lightweight a11y checks using Playwright's aria snapshot
+ * and manual DOM queries. No axe-core dependency needed.
+ *
+ * Checks:
+ * 1. Images without alt text
+ * 2. Form inputs without associated labels
+ * 3. Buttons/links with no accessible name
+ * 4. Missing lang attribute on html element
+ * 5. Missing page title
+ * 6. Empty headings
+ * 7. Color contrast (limited — detects text with inline style color only)
+ * 8. Keyboard-unfocusable interactive elements (tabIndex=-1 on buttons)
+ */
+import type { Page } from 'playwright';
+export interface A11yViolation {
+    rule: string;
+    severity: 'critical' | 'serious' | 'moderate' | 'minor';
+    count: number;
+    examples: string[];
+}
+export interface A11yAuditResult {
+    url: string;
+    violations: A11yViolation[];
+    passCount: number;
+    score: number;
+    auditedAt: string;
+}
+export declare function auditAccessibility(page: Page, url: string): Promise<A11yAuditResult>;

package/dist/accessibility-auditor.js

@@ -0,0 +1,57 @@
+async function queryAll(page, selector) {
+  try {
+    return await page.$$eval(selector, (els) => els.map((el) => ({
+      outerHTML: el.outerHTML?.slice(0, 200) ?? "",
+      textContent: (el.textContent ?? "").trim().slice(0, 100),
+      tabIndex: el.tabIndex,
+      getAttribute: (a) => el.getAttribute(a)
+    })));
+  } catch {
+    return [];
+  }
+}
+async function auditAccessibility(page, url) {
+  const violations = [];
+  let passCount = 0;
+  const auditedAt = (/* @__PURE__ */ new Date()).toISOString();
+  const imgs = await page.$$eval("img:not([alt])", (els) => els.map((e) => e.outerHTML?.slice(0, 120) ?? ""));
+  if (imgs.length > 0) violations.push({ rule: "image-alt", severity: "critical", count: imgs.length, examples: imgs.slice(0, 3) });
+  else passCount++;
+  const unlabeledInputs = await page.$$eval(
+    'input:not([type="hidden"]):not([type="submit"]):not([type="button"]):not([aria-label]):not([aria-labelledby]):not([id])',
+    (els) => els.map((e) => e.outerHTML?.slice(0, 120) ?? "")
+  );
+  if (unlabeledInputs.length > 0) violations.push({ rule: "label", severity: "serious", count: unlabeledInputs.length, examples: unlabeledInputs.slice(0, 3) });
+  else passCount++;
+  const emptyButtons = await page.$$eval(
+    "button:not([aria-label]):not([aria-labelledby])",
+    (els) => els.filter((e) => !(e.textContent ?? "").trim()).map((e) => e.outerHTML?.slice(0, 120) ?? "")
+  );
+  if (emptyButtons.length > 0) violations.push({ rule: "button-name", severity: "critical", count: emptyButtons.length, examples: emptyButtons.slice(0, 3) });
+  else passCount++;
+  const htmlLang = await page.$eval("html", (el) => el.getAttribute("lang") ?? "").catch(() => "");
+  if (!htmlLang) violations.push({ rule: "html-has-lang", severity: "serious", count: 1, examples: ["<html> missing lang attribute"] });
+  else passCount++;
+  const title = await page.title().catch(() => "");
+  if (!title) violations.push({ rule: "document-title", severity: "serious", count: 1, examples: ["Page has no <title>"] });
+  else passCount++;
+  const emptyHeadings = await page.$$eval(
+    "h1,h2,h3,h4,h5,h6",
+    (els) => els.filter((e) => !(e.textContent ?? "").trim()).map((e) => e.tagName.toLowerCase())
+  );
+  if (emptyHeadings.length > 0) violations.push({ rule: "empty-heading", severity: "moderate", count: emptyHeadings.length, examples: emptyHeadings.slice(0, 3) });
+  else passCount++;
+  const emptyLinks = await page.$$eval(
+    "a[href]:not([aria-label]):not([aria-labelledby])",
+    (els) => els.filter((e) => !(e.textContent ?? "").trim() && !e.querySelector("img[alt]")).map((e) => e.outerHTML?.slice(0, 120) ?? "")
+  );
+  if (emptyLinks.length > 0) violations.push({ rule: "link-name", severity: "serious", count: emptyLinks.length, examples: emptyLinks.slice(0, 3) });
+  else passCount++;
+  const totalChecks = passCount + violations.length;
+  const score = totalChecks > 0 ? Math.round(passCount / totalChecks * 100) : 100;
+  return { url, violations, passCount, score, auditedAt };
+}
+export {
+  auditAccessibility
+};
+//# sourceMappingURL=accessibility-auditor.js.map

package/dist/accessibility-auditor.js.map

@@ -0,0 +1,7 @@
+{
+  "version": 3,
+  "sources": ["../src/accessibility-auditor.ts"],
+  "sourcesContent": ["/**\n * Accessibility auditor \u2014 lightweight a11y checks using Playwright's aria snapshot\n * and manual DOM queries. No axe-core dependency needed.\n *\n * Checks:\n * 1. Images without alt text\n * 2. Form inputs without associated labels\n * 3. Buttons/links with no accessible name\n * 4. Missing lang attribute on html element\n * 5. Missing page title\n * 6. Empty headings\n * 7. Color contrast (limited \u2014 detects text with inline style color only)\n * 8. Keyboard-unfocusable interactive elements (tabIndex=-1 on buttons)\n */\n\nimport type { Page } from 'playwright';\n\nexport interface A11yViolation {\n  rule: string;\n  severity: 'critical' | 'serious' | 'moderate' | 'minor';\n  count: number;\n  examples: string[];\n}\n\nexport interface A11yAuditResult {\n  url: string;\n  violations: A11yViolation[];\n  passCount: number;\n  score: number; // 0-100\n  auditedAt: string;\n}\n\nasync function queryAll(page: Page, selector: string): Promise<any[]> {\n  try {\n    return await page.$$eval(selector, els => els.map((el: any) => ({\n      outerHTML: el.outerHTML?.slice(0, 200) ?? '',\n      textContent: (el.textContent ?? '').trim().slice(0, 100),\n      tabIndex: el.tabIndex,\n      getAttribute: (a: string) => el.getAttribute(a),\n    })));\n  } catch {\n    return [];\n  }\n}\n\nexport async function auditAccessibility(page: Page, url: string): Promise<A11yAuditResult> {\n  const violations: A11yViolation[] = [];\n  let passCount = 0;\n  const auditedAt = new Date().toISOString();\n\n  // 1. Images without alt\n  const imgs = await page.$$eval('img:not([alt])', els => els.map((e: any) => e.outerHTML?.slice(0, 120) ?? ''));\n  if (imgs.length > 0) violations.push({ rule: 'image-alt', severity: 'critical', count: imgs.length, examples: imgs.slice(0, 3) });\n  else passCount++;\n\n  // 2. Inputs without labels\n  const unlabeledInputs = await page.$$eval(\n    'input:not([type=\"hidden\"]):not([type=\"submit\"]):not([type=\"button\"]):not([aria-label]):not([aria-labelledby]):not([id])',\n    els => els.map((e: any) => e.outerHTML?.slice(0, 120) ?? ''),\n  );\n  if (unlabeledInputs.length > 0) violations.push({ rule: 'label', severity: 'serious', count: unlabeledInputs.length, examples: unlabeledInputs.slice(0, 3) });\n  else passCount++;\n\n  // 3. Buttons with no text\n  const emptyButtons = await page.$$eval(\n    'button:not([aria-label]):not([aria-labelledby])',\n    els => els.filter((e: any) => !(e.textContent ?? '').trim()).map((e: any) => e.outerHTML?.slice(0, 120) ?? ''),\n  );\n  if (emptyButtons.length > 0) violations.push({ rule: 'button-name', severity: 'critical', count: emptyButtons.length, examples: emptyButtons.slice(0, 3) });\n  else passCount++;\n\n  // 4. Missing html lang\n  const htmlLang = await page.$eval('html', (el: any) => el.getAttribute('lang') ?? '').catch(() => '');\n  if (!htmlLang) violations.push({ rule: 'html-has-lang', severity: 'serious', count: 1, examples: ['<html> missing lang attribute'] });\n  else passCount++;\n\n  // 5. Missing page title\n  const title = await page.title().catch(() => '');\n  if (!title) violations.push({ rule: 'document-title', severity: 'serious', count: 1, examples: ['Page has no <title>'] });\n  else passCount++;\n\n  // 6. Empty headings\n  const emptyHeadings = await page.$$eval(\n    'h1,h2,h3,h4,h5,h6',\n    els => els.filter((e: any) => !(e.textContent ?? '').trim()).map((e: any) => e.tagName.toLowerCase()),\n  );\n  if (emptyHeadings.length > 0) violations.push({ rule: 'empty-heading', severity: 'moderate', count: emptyHeadings.length, examples: emptyHeadings.slice(0, 3) });\n  else passCount++;\n\n  // 7. Links with no accessible name\n  const emptyLinks = await page.$$eval(\n    'a[href]:not([aria-label]):not([aria-labelledby])',\n    els => els.filter((e: any) => !(e.textContent ?? '').trim() && !e.querySelector('img[alt]')).map((e: any) => e.outerHTML?.slice(0, 120) ?? ''),\n  );\n  if (emptyLinks.length > 0) violations.push({ rule: 'link-name', severity: 'serious', count: emptyLinks.length, examples: emptyLinks.slice(0, 3) });\n  else passCount++;\n\n  const totalChecks = passCount + violations.length;\n  const score = totalChecks > 0 ? Math.round((passCount / totalChecks) * 100) : 100;\n\n  return { url, violations, passCount, score, auditedAt };\n}\n"],
+  "mappings": "AAgCA,eAAe,SAAS,MAAY,UAAkC;AACpE,MAAI;AACF,WAAO,MAAM,KAAK,OAAO,UAAU,SAAO,IAAI,IAAI,CAAC,QAAa;AAAA,MAC9D,WAAW,GAAG,WAAW,MAAM,GAAG,GAAG,KAAK;AAAA,MAC1C,cAAc,GAAG,eAAe,IAAI,KAAK,EAAE,MAAM,GAAG,GAAG;AAAA,MACvD,UAAU,GAAG;AAAA,MACb,cAAc,CAAC,MAAc,GAAG,aAAa,CAAC;AAAA,IAChD,EAAE,CAAC;AAAA,EACL,QAAQ;AACN,WAAO,CAAC;AAAA,EACV;AACF;AAEA,eAAsB,mBAAmB,MAAY,KAAuC;AAC1F,QAAM,aAA8B,CAAC;AACrC,MAAI,YAAY;AAChB,QAAM,aAAY,oBAAI,KAAK,GAAE,YAAY;AAGzC,QAAM,OAAO,MAAM,KAAK,OAAO,kBAAkB,SAAO,IAAI,IAAI,CAAC,MAAW,EAAE,WAAW,MAAM,GAAG,GAAG,KAAK,EAAE,CAAC;AAC7G,MAAI,KAAK,SAAS,EAAG,YAAW,KAAK,EAAE,MAAM,aAAa,UAAU,YAAY,OAAO,KAAK,QAAQ,UAAU,KAAK,MAAM,GAAG,CAAC,EAAE,CAAC;AAAA,MAC3H;AAGL,QAAM,kBAAkB,MAAM,KAAK;AAAA,IACjC;AAAA,IACA,SAAO,IAAI,IAAI,CAAC,MAAW,EAAE,WAAW,MAAM,GAAG,GAAG,KAAK,EAAE;AAAA,EAC7D;AACA,MAAI,gBAAgB,SAAS,EAAG,YAAW,KAAK,EAAE,MAAM,SAAS,UAAU,WAAW,OAAO,gBAAgB,QAAQ,UAAU,gBAAgB,MAAM,GAAG,CAAC,EAAE,CAAC;AAAA,MACvJ;AAGL,QAAM,eAAe,MAAM,KAAK;AAAA,IAC9B;AAAA,IACA,SAAO,IAAI,OAAO,CAAC,MAAW,EAAE,EAAE,eAAe,IAAI,KAAK,CAAC,EAAE,IAAI,CAAC,MAAW,EAAE,WAAW,MAAM,GAAG,GAAG,KAAK,EAAE;AAAA,EAC/G;AACA,MAAI,aAAa,SAAS,EAAG,YAAW,KAAK,EAAE,MAAM,eAAe,UAAU,YAAY,OAAO,aAAa,QAAQ,UAAU,aAAa,MAAM,GAAG,CAAC,EAAE,CAAC;AAAA,MACrJ;AAGL,QAAM,WAAW,MAAM,KAAK,MAAM,QAAQ,CAAC,OAAY,GAAG,aAAa,MAAM,KAAK,EAAE,EAAE,MAAM,MAAM,EAAE;AACpG,MAAI,CAAC,SAAU,YAAW,KAAK,EAAE,MAAM,iBAAiB,UAAU,WAAW,OAAO,GAAG,UAAU,CAAC,+BAA+B,EAAE,CAAC;AAAA,MAC/H;AAGL,QAAM,QAAQ,MAAM,KAAK,MAAM,EAAE,MAAM,MAAM,EAAE;AAC/C,MAAI,CAAC,MAAO,YAAW,KAAK,EAAE,MAAM,kBAAkB,UAAU,WAAW,OAAO,GAAG,UAAU,CAAC,qBAAqB,EAAE,CAAC;AAAA,MACnH;AAGL,QAAM,gBAAgB,MAAM,KAAK;AAAA,IAC/B;AAAA,IACA,SAAO,IAAI,OAAO,CAAC,MAAW,EAAE,EAAE,eAAe,IAAI,KAAK,CAAC,EAAE,IAAI,CAAC,MAAW,EAAE,QAAQ,YAAY,CAAC;AAAA,EACtG;AACA,MAAI,cAAc,SAAS,EAAG,YAAW,KAAK,EAAE,MAAM,iBAAiB,UAAU,YAAY,OAAO,cAAc,QAAQ,UAAU,cAAc,MAAM,GAAG,CAAC,EAAE,CAAC;AAAA,MAC1J;AAGL,QAAM,aAAa,MAAM,KAAK;AAAA,IAC5B;AAAA,IACA,SAAO,IAAI,OAAO,CAAC,MAAW,EAAE,EAAE,eAAe,IAAI,KAAK,KAAK,CAAC,EAAE,cAAc,UAAU,CAAC,EAAE,IAAI,CAAC,MAAW,EAAE,WAAW,MAAM,GAAG,GAAG,KAAK,EAAE;AAAA,EAC/I;AACA,MAAI,WAAW,SAAS,EAAG,YAAW,KAAK,EAAE,MAAM,aAAa,UAAU,WAAW,OAAO,WAAW,QAAQ,UAAU,WAAW,MAAM,GAAG,CAAC,EAAE,CAAC;AAAA,MAC5I;AAEL,QAAM,cAAc,YAAY,WAAW;AAC3C,QAAM,QAAQ,cAAc,IAAI,KAAK,MAAO,YAAY,cAAe,GAAG,IAAI;AAE9E,SAAO,EAAE,KAAK,YAAY,WAAW,OAAO,UAAU;AACxD;",
+  "names": []
+}

package/dist/ai-login.d.ts

@@ -0,0 +1,32 @@
+import type { Stagehand } from '@browserbasehq/stagehand';
+import type { Credentials } from './auth.js';
+/**
+ * Recorded login step from the Chrome extension's step recorder.
+ * {username} and {password} in `value` are substituted at replay time.
+ */
+export interface LoginStep {
+    action: 'fill' | 'click' | 'wait';
+    selector: string;
+    value?: string;
+    ms?: number;
+}
+/**
+ * AI-guided login using Stagehand act/extract.
+ *
+ * @param loginInstructions - Optional natural language description of the login flow.
+ *   When provided, an initial stagehand.act() is called with these instructions to
+ *   navigate the UI (e.g. "click Sign In in the nav, fill email, click Continue").
+ *   The standard 3-step credential fill follows immediately after.
+ * @param opts - Optional tenant/project/job IDs for writing AUTH_STEP CrawlEvidence.
+ */
+export declare function loginWithStagehand(stagehand: Stagehand, creds: Credentials, startUrl: string, captchaOpts?: {
+    apiKey?: string;
+    provider?: '2captcha' | 'capsolver';
+}, emailOtpOpts?: {
+    apiKey?: string;
+    inboxId?: string;
+}, loginInstructions?: string, opts?: {
+    tenantId?: string;
+    projectId?: string;
+    jobId?: string;
+}): Promise<boolean>;

package/dist/ai-login.js

@@ -0,0 +1,177 @@
+import { z } from "zod";
+import { AppBrain } from "@qip/app-brain";
+import { solveCaptcha } from "./captcha-solver.js";
+import { waitForEmailContent } from "./email-otp.js";
+import { logger } from "./logger.js";
+import { simulateMouseMovement } from "./stealth/human-behavior.js";
+const CLICK_INTENT = /\b(click|tap|press|submit|select|choose|toggle)\b/i;
+async function actWithMouseSim(stagehand, instruction) {
+  if (CLICK_INTENT.test(instruction)) {
+    try {
+      const pg = await stagehand.resolvePage();
+      const viewport = pg.viewportSize() ?? { width: 1280, height: 720 };
+      const targetX = viewport.width * 0.3 + Math.random() * viewport.width * 0.4;
+      const targetY = viewport.height * 0.2 + Math.random() * viewport.height * 0.5;
+      await simulateMouseMovement(pg, targetX, targetY);
+    } catch {
+    }
+  }
+  return stagehand.act(instruction);
+}
+const TwoFASchema = z.object({
+  has2FA: z.boolean(),
+  fieldLabel: z.string().optional()
+});
+const CAPTCHA_SELECTORS = [
+  'iframe[src*="recaptcha"]',
+  'iframe[src*="hcaptcha"]',
+  'iframe[src*="challenges.cloudflare.com"]',
+  ".cf-turnstile",
+  "[data-sitekey]",
+  "#captcha",
+  ".captcha",
+  '[id*="captcha"]',
+  '[class*="captcha"]'
+];
+async function loginWithStagehand(stagehand, creds, startUrl, captchaOpts, emailOtpOpts, loginInstructions, opts) {
+  async function logStep(step, fn) {
+    const t0 = Date.now();
+    try {
+      const r = await fn();
+      const result = { success: true, detail: r?.description ?? r?.message ?? step };
+      if (opts?.tenantId && opts?.projectId) {
+        AppBrain.saveCrawlEvidence(opts.tenantId, opts.projectId, {
+          jobId: opts?.jobId,
+          type: "AUTH_STEP",
+          payload: { step, ...result, ms: Date.now() - t0 }
+        }).catch(() => {
+        });
+      }
+      return result;
+    } catch (e) {
+      const result = { success: false, detail: e.message ?? "failed" };
+      if (opts?.tenantId && opts?.projectId) {
+        AppBrain.saveCrawlEvidence(opts.tenantId, opts.projectId, {
+          jobId: opts?.jobId,
+          type: "AUTH_STEP",
+          payload: { step, ...result, ms: Date.now() - t0 }
+        }).catch(() => {
+        });
+      }
+      throw e;
+    }
+  }
+  try {
+    const pg = await stagehand.resolvePage();
+    for (const sel of CAPTCHA_SELECTORS) {
+      const count = await pg.locator(sel).count().catch(() => 0);
+      if (count > 0) {
+        logger.warn({ selector: sel }, "[crawler] CAPTCHA detected on login page");
+        if (captchaOpts?.apiKey) {
+          logger.info({ provider: captchaOpts.provider ?? "2captcha" }, "[crawler] Attempting auto-solve");
+          try {
+            const result = await solveCaptcha(pg, startUrl, captchaOpts.apiKey, captchaOpts.provider ?? "2captcha");
+            if (result) {
+              logger.info({ provider: result.provider }, "[crawler] CAPTCHA solved \u2014 proceeding with login");
+              break;
+            }
+          } catch (solveErr) {
+            logger.warn({ err: solveErr }, "[crawler] CAPTCHA solve failed \u2014 falling back to manual error");
+            throw new Error(`CAPTCHA_DETECTED: Auto-solve failed (${String(solveErr)}). Use session capture instead.`);
+          }
+        } else {
+          throw new Error(
+            'CAPTCHA_DETECTED: Login page has a CAPTCHA. Use "Session Capture \u2192 Launch Browser" to log in manually, then capture the session. Alternatively, disable CAPTCHA for the ZeTa crawler IP.'
+          );
+        }
+        break;
+      }
+    }
+    if (loginInstructions) {
+      await logStep("custom_recipe", () => actWithMouseSim(stagehand, loginInstructions));
+      await pg.waitForLoadState("networkidle").catch(() => pg.waitForTimeout(1500));
+    }
+    await logStep("fill_username", () => actWithMouseSim(stagehand, `Type "${creds.username}" into the username or email input field`));
+    await logStep("fill_password", () => actWithMouseSim(stagehand, `Type "${creds.password}" into the password input field`));
+    await logStep("click_submit", () => actWithMouseSim(stagehand, "Click the sign in, log in, login, or continue button to submit credentials"));
+    await pg.waitForLoadState("networkidle").catch(() => pg.waitForTimeout(2e3));
+    const twoFA = await stagehand.extract(
+      "Is there a two-factor authentication, OTP, or verification code input visible right now?",
+      TwoFASchema
+    );
+    if (twoFA.has2FA) {
+      if (opts?.tenantId && opts?.projectId) {
+        AppBrain.saveCrawlEvidence(opts.tenantId, opts.projectId, {
+          jobId: opts?.jobId,
+          type: "AUTH_STEP",
+          payload: { step: "2fa_detect", success: true, detail: twoFA.fieldLabel ?? "2FA detected", ms: 0 }
+        }).catch(() => {
+        });
+      }
+      let code = creds.oneTimeCode ?? await creds.getOneTimeCode?.();
+      if (!code && emailOtpOpts?.apiKey && emailOtpOpts?.inboxId) {
+        const since = (/* @__PURE__ */ new Date()).toISOString();
+        logger.info("[crawler] 2FA detected \u2014 waiting for email OTP or magic link via MailSlurp");
+        const emailContent = await waitForEmailContent(
+          { provider: "mailslurp", apiKey: emailOtpOpts.apiKey, inboxId: emailOtpOpts.inboxId },
+          since
+        );
+        if (emailContent?.magicLink) {
+          logger.info({ url: emailContent.magicLink }, "[crawler] Magic link received \u2014 navigating");
+          await pg.goto(emailContent.magicLink, { waitUntil: "domcontentloaded" });
+          await pg.waitForLoadState("networkidle").catch(() => {
+          });
+          return true;
+        }
+        code = emailContent?.otp ?? void 0;
+        if (code) {
+          logger.info("[crawler] Email OTP received");
+        } else {
+          logger.warn("[crawler] Email OTP timed out after 60s");
+        }
+      }
+      if (!code) {
+        console.warn("[crawler] 2FA detected but no oneTimeCode provided");
+        return false;
+      }
+      await logStep("2fa_inject", () => actWithMouseSim(stagehand, `Enter "${code}" into the ${twoFA.fieldLabel ?? "verification code"} field`));
+      await logStep("2fa_submit", () => actWithMouseSim(stagehand, "Click verify or confirm to submit the verification code"));
+      await pg.waitForLoadState("networkidle").catch(() => pg.waitForTimeout(2e3));
+    }
+    const currentUrl = pg.url();
+    const loginPathPattern = /\/(login|signin|sign-in|auth|authenticate|log-in|account\/login)(\?|$|\/)/i;
+    try {
+      const { pathname } = new URL(currentUrl);
+      const wasRedirected = currentUrl !== startUrl;
+      const stillOnLoginPath = loginPathPattern.test(pathname);
+      if (wasRedirected && !stillOnLoginPath) {
+        logger.info({ url: currentUrl }, "Login confirmed via URL change");
+        return true;
+      }
+    } catch {
+    }
+    const passwordCount = await pg.locator('input[type="password"]').count().catch(() => 1);
+    const bodyText = await pg.evaluate(() => document.body?.innerText?.trim()?.length ?? 0).catch(() => 0);
+    if (passwordCount === 0 && bodyText > 30) {
+      logger.info("Login confirmed: password field no longer visible");
+      return true;
+    }
+    logger.warn({ url: currentUrl }, "Login appears failed");
+    try {
+      const screenshotPath = `/tmp/zeta-auth-failure-${Date.now()}.jpg`;
+      await pg.screenshot({ path: screenshotPath, type: "jpeg", quality: 70, fullPage: false });
+      logger.warn({ screenshotPath }, "[crawler] Auth failure screenshot saved");
+    } catch {
+    }
+    return false;
+  } catch (err) {
+    const msg = String(err.message ?? err);
+    if (msg.includes("CAPTCHA_DETECTED")) throw err;
+    logger.error({ err }, "Login error");
+    return false;
+  }
+}
+export {
+  loginWithStagehand
+};
+//# sourceMappingURL=ai-login.js.map

package/dist/ai-login.js.map

@@ -0,0 +1,7 @@
+{
+  "version": 3,
+  "sources": ["../src/ai-login.ts"],
+  "sourcesContent": ["import { z } from 'zod';\nimport type { Stagehand } from '@browserbasehq/stagehand';\nimport { AppBrain } from '@qip/app-brain';\nimport { solveCaptcha } from './captcha-solver.js';\nimport { waitForEmailContent } from './email-otp.js';\nimport { logger } from './logger.js';\nimport type { Credentials } from './auth.js';\nimport { simulateMouseMovement } from './stealth/human-behavior.js';\n\nconst CLICK_INTENT = /\\b(click|tap|press|submit|select|choose|toggle)\\b/i;\n\n/** Simulate human mouse movement before click-type actions, then call stagehand.act(). */\nasync function actWithMouseSim(stagehand: Stagehand, instruction: string): Promise<any> {\n  if (CLICK_INTENT.test(instruction)) {\n    try {\n      const pg = await (stagehand as any).resolvePage();\n      const viewport = pg.viewportSize() ?? { width: 1280, height: 720 };\n      // Move mouse toward center of page with some randomness before Stagehand clicks\n      const targetX = viewport.width * 0.3 + Math.random() * viewport.width * 0.4;\n      const targetY = viewport.height * 0.2 + Math.random() * viewport.height * 0.5;\n      await simulateMouseMovement(pg, targetX, targetY);\n    } catch { /* non-fatal \u2014 don't block the action */ }\n  }\n  return stagehand.act(instruction);\n}\n\n/**\n * Recorded login step from the Chrome extension's step recorder.\n * {username} and {password} in `value` are substituted at replay time.\n */\nexport interface LoginStep {\n  action: 'fill' | 'click' | 'wait';\n  selector: string;\n  value?: string;\n  ms?: number;\n}\n\nconst TwoFASchema = z.object({\n  has2FA: z.boolean(),\n  fieldLabel: z.string().optional(),\n});\n\nconst CAPTCHA_SELECTORS = [\n  'iframe[src*=\"recaptcha\"]',\n  'iframe[src*=\"hcaptcha\"]',\n  'iframe[src*=\"challenges.cloudflare.com\"]',\n  '.cf-turnstile',\n  '[data-sitekey]',\n  '#captcha',\n  '.captcha',\n  '[id*=\"captcha\"]',\n  '[class*=\"captcha\"]',\n];\n\n/**\n * AI-guided login using Stagehand act/extract.\n *\n * @param loginInstructions - Optional natural language description of the login flow.\n *   When provided, an initial stagehand.act() is called with these instructions to\n *   navigate the UI (e.g. \"click Sign In in the nav, fill email, click Continue\").\n *   The standard 3-step credential fill follows immediately after.\n * @param opts - Optional tenant/project/job IDs for writing AUTH_STEP CrawlEvidence.\n */\nexport async function loginWithStagehand(\n  stagehand: Stagehand,\n  creds: Credentials,\n  startUrl: string,\n  captchaOpts?: { apiKey?: string; provider?: '2captcha' | 'capsolver' },\n  emailOtpOpts?: { apiKey?: string; inboxId?: string },\n  loginInstructions?: string,\n  opts?: { tenantId?: string; projectId?: string; jobId?: string },\n): Promise<boolean> {\n  async function logStep(step: string, fn: () => Promise<any>): Promise<{ success: boolean; detail: string }> {\n    const t0 = Date.now();\n    try {\n      const r = await fn();\n      const result = { success: true, detail: r?.description ?? r?.message ?? step };\n      if (opts?.tenantId && opts?.projectId) {\n        AppBrain.saveCrawlEvidence(opts.tenantId, opts.projectId, {\n          jobId: opts?.jobId,\n          type: 'AUTH_STEP',\n          payload: { step, ...result, ms: Date.now() - t0 },\n        }).catch(() => {});\n      }\n      return result;\n    } catch (e: any) {\n      const result = { success: false, detail: e.message ?? 'failed' };\n      if (opts?.tenantId && opts?.projectId) {\n        AppBrain.saveCrawlEvidence(opts.tenantId, opts.projectId, {\n          jobId: opts?.jobId,\n          type: 'AUTH_STEP',\n          payload: { step, ...result, ms: Date.now() - t0 },\n        }).catch(() => {});\n      }\n      throw e;\n    }\n  }\n\n  try {\n    const pg = await (stagehand as any).resolvePage();\n\n    // CAPTCHA detection\n    for (const sel of CAPTCHA_SELECTORS) {\n      const count = await pg.locator(sel).count().catch(() => 0);\n      if (count > 0) {\n        logger.warn({ selector: sel }, '[crawler] CAPTCHA detected on login page');\n        if (captchaOpts?.apiKey) {\n          logger.info({ provider: captchaOpts.provider ?? '2captcha' }, '[crawler] Attempting auto-solve');\n          try {\n            const result = await solveCaptcha(pg, startUrl, captchaOpts.apiKey, captchaOpts.provider ?? '2captcha');\n            if (result) {\n              logger.info({ provider: result.provider }, '[crawler] CAPTCHA solved \u2014 proceeding with login');\n              break;\n            }\n          } catch (solveErr) {\n            logger.warn({ err: solveErr }, '[crawler] CAPTCHA solve failed \u2014 falling back to manual error');\n            throw new Error(`CAPTCHA_DETECTED: Auto-solve failed (${String(solveErr)}). Use session capture instead.`);\n          }\n        } else {\n          throw new Error(\n            'CAPTCHA_DETECTED: Login page has a CAPTCHA. Use \"Session Capture \u2192 Launch Browser\" to log in manually, then capture the session. Alternatively, disable CAPTCHA for the ZeTa crawler IP.'\n          );\n        }\n        break;\n      }\n    }\n\n    // If caller provided natural language login instructions, use them to navigate to the login form\n    if (loginInstructions) {\n      await logStep('custom_recipe', () => actWithMouseSim(stagehand, loginInstructions));\n      await pg.waitForLoadState('networkidle').catch(() => pg.waitForTimeout(1500));\n    }\n\n    await logStep('fill_username', () => actWithMouseSim(stagehand, `Type \"${creds.username}\" into the username or email input field`));\n    await logStep('fill_password', () => actWithMouseSim(stagehand, `Type \"${creds.password}\" into the password input field`));\n    await logStep('click_submit', () => actWithMouseSim(stagehand, 'Click the sign in, log in, login, or continue button to submit credentials'));\n    await pg.waitForLoadState('networkidle').catch(() => pg.waitForTimeout(2000));\n\n    const twoFA = await stagehand.extract(\n      'Is there a two-factor authentication, OTP, or verification code input visible right now?',\n      TwoFASchema as any\n    );\n\n    if (twoFA.has2FA) {\n      if (opts?.tenantId && opts?.projectId) {\n        AppBrain.saveCrawlEvidence(opts.tenantId, opts.projectId, {\n          jobId: opts?.jobId,\n          type: 'AUTH_STEP',\n          payload: { step: '2fa_detect', success: true, detail: twoFA.fieldLabel ?? '2FA detected', ms: 0 },\n        }).catch(() => {});\n      }\n\n      let code = creds.oneTimeCode ?? await creds.getOneTimeCode?.();\n\n      if (!code && emailOtpOpts?.apiKey && emailOtpOpts?.inboxId) {\n        const since = new Date().toISOString();\n        logger.info('[crawler] 2FA detected \u2014 waiting for email OTP or magic link via MailSlurp');\n        const emailContent = await waitForEmailContent(\n          { provider: 'mailslurp', apiKey: emailOtpOpts.apiKey, inboxId: emailOtpOpts.inboxId },\n          since\n        );\n        if (emailContent?.magicLink) {\n          logger.info({ url: emailContent.magicLink }, '[crawler] Magic link received \u2014 navigating');\n          await pg.goto(emailContent.magicLink, { waitUntil: 'domcontentloaded' });\n          await pg.waitForLoadState('networkidle').catch(() => {});\n          return true; // magic link navigation = login complete\n        }\n        code = emailContent?.otp ?? undefined;\n        if (code) {\n          logger.info('[crawler] Email OTP received');\n        } else {\n          logger.warn('[crawler] Email OTP timed out after 60s');\n        }\n      }\n\n      if (!code) {\n        console.warn('[crawler] 2FA detected but no oneTimeCode provided');\n        return false;\n      }\n      await logStep('2fa_inject', () => actWithMouseSim(stagehand, `Enter \"${code}\" into the ${twoFA.fieldLabel ?? 'verification code'} field`));\n      await logStep('2fa_submit', () => actWithMouseSim(stagehand, 'Click verify or confirm to submit the verification code'));\n      await pg.waitForLoadState('networkidle').catch(() => pg.waitForTimeout(2000));\n    }\n\n    // URL-based success check\n    const currentUrl = pg.url();\n    const loginPathPattern = /\\/(login|signin|sign-in|auth|authenticate|log-in|account\\/login)(\\?|$|\\/)/i;\n    try {\n      const { pathname } = new URL(currentUrl);\n      const wasRedirected = currentUrl !== startUrl;\n      const stillOnLoginPath = loginPathPattern.test(pathname);\n      if (wasRedirected && !stillOnLoginPath) {\n        logger.info({ url: currentUrl }, 'Login confirmed via URL change');\n        return true;\n      }\n    } catch { /* invalid URL \u2014 fall through */ }\n\n    const passwordCount = await pg.locator('input[type=\"password\"]').count().catch(() => 1);\n    const bodyText = await pg.evaluate(() => (document as any).body?.innerText?.trim()?.length ?? 0).catch(() => 0);\n    if (passwordCount === 0 && bodyText > 30) {\n      logger.info('Login confirmed: password field no longer visible');\n      return true;\n    }\n\n    logger.warn({ url: currentUrl }, 'Login appears failed');\n    try {\n      const screenshotPath = `/tmp/zeta-auth-failure-${Date.now()}.jpg`;\n      await pg.screenshot({ path: screenshotPath, type: 'jpeg', quality: 70, fullPage: false });\n      logger.warn({ screenshotPath }, '[crawler] Auth failure screenshot saved');\n    } catch { /* non-fatal */ }\n    return false;\n  } catch (err) {\n    const msg = String((err as Error).message ?? err);\n    if (msg.includes('CAPTCHA_DETECTED')) throw err;\n    logger.error({ err }, 'Login error');\n    return false;\n  }\n}\n"],
+  "mappings": "AAAA,SAAS,SAAS;AAElB,SAAS,gBAAgB;AACzB,SAAS,oBAAoB;AAC7B,SAAS,2BAA2B;AACpC,SAAS,cAAc;AAEvB,SAAS,6BAA6B;AAEtC,MAAM,eAAe;AAGrB,eAAe,gBAAgB,WAAsB,aAAmC;AACtF,MAAI,aAAa,KAAK,WAAW,GAAG;AAClC,QAAI;AACF,YAAM,KAAK,MAAO,UAAkB,YAAY;AAChD,YAAM,WAAW,GAAG,aAAa,KAAK,EAAE,OAAO,MAAM,QAAQ,IAAI;AAEjE,YAAM,UAAU,SAAS,QAAQ,MAAM,KAAK,OAAO,IAAI,SAAS,QAAQ;AACxE,YAAM,UAAU,SAAS,SAAS,MAAM,KAAK,OAAO,IAAI,SAAS,SAAS;AAC1E,YAAM,sBAAsB,IAAI,SAAS,OAAO;AAAA,IAClD,QAAQ;AAAA,IAA2C;AAAA,EACrD;AACA,SAAO,UAAU,IAAI,WAAW;AAClC;AAaA,MAAM,cAAc,EAAE,OAAO;AAAA,EAC3B,QAAQ,EAAE,QAAQ;AAAA,EAClB,YAAY,EAAE,OAAO,EAAE,SAAS;AAClC,CAAC;AAED,MAAM,oBAAoB;AAAA,EACxB;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AACF;AAWA,eAAsB,mBACpB,WACA,OACA,UACA,aACA,cACA,mBACA,MACkB;AAClB,iBAAe,QAAQ,MAAc,IAAuE;AAC1G,UAAM,KAAK,KAAK,IAAI;AACpB,QAAI;AACF,YAAM,IAAI,MAAM,GAAG;AACnB,YAAM,SAAS,EAAE,SAAS,MAAM,QAAQ,GAAG,eAAe,GAAG,WAAW,KAAK;AAC7E,UAAI,MAAM,YAAY,MAAM,WAAW;AACrC,iBAAS,kBAAkB,KAAK,UAAU,KAAK,WAAW;AAAA,UACxD,OAAO,MAAM;AAAA,UACb,MAAM;AAAA,UACN,SAAS,EAAE,MAAM,GAAG,QAAQ,IAAI,KAAK,IAAI,IAAI,GAAG;AAAA,QAClD,CAAC,EAAE,MAAM,MAAM;AAAA,QAAC,CAAC;AAAA,MACnB;AACA,aAAO;AAAA,IACT,SAAS,GAAQ;AACf,YAAM,SAAS,EAAE,SAAS,OAAO,QAAQ,EAAE,WAAW,SAAS;AAC/D,UAAI,MAAM,YAAY,MAAM,WAAW;AACrC,iBAAS,kBAAkB,KAAK,UAAU,KAAK,WAAW;AAAA,UACxD,OAAO,MAAM;AAAA,UACb,MAAM;AAAA,UACN,SAAS,EAAE,MAAM,GAAG,QAAQ,IAAI,KAAK,IAAI,IAAI,GAAG;AAAA,QAClD,CAAC,EAAE,MAAM,MAAM;AAAA,QAAC,CAAC;AAAA,MACnB;AACA,YAAM;AAAA,IACR;AAAA,EACF;AAEA,MAAI;AACF,UAAM,KAAK,MAAO,UAAkB,YAAY;AAGhD,eAAW,OAAO,mBAAmB;AACnC,YAAM,QAAQ,MAAM,GAAG,QAAQ,GAAG,EAAE,MAAM,EAAE,MAAM,MAAM,CAAC;AACzD,UAAI,QAAQ,GAAG;AACb,eAAO,KAAK,EAAE,UAAU,IAAI,GAAG,0CAA0C;AACzE,YAAI,aAAa,QAAQ;AACvB,iBAAO,KAAK,EAAE,UAAU,YAAY,YAAY,WAAW,GAAG,iCAAiC;AAC/F,cAAI;AACF,kBAAM,SAAS,MAAM,aAAa,IAAI,UAAU,YAAY,QAAQ,YAAY,YAAY,UAAU;AACtG,gBAAI,QAAQ;AACV,qBAAO,KAAK,EAAE,UAAU,OAAO,SAAS,GAAG,uDAAkD;AAC7F;AAAA,YACF;AAAA,UACF,SAAS,UAAU;AACjB,mBAAO,KAAK,EAAE,KAAK,SAAS,GAAG,oEAA+D;AAC9F,kBAAM,IAAI,MAAM,wCAAwC,OAAO,QAAQ,CAAC,iCAAiC;AAAA,UAC3G;AAAA,QACF,OAAO;AACL,gBAAM,IAAI;AAAA,YACR;AAAA,UACF;AAAA,QACF;AACA;AAAA,MACF;AAAA,IACF;AAGA,QAAI,mBAAmB;AACrB,YAAM,QAAQ,iBAAiB,MAAM,gBAAgB,WAAW,iBAAiB,CAAC;AAClF,YAAM,GAAG,iBAAiB,aAAa,EAAE,MAAM,MAAM,GAAG,eAAe,IAAI,CAAC;AAAA,IAC9E;AAEA,UAAM,QAAQ,iBAAiB,MAAM,gBAAgB,WAAW,SAAS,MAAM,QAAQ,0CAA0C,CAAC;AAClI,UAAM,QAAQ,iBAAiB,MAAM,gBAAgB,WAAW,SAAS,MAAM,QAAQ,iCAAiC,CAAC;AACzH,UAAM,QAAQ,gBAAgB,MAAM,gBAAgB,WAAW,4EAA4E,CAAC;AAC5I,UAAM,GAAG,iBAAiB,aAAa,EAAE,MAAM,MAAM,GAAG,eAAe,GAAI,CAAC;AAE5E,UAAM,QAAQ,MAAM,UAAU;AAAA,MAC5B;AAAA,MACA;AAAA,IACF;AAEA,QAAI,MAAM,QAAQ;AAChB,UAAI,MAAM,YAAY,MAAM,WAAW;AACrC,iBAAS,kBAAkB,KAAK,UAAU,KAAK,WAAW;AAAA,UACxD,OAAO,MAAM;AAAA,UACb,MAAM;AAAA,UACN,SAAS,EAAE,MAAM,cAAc,SAAS,MAAM,QAAQ,MAAM,cAAc,gBAAgB,IAAI,EAAE;AAAA,QAClG,CAAC,EAAE,MAAM,MAAM;AAAA,QAAC,CAAC;AAAA,MACnB;AAEA,UAAI,OAAO,MAAM,eAAe,MAAM,MAAM,iBAAiB;AAE7D,UAAI,CAAC,QAAQ,cAAc,UAAU,cAAc,SAAS;AAC1D,cAAM,SAAQ,oBAAI,KAAK,GAAE,YAAY;AACrC,eAAO,KAAK,iFAA4E;AACxF,cAAM,eAAe,MAAM;AAAA,UACzB,EAAE,UAAU,aAAa,QAAQ,aAAa,QAAQ,SAAS,aAAa,QAAQ;AAAA,UACpF;AAAA,QACF;AACA,YAAI,cAAc,WAAW;AAC3B,iBAAO,KAAK,EAAE,KAAK,aAAa,UAAU,GAAG,iDAA4C;AACzF,gBAAM,GAAG,KAAK,aAAa,WAAW,EAAE,WAAW,mBAAmB,CAAC;AACvE,gBAAM,GAAG,iBAAiB,aAAa,EAAE,MAAM,MAAM;AAAA,UAAC,CAAC;AACvD,iBAAO;AAAA,QACT;AACA,eAAO,cAAc,OAAO;AAC5B,YAAI,MAAM;AACR,iBAAO,KAAK,8BAA8B;AAAA,QAC5C,OAAO;AACL,iBAAO,KAAK,yCAAyC;AAAA,QACvD;AAAA,MACF;AAEA,UAAI,CAAC,MAAM;AACT,gBAAQ,KAAK,oDAAoD;AACjE,eAAO;AAAA,MACT;AACA,YAAM,QAAQ,cAAc,MAAM,gBAAgB,WAAW,UAAU,IAAI,cAAc,MAAM,cAAc,mBAAmB,QAAQ,CAAC;AACzI,YAAM,QAAQ,cAAc,MAAM,gBAAgB,WAAW,yDAAyD,CAAC;AACvH,YAAM,GAAG,iBAAiB,aAAa,EAAE,MAAM,MAAM,GAAG,eAAe,GAAI,CAAC;AAAA,IAC9E;AAGA,UAAM,aAAa,GAAG,IAAI;AAC1B,UAAM,mBAAmB;AACzB,QAAI;AACF,YAAM,EAAE,SAAS,IAAI,IAAI,IAAI,UAAU;AACvC,YAAM,gBAAgB,eAAe;AACrC,YAAM,mBAAmB,iBAAiB,KAAK,QAAQ;AACvD,UAAI,iBAAiB,CAAC,kBAAkB;AACtC,eAAO,KAAK,EAAE,KAAK,WAAW,GAAG,gCAAgC;AACjE,eAAO;AAAA,MACT;AAAA,IACF,QAAQ;AAAA,IAAmC;AAE3C,UAAM,gBAAgB,MAAM,GAAG,QAAQ,wBAAwB,EAAE,MAAM,EAAE,MAAM,MAAM,CAAC;AACtF,UAAM,WAAW,MAAM,GAAG,SAAS,MAAO,SAAiB,MAAM,WAAW,KAAK,GAAG,UAAU,CAAC,EAAE,MAAM,MAAM,CAAC;AAC9G,QAAI,kBAAkB,KAAK,WAAW,IAAI;AACxC,aAAO,KAAK,mDAAmD;AAC/D,aAAO;AAAA,IACT;AAEA,WAAO,KAAK,EAAE,KAAK,WAAW,GAAG,sBAAsB;AACvD,QAAI;AACF,YAAM,iBAAiB,0BAA0B,KAAK,IAAI,CAAC;AAC3D,YAAM,GAAG,WAAW,EAAE,MAAM,gBAAgB,MAAM,QAAQ,SAAS,IAAI,UAAU,MAAM,CAAC;AACxF,aAAO,KAAK,EAAE,eAAe,GAAG,yCAAyC;AAAA,IAC3E,QAAQ;AAAA,IAAkB;AAC1B,WAAO;AAAA,EACT,SAAS,KAAK;AACZ,UAAM,MAAM,OAAQ,IAAc,WAAW,GAAG;AAChD,QAAI,IAAI,SAAS,kBAAkB,EAAG,OAAM;AAC5C,WAAO,MAAM,EAAE,IAAI,GAAG,aAAa;AACnC,WAAO;AAAA,EACT;AACF;",
+  "names": []
+}

package/dist/auth-token-rotator.d.ts

@@ -0,0 +1,47 @@
+/**
+ * Auth token rotator — manage a pool of authenticated sessions for multi-account crawls.
+ *
+ * Use case: crawl the same app as admin+user+guest in one crawl job to discover
+ * role-specific screens that only appear when authenticated with different privileges.
+ * Each session is a Playwright storageState JSON string.
+ *
+ * Sessions are checked out (locked) before use and returned after the page is done.
+ * Round-robin assignment with stale-session eviction after maxAgeMs.
+ */
+export interface SessionEntry {
+    id: string;
+    role: string;
+    storageState: string;
+    createdAt: number;
+    checkedOutAt: number | null;
+    requestCount: number;
+}
+export interface SessionPoolOpts {
+    sessions: Array<{
+        role: string;
+        storageState: string;
+    }>;
+    maxAgeMs?: number;
+    maxRequestsPerSession?: number;
+}
+export declare class AuthTokenRotator {
+    private pool;
+    private readonly maxAgeMs;
+    private readonly maxRequests;
+    constructor(opts: SessionPoolOpts);
+    /** Get the next available session. Returns null if all are checked out. */
+    checkout(): SessionEntry | null;
+    /** Return a session to the pool after use. */
+    checkin(sessionId: string, updatedStorageState?: string): void;
+    /** Get all available sessions (for parallel crawl setup). */
+    getAll(): ReadonlyArray<SessionEntry>;
+    /** Stats for monitoring. */
+    stats(): {
+        total: number;
+        available: number;
+        checkedOut: number;
+        stale: number;
+    };
+    /** Add a new session to the pool at runtime. */
+    addSession(role: string, storageState: string): string;
+}

package/dist/auth-token-rotator.js

@@ -0,0 +1,58 @@
+class AuthTokenRotator {
+  pool;
+  maxAgeMs;
+  maxRequests;
+  constructor(opts) {
+    this.maxAgeMs = opts.maxAgeMs ?? 4 * 60 * 60 * 1e3;
+    this.maxRequests = opts.maxRequestsPerSession ?? 100;
+    this.pool = opts.sessions.map((s, i) => ({
+      id: `session-${i}-${s.role}`,
+      role: s.role,
+      storageState: s.storageState,
+      createdAt: Date.now(),
+      checkedOutAt: null,
+      requestCount: 0
+    }));
+  }
+  /** Get the next available session. Returns null if all are checked out. */
+  checkout() {
+    const now = Date.now();
+    const available = this.pool.filter(
+      (s) => !s.checkedOutAt && now - s.createdAt < this.maxAgeMs && s.requestCount < this.maxRequests
+    );
+    if (available.length === 0) return null;
+    available.sort((a, b) => a.requestCount - b.requestCount);
+    const session = available[0];
+    session.checkedOutAt = now;
+    return session;
+  }
+  /** Return a session to the pool after use. */
+  checkin(sessionId, updatedStorageState) {
+    const s = this.pool.find((e) => e.id === sessionId);
+    if (!s) return;
+    s.checkedOutAt = null;
+    s.requestCount++;
+    if (updatedStorageState) s.storageState = updatedStorageState;
+  }
+  /** Get all available sessions (for parallel crawl setup). */
+  getAll() {
+    return this.pool;
+  }
+  /** Stats for monitoring. */
+  stats() {
+    const now = Date.now();
+    const stale = this.pool.filter((s) => now - s.createdAt >= this.maxAgeMs || s.requestCount >= this.maxRequests).length;
+    const checkedOut = this.pool.filter((s) => !!s.checkedOutAt).length;
+    return { total: this.pool.length, available: this.pool.length - checkedOut - stale, checkedOut, stale };
+  }
+  /** Add a new session to the pool at runtime. */
+  addSession(role, storageState) {
+    const id = `session-${this.pool.length}-${role}`;
+    this.pool.push({ id, role, storageState, createdAt: Date.now(), checkedOutAt: null, requestCount: 0 });
+    return id;
+  }
+}
+export {
+  AuthTokenRotator
+};
+//# sourceMappingURL=auth-token-rotator.js.map

package/dist/auth-token-rotator.js.map

@@ -0,0 +1,7 @@
+{
+  "version": 3,
+  "sources": ["../src/auth-token-rotator.ts"],
+  "sourcesContent": ["/**\n * Auth token rotator \u2014 manage a pool of authenticated sessions for multi-account crawls.\n *\n * Use case: crawl the same app as admin+user+guest in one crawl job to discover\n * role-specific screens that only appear when authenticated with different privileges.\n * Each session is a Playwright storageState JSON string.\n *\n * Sessions are checked out (locked) before use and returned after the page is done.\n * Round-robin assignment with stale-session eviction after maxAgeMs.\n */\n\nexport interface SessionEntry {\n  id: string;\n  role: string;          // 'admin' | 'user' | 'guest' | custom label\n  storageState: string;  // Playwright StorageState JSON\n  createdAt: number;     // Date.now()\n  checkedOutAt: number | null;\n  requestCount: number;\n}\n\nexport interface SessionPoolOpts {\n  sessions: Array<{ role: string; storageState: string }>;\n  maxAgeMs?: number;        // evict sessions older than this (default: 4h)\n  maxRequestsPerSession?: number; // rotate after N requests (default: 100)\n}\n\nexport class AuthTokenRotator {\n  private pool: SessionEntry[];\n  private readonly maxAgeMs: number;\n  private readonly maxRequests: number;\n\n  constructor(opts: SessionPoolOpts) {\n    this.maxAgeMs = opts.maxAgeMs ?? 4 * 60 * 60 * 1000;\n    this.maxRequests = opts.maxRequestsPerSession ?? 100;\n    this.pool = opts.sessions.map((s, i) => ({\n      id: `session-${i}-${s.role}`,\n      role: s.role,\n      storageState: s.storageState,\n      createdAt: Date.now(),\n      checkedOutAt: null,\n      requestCount: 0,\n    }));\n  }\n\n  /** Get the next available session. Returns null if all are checked out. */\n  checkout(): SessionEntry | null {\n    const now = Date.now();\n    const available = this.pool.filter(s =>\n      !s.checkedOutAt &&\n      now - s.createdAt < this.maxAgeMs &&\n      s.requestCount < this.maxRequests,\n    );\n    if (available.length === 0) return null;\n    // Round-robin: pick lowest requestCount\n    available.sort((a, b) => a.requestCount - b.requestCount);\n    const session = available[0];\n    session.checkedOutAt = now;\n    return session;\n  }\n\n  /** Return a session to the pool after use. */\n  checkin(sessionId: string, updatedStorageState?: string): void {\n    const s = this.pool.find(e => e.id === sessionId);\n    if (!s) return;\n    s.checkedOutAt = null;\n    s.requestCount++;\n    if (updatedStorageState) s.storageState = updatedStorageState;\n  }\n\n  /** Get all available sessions (for parallel crawl setup). */\n  getAll(): ReadonlyArray<SessionEntry> {\n    return this.pool;\n  }\n\n  /** Stats for monitoring. */\n  stats(): { total: number; available: number; checkedOut: number; stale: number } {\n    const now = Date.now();\n    const stale = this.pool.filter(s => now - s.createdAt >= this.maxAgeMs || s.requestCount >= this.maxRequests).length;\n    const checkedOut = this.pool.filter(s => !!s.checkedOutAt).length;\n    return { total: this.pool.length, available: this.pool.length - checkedOut - stale, checkedOut, stale };\n  }\n\n  /** Add a new session to the pool at runtime. */\n  addSession(role: string, storageState: string): string {\n    const id = `session-${this.pool.length}-${role}`;\n    this.pool.push({ id, role, storageState, createdAt: Date.now(), checkedOutAt: null, requestCount: 0 });\n    return id;\n  }\n}\n"],
+  "mappings": "AA0BO,MAAM,iBAAiB;AAAA,EACpB;AAAA,EACS;AAAA,EACA;AAAA,EAEjB,YAAY,MAAuB;AACjC,SAAK,WAAW,KAAK,YAAY,IAAI,KAAK,KAAK;AAC/C,SAAK,cAAc,KAAK,yBAAyB;AACjD,SAAK,OAAO,KAAK,SAAS,IAAI,CAAC,GAAG,OAAO;AAAA,MACvC,IAAI,WAAW,CAAC,IAAI,EAAE,IAAI;AAAA,MAC1B,MAAM,EAAE;AAAA,MACR,cAAc,EAAE;AAAA,MAChB,WAAW,KAAK,IAAI;AAAA,MACpB,cAAc;AAAA,MACd,cAAc;AAAA,IAChB,EAAE;AAAA,EACJ;AAAA;AAAA,EAGA,WAAgC;AAC9B,UAAM,MAAM,KAAK,IAAI;AACrB,UAAM,YAAY,KAAK,KAAK;AAAA,MAAO,OACjC,CAAC,EAAE,gBACH,MAAM,EAAE,YAAY,KAAK,YACzB,EAAE,eAAe,KAAK;AAAA,IACxB;AACA,QAAI,UAAU,WAAW,EAAG,QAAO;AAEnC,cAAU,KAAK,CAAC,GAAG,MAAM,EAAE,eAAe,EAAE,YAAY;AACxD,UAAM,UAAU,UAAU,CAAC;AAC3B,YAAQ,eAAe;AACvB,WAAO;AAAA,EACT;AAAA;AAAA,EAGA,QAAQ,WAAmB,qBAAoC;AAC7D,UAAM,IAAI,KAAK,KAAK,KAAK,OAAK,EAAE,OAAO,SAAS;AAChD,QAAI,CAAC,EAAG;AACR,MAAE,eAAe;AACjB,MAAE;AACF,QAAI,oBAAqB,GAAE,eAAe;AAAA,EAC5C;AAAA;AAAA,EAGA,SAAsC;AACpC,WAAO,KAAK;AAAA,EACd;AAAA;AAAA,EAGA,QAAiF;AAC/E,UAAM,MAAM,KAAK,IAAI;AACrB,UAAM,QAAQ,KAAK,KAAK,OAAO,OAAK,MAAM,EAAE,aAAa,KAAK,YAAY,EAAE,gBAAgB,KAAK,WAAW,EAAE;AAC9G,UAAM,aAAa,KAAK,KAAK,OAAO,OAAK,CAAC,CAAC,EAAE,YAAY,EAAE;AAC3D,WAAO,EAAE,OAAO,KAAK,KAAK,QAAQ,WAAW,KAAK,KAAK,SAAS,aAAa,OAAO,YAAY,MAAM;AAAA,EACxG;AAAA;AAAA,EAGA,WAAW,MAAc,cAA8B;AACrD,UAAM,KAAK,WAAW,KAAK,KAAK,MAAM,IAAI,IAAI;AAC9C,SAAK,KAAK,KAAK,EAAE,IAAI,MAAM,cAAc,WAAW,KAAK,IAAI,GAAG,cAAc,MAAM,cAAc,EAAE,CAAC;AACrG,WAAO;AAAA,EACT;AACF;",
+  "names": []
+}

package/dist/auth.d.ts

@@ -0,0 +1,20 @@
+import type { Page } from 'playwright';
+export interface Credentials {
+    username: string;
+    password: string;
+    oneTimeCode?: string;
+    getOneTimeCode?: () => Promise<string>;
+}
+/**
+ * Recorded login step captured by the Chrome extension.
+ * {username} and {password} in `value` are substituted at replay time.
+ */
+export interface LoginStep {
+    action: 'fill' | 'click' | 'wait';
+    selector: string;
+    value?: string;
+    ms?: number;
+}
+export declare function login(page: Page, appUrl: string, creds: Credentials & {
+    loginRecipe?: LoginStep[];
+}): Promise<boolean>;