@detiq/crawler 0.1.6 → 0.1.8

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/ai-login.js CHANGED
@@ -99,7 +99,18 @@ async function loginWithStagehand(stagehand, creds, startUrl, captchaOpts, email
99
99
  await logStep("fill_password", async () => {
100
100
  const observations = await stagehand.observe("Find the password input field");
101
101
  const selector = observations[0]?.selector || 'input[type="password"]';
102
- await pg.locator(selector).fill(creds.password);
102
+ try {
103
+ await pg.locator(selector).fill(creds.password);
104
+ } catch {
105
+ await pg.evaluate((pwd) => {
106
+ const input = document.querySelector('input[type="password"]');
107
+ if (!input) throw new Error("No password field found on page");
108
+ const setter = Object.getOwnPropertyDescriptor(window.HTMLInputElement.prototype, "value")?.set;
109
+ setter?.call(input, pwd);
110
+ input.dispatchEvent(new Event("input", { bubbles: true }));
111
+ input.dispatchEvent(new Event("change", { bubbles: true }));
112
+ }, creds.password);
113
+ }
103
114
  });
104
115
  await logStep("click_submit", () => actWithMouseSim(stagehand, "Click the sign in, log in, login, or continue button to submit credentials"));
105
116
  await pg.waitForLoadState("networkidle").catch(() => pg.waitForTimeout(2e3));
@@ -1,7 +1,7 @@
1
1
  {
2
2
  "version": 3,
3
3
  "sources": ["../src/ai-login.ts"],
4
- "sourcesContent": ["import { z } from 'zod';\nimport type { Stagehand } from '@browserbasehq/stagehand';\nimport { AppBrain } from '@detiq/app-brain';\nimport { solveCaptcha } from './captcha-solver.js';\nimport { waitForEmailContent } from './email-otp.js';\nimport { logger } from './logger.js';\nimport type { Credentials } from './auth.js';\nimport { simulateMouseMovement } from './stealth/human-behavior.js';\n\nconst CLICK_INTENT = /\\b(click|tap|press|submit|select|choose|toggle)\\b/i;\n\n/** Simulate human mouse movement before click-type actions, then call stagehand.act(). */\nasync function actWithMouseSim(stagehand: Stagehand, instruction: string): Promise<any> {\n if (CLICK_INTENT.test(instruction)) {\n try {\n const pg = await (stagehand as any).resolvePage();\n const viewport = pg.viewportSize() ?? { width: 1280, height: 720 };\n // Move mouse toward center of page with some randomness before Stagehand clicks\n const targetX = viewport.width * 0.3 + Math.random() * viewport.width * 0.4;\n const targetY = viewport.height * 0.2 + Math.random() * viewport.height * 0.5;\n await simulateMouseMovement(pg, targetX, targetY);\n } catch { /* non-fatal \u2014 don't block the action */ }\n }\n return stagehand.act(instruction);\n}\n\n/**\n * Recorded login step from the Chrome extension's step recorder.\n * {username} and {password} in `value` are substituted at replay time.\n */\nexport interface LoginStep {\n action: 'fill' | 'click' | 'wait';\n selector: string;\n value?: string;\n ms?: number;\n}\n\nconst TwoFASchema = z.object({\n has2FA: z.boolean(),\n fieldLabel: z.string().optional(),\n});\n\nconst CAPTCHA_SELECTORS = [\n 'iframe[src*=\"recaptcha\"]',\n 'iframe[src*=\"hcaptcha\"]',\n 'iframe[src*=\"challenges.cloudflare.com\"]',\n '.cf-turnstile',\n '[data-sitekey]',\n '#captcha',\n '.captcha',\n '[id*=\"captcha\"]',\n '[class*=\"captcha\"]',\n];\n\n/**\n * AI-guided login using Stagehand act/extract.\n *\n * @param loginInstructions - Optional natural language description of the login flow.\n * When provided, an initial stagehand.act() is called with these instructions to\n * navigate the UI (e.g. \"click Sign In in the nav, fill email, click Continue\").\n * The standard 3-step credential fill follows immediately after.\n * @param opts - Optional tenant/project/job IDs for writing AUTH_STEP CrawlEvidence.\n */\nexport async function loginWithStagehand(\n stagehand: Stagehand,\n creds: Credentials,\n startUrl: string,\n captchaOpts?: { apiKey?: string; provider?: '2captcha' | 'capsolver' },\n emailOtpOpts?: { apiKey?: string; inboxId?: string },\n loginInstructions?: string,\n opts?: { tenantId?: string; projectId?: string; jobId?: string },\n): Promise<boolean> {\n async function logStep(step: string, fn: () => Promise<any>): Promise<{ success: boolean; detail: string }> {\n const t0 = Date.now();\n try {\n const r = await fn();\n const result = { success: true, detail: r?.description ?? r?.message ?? step };\n if (opts?.tenantId && opts?.projectId) {\n AppBrain.saveCrawlEvidence(opts.tenantId, opts.projectId, {\n jobId: opts?.jobId,\n type: 'AUTH_STEP',\n payload: { step, ...result, ms: Date.now() - t0 },\n }).catch(() => {});\n }\n return result;\n } catch (e: any) {\n const result = { success: false, detail: e.message ?? 'failed' };\n if (opts?.tenantId && opts?.projectId) {\n AppBrain.saveCrawlEvidence(opts.tenantId, opts.projectId, {\n jobId: opts?.jobId,\n type: 'AUTH_STEP',\n payload: { step, ...result, ms: Date.now() - t0 },\n }).catch(() => {});\n }\n throw e;\n }\n }\n\n try {\n const pg = await (stagehand as any).resolvePage();\n\n // CAPTCHA detection\n for (const sel of CAPTCHA_SELECTORS) {\n const count = await pg.locator(sel).count().catch(() => 0);\n if (count > 0) {\n logger.warn({ selector: sel }, '[crawler] CAPTCHA detected on login page');\n if (captchaOpts?.apiKey) {\n logger.info({ provider: captchaOpts.provider ?? '2captcha' }, '[crawler] Attempting auto-solve');\n try {\n const result = await solveCaptcha(pg, startUrl, captchaOpts.apiKey, captchaOpts.provider ?? '2captcha');\n if (result) {\n logger.info({ provider: result.provider }, '[crawler] CAPTCHA solved \u2014 proceeding with login');\n break;\n }\n } catch (solveErr) {\n logger.warn({ err: solveErr }, '[crawler] CAPTCHA solve failed \u2014 falling back to manual error');\n throw new Error(`CAPTCHA_DETECTED: Auto-solve failed (${String(solveErr)}). Use session capture instead.`);\n }\n } else {\n throw new Error(\n 'CAPTCHA_DETECTED: Login page has a CAPTCHA. Use \"Session Capture \u2192 Launch Browser\" to log in manually, then capture the session. Alternatively, disable CAPTCHA for the ZeTa crawler IP.'\n );\n }\n break;\n }\n }\n\n // If caller provided natural language login instructions, use them to navigate to the login form\n const MAX_LOGIN_INSTRUCTIONS = 2000;\n if (loginInstructions && loginInstructions.length > MAX_LOGIN_INSTRUCTIONS) {\n throw new Error(`loginInstructions exceeds ${MAX_LOGIN_INSTRUCTIONS} character limit`);\n }\n if (loginInstructions) {\n await logStep('custom_recipe', () => actWithMouseSim(stagehand, loginInstructions));\n await pg.waitForLoadState('networkidle').catch(() => pg.waitForTimeout(1500));\n }\n\n await logStep('fill_username', () => actWithMouseSim(stagehand, `Type \"${creds.username}\" into the username or email input field`));\n // Don't send password to LLM \u2014 use direct Playwright fill to avoid\n // leaking credentials into the Stagehand/LLM provider's request logs.\n await logStep('fill_password', async () => {\n const observations = await stagehand.observe('Find the password input field');\n const selector = observations[0]?.selector || 'input[type=\"password\"]';\n await pg.locator(selector).fill(creds.password);\n });\n await logStep('click_submit', () => actWithMouseSim(stagehand, 'Click the sign in, log in, login, or continue button to submit credentials'));\n await pg.waitForLoadState('networkidle').catch(() => pg.waitForTimeout(2000));\n\n const twoFA = await stagehand.extract(\n 'Is there a two-factor authentication, OTP, or verification code input visible right now?',\n TwoFASchema as any\n );\n\n if (twoFA.has2FA) {\n if (opts?.tenantId && opts?.projectId) {\n AppBrain.saveCrawlEvidence(opts.tenantId, opts.projectId, {\n jobId: opts?.jobId,\n type: 'AUTH_STEP',\n payload: { step: '2fa_detect', success: true, detail: twoFA.fieldLabel ?? '2FA detected', ms: 0 },\n }).catch(() => {});\n }\n\n let code = creds.oneTimeCode ?? await creds.getOneTimeCode?.();\n\n if (!code && emailOtpOpts?.apiKey && emailOtpOpts?.inboxId) {\n const since = new Date().toISOString();\n logger.info('[crawler] 2FA detected \u2014 waiting for email OTP or magic link via MailSlurp');\n const emailContent = await waitForEmailContent(\n { provider: 'mailslurp', apiKey: emailOtpOpts.apiKey, inboxId: emailOtpOpts.inboxId },\n since\n );\n if (emailContent?.magicLink) {\n logger.info({ url: emailContent.magicLink }, '[crawler] Magic link received \u2014 navigating');\n await pg.goto(emailContent.magicLink, { waitUntil: 'domcontentloaded' });\n await pg.waitForLoadState('networkidle').catch(() => {});\n return true; // magic link navigation = login complete\n }\n code = emailContent?.otp ?? undefined;\n if (code) {\n logger.info('[crawler] Email OTP received');\n } else {\n logger.warn('[crawler] Email OTP timed out after 60s');\n }\n }\n\n if (!code) {\n console.warn('[crawler] 2FA detected but no oneTimeCode provided');\n return false;\n }\n await logStep('2fa_inject', () => actWithMouseSim(stagehand, `Enter \"${code}\" into the ${twoFA.fieldLabel ?? 'verification code'} field`));\n await logStep('2fa_submit', () => actWithMouseSim(stagehand, 'Click verify or confirm to submit the verification code'));\n await pg.waitForLoadState('networkidle').catch(() => pg.waitForTimeout(2000));\n }\n\n // URL-based success check\n const currentUrl = pg.url();\n const loginPathPattern = /\\/(login|signin|sign-in|auth|authenticate|log-in|account\\/login)(\\?|$|\\/)/i;\n try {\n const { pathname } = new URL(currentUrl);\n const wasRedirected = currentUrl !== startUrl;\n const stillOnLoginPath = loginPathPattern.test(pathname);\n if (wasRedirected && !stillOnLoginPath) {\n logger.info({ url: currentUrl }, 'Login confirmed via URL change');\n return true;\n }\n } catch { /* invalid URL \u2014 fall through */ }\n\n const passwordCount = await pg.locator('input[type=\"password\"]').count().catch(() => 1);\n const bodyText = await pg.evaluate(() => (document as any).body?.innerText?.trim()?.length ?? 0).catch(() => 0);\n if (passwordCount === 0 && bodyText > 30) {\n logger.info('Login confirmed: password field no longer visible');\n return true;\n }\n\n logger.warn({ url: currentUrl }, 'Login appears failed');\n try {\n const screenshotPath = `/tmp/zeta-auth-failure-${Date.now()}.jpg`;\n await pg.screenshot({ path: screenshotPath, type: 'jpeg', quality: 70, fullPage: false });\n logger.warn({ screenshotPath }, '[crawler] Auth failure screenshot saved');\n } catch { /* non-fatal */ }\n return false;\n } catch (err) {\n const msg = String((err as Error).message ?? err);\n if (msg.includes('CAPTCHA_DETECTED')) throw err;\n logger.error({ err }, 'Login error');\n return false;\n }\n}\n"],
5
- "mappings": "AAAA,SAAS,SAAS;AAElB,SAAS,gBAAgB;AACzB,SAAS,oBAAoB;AAC7B,SAAS,2BAA2B;AACpC,SAAS,cAAc;AAEvB,SAAS,6BAA6B;AAEtC,MAAM,eAAe;AAGrB,eAAe,gBAAgB,WAAsB,aAAmC;AACtF,MAAI,aAAa,KAAK,WAAW,GAAG;AAClC,QAAI;AACF,YAAM,KAAK,MAAO,UAAkB,YAAY;AAChD,YAAM,WAAW,GAAG,aAAa,KAAK,EAAE,OAAO,MAAM,QAAQ,IAAI;AAEjE,YAAM,UAAU,SAAS,QAAQ,MAAM,KAAK,OAAO,IAAI,SAAS,QAAQ;AACxE,YAAM,UAAU,SAAS,SAAS,MAAM,KAAK,OAAO,IAAI,SAAS,SAAS;AAC1E,YAAM,sBAAsB,IAAI,SAAS,OAAO;AAAA,IAClD,QAAQ;AAAA,IAA2C;AAAA,EACrD;AACA,SAAO,UAAU,IAAI,WAAW;AAClC;AAaA,MAAM,cAAc,EAAE,OAAO;AAAA,EAC3B,QAAQ,EAAE,QAAQ;AAAA,EAClB,YAAY,EAAE,OAAO,EAAE,SAAS;AAClC,CAAC;AAED,MAAM,oBAAoB;AAAA,EACxB;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AACF;AAWA,eAAsB,mBACpB,WACA,OACA,UACA,aACA,cACA,mBACA,MACkB;AAClB,iBAAe,QAAQ,MAAc,IAAuE;AAC1G,UAAM,KAAK,KAAK,IAAI;AACpB,QAAI;AACF,YAAM,IAAI,MAAM,GAAG;AACnB,YAAM,SAAS,EAAE,SAAS,MAAM,QAAQ,GAAG,eAAe,GAAG,WAAW,KAAK;AAC7E,UAAI,MAAM,YAAY,MAAM,WAAW;AACrC,iBAAS,kBAAkB,KAAK,UAAU,KAAK,WAAW;AAAA,UACxD,OAAO,MAAM;AAAA,UACb,MAAM;AAAA,UACN,SAAS,EAAE,MAAM,GAAG,QAAQ,IAAI,KAAK,IAAI,IAAI,GAAG;AAAA,QAClD,CAAC,EAAE,MAAM,MAAM;AAAA,QAAC,CAAC;AAAA,MACnB;AACA,aAAO;AAAA,IACT,SAAS,GAAQ;AACf,YAAM,SAAS,EAAE,SAAS,OAAO,QAAQ,EAAE,WAAW,SAAS;AAC/D,UAAI,MAAM,YAAY,MAAM,WAAW;AACrC,iBAAS,kBAAkB,KAAK,UAAU,KAAK,WAAW;AAAA,UACxD,OAAO,MAAM;AAAA,UACb,MAAM;AAAA,UACN,SAAS,EAAE,MAAM,GAAG,QAAQ,IAAI,KAAK,IAAI,IAAI,GAAG;AAAA,QAClD,CAAC,EAAE,MAAM,MAAM;AAAA,QAAC,CAAC;AAAA,MACnB;AACA,YAAM;AAAA,IACR;AAAA,EACF;AAEA,MAAI;AACF,UAAM,KAAK,MAAO,UAAkB,YAAY;AAGhD,eAAW,OAAO,mBAAmB;AACnC,YAAM,QAAQ,MAAM,GAAG,QAAQ,GAAG,EAAE,MAAM,EAAE,MAAM,MAAM,CAAC;AACzD,UAAI,QAAQ,GAAG;AACb,eAAO,KAAK,EAAE,UAAU,IAAI,GAAG,0CAA0C;AACzE,YAAI,aAAa,QAAQ;AACvB,iBAAO,KAAK,EAAE,UAAU,YAAY,YAAY,WAAW,GAAG,iCAAiC;AAC/F,cAAI;AACF,kBAAM,SAAS,MAAM,aAAa,IAAI,UAAU,YAAY,QAAQ,YAAY,YAAY,UAAU;AACtG,gBAAI,QAAQ;AACV,qBAAO,KAAK,EAAE,UAAU,OAAO,SAAS,GAAG,uDAAkD;AAC7F;AAAA,YACF;AAAA,UACF,SAAS,UAAU;AACjB,mBAAO,KAAK,EAAE,KAAK,SAAS,GAAG,oEAA+D;AAC9F,kBAAM,IAAI,MAAM,wCAAwC,OAAO,QAAQ,CAAC,iCAAiC;AAAA,UAC3G;AAAA,QACF,OAAO;AACL,gBAAM,IAAI;AAAA,YACR;AAAA,UACF;AAAA,QACF;AACA;AAAA,MACF;AAAA,IACF;AAGA,UAAM,yBAAyB;AAC/B,QAAI,qBAAqB,kBAAkB,SAAS,wBAAwB;AAC1E,YAAM,IAAI,MAAM,6BAA6B,sBAAsB,kBAAkB;AAAA,IACvF;AACA,QAAI,mBAAmB;AACrB,YAAM,QAAQ,iBAAiB,MAAM,gBAAgB,WAAW,iBAAiB,CAAC;AAClF,YAAM,GAAG,iBAAiB,aAAa,EAAE,MAAM,MAAM,GAAG,eAAe,IAAI,CAAC;AAAA,IAC9E;AAEA,UAAM,QAAQ,iBAAiB,MAAM,gBAAgB,WAAW,SAAS,MAAM,QAAQ,0CAA0C,CAAC;AAGlI,UAAM,QAAQ,iBAAiB,YAAY;AACzC,YAAM,eAAe,MAAM,UAAU,QAAQ,+BAA+B;AAC5E,YAAM,WAAW,aAAa,CAAC,GAAG,YAAY;AAC9C,YAAM,GAAG,QAAQ,QAAQ,EAAE,KAAK,MAAM,QAAQ;AAAA,IAChD,CAAC;AACD,UAAM,QAAQ,gBAAgB,MAAM,gBAAgB,WAAW,4EAA4E,CAAC;AAC5I,UAAM,GAAG,iBAAiB,aAAa,EAAE,MAAM,MAAM,GAAG,eAAe,GAAI,CAAC;AAE5E,UAAM,QAAQ,MAAM,UAAU;AAAA,MAC5B;AAAA,MACA;AAAA,IACF;AAEA,QAAI,MAAM,QAAQ;AAChB,UAAI,MAAM,YAAY,MAAM,WAAW;AACrC,iBAAS,kBAAkB,KAAK,UAAU,KAAK,WAAW;AAAA,UACxD,OAAO,MAAM;AAAA,UACb,MAAM;AAAA,UACN,SAAS,EAAE,MAAM,cAAc,SAAS,MAAM,QAAQ,MAAM,cAAc,gBAAgB,IAAI,EAAE;AAAA,QAClG,CAAC,EAAE,MAAM,MAAM;AAAA,QAAC,CAAC;AAAA,MACnB;AAEA,UAAI,OAAO,MAAM,eAAe,MAAM,MAAM,iBAAiB;AAE7D,UAAI,CAAC,QAAQ,cAAc,UAAU,cAAc,SAAS;AAC1D,cAAM,SAAQ,oBAAI,KAAK,GAAE,YAAY;AACrC,eAAO,KAAK,iFAA4E;AACxF,cAAM,eAAe,MAAM;AAAA,UACzB,EAAE,UAAU,aAAa,QAAQ,aAAa,QAAQ,SAAS,aAAa,QAAQ;AAAA,UACpF;AAAA,QACF;AACA,YAAI,cAAc,WAAW;AAC3B,iBAAO,KAAK,EAAE,KAAK,aAAa,UAAU,GAAG,iDAA4C;AACzF,gBAAM,GAAG,KAAK,aAAa,WAAW,EAAE,WAAW,mBAAmB,CAAC;AACvE,gBAAM,GAAG,iBAAiB,aAAa,EAAE,MAAM,MAAM;AAAA,UAAC,CAAC;AACvD,iBAAO;AAAA,QACT;AACA,eAAO,cAAc,OAAO;AAC5B,YAAI,MAAM;AACR,iBAAO,KAAK,8BAA8B;AAAA,QAC5C,OAAO;AACL,iBAAO,KAAK,yCAAyC;AAAA,QACvD;AAAA,MACF;AAEA,UAAI,CAAC,MAAM;AACT,gBAAQ,KAAK,oDAAoD;AACjE,eAAO;AAAA,MACT;AACA,YAAM,QAAQ,cAAc,MAAM,gBAAgB,WAAW,UAAU,IAAI,cAAc,MAAM,cAAc,mBAAmB,QAAQ,CAAC;AACzI,YAAM,QAAQ,cAAc,MAAM,gBAAgB,WAAW,yDAAyD,CAAC;AACvH,YAAM,GAAG,iBAAiB,aAAa,EAAE,MAAM,MAAM,GAAG,eAAe,GAAI,CAAC;AAAA,IAC9E;AAGA,UAAM,aAAa,GAAG,IAAI;AAC1B,UAAM,mBAAmB;AACzB,QAAI;AACF,YAAM,EAAE,SAAS,IAAI,IAAI,IAAI,UAAU;AACvC,YAAM,gBAAgB,eAAe;AACrC,YAAM,mBAAmB,iBAAiB,KAAK,QAAQ;AACvD,UAAI,iBAAiB,CAAC,kBAAkB;AACtC,eAAO,KAAK,EAAE,KAAK,WAAW,GAAG,gCAAgC;AACjE,eAAO;AAAA,MACT;AAAA,IACF,QAAQ;AAAA,IAAmC;AAE3C,UAAM,gBAAgB,MAAM,GAAG,QAAQ,wBAAwB,EAAE,MAAM,EAAE,MAAM,MAAM,CAAC;AACtF,UAAM,WAAW,MAAM,GAAG,SAAS,MAAO,SAAiB,MAAM,WAAW,KAAK,GAAG,UAAU,CAAC,EAAE,MAAM,MAAM,CAAC;AAC9G,QAAI,kBAAkB,KAAK,WAAW,IAAI;AACxC,aAAO,KAAK,mDAAmD;AAC/D,aAAO;AAAA,IACT;AAEA,WAAO,KAAK,EAAE,KAAK,WAAW,GAAG,sBAAsB;AACvD,QAAI;AACF,YAAM,iBAAiB,0BAA0B,KAAK,IAAI,CAAC;AAC3D,YAAM,GAAG,WAAW,EAAE,MAAM,gBAAgB,MAAM,QAAQ,SAAS,IAAI,UAAU,MAAM,CAAC;AACxF,aAAO,KAAK,EAAE,eAAe,GAAG,yCAAyC;AAAA,IAC3E,QAAQ;AAAA,IAAkB;AAC1B,WAAO;AAAA,EACT,SAAS,KAAK;AACZ,UAAM,MAAM,OAAQ,IAAc,WAAW,GAAG;AAChD,QAAI,IAAI,SAAS,kBAAkB,EAAG,OAAM;AAC5C,WAAO,MAAM,EAAE,IAAI,GAAG,aAAa;AACnC,WAAO;AAAA,EACT;AACF;",
4
+ "sourcesContent": ["import { z } from 'zod';\nimport type { Stagehand } from '@browserbasehq/stagehand';\nimport { AppBrain } from '@detiq/app-brain';\nimport { solveCaptcha } from './captcha-solver.js';\nimport { waitForEmailContent } from './email-otp.js';\nimport { logger } from './logger.js';\nimport type { Credentials } from './auth.js';\nimport { simulateMouseMovement } from './stealth/human-behavior.js';\n\nconst CLICK_INTENT = /\\b(click|tap|press|submit|select|choose|toggle)\\b/i;\n\n/** Simulate human mouse movement before click-type actions, then call stagehand.act(). */\nasync function actWithMouseSim(stagehand: Stagehand, instruction: string): Promise<any> {\n if (CLICK_INTENT.test(instruction)) {\n try {\n const pg = await (stagehand as any).resolvePage();\n const viewport = pg.viewportSize() ?? { width: 1280, height: 720 };\n // Move mouse toward center of page with some randomness before Stagehand clicks\n const targetX = viewport.width * 0.3 + Math.random() * viewport.width * 0.4;\n const targetY = viewport.height * 0.2 + Math.random() * viewport.height * 0.5;\n await simulateMouseMovement(pg, targetX, targetY);\n } catch { /* non-fatal \u2014 don't block the action */ }\n }\n return stagehand.act(instruction);\n}\n\n/**\n * Recorded login step from the Chrome extension's step recorder.\n * {username} and {password} in `value` are substituted at replay time.\n */\nexport interface LoginStep {\n action: 'fill' | 'click' | 'wait';\n selector: string;\n value?: string;\n ms?: number;\n}\n\nconst TwoFASchema = z.object({\n has2FA: z.boolean(),\n fieldLabel: z.string().optional(),\n});\n\nconst CAPTCHA_SELECTORS = [\n 'iframe[src*=\"recaptcha\"]',\n 'iframe[src*=\"hcaptcha\"]',\n 'iframe[src*=\"challenges.cloudflare.com\"]',\n '.cf-turnstile',\n '[data-sitekey]',\n '#captcha',\n '.captcha',\n '[id*=\"captcha\"]',\n '[class*=\"captcha\"]',\n];\n\n/**\n * AI-guided login using Stagehand act/extract.\n *\n * @param loginInstructions - Optional natural language description of the login flow.\n * When provided, an initial stagehand.act() is called with these instructions to\n * navigate the UI (e.g. \"click Sign In in the nav, fill email, click Continue\").\n * The standard 3-step credential fill follows immediately after.\n * @param opts - Optional tenant/project/job IDs for writing AUTH_STEP CrawlEvidence.\n */\nexport async function loginWithStagehand(\n stagehand: Stagehand,\n creds: Credentials,\n startUrl: string,\n captchaOpts?: { apiKey?: string; provider?: '2captcha' | 'capsolver' },\n emailOtpOpts?: { apiKey?: string; inboxId?: string },\n loginInstructions?: string,\n opts?: { tenantId?: string; projectId?: string; jobId?: string },\n): Promise<boolean> {\n async function logStep(step: string, fn: () => Promise<any>): Promise<{ success: boolean; detail: string }> {\n const t0 = Date.now();\n try {\n const r = await fn();\n const result = { success: true, detail: r?.description ?? r?.message ?? step };\n if (opts?.tenantId && opts?.projectId) {\n AppBrain.saveCrawlEvidence(opts.tenantId, opts.projectId, {\n jobId: opts?.jobId,\n type: 'AUTH_STEP',\n payload: { step, ...result, ms: Date.now() - t0 },\n }).catch(() => {});\n }\n return result;\n } catch (e: any) {\n const result = { success: false, detail: e.message ?? 'failed' };\n if (opts?.tenantId && opts?.projectId) {\n AppBrain.saveCrawlEvidence(opts.tenantId, opts.projectId, {\n jobId: opts?.jobId,\n type: 'AUTH_STEP',\n payload: { step, ...result, ms: Date.now() - t0 },\n }).catch(() => {});\n }\n throw e;\n }\n }\n\n try {\n const pg = await (stagehand as any).resolvePage();\n\n // CAPTCHA detection\n for (const sel of CAPTCHA_SELECTORS) {\n const count = await pg.locator(sel).count().catch(() => 0);\n if (count > 0) {\n logger.warn({ selector: sel }, '[crawler] CAPTCHA detected on login page');\n if (captchaOpts?.apiKey) {\n logger.info({ provider: captchaOpts.provider ?? '2captcha' }, '[crawler] Attempting auto-solve');\n try {\n const result = await solveCaptcha(pg, startUrl, captchaOpts.apiKey, captchaOpts.provider ?? '2captcha');\n if (result) {\n logger.info({ provider: result.provider }, '[crawler] CAPTCHA solved \u2014 proceeding with login');\n break;\n }\n } catch (solveErr) {\n logger.warn({ err: solveErr }, '[crawler] CAPTCHA solve failed \u2014 falling back to manual error');\n throw new Error(`CAPTCHA_DETECTED: Auto-solve failed (${String(solveErr)}). Use session capture instead.`);\n }\n } else {\n throw new Error(\n 'CAPTCHA_DETECTED: Login page has a CAPTCHA. Use \"Session Capture \u2192 Launch Browser\" to log in manually, then capture the session. Alternatively, disable CAPTCHA for the ZeTa crawler IP.'\n );\n }\n break;\n }\n }\n\n // If caller provided natural language login instructions, use them to navigate to the login form\n const MAX_LOGIN_INSTRUCTIONS = 2000;\n if (loginInstructions && loginInstructions.length > MAX_LOGIN_INSTRUCTIONS) {\n throw new Error(`loginInstructions exceeds ${MAX_LOGIN_INSTRUCTIONS} character limit`);\n }\n if (loginInstructions) {\n await logStep('custom_recipe', () => actWithMouseSim(stagehand, loginInstructions));\n await pg.waitForLoadState('networkidle').catch(() => pg.waitForTimeout(1500));\n }\n\n await logStep('fill_username', () => actWithMouseSim(stagehand, `Type \"${creds.username}\" into the username or email input field`));\n // Don't send password to LLM \u2014 use direct Playwright fill to avoid\n // leaking credentials into the Stagehand/LLM provider's request logs.\n await logStep('fill_password', async () => {\n const observations = await stagehand.observe('Find the password input field');\n const selector = observations[0]?.selector || 'input[type=\"password\"]';\n try {\n await pg.locator(selector).fill(creds.password);\n } catch {\n // Stagehand locator failed (e.g. form re-rendered, proxy can't locate element).\n // Fall back to direct DOM fill via evaluate so the password never touches the LLM.\n await pg.evaluate((pwd: string) => {\n const input = document.querySelector('input[type=\"password\"]') as HTMLInputElement | null;\n if (!input) throw new Error('No password field found on page');\n const setter = Object.getOwnPropertyDescriptor(window.HTMLInputElement.prototype, 'value')?.set;\n setter?.call(input, pwd);\n input.dispatchEvent(new Event('input', { bubbles: true }));\n input.dispatchEvent(new Event('change', { bubbles: true }));\n }, creds.password);\n }\n });\n await logStep('click_submit', () => actWithMouseSim(stagehand, 'Click the sign in, log in, login, or continue button to submit credentials'));\n await pg.waitForLoadState('networkidle').catch(() => pg.waitForTimeout(2000));\n\n const twoFA = await stagehand.extract(\n 'Is there a two-factor authentication, OTP, or verification code input visible right now?',\n TwoFASchema as any\n );\n\n if (twoFA.has2FA) {\n if (opts?.tenantId && opts?.projectId) {\n AppBrain.saveCrawlEvidence(opts.tenantId, opts.projectId, {\n jobId: opts?.jobId,\n type: 'AUTH_STEP',\n payload: { step: '2fa_detect', success: true, detail: twoFA.fieldLabel ?? '2FA detected', ms: 0 },\n }).catch(() => {});\n }\n\n let code = creds.oneTimeCode ?? await creds.getOneTimeCode?.();\n\n if (!code && emailOtpOpts?.apiKey && emailOtpOpts?.inboxId) {\n const since = new Date().toISOString();\n logger.info('[crawler] 2FA detected \u2014 waiting for email OTP or magic link via MailSlurp');\n const emailContent = await waitForEmailContent(\n { provider: 'mailslurp', apiKey: emailOtpOpts.apiKey, inboxId: emailOtpOpts.inboxId },\n since\n );\n if (emailContent?.magicLink) {\n logger.info({ url: emailContent.magicLink }, '[crawler] Magic link received \u2014 navigating');\n await pg.goto(emailContent.magicLink, { waitUntil: 'domcontentloaded' });\n await pg.waitForLoadState('networkidle').catch(() => {});\n return true; // magic link navigation = login complete\n }\n code = emailContent?.otp ?? undefined;\n if (code) {\n logger.info('[crawler] Email OTP received');\n } else {\n logger.warn('[crawler] Email OTP timed out after 60s');\n }\n }\n\n if (!code) {\n console.warn('[crawler] 2FA detected but no oneTimeCode provided');\n return false;\n }\n await logStep('2fa_inject', () => actWithMouseSim(stagehand, `Enter \"${code}\" into the ${twoFA.fieldLabel ?? 'verification code'} field`));\n await logStep('2fa_submit', () => actWithMouseSim(stagehand, 'Click verify or confirm to submit the verification code'));\n await pg.waitForLoadState('networkidle').catch(() => pg.waitForTimeout(2000));\n }\n\n // URL-based success check\n const currentUrl = pg.url();\n const loginPathPattern = /\\/(login|signin|sign-in|auth|authenticate|log-in|account\\/login)(\\?|$|\\/)/i;\n try {\n const { pathname } = new URL(currentUrl);\n const wasRedirected = currentUrl !== startUrl;\n const stillOnLoginPath = loginPathPattern.test(pathname);\n if (wasRedirected && !stillOnLoginPath) {\n logger.info({ url: currentUrl }, 'Login confirmed via URL change');\n return true;\n }\n } catch { /* invalid URL \u2014 fall through */ }\n\n const passwordCount = await pg.locator('input[type=\"password\"]').count().catch(() => 1);\n const bodyText = await pg.evaluate(() => (document as any).body?.innerText?.trim()?.length ?? 0).catch(() => 0);\n if (passwordCount === 0 && bodyText > 30) {\n logger.info('Login confirmed: password field no longer visible');\n return true;\n }\n\n logger.warn({ url: currentUrl }, 'Login appears failed');\n try {\n const screenshotPath = `/tmp/zeta-auth-failure-${Date.now()}.jpg`;\n await pg.screenshot({ path: screenshotPath, type: 'jpeg', quality: 70, fullPage: false });\n logger.warn({ screenshotPath }, '[crawler] Auth failure screenshot saved');\n } catch { /* non-fatal */ }\n return false;\n } catch (err) {\n const msg = String((err as Error).message ?? err);\n if (msg.includes('CAPTCHA_DETECTED')) throw err;\n logger.error({ err }, 'Login error');\n return false;\n }\n}\n"],
5
+ "mappings": "AAAA,SAAS,SAAS;AAElB,SAAS,gBAAgB;AACzB,SAAS,oBAAoB;AAC7B,SAAS,2BAA2B;AACpC,SAAS,cAAc;AAEvB,SAAS,6BAA6B;AAEtC,MAAM,eAAe;AAGrB,eAAe,gBAAgB,WAAsB,aAAmC;AACtF,MAAI,aAAa,KAAK,WAAW,GAAG;AAClC,QAAI;AACF,YAAM,KAAK,MAAO,UAAkB,YAAY;AAChD,YAAM,WAAW,GAAG,aAAa,KAAK,EAAE,OAAO,MAAM,QAAQ,IAAI;AAEjE,YAAM,UAAU,SAAS,QAAQ,MAAM,KAAK,OAAO,IAAI,SAAS,QAAQ;AACxE,YAAM,UAAU,SAAS,SAAS,MAAM,KAAK,OAAO,IAAI,SAAS,SAAS;AAC1E,YAAM,sBAAsB,IAAI,SAAS,OAAO;AAAA,IAClD,QAAQ;AAAA,IAA2C;AAAA,EACrD;AACA,SAAO,UAAU,IAAI,WAAW;AAClC;AAaA,MAAM,cAAc,EAAE,OAAO;AAAA,EAC3B,QAAQ,EAAE,QAAQ;AAAA,EAClB,YAAY,EAAE,OAAO,EAAE,SAAS;AAClC,CAAC;AAED,MAAM,oBAAoB;AAAA,EACxB;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AACF;AAWA,eAAsB,mBACpB,WACA,OACA,UACA,aACA,cACA,mBACA,MACkB;AAClB,iBAAe,QAAQ,MAAc,IAAuE;AAC1G,UAAM,KAAK,KAAK,IAAI;AACpB,QAAI;AACF,YAAM,IAAI,MAAM,GAAG;AACnB,YAAM,SAAS,EAAE,SAAS,MAAM,QAAQ,GAAG,eAAe,GAAG,WAAW,KAAK;AAC7E,UAAI,MAAM,YAAY,MAAM,WAAW;AACrC,iBAAS,kBAAkB,KAAK,UAAU,KAAK,WAAW;AAAA,UACxD,OAAO,MAAM;AAAA,UACb,MAAM;AAAA,UACN,SAAS,EAAE,MAAM,GAAG,QAAQ,IAAI,KAAK,IAAI,IAAI,GAAG;AAAA,QAClD,CAAC,EAAE,MAAM,MAAM;AAAA,QAAC,CAAC;AAAA,MACnB;AACA,aAAO;AAAA,IACT,SAAS,GAAQ;AACf,YAAM,SAAS,EAAE,SAAS,OAAO,QAAQ,EAAE,WAAW,SAAS;AAC/D,UAAI,MAAM,YAAY,MAAM,WAAW;AACrC,iBAAS,kBAAkB,KAAK,UAAU,KAAK,WAAW;AAAA,UACxD,OAAO,MAAM;AAAA,UACb,MAAM;AAAA,UACN,SAAS,EAAE,MAAM,GAAG,QAAQ,IAAI,KAAK,IAAI,IAAI,GAAG;AAAA,QAClD,CAAC,EAAE,MAAM,MAAM;AAAA,QAAC,CAAC;AAAA,MACnB;AACA,YAAM;AAAA,IACR;AAAA,EACF;AAEA,MAAI;AACF,UAAM,KAAK,MAAO,UAAkB,YAAY;AAGhD,eAAW,OAAO,mBAAmB;AACnC,YAAM,QAAQ,MAAM,GAAG,QAAQ,GAAG,EAAE,MAAM,EAAE,MAAM,MAAM,CAAC;AACzD,UAAI,QAAQ,GAAG;AACb,eAAO,KAAK,EAAE,UAAU,IAAI,GAAG,0CAA0C;AACzE,YAAI,aAAa,QAAQ;AACvB,iBAAO,KAAK,EAAE,UAAU,YAAY,YAAY,WAAW,GAAG,iCAAiC;AAC/F,cAAI;AACF,kBAAM,SAAS,MAAM,aAAa,IAAI,UAAU,YAAY,QAAQ,YAAY,YAAY,UAAU;AACtG,gBAAI,QAAQ;AACV,qBAAO,KAAK,EAAE,UAAU,OAAO,SAAS,GAAG,uDAAkD;AAC7F;AAAA,YACF;AAAA,UACF,SAAS,UAAU;AACjB,mBAAO,KAAK,EAAE,KAAK,SAAS,GAAG,oEAA+D;AAC9F,kBAAM,IAAI,MAAM,wCAAwC,OAAO,QAAQ,CAAC,iCAAiC;AAAA,UAC3G;AAAA,QACF,OAAO;AACL,gBAAM,IAAI;AAAA,YACR;AAAA,UACF;AAAA,QACF;AACA;AAAA,MACF;AAAA,IACF;AAGA,UAAM,yBAAyB;AAC/B,QAAI,qBAAqB,kBAAkB,SAAS,wBAAwB;AAC1E,YAAM,IAAI,MAAM,6BAA6B,sBAAsB,kBAAkB;AAAA,IACvF;AACA,QAAI,mBAAmB;AACrB,YAAM,QAAQ,iBAAiB,MAAM,gBAAgB,WAAW,iBAAiB,CAAC;AAClF,YAAM,GAAG,iBAAiB,aAAa,EAAE,MAAM,MAAM,GAAG,eAAe,IAAI,CAAC;AAAA,IAC9E;AAEA,UAAM,QAAQ,iBAAiB,MAAM,gBAAgB,WAAW,SAAS,MAAM,QAAQ,0CAA0C,CAAC;AAGlI,UAAM,QAAQ,iBAAiB,YAAY;AACzC,YAAM,eAAe,MAAM,UAAU,QAAQ,+BAA+B;AAC5E,YAAM,WAAW,aAAa,CAAC,GAAG,YAAY;AAC9C,UAAI;AACF,cAAM,GAAG,QAAQ,QAAQ,EAAE,KAAK,MAAM,QAAQ;AAAA,MAChD,QAAQ;AAGN,cAAM,GAAG,SAAS,CAAC,QAAgB;AACjC,gBAAM,QAAQ,SAAS,cAAc,wBAAwB;AAC7D,cAAI,CAAC,MAAO,OAAM,IAAI,MAAM,iCAAiC;AAC7D,gBAAM,SAAS,OAAO,yBAAyB,OAAO,iBAAiB,WAAW,OAAO,GAAG;AAC5F,kBAAQ,KAAK,OAAO,GAAG;AACvB,gBAAM,cAAc,IAAI,MAAM,SAAS,EAAE,SAAS,KAAK,CAAC,CAAC;AACzD,gBAAM,cAAc,IAAI,MAAM,UAAU,EAAE,SAAS,KAAK,CAAC,CAAC;AAAA,QAC5D,GAAG,MAAM,QAAQ;AAAA,MACnB;AAAA,IACF,CAAC;AACD,UAAM,QAAQ,gBAAgB,MAAM,gBAAgB,WAAW,4EAA4E,CAAC;AAC5I,UAAM,GAAG,iBAAiB,aAAa,EAAE,MAAM,MAAM,GAAG,eAAe,GAAI,CAAC;AAE5E,UAAM,QAAQ,MAAM,UAAU;AAAA,MAC5B;AAAA,MACA;AAAA,IACF;AAEA,QAAI,MAAM,QAAQ;AAChB,UAAI,MAAM,YAAY,MAAM,WAAW;AACrC,iBAAS,kBAAkB,KAAK,UAAU,KAAK,WAAW;AAAA,UACxD,OAAO,MAAM;AAAA,UACb,MAAM;AAAA,UACN,SAAS,EAAE,MAAM,cAAc,SAAS,MAAM,QAAQ,MAAM,cAAc,gBAAgB,IAAI,EAAE;AAAA,QAClG,CAAC,EAAE,MAAM,MAAM;AAAA,QAAC,CAAC;AAAA,MACnB;AAEA,UAAI,OAAO,MAAM,eAAe,MAAM,MAAM,iBAAiB;AAE7D,UAAI,CAAC,QAAQ,cAAc,UAAU,cAAc,SAAS;AAC1D,cAAM,SAAQ,oBAAI,KAAK,GAAE,YAAY;AACrC,eAAO,KAAK,iFAA4E;AACxF,cAAM,eAAe,MAAM;AAAA,UACzB,EAAE,UAAU,aAAa,QAAQ,aAAa,QAAQ,SAAS,aAAa,QAAQ;AAAA,UACpF;AAAA,QACF;AACA,YAAI,cAAc,WAAW;AAC3B,iBAAO,KAAK,EAAE,KAAK,aAAa,UAAU,GAAG,iDAA4C;AACzF,gBAAM,GAAG,KAAK,aAAa,WAAW,EAAE,WAAW,mBAAmB,CAAC;AACvE,gBAAM,GAAG,iBAAiB,aAAa,EAAE,MAAM,MAAM;AAAA,UAAC,CAAC;AACvD,iBAAO;AAAA,QACT;AACA,eAAO,cAAc,OAAO;AAC5B,YAAI,MAAM;AACR,iBAAO,KAAK,8BAA8B;AAAA,QAC5C,OAAO;AACL,iBAAO,KAAK,yCAAyC;AAAA,QACvD;AAAA,MACF;AAEA,UAAI,CAAC,MAAM;AACT,gBAAQ,KAAK,oDAAoD;AACjE,eAAO;AAAA,MACT;AACA,YAAM,QAAQ,cAAc,MAAM,gBAAgB,WAAW,UAAU,IAAI,cAAc,MAAM,cAAc,mBAAmB,QAAQ,CAAC;AACzI,YAAM,QAAQ,cAAc,MAAM,gBAAgB,WAAW,yDAAyD,CAAC;AACvH,YAAM,GAAG,iBAAiB,aAAa,EAAE,MAAM,MAAM,GAAG,eAAe,GAAI,CAAC;AAAA,IAC9E;AAGA,UAAM,aAAa,GAAG,IAAI;AAC1B,UAAM,mBAAmB;AACzB,QAAI;AACF,YAAM,EAAE,SAAS,IAAI,IAAI,IAAI,UAAU;AACvC,YAAM,gBAAgB,eAAe;AACrC,YAAM,mBAAmB,iBAAiB,KAAK,QAAQ;AACvD,UAAI,iBAAiB,CAAC,kBAAkB;AACtC,eAAO,KAAK,EAAE,KAAK,WAAW,GAAG,gCAAgC;AACjE,eAAO;AAAA,MACT;AAAA,IACF,QAAQ;AAAA,IAAmC;AAE3C,UAAM,gBAAgB,MAAM,GAAG,QAAQ,wBAAwB,EAAE,MAAM,EAAE,MAAM,MAAM,CAAC;AACtF,UAAM,WAAW,MAAM,GAAG,SAAS,MAAO,SAAiB,MAAM,WAAW,KAAK,GAAG,UAAU,CAAC,EAAE,MAAM,MAAM,CAAC;AAC9G,QAAI,kBAAkB,KAAK,WAAW,IAAI;AACxC,aAAO,KAAK,mDAAmD;AAC/D,aAAO;AAAA,IACT;AAEA,WAAO,KAAK,EAAE,KAAK,WAAW,GAAG,sBAAsB;AACvD,QAAI;AACF,YAAM,iBAAiB,0BAA0B,KAAK,IAAI,CAAC;AAC3D,YAAM,GAAG,WAAW,EAAE,MAAM,gBAAgB,MAAM,QAAQ,SAAS,IAAI,UAAU,MAAM,CAAC;AACxF,aAAO,KAAK,EAAE,eAAe,GAAG,yCAAyC;AAAA,IAC3E,QAAQ;AAAA,IAAkB;AAC1B,WAAO;AAAA,EACT,SAAS,KAAK;AACZ,UAAM,MAAM,OAAQ,IAAc,WAAW,GAAG;AAChD,QAAI,IAAI,SAAS,kBAAkB,EAAG,OAAM;AAC5C,WAAO,MAAM,EAAE,IAAI,GAAG,aAAa;AACnC,WAAO;AAAA,EACT;AACF;",
6
6
  "names": []
7
7
  }
package/dist/crawler.js CHANGED
@@ -736,7 +736,8 @@ async function crawlProject(opts, onProgress) {
736
736
  }, state.origins);
737
737
  }
738
738
  await onProgress?.(12, "Restoring session");
739
- await rawPage.reload({ waitUntil: "domcontentloaded" });
739
+ await rawPage.goto(opts.appUrl, { waitUntil: "domcontentloaded" }).catch(() => {
740
+ });
740
741
  await waitForStableDOM(page);
741
742
  if (page.url() === opts.appUrl || page.url() === opts.appUrl.replace(/\/$/, "")) {
742
743
  await waitForURLChange(page, opts.appUrl).catch(() => {
@@ -1732,8 +1733,11 @@ async function crawlProject(opts, onProgress) {
1732
1733
  }
1733
1734
  })()}`);
1734
1735
  const [, elements, nextLinks, atElements] = await Promise.all([
1735
- page.screenshot({ path: shotPath, fullPage: true }).then(() => AppBrain.saveScreenshot(opts.tenantId, opts.projectId, screen.id, shotPath, screen.version, "DESKTOP")),
1736
- extractElements(stagehand, page, elementLLM ?? void 0).catch(() => []),
1736
+ page.screenshot({ path: shotPath, fullPage: true }).then(() => AppBrain.saveScreenshot(opts.tenantId, opts.projectId, screen.id, shotPath, screen.version, "DESKTOP")).catch((err) => console.error("[crawler] page.screenshot failed:", err)),
1737
+ extractElements(stagehand, page, elementLLM ?? void 0).catch((err) => {
1738
+ console.error("[crawler] extractElements failed:", err);
1739
+ return [];
1740
+ }),
1737
1741
  // Always run AI link discovery (even when cache exists) — cache is merged in, not used as a replacement.
1738
1742
  // Skipping AI when cache exists was a bug: stale cache from broken crawls permanently blocked discovery.
1739
1743
  discoverLinks(stagehand, origin, page, linkLLM ?? void 0, aiSkipCount > 0).then((freshLinks) => {
@@ -2155,7 +2159,7 @@ ${snippet}`,
2155
2159
  fcp: rawPerf?.fcp,
2156
2160
  cls: rawPerf?.cls,
2157
2161
  ttfb: rawPerf?.ttfb,
2158
- viewport: "DESKTOP",
2162
+ viewport: opts.viewport ?? "DESKTOP",
2159
2163
  ...lhScores ? {
2160
2164
  lighthousePerf: lhScores.perf,
2161
2165
  lighthouseA11y: lhScores.a11y,
@@ -2555,8 +2559,7 @@ ${snippet}`,
2555
2559
  logger.info({ url: sessionCheckUrl }, "[P5] Session alive \u2014 proceeding with viewport pass");
2556
2560
  }
2557
2561
  } catch (err) {
2558
- logger.warn({ err: String(err) }, "[P5] Session health check failed \u2014 skipping viewport pass to avoid bad screenshots");
2559
- sessionAliveForViewports = false;
2562
+ logger.warn({ err: String(err) }, "[P5] Session health check navigation failed \u2014 proceeding with viewport pass anyway");
2560
2563
  }
2561
2564
  const extraProfiles = (opts.deviceProfiles ?? []).filter((p) => p !== "DESKTOP");
2562
2565
  if (sessionAliveForViewports && extraProfiles.length > 0 && discovered.length > 0) {