@detiq/crawler 1.1.9 → 1.1.10

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (82) hide show
  1. package/dist/accessibility-auditor.js +54 -74
  2. package/dist/ai-login.js +227 -249
  3. package/dist/auth-token-rotator.js +57 -67
  4. package/dist/auth.js +299 -336
  5. package/dist/browser-extension.js +104 -142
  6. package/dist/captcha-solver.js +98 -125
  7. package/dist/color-contrast.js +77 -89
  8. package/dist/content-chunker.js +126 -126
  9. package/dist/content-dedup.js +54 -71
  10. package/dist/cookie-consent.js +68 -83
  11. package/dist/cookie-jar.js +53 -60
  12. package/dist/crawler.js +4494 -5353
  13. package/dist/custom-js-injector.js +45 -61
  14. package/dist/dark-mode-capture.js +67 -81
  15. package/dist/dead-link-detector.js +47 -55
  16. package/dist/element-hash.js +8 -12
  17. package/dist/email-otp.js +154 -195
  18. package/dist/engine-router.js +87 -98
  19. package/dist/export/bulk-export.js +70 -55
  20. package/dist/export/vector-export.js +77 -105
  21. package/dist/extractors/advanced.js +249 -266
  22. package/dist/extractors/artifact-envelope.js +59 -58
  23. package/dist/failure-analysis-agent.js +37 -33
  24. package/dist/feature-flag-detector.js +51 -66
  25. package/dist/form-detector.js +55 -75
  26. package/dist/frontier.js +76 -101
  27. package/dist/gap-pass.js +68 -89
  28. package/dist/graphql-detector.js +62 -72
  29. package/dist/http-engine.js +282 -330
  30. package/dist/incremental-diff.js +54 -87
  31. package/dist/index.js +67 -28
  32. package/dist/input-fuzzer.js +93 -97
  33. package/dist/interactive-probe.js +715 -827
  34. package/dist/job-queue.js +2151 -2169
  35. package/dist/js-coverage.js +52 -56
  36. package/dist/js-router-detector.js +86 -117
  37. package/dist/language-detector.js +51 -63
  38. package/dist/link-classifier.js +32 -50
  39. package/dist/live-view.js +91 -108
  40. package/dist/locator-extractor.js +151 -187
  41. package/dist/logger.js +9 -7
  42. package/dist/markdown-extract.js +48 -61
  43. package/dist/native-crawler.js +780 -946
  44. package/dist/network-observer.js +135 -156
  45. package/dist/network-throttle.js +29 -40
  46. package/dist/openapi-generator.js +77 -92
  47. package/dist/pdf-extractor.js +72 -90
  48. package/dist/performance-metrics.js +80 -102
  49. package/dist/phash.js +125 -144
  50. package/dist/pii-scrubber.js +65 -68
  51. package/dist/playwright-recorder.js +70 -58
  52. package/dist/policies/crawl-safety.js +31 -34
  53. package/dist/profiles/domain-profile.js +130 -125
  54. package/dist/proxy.js +146 -172
  55. package/dist/rate-limiter.js +58 -65
  56. package/dist/redirect-tracker.js +60 -64
  57. package/dist/redis-frontier.js +116 -136
  58. package/dist/redis-queue.js +157 -144
  59. package/dist/response-cache.js +127 -140
  60. package/dist/responsive-detector.js +42 -46
  61. package/dist/retry-policy.js +46 -50
  62. package/dist/robots-txt.js +69 -83
  63. package/dist/screen-reader.js +45 -70
  64. package/dist/screenshot-masker.js +59 -77
  65. package/dist/seo-checker.js +88 -107
  66. package/dist/session-capture.js +192 -212
  67. package/dist/session-export.js +107 -133
  68. package/dist/sitemap-writer.js +38 -41
  69. package/dist/sitemap.js +141 -202
  70. package/dist/skyvern.js +123 -148
  71. package/dist/sqs-queue.js +42 -42
  72. package/dist/sso-token.js +74 -90
  73. package/dist/stealth/human-behavior.js +156 -173
  74. package/dist/telemetry.js +68 -82
  75. package/dist/test-executor.js +309 -335
  76. package/dist/tls-fingerprint.js +77 -88
  77. package/dist/url-pattern-completer.js +71 -90
  78. package/dist/visual-diff-verdict.js +23 -38
  79. package/dist/webhook-notifier.js +71 -69
  80. package/dist/websocket-detector.js +56 -68
  81. package/dist/worker.js +29 -38
  82. package/package.json +1 -1
package/dist/ai-login.js CHANGED
@@ -1,272 +1,250 @@
1
- import { z } from 'zod';
2
- import { AppBrain } from '@detiq/app-brain';
3
- import { solveCaptcha } from './captcha-solver.js';
4
- import { waitForEmailContent } from './email-otp.js';
5
- import { logger } from './logger.js';
6
- import { simulateMouseMovement } from './stealth/human-behavior.js';
1
+ import { z } from "zod";
2
+ import { AppBrain } from "@detiq/app-brain";
3
+ import { solveCaptcha } from "./captcha-solver.js";
4
+ import { waitForEmailContent } from "./email-otp.js";
5
+ import { logger } from "./logger.js";
6
+ import { simulateMouseMovement } from "./stealth/human-behavior.js";
7
7
  const CLICK_INTENT = /\b(click|tap|press|submit|select|choose|toggle)\b/i;
8
- /** Stagehand V3 enforces elementId as /^\d+-\d+$/ — LLM sometimes returns bare integers.
9
- * Catch the resulting AI_NoObjectGeneratedError so callers can fall back to DOM. */
10
8
  function isNoObjectGeneratedError(err) {
11
- const name = String(err?.name ?? '');
12
- const msg = String(err?.message ?? '');
13
- return name.includes('NoObjectGenerated') || msg.includes('NoObjectGenerated') || msg.includes('No object generated');
9
+ const name = String(err?.name ?? "");
10
+ const msg = String(err?.message ?? "");
11
+ return name.includes("NoObjectGenerated") || msg.includes("NoObjectGenerated") || msg.includes("No object generated");
14
12
  }
15
- /** Simulate human mouse movement before click-type actions, then call stagehand.act(). */
16
13
  async function actWithMouseSim(stagehand, instruction) {
17
- if (CLICK_INTENT.test(instruction)) {
18
- try {
19
- const pg = await stagehand.resolvePage();
20
- const viewport = pg.viewportSize() ?? { width: 1280, height: 720 };
21
- // Move mouse toward center of page with some randomness before Stagehand clicks
22
- const targetX = viewport.width * 0.3 + Math.random() * viewport.width * 0.4;
23
- const targetY = viewport.height * 0.2 + Math.random() * viewport.height * 0.5;
24
- await simulateMouseMovement(pg, targetX, targetY);
25
- }
26
- catch { /* non-fatal — don't block the action */ }
14
+ if (CLICK_INTENT.test(instruction)) {
15
+ try {
16
+ const pg = await stagehand.resolvePage();
17
+ const viewport = pg.viewportSize() ?? { width: 1280, height: 720 };
18
+ const targetX = viewport.width * 0.3 + Math.random() * viewport.width * 0.4;
19
+ const targetY = viewport.height * 0.2 + Math.random() * viewport.height * 0.5;
20
+ await simulateMouseMovement(pg, targetX, targetY);
21
+ } catch {
27
22
  }
28
- return stagehand.act(instruction);
23
+ }
24
+ return stagehand.act(instruction);
29
25
  }
30
26
  const TwoFASchema = z.object({
31
- has2FA: z.boolean(),
32
- fieldLabel: z.string().optional(),
27
+ has2FA: z.boolean(),
28
+ fieldLabel: z.string().optional()
33
29
  });
34
30
  const CAPTCHA_SELECTORS = [
35
- 'iframe[src*="recaptcha"]',
36
- 'iframe[src*="hcaptcha"]',
37
- 'iframe[src*="challenges.cloudflare.com"]',
38
- '.cf-turnstile',
39
- '[data-sitekey]',
40
- '#captcha',
41
- '.captcha',
42
- '[id*="captcha"]',
43
- '[class*="captcha"]',
31
+ 'iframe[src*="recaptcha"]',
32
+ 'iframe[src*="hcaptcha"]',
33
+ 'iframe[src*="challenges.cloudflare.com"]',
34
+ ".cf-turnstile",
35
+ "[data-sitekey]",
36
+ "#captcha",
37
+ ".captcha",
38
+ '[id*="captcha"]',
39
+ '[class*="captcha"]'
44
40
  ];
45
- /**
46
- * AI-guided login using Stagehand act/extract.
47
- *
48
- * @param loginInstructions - Optional natural language description of the login flow.
49
- * When provided, an initial stagehand.act() is called with these instructions to
50
- * navigate the UI (e.g. "click Sign In in the nav, fill email, click Continue").
51
- * The standard 3-step credential fill follows immediately after.
52
- * @param opts - Optional tenant/project/job IDs for writing AUTH_STEP CrawlEvidence.
53
- */
54
- export async function loginWithStagehand(stagehand, creds, startUrl, captchaOpts, emailOtpOpts, loginInstructions, opts) {
55
- async function logStep(step, fn) {
56
- const t0 = Date.now();
57
- try {
58
- const r = await fn();
59
- const result = { success: true, detail: r?.description ?? r?.message ?? step };
60
- if (opts?.tenantId && opts?.projectId) {
61
- AppBrain.saveCrawlEvidence(opts.tenantId, opts.projectId, {
62
- jobId: opts?.jobId,
63
- type: 'AUTH_STEP',
64
- payload: { step, ...result, ms: Date.now() - t0 },
65
- }).catch(() => { });
66
- }
67
- return result;
68
- }
69
- catch (e) {
70
- const result = { success: false, detail: e.message ?? 'failed' };
71
- if (opts?.tenantId && opts?.projectId) {
72
- AppBrain.saveCrawlEvidence(opts.tenantId, opts.projectId, {
73
- jobId: opts?.jobId,
74
- type: 'AUTH_STEP',
75
- payload: { step, ...result, ms: Date.now() - t0 },
76
- }).catch(() => { });
77
- }
78
- throw e;
79
- }
80
- }
41
+ async function loginWithStagehand(stagehand, creds, startUrl, captchaOpts, emailOtpOpts, loginInstructions, opts) {
42
+ async function logStep(step, fn) {
43
+ const t0 = Date.now();
81
44
  try {
82
- const pg = await stagehand.resolvePage();
83
- // CAPTCHA detection
84
- for (const sel of CAPTCHA_SELECTORS) {
85
- const count = await pg.locator(sel).count().catch(() => 0);
86
- if (count > 0) {
87
- logger.warn({ selector: sel }, '[crawler] CAPTCHA detected on login page');
88
- if (captchaOpts?.apiKey) {
89
- logger.info({ provider: captchaOpts.provider ?? '2captcha' }, '[crawler] Attempting auto-solve');
90
- try {
91
- const result = await solveCaptcha(pg, startUrl, captchaOpts.apiKey, captchaOpts.provider ?? '2captcha');
92
- if (result) {
93
- logger.info({ provider: result.provider }, '[crawler] CAPTCHA solved — proceeding with login');
94
- break;
95
- }
96
- }
97
- catch (solveErr) {
98
- logger.warn({ err: solveErr }, '[crawler] CAPTCHA solve failed — falling back to manual error');
99
- throw new Error(`CAPTCHA_DETECTED: Auto-solve failed (${String(solveErr)}). Use session capture instead.`);
100
- }
101
- }
102
- else {
103
- throw new Error('CAPTCHA_DETECTED: Login page has a CAPTCHA. Use "Session Capture → Launch Browser" to log in manually, then capture the session. Alternatively, disable CAPTCHA for the ZeTa crawler IP.');
104
- }
105
- break;
106
- }
107
- }
108
- // If caller provided natural language login instructions, use them to navigate to the login form
109
- const MAX_LOGIN_INSTRUCTIONS = 2000;
110
- if (loginInstructions && loginInstructions.length > MAX_LOGIN_INSTRUCTIONS) {
111
- throw new Error(`loginInstructions exceeds ${MAX_LOGIN_INSTRUCTIONS} character limit`);
112
- }
113
- if (loginInstructions) {
114
- await logStep('custom_recipe', async () => {
115
- try {
116
- return await actWithMouseSim(stagehand, loginInstructions);
117
- }
118
- catch (err) {
119
- if (!isNoObjectGeneratedError(err))
120
- throw err;
121
- logger.warn('[crawler] custom_recipe act() failed — Stagehand V3 elementId schema error, skipping to standard fill');
122
- return { description: 'custom_recipe skipped (schema error)' };
123
- }
124
- });
125
- await pg.waitForLoadState('networkidle').catch(() => pg.waitForTimeout(1500));
126
- }
127
- await logStep('fill_username', async () => {
128
- try {
129
- return await actWithMouseSim(stagehand, `Type "${creds.username}" into the username or email input field`);
130
- }
131
- catch (err) {
132
- if (!isNoObjectGeneratedError(err))
133
- throw err;
134
- logger.warn('[crawler] fill_username act() failed — falling back to DOM');
135
- const sel = 'input[type="email"], input[autocomplete="email"], input[autocomplete="username"], input[name*="email" i], input[name*="user" i], input[placeholder*="email" i], input[placeholder*="user" i], input[id*="email" i], input[id*="user" i]';
136
- await pg.locator(sel).first().fill(creds.username);
137
- return { description: 'fill_username via DOM fallback' };
138
- }
45
+ const r = await fn();
46
+ const result = { success: true, detail: r?.description ?? r?.message ?? step };
47
+ if (opts?.tenantId && opts?.projectId) {
48
+ AppBrain.saveCrawlEvidence(opts.tenantId, opts.projectId, {
49
+ jobId: opts?.jobId,
50
+ type: "AUTH_STEP",
51
+ payload: { step, ...result, ms: Date.now() - t0 }
52
+ }).catch(() => {
139
53
  });
140
- // Don't send password to LLM — use direct Playwright fill to avoid
141
- // leaking credentials into the Stagehand/LLM provider's request logs.
142
- await logStep('fill_password', async () => {
143
- const observations = await stagehand.observe('Find the password input field');
144
- const selector = observations[0]?.selector || 'input[type="password"]';
145
- try {
146
- await pg.locator(selector).fill(creds.password);
147
- }
148
- catch {
149
- // Stagehand locator failed (e.g. form re-rendered, proxy can't locate element).
150
- // Fall back to direct DOM fill via evaluate so the password never touches the LLM.
151
- await pg.evaluate((pwd) => {
152
- const input = document.querySelector('input[type="password"]');
153
- if (!input)
154
- throw new Error('No password field found on page');
155
- const setter = Object.getOwnPropertyDescriptor(window.HTMLInputElement.prototype, 'value')?.set;
156
- setter?.call(input, pwd);
157
- input.dispatchEvent(new Event('input', { bubbles: true }));
158
- input.dispatchEvent(new Event('change', { bubbles: true }));
159
- }, creds.password);
160
- }
161
- });
162
- await logStep('click_submit', async () => {
163
- try {
164
- return await actWithMouseSim(stagehand, 'Click the sign in, log in, login, or continue button to submit credentials');
165
- }
166
- catch (err) {
167
- if (!isNoObjectGeneratedError(err))
168
- throw err;
169
- logger.warn('[crawler] click_submit act() failed — falling back to DOM');
170
- const sel = 'button[type="submit"], input[type="submit"], button:has-text("Sign in"), button:has-text("Log in"), button:has-text("Login"), button:has-text("Continue"), button:has-text("Next"), button:has-text("Sign In"), [role="button"]:has-text("Sign in"), [role="button"]:has-text("Login")';
171
- await pg.locator(sel).first().click();
172
- await pg.waitForTimeout(500);
173
- return { description: 'click_submit via DOM fallback' };
174
- }
54
+ }
55
+ return result;
56
+ } catch (e) {
57
+ const result = { success: false, detail: e.message ?? "failed" };
58
+ if (opts?.tenantId && opts?.projectId) {
59
+ AppBrain.saveCrawlEvidence(opts.tenantId, opts.projectId, {
60
+ jobId: opts?.jobId,
61
+ type: "AUTH_STEP",
62
+ payload: { step, ...result, ms: Date.now() - t0 }
63
+ }).catch(() => {
175
64
  });
176
- await pg.waitForLoadState('networkidle').catch(() => pg.waitForTimeout(2000));
177
- const twoFA = await stagehand.extract('Is there a two-factor authentication, OTP, or verification code input visible right now?', TwoFASchema);
178
- if (twoFA.has2FA) {
179
- if (opts?.tenantId && opts?.projectId) {
180
- AppBrain.saveCrawlEvidence(opts.tenantId, opts.projectId, {
181
- jobId: opts?.jobId,
182
- type: 'AUTH_STEP',
183
- payload: { step: '2fa_detect', success: true, detail: twoFA.fieldLabel ?? '2FA detected', ms: 0 },
184
- }).catch(() => { });
185
- }
186
- let code = creds.oneTimeCode ?? await creds.getOneTimeCode?.();
187
- if (!code && emailOtpOpts?.apiKey && emailOtpOpts?.inboxId) {
188
- const since = new Date().toISOString();
189
- logger.info('[crawler] 2FA detected — waiting for email OTP or magic link via MailSlurp');
190
- const emailContent = await waitForEmailContent({ provider: 'mailslurp', apiKey: emailOtpOpts.apiKey, inboxId: emailOtpOpts.inboxId }, since);
191
- if (emailContent?.magicLink) {
192
- logger.info({ url: emailContent.magicLink }, '[crawler] Magic link received — navigating');
193
- await pg.goto(emailContent.magicLink, { waitUntil: 'domcontentloaded' });
194
- await pg.waitForLoadState('networkidle').catch(() => { });
195
- return true; // magic link navigation = login complete
196
- }
197
- code = emailContent?.otp ?? undefined;
198
- if (code) {
199
- logger.info('[crawler] Email OTP received');
200
- }
201
- else {
202
- logger.warn('[crawler] Email OTP timed out after 60s');
203
- }
204
- }
205
- if (!code) {
206
- console.warn('[crawler] 2FA detected but no oneTimeCode provided');
207
- return false;
65
+ }
66
+ throw e;
67
+ }
68
+ }
69
+ try {
70
+ const pg = await stagehand.resolvePage();
71
+ for (const sel of CAPTCHA_SELECTORS) {
72
+ const count = await pg.locator(sel).count().catch(() => 0);
73
+ if (count > 0) {
74
+ logger.warn({ selector: sel }, "[crawler] CAPTCHA detected on login page");
75
+ if (captchaOpts?.apiKey) {
76
+ logger.info({ provider: captchaOpts.provider ?? "2captcha" }, "[crawler] Attempting auto-solve");
77
+ try {
78
+ const result = await solveCaptcha(pg, startUrl, captchaOpts.apiKey, captchaOpts.provider ?? "2captcha");
79
+ if (result) {
80
+ logger.info({ provider: result.provider }, "[crawler] CAPTCHA solved \u2014 proceeding with login");
81
+ break;
208
82
  }
209
- await logStep('2fa_inject', async () => {
210
- try {
211
- return await actWithMouseSim(stagehand, `Enter "${code}" into the ${twoFA.fieldLabel ?? 'verification code'} field`);
212
- }
213
- catch (err) {
214
- if (!isNoObjectGeneratedError(err))
215
- throw err;
216
- logger.warn('[crawler] 2fa_inject act() failed — falling back to DOM');
217
- const sel = 'input[autocomplete="one-time-code"], input[name*="otp" i], input[name*="code" i], input[name*="token" i], input[inputmode="numeric"], input[type="number"]';
218
- await pg.locator(sel).first().fill(code);
219
- return { description: '2fa_inject via DOM fallback' };
220
- }
221
- });
222
- await logStep('2fa_submit', async () => {
223
- try {
224
- return await actWithMouseSim(stagehand, 'Click verify or confirm to submit the verification code');
225
- }
226
- catch (err) {
227
- if (!isNoObjectGeneratedError(err))
228
- throw err;
229
- logger.warn('[crawler] 2fa_submit act() failed — falling back to DOM');
230
- const sel = 'button[type="submit"], button:has-text("Verify"), button:has-text("Confirm"), button:has-text("Submit"), button:has-text("Continue")';
231
- await pg.locator(sel).first().click();
232
- return { description: '2fa_submit via DOM fallback' };
233
- }
234
- });
235
- await pg.waitForLoadState('networkidle').catch(() => pg.waitForTimeout(2000));
83
+ } catch (solveErr) {
84
+ logger.warn({ err: solveErr }, "[crawler] CAPTCHA solve failed \u2014 falling back to manual error");
85
+ throw new Error(`CAPTCHA_DETECTED: Auto-solve failed (${String(solveErr)}). Use session capture instead.`);
86
+ }
87
+ } else {
88
+ throw new Error(
89
+ 'CAPTCHA_DETECTED: Login page has a CAPTCHA. Use "Session Capture \u2192 Launch Browser" to log in manually, then capture the session. Alternatively, disable CAPTCHA for the ZeTa crawler IP.'
90
+ );
236
91
  }
237
- // URL-based success check
238
- const currentUrl = pg.url();
239
- const loginPathPattern = /\/(login|signin|sign-in|auth|authenticate|log-in|account\/login)(\?|$|\/)/i;
92
+ break;
93
+ }
94
+ }
95
+ const MAX_LOGIN_INSTRUCTIONS = 2e3;
96
+ if (loginInstructions && loginInstructions.length > MAX_LOGIN_INSTRUCTIONS) {
97
+ throw new Error(`loginInstructions exceeds ${MAX_LOGIN_INSTRUCTIONS} character limit`);
98
+ }
99
+ if (loginInstructions) {
100
+ await logStep("custom_recipe", async () => {
240
101
  try {
241
- const { pathname } = new URL(currentUrl);
242
- const wasRedirected = currentUrl !== startUrl;
243
- const stillOnLoginPath = loginPathPattern.test(pathname);
244
- if (wasRedirected && !stillOnLoginPath) {
245
- logger.info({ url: currentUrl }, 'Login confirmed via URL change');
246
- return true;
247
- }
102
+ return await actWithMouseSim(stagehand, loginInstructions);
103
+ } catch (err) {
104
+ if (!isNoObjectGeneratedError(err)) throw err;
105
+ logger.warn("[crawler] custom_recipe act() failed \u2014 Stagehand V3 elementId schema error, skipping to standard fill");
106
+ return { description: "custom_recipe skipped (schema error)" };
248
107
  }
249
- catch { /* invalid URL — fall through */ }
250
- const passwordCount = await pg.locator('input[type="password"]').count().catch(() => 1);
251
- const bodyText = await pg.evaluate(() => document.body?.innerText?.trim()?.length ?? 0).catch(() => 0);
252
- if (passwordCount === 0 && bodyText > 30) {
253
- logger.info('Login confirmed: password field no longer visible');
254
- return true;
108
+ });
109
+ await pg.waitForLoadState("networkidle").catch(() => pg.waitForTimeout(1500));
110
+ }
111
+ await logStep("fill_username", async () => {
112
+ try {
113
+ return await actWithMouseSim(stagehand, `Type "${creds.username}" into the username or email input field`);
114
+ } catch (err) {
115
+ if (!isNoObjectGeneratedError(err)) throw err;
116
+ logger.warn("[crawler] fill_username act() failed \u2014 falling back to DOM");
117
+ const sel = 'input[type="email"], input[autocomplete="email"], input[autocomplete="username"], input[name*="email" i], input[name*="user" i], input[placeholder*="email" i], input[placeholder*="user" i], input[id*="email" i], input[id*="user" i]';
118
+ await pg.locator(sel).first().fill(creds.username);
119
+ return { description: "fill_username via DOM fallback" };
120
+ }
121
+ });
122
+ await logStep("fill_password", async () => {
123
+ const observations = await stagehand.observe("Find the password input field");
124
+ const selector = observations[0]?.selector || 'input[type="password"]';
125
+ try {
126
+ await pg.locator(selector).fill(creds.password);
127
+ } catch {
128
+ await pg.evaluate((pwd) => {
129
+ const input = document.querySelector('input[type="password"]');
130
+ if (!input) throw new Error("No password field found on page");
131
+ const setter = Object.getOwnPropertyDescriptor(window.HTMLInputElement.prototype, "value")?.set;
132
+ setter?.call(input, pwd);
133
+ input.dispatchEvent(new Event("input", { bubbles: true }));
134
+ input.dispatchEvent(new Event("change", { bubbles: true }));
135
+ }, creds.password);
136
+ }
137
+ });
138
+ await logStep("click_submit", async () => {
139
+ try {
140
+ return await actWithMouseSim(stagehand, "Click the sign in, log in, login, or continue button to submit credentials");
141
+ } catch (err) {
142
+ if (!isNoObjectGeneratedError(err)) throw err;
143
+ logger.warn("[crawler] click_submit act() failed \u2014 falling back to DOM");
144
+ const sel = 'button[type="submit"], input[type="submit"], button:has-text("Sign in"), button:has-text("Log in"), button:has-text("Login"), button:has-text("Continue"), button:has-text("Next"), button:has-text("Sign In"), [role="button"]:has-text("Sign in"), [role="button"]:has-text("Login")';
145
+ await pg.locator(sel).first().click();
146
+ await pg.waitForTimeout(500);
147
+ return { description: "click_submit via DOM fallback" };
148
+ }
149
+ });
150
+ await pg.waitForLoadState("networkidle").catch(() => pg.waitForTimeout(2e3));
151
+ const twoFA = await stagehand.extract(
152
+ "Is there a two-factor authentication, OTP, or verification code input visible right now?",
153
+ TwoFASchema
154
+ );
155
+ if (twoFA.has2FA) {
156
+ if (opts?.tenantId && opts?.projectId) {
157
+ AppBrain.saveCrawlEvidence(opts.tenantId, opts.projectId, {
158
+ jobId: opts?.jobId,
159
+ type: "AUTH_STEP",
160
+ payload: { step: "2fa_detect", success: true, detail: twoFA.fieldLabel ?? "2FA detected", ms: 0 }
161
+ }).catch(() => {
162
+ });
163
+ }
164
+ let code = creds.oneTimeCode ?? await creds.getOneTimeCode?.();
165
+ if (!code && emailOtpOpts?.apiKey && emailOtpOpts?.inboxId) {
166
+ const since = (/* @__PURE__ */ new Date()).toISOString();
167
+ logger.info("[crawler] 2FA detected \u2014 waiting for email OTP or magic link via MailSlurp");
168
+ const emailContent = await waitForEmailContent(
169
+ { provider: "mailslurp", apiKey: emailOtpOpts.apiKey, inboxId: emailOtpOpts.inboxId },
170
+ since
171
+ );
172
+ if (emailContent?.magicLink) {
173
+ logger.info({ url: emailContent.magicLink }, "[crawler] Magic link received \u2014 navigating");
174
+ await pg.goto(emailContent.magicLink, { waitUntil: "domcontentloaded" });
175
+ await pg.waitForLoadState("networkidle").catch(() => {
176
+ });
177
+ return true;
255
178
  }
256
- logger.warn({ url: currentUrl }, 'Login appears failed');
257
- try {
258
- const screenshotPath = `/tmp/zeta-auth-failure-${Date.now()}.jpg`;
259
- await pg.screenshot({ path: screenshotPath, type: 'jpeg', quality: 70, fullPage: false });
260
- logger.warn({ screenshotPath }, '[crawler] Auth failure screenshot saved');
179
+ code = emailContent?.otp ?? void 0;
180
+ if (code) {
181
+ logger.info("[crawler] Email OTP received");
182
+ } else {
183
+ logger.warn("[crawler] Email OTP timed out after 60s");
261
184
  }
262
- catch { /* non-fatal */ }
185
+ }
186
+ if (!code) {
187
+ console.warn("[crawler] 2FA detected but no oneTimeCode provided");
263
188
  return false;
189
+ }
190
+ await logStep("2fa_inject", async () => {
191
+ try {
192
+ return await actWithMouseSim(stagehand, `Enter "${code}" into the ${twoFA.fieldLabel ?? "verification code"} field`);
193
+ } catch (err) {
194
+ if (!isNoObjectGeneratedError(err)) throw err;
195
+ logger.warn("[crawler] 2fa_inject act() failed \u2014 falling back to DOM");
196
+ const sel = 'input[autocomplete="one-time-code"], input[name*="otp" i], input[name*="code" i], input[name*="token" i], input[inputmode="numeric"], input[type="number"]';
197
+ await pg.locator(sel).first().fill(code);
198
+ return { description: "2fa_inject via DOM fallback" };
199
+ }
200
+ });
201
+ await logStep("2fa_submit", async () => {
202
+ try {
203
+ return await actWithMouseSim(stagehand, "Click verify or confirm to submit the verification code");
204
+ } catch (err) {
205
+ if (!isNoObjectGeneratedError(err)) throw err;
206
+ logger.warn("[crawler] 2fa_submit act() failed \u2014 falling back to DOM");
207
+ const sel = 'button[type="submit"], button:has-text("Verify"), button:has-text("Confirm"), button:has-text("Submit"), button:has-text("Continue")';
208
+ await pg.locator(sel).first().click();
209
+ return { description: "2fa_submit via DOM fallback" };
210
+ }
211
+ });
212
+ await pg.waitForLoadState("networkidle").catch(() => pg.waitForTimeout(2e3));
264
213
  }
265
- catch (err) {
266
- const msg = String(err.message ?? err);
267
- if (msg.includes('CAPTCHA_DETECTED'))
268
- throw err;
269
- logger.error({ err }, 'Login error');
270
- return false;
214
+ const currentUrl = pg.url();
215
+ const loginPathPattern = /\/(login|signin|sign-in|auth|authenticate|log-in|account\/login)(\?|$|\/)/i;
216
+ try {
217
+ const { pathname } = new URL(currentUrl);
218
+ const wasRedirected = currentUrl !== startUrl;
219
+ const stillOnLoginPath = loginPathPattern.test(pathname);
220
+ if (wasRedirected && !stillOnLoginPath) {
221
+ logger.info({ url: currentUrl }, "Login confirmed via URL change");
222
+ return true;
223
+ }
224
+ } catch {
225
+ }
226
+ const passwordCount = await pg.locator('input[type="password"]').count().catch(() => 1);
227
+ const bodyText = await pg.evaluate(() => document.body?.innerText?.trim()?.length ?? 0).catch(() => 0);
228
+ if (passwordCount === 0 && bodyText > 30) {
229
+ logger.info("Login confirmed: password field no longer visible");
230
+ return true;
231
+ }
232
+ logger.warn({ url: currentUrl }, "Login appears failed");
233
+ try {
234
+ const screenshotPath = `/tmp/zeta-auth-failure-${Date.now()}.jpg`;
235
+ await pg.screenshot({ path: screenshotPath, type: "jpeg", quality: 70, fullPage: false });
236
+ logger.warn({ screenshotPath }, "[crawler] Auth failure screenshot saved");
237
+ } catch {
271
238
  }
239
+ return false;
240
+ } catch (err) {
241
+ const msg = String(err.message ?? err);
242
+ if (msg.includes("CAPTCHA_DETECTED")) throw err;
243
+ logger.error({ err }, "Login error");
244
+ return false;
245
+ }
272
246
  }
247
+ export {
248
+ loginWithStagehand
249
+ };
250
+ //# sourceMappingURL=ai-login.js.map
@@ -1,68 +1,58 @@
1
- /**
2
- * Auth token rotator — manage a pool of authenticated sessions for multi-account crawls.
3
- *
4
- * Use case: crawl the same app as admin+user+guest in one crawl job to discover
5
- * role-specific screens that only appear when authenticated with different privileges.
6
- * Each session is a Playwright storageState JSON string.
7
- *
8
- * Sessions are checked out (locked) before use and returned after the page is done.
9
- * Round-robin assignment with stale-session eviction after maxAgeMs.
10
- */
11
- export class AuthTokenRotator {
12
- pool;
13
- maxAgeMs;
14
- maxRequests;
15
- constructor(opts) {
16
- this.maxAgeMs = opts.maxAgeMs ?? 4 * 60 * 60 * 1000;
17
- this.maxRequests = opts.maxRequestsPerSession ?? 100;
18
- this.pool = opts.sessions.map((s, i) => ({
19
- id: `session-${i}-${s.role}`,
20
- role: s.role,
21
- storageState: s.storageState,
22
- createdAt: Date.now(),
23
- checkedOutAt: null,
24
- requestCount: 0,
25
- }));
26
- }
27
- /** Get the next available session. Returns null if all are checked out. */
28
- checkout() {
29
- const now = Date.now();
30
- const available = this.pool.filter(s => !s.checkedOutAt &&
31
- now - s.createdAt < this.maxAgeMs &&
32
- s.requestCount < this.maxRequests);
33
- if (available.length === 0)
34
- return null;
35
- // Round-robin: pick lowest requestCount
36
- available.sort((a, b) => a.requestCount - b.requestCount);
37
- const session = available[0];
38
- session.checkedOutAt = now;
39
- return session;
40
- }
41
- /** Return a session to the pool after use. */
42
- checkin(sessionId, updatedStorageState) {
43
- const s = this.pool.find(e => e.id === sessionId);
44
- if (!s)
45
- return;
46
- s.checkedOutAt = null;
47
- s.requestCount++;
48
- if (updatedStorageState)
49
- s.storageState = updatedStorageState;
50
- }
51
- /** Get all available sessions (for parallel crawl setup). */
52
- getAll() {
53
- return this.pool;
54
- }
55
- /** Stats for monitoring. */
56
- stats() {
57
- const now = Date.now();
58
- const stale = this.pool.filter(s => now - s.createdAt >= this.maxAgeMs || s.requestCount >= this.maxRequests).length;
59
- const checkedOut = this.pool.filter(s => !!s.checkedOutAt).length;
60
- return { total: this.pool.length, available: this.pool.length - checkedOut - stale, checkedOut, stale };
61
- }
62
- /** Add a new session to the pool at runtime. */
63
- addSession(role, storageState) {
64
- const id = `session-${this.pool.length}-${role}`;
65
- this.pool.push({ id, role, storageState, createdAt: Date.now(), checkedOutAt: null, requestCount: 0 });
66
- return id;
67
- }
1
+ class AuthTokenRotator {
2
+ pool;
3
+ maxAgeMs;
4
+ maxRequests;
5
+ constructor(opts) {
6
+ this.maxAgeMs = opts.maxAgeMs ?? 4 * 60 * 60 * 1e3;
7
+ this.maxRequests = opts.maxRequestsPerSession ?? 100;
8
+ this.pool = opts.sessions.map((s, i) => ({
9
+ id: `session-${i}-${s.role}`,
10
+ role: s.role,
11
+ storageState: s.storageState,
12
+ createdAt: Date.now(),
13
+ checkedOutAt: null,
14
+ requestCount: 0
15
+ }));
16
+ }
17
+ /** Get the next available session. Returns null if all are checked out. */
18
+ checkout() {
19
+ const now = Date.now();
20
+ const available = this.pool.filter(
21
+ (s) => !s.checkedOutAt && now - s.createdAt < this.maxAgeMs && s.requestCount < this.maxRequests
22
+ );
23
+ if (available.length === 0) return null;
24
+ available.sort((a, b) => a.requestCount - b.requestCount);
25
+ const session = available[0];
26
+ session.checkedOutAt = now;
27
+ return session;
28
+ }
29
+ /** Return a session to the pool after use. */
30
+ checkin(sessionId, updatedStorageState) {
31
+ const s = this.pool.find((e) => e.id === sessionId);
32
+ if (!s) return;
33
+ s.checkedOutAt = null;
34
+ s.requestCount++;
35
+ if (updatedStorageState) s.storageState = updatedStorageState;
36
+ }
37
+ /** Get all available sessions (for parallel crawl setup). */
38
+ getAll() {
39
+ return this.pool;
40
+ }
41
+ /** Stats for monitoring. */
42
+ stats() {
43
+ const now = Date.now();
44
+ const stale = this.pool.filter((s) => now - s.createdAt >= this.maxAgeMs || s.requestCount >= this.maxRequests).length;
45
+ const checkedOut = this.pool.filter((s) => !!s.checkedOutAt).length;
46
+ return { total: this.pool.length, available: this.pool.length - checkedOut - stale, checkedOut, stale };
47
+ }
48
+ /** Add a new session to the pool at runtime. */
49
+ addSession(role, storageState) {
50
+ const id = `session-${this.pool.length}-${role}`;
51
+ this.pool.push({ id, role, storageState, createdAt: Date.now(), checkedOutAt: null, requestCount: 0 });
52
+ return id;
53
+ }
68
54
  }
55
+ export {
56
+ AuthTokenRotator
57
+ };
58
+ //# sourceMappingURL=auth-token-rotator.js.map