@detiq/crawler 1.1.9 → 1.1.10
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/accessibility-auditor.js +54 -74
- package/dist/ai-login.js +227 -249
- package/dist/auth-token-rotator.js +57 -67
- package/dist/auth.js +299 -336
- package/dist/browser-extension.js +104 -142
- package/dist/captcha-solver.js +98 -125
- package/dist/color-contrast.js +77 -89
- package/dist/content-chunker.js +126 -126
- package/dist/content-dedup.js +54 -71
- package/dist/cookie-consent.js +68 -83
- package/dist/cookie-jar.js +53 -60
- package/dist/crawler.js +4494 -5353
- package/dist/custom-js-injector.js +45 -61
- package/dist/dark-mode-capture.js +67 -81
- package/dist/dead-link-detector.js +47 -55
- package/dist/element-hash.js +8 -12
- package/dist/email-otp.js +154 -195
- package/dist/engine-router.js +87 -98
- package/dist/export/bulk-export.js +70 -55
- package/dist/export/vector-export.js +77 -105
- package/dist/extractors/advanced.js +249 -266
- package/dist/extractors/artifact-envelope.js +59 -58
- package/dist/failure-analysis-agent.js +37 -33
- package/dist/feature-flag-detector.js +51 -66
- package/dist/form-detector.js +55 -75
- package/dist/frontier.js +76 -101
- package/dist/gap-pass.js +68 -89
- package/dist/graphql-detector.js +62 -72
- package/dist/http-engine.js +282 -330
- package/dist/incremental-diff.js +54 -87
- package/dist/index.js +67 -28
- package/dist/input-fuzzer.js +93 -97
- package/dist/interactive-probe.js +715 -827
- package/dist/job-queue.js +2151 -2169
- package/dist/js-coverage.js +52 -56
- package/dist/js-router-detector.js +86 -117
- package/dist/language-detector.js +51 -63
- package/dist/link-classifier.js +32 -50
- package/dist/live-view.js +91 -108
- package/dist/locator-extractor.js +151 -187
- package/dist/logger.js +9 -7
- package/dist/markdown-extract.js +48 -61
- package/dist/native-crawler.js +780 -946
- package/dist/network-observer.js +135 -156
- package/dist/network-throttle.js +29 -40
- package/dist/openapi-generator.js +77 -92
- package/dist/pdf-extractor.js +72 -90
- package/dist/performance-metrics.js +80 -102
- package/dist/phash.js +125 -144
- package/dist/pii-scrubber.js +65 -68
- package/dist/playwright-recorder.js +70 -58
- package/dist/policies/crawl-safety.js +31 -34
- package/dist/profiles/domain-profile.js +130 -125
- package/dist/proxy.js +146 -172
- package/dist/rate-limiter.js +58 -65
- package/dist/redirect-tracker.js +60 -64
- package/dist/redis-frontier.js +116 -136
- package/dist/redis-queue.js +157 -144
- package/dist/response-cache.js +127 -140
- package/dist/responsive-detector.js +42 -46
- package/dist/retry-policy.js +46 -50
- package/dist/robots-txt.js +69 -83
- package/dist/screen-reader.js +45 -70
- package/dist/screenshot-masker.js +59 -77
- package/dist/seo-checker.js +88 -107
- package/dist/session-capture.js +192 -212
- package/dist/session-export.js +107 -133
- package/dist/sitemap-writer.js +38 -41
- package/dist/sitemap.js +141 -202
- package/dist/skyvern.js +123 -148
- package/dist/sqs-queue.js +42 -42
- package/dist/sso-token.js +74 -90
- package/dist/stealth/human-behavior.js +156 -173
- package/dist/telemetry.js +68 -82
- package/dist/test-executor.js +309 -335
- package/dist/tls-fingerprint.js +77 -88
- package/dist/url-pattern-completer.js +71 -90
- package/dist/visual-diff-verdict.js +23 -38
- package/dist/webhook-notifier.js +71 -69
- package/dist/websocket-detector.js +56 -68
- package/dist/worker.js +29 -38
- package/package.json +1 -1
package/dist/ai-login.js
CHANGED
|
@@ -1,272 +1,250 @@
|
|
|
1
|
-
import { z } from
|
|
2
|
-
import { AppBrain } from
|
|
3
|
-
import { solveCaptcha } from
|
|
4
|
-
import { waitForEmailContent } from
|
|
5
|
-
import { logger } from
|
|
6
|
-
import { simulateMouseMovement } from
|
|
1
|
+
import { z } from "zod";
|
|
2
|
+
import { AppBrain } from "@detiq/app-brain";
|
|
3
|
+
import { solveCaptcha } from "./captcha-solver.js";
|
|
4
|
+
import { waitForEmailContent } from "./email-otp.js";
|
|
5
|
+
import { logger } from "./logger.js";
|
|
6
|
+
import { simulateMouseMovement } from "./stealth/human-behavior.js";
|
|
7
7
|
const CLICK_INTENT = /\b(click|tap|press|submit|select|choose|toggle)\b/i;
|
|
8
|
-
/** Stagehand V3 enforces elementId as /^\d+-\d+$/ — LLM sometimes returns bare integers.
|
|
9
|
-
* Catch the resulting AI_NoObjectGeneratedError so callers can fall back to DOM. */
|
|
10
8
|
function isNoObjectGeneratedError(err) {
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
9
|
+
const name = String(err?.name ?? "");
|
|
10
|
+
const msg = String(err?.message ?? "");
|
|
11
|
+
return name.includes("NoObjectGenerated") || msg.includes("NoObjectGenerated") || msg.includes("No object generated");
|
|
14
12
|
}
|
|
15
|
-
/** Simulate human mouse movement before click-type actions, then call stagehand.act(). */
|
|
16
13
|
async function actWithMouseSim(stagehand, instruction) {
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
}
|
|
26
|
-
catch { /* non-fatal — don't block the action */ }
|
|
14
|
+
if (CLICK_INTENT.test(instruction)) {
|
|
15
|
+
try {
|
|
16
|
+
const pg = await stagehand.resolvePage();
|
|
17
|
+
const viewport = pg.viewportSize() ?? { width: 1280, height: 720 };
|
|
18
|
+
const targetX = viewport.width * 0.3 + Math.random() * viewport.width * 0.4;
|
|
19
|
+
const targetY = viewport.height * 0.2 + Math.random() * viewport.height * 0.5;
|
|
20
|
+
await simulateMouseMovement(pg, targetX, targetY);
|
|
21
|
+
} catch {
|
|
27
22
|
}
|
|
28
|
-
|
|
23
|
+
}
|
|
24
|
+
return stagehand.act(instruction);
|
|
29
25
|
}
|
|
30
26
|
const TwoFASchema = z.object({
|
|
31
|
-
|
|
32
|
-
|
|
27
|
+
has2FA: z.boolean(),
|
|
28
|
+
fieldLabel: z.string().optional()
|
|
33
29
|
});
|
|
34
30
|
const CAPTCHA_SELECTORS = [
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
31
|
+
'iframe[src*="recaptcha"]',
|
|
32
|
+
'iframe[src*="hcaptcha"]',
|
|
33
|
+
'iframe[src*="challenges.cloudflare.com"]',
|
|
34
|
+
".cf-turnstile",
|
|
35
|
+
"[data-sitekey]",
|
|
36
|
+
"#captcha",
|
|
37
|
+
".captcha",
|
|
38
|
+
'[id*="captcha"]',
|
|
39
|
+
'[class*="captcha"]'
|
|
44
40
|
];
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
* @param loginInstructions - Optional natural language description of the login flow.
|
|
49
|
-
* When provided, an initial stagehand.act() is called with these instructions to
|
|
50
|
-
* navigate the UI (e.g. "click Sign In in the nav, fill email, click Continue").
|
|
51
|
-
* The standard 3-step credential fill follows immediately after.
|
|
52
|
-
* @param opts - Optional tenant/project/job IDs for writing AUTH_STEP CrawlEvidence.
|
|
53
|
-
*/
|
|
54
|
-
export async function loginWithStagehand(stagehand, creds, startUrl, captchaOpts, emailOtpOpts, loginInstructions, opts) {
|
|
55
|
-
async function logStep(step, fn) {
|
|
56
|
-
const t0 = Date.now();
|
|
57
|
-
try {
|
|
58
|
-
const r = await fn();
|
|
59
|
-
const result = { success: true, detail: r?.description ?? r?.message ?? step };
|
|
60
|
-
if (opts?.tenantId && opts?.projectId) {
|
|
61
|
-
AppBrain.saveCrawlEvidence(opts.tenantId, opts.projectId, {
|
|
62
|
-
jobId: opts?.jobId,
|
|
63
|
-
type: 'AUTH_STEP',
|
|
64
|
-
payload: { step, ...result, ms: Date.now() - t0 },
|
|
65
|
-
}).catch(() => { });
|
|
66
|
-
}
|
|
67
|
-
return result;
|
|
68
|
-
}
|
|
69
|
-
catch (e) {
|
|
70
|
-
const result = { success: false, detail: e.message ?? 'failed' };
|
|
71
|
-
if (opts?.tenantId && opts?.projectId) {
|
|
72
|
-
AppBrain.saveCrawlEvidence(opts.tenantId, opts.projectId, {
|
|
73
|
-
jobId: opts?.jobId,
|
|
74
|
-
type: 'AUTH_STEP',
|
|
75
|
-
payload: { step, ...result, ms: Date.now() - t0 },
|
|
76
|
-
}).catch(() => { });
|
|
77
|
-
}
|
|
78
|
-
throw e;
|
|
79
|
-
}
|
|
80
|
-
}
|
|
41
|
+
async function loginWithStagehand(stagehand, creds, startUrl, captchaOpts, emailOtpOpts, loginInstructions, opts) {
|
|
42
|
+
async function logStep(step, fn) {
|
|
43
|
+
const t0 = Date.now();
|
|
81
44
|
try {
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
try {
|
|
91
|
-
const result = await solveCaptcha(pg, startUrl, captchaOpts.apiKey, captchaOpts.provider ?? '2captcha');
|
|
92
|
-
if (result) {
|
|
93
|
-
logger.info({ provider: result.provider }, '[crawler] CAPTCHA solved — proceeding with login');
|
|
94
|
-
break;
|
|
95
|
-
}
|
|
96
|
-
}
|
|
97
|
-
catch (solveErr) {
|
|
98
|
-
logger.warn({ err: solveErr }, '[crawler] CAPTCHA solve failed — falling back to manual error');
|
|
99
|
-
throw new Error(`CAPTCHA_DETECTED: Auto-solve failed (${String(solveErr)}). Use session capture instead.`);
|
|
100
|
-
}
|
|
101
|
-
}
|
|
102
|
-
else {
|
|
103
|
-
throw new Error('CAPTCHA_DETECTED: Login page has a CAPTCHA. Use "Session Capture → Launch Browser" to log in manually, then capture the session. Alternatively, disable CAPTCHA for the ZeTa crawler IP.');
|
|
104
|
-
}
|
|
105
|
-
break;
|
|
106
|
-
}
|
|
107
|
-
}
|
|
108
|
-
// If caller provided natural language login instructions, use them to navigate to the login form
|
|
109
|
-
const MAX_LOGIN_INSTRUCTIONS = 2000;
|
|
110
|
-
if (loginInstructions && loginInstructions.length > MAX_LOGIN_INSTRUCTIONS) {
|
|
111
|
-
throw new Error(`loginInstructions exceeds ${MAX_LOGIN_INSTRUCTIONS} character limit`);
|
|
112
|
-
}
|
|
113
|
-
if (loginInstructions) {
|
|
114
|
-
await logStep('custom_recipe', async () => {
|
|
115
|
-
try {
|
|
116
|
-
return await actWithMouseSim(stagehand, loginInstructions);
|
|
117
|
-
}
|
|
118
|
-
catch (err) {
|
|
119
|
-
if (!isNoObjectGeneratedError(err))
|
|
120
|
-
throw err;
|
|
121
|
-
logger.warn('[crawler] custom_recipe act() failed — Stagehand V3 elementId schema error, skipping to standard fill');
|
|
122
|
-
return { description: 'custom_recipe skipped (schema error)' };
|
|
123
|
-
}
|
|
124
|
-
});
|
|
125
|
-
await pg.waitForLoadState('networkidle').catch(() => pg.waitForTimeout(1500));
|
|
126
|
-
}
|
|
127
|
-
await logStep('fill_username', async () => {
|
|
128
|
-
try {
|
|
129
|
-
return await actWithMouseSim(stagehand, `Type "${creds.username}" into the username or email input field`);
|
|
130
|
-
}
|
|
131
|
-
catch (err) {
|
|
132
|
-
if (!isNoObjectGeneratedError(err))
|
|
133
|
-
throw err;
|
|
134
|
-
logger.warn('[crawler] fill_username act() failed — falling back to DOM');
|
|
135
|
-
const sel = 'input[type="email"], input[autocomplete="email"], input[autocomplete="username"], input[name*="email" i], input[name*="user" i], input[placeholder*="email" i], input[placeholder*="user" i], input[id*="email" i], input[id*="user" i]';
|
|
136
|
-
await pg.locator(sel).first().fill(creds.username);
|
|
137
|
-
return { description: 'fill_username via DOM fallback' };
|
|
138
|
-
}
|
|
45
|
+
const r = await fn();
|
|
46
|
+
const result = { success: true, detail: r?.description ?? r?.message ?? step };
|
|
47
|
+
if (opts?.tenantId && opts?.projectId) {
|
|
48
|
+
AppBrain.saveCrawlEvidence(opts.tenantId, opts.projectId, {
|
|
49
|
+
jobId: opts?.jobId,
|
|
50
|
+
type: "AUTH_STEP",
|
|
51
|
+
payload: { step, ...result, ms: Date.now() - t0 }
|
|
52
|
+
}).catch(() => {
|
|
139
53
|
});
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
// Fall back to direct DOM fill via evaluate so the password never touches the LLM.
|
|
151
|
-
await pg.evaluate((pwd) => {
|
|
152
|
-
const input = document.querySelector('input[type="password"]');
|
|
153
|
-
if (!input)
|
|
154
|
-
throw new Error('No password field found on page');
|
|
155
|
-
const setter = Object.getOwnPropertyDescriptor(window.HTMLInputElement.prototype, 'value')?.set;
|
|
156
|
-
setter?.call(input, pwd);
|
|
157
|
-
input.dispatchEvent(new Event('input', { bubbles: true }));
|
|
158
|
-
input.dispatchEvent(new Event('change', { bubbles: true }));
|
|
159
|
-
}, creds.password);
|
|
160
|
-
}
|
|
161
|
-
});
|
|
162
|
-
await logStep('click_submit', async () => {
|
|
163
|
-
try {
|
|
164
|
-
return await actWithMouseSim(stagehand, 'Click the sign in, log in, login, or continue button to submit credentials');
|
|
165
|
-
}
|
|
166
|
-
catch (err) {
|
|
167
|
-
if (!isNoObjectGeneratedError(err))
|
|
168
|
-
throw err;
|
|
169
|
-
logger.warn('[crawler] click_submit act() failed — falling back to DOM');
|
|
170
|
-
const sel = 'button[type="submit"], input[type="submit"], button:has-text("Sign in"), button:has-text("Log in"), button:has-text("Login"), button:has-text("Continue"), button:has-text("Next"), button:has-text("Sign In"), [role="button"]:has-text("Sign in"), [role="button"]:has-text("Login")';
|
|
171
|
-
await pg.locator(sel).first().click();
|
|
172
|
-
await pg.waitForTimeout(500);
|
|
173
|
-
return { description: 'click_submit via DOM fallback' };
|
|
174
|
-
}
|
|
54
|
+
}
|
|
55
|
+
return result;
|
|
56
|
+
} catch (e) {
|
|
57
|
+
const result = { success: false, detail: e.message ?? "failed" };
|
|
58
|
+
if (opts?.tenantId && opts?.projectId) {
|
|
59
|
+
AppBrain.saveCrawlEvidence(opts.tenantId, opts.projectId, {
|
|
60
|
+
jobId: opts?.jobId,
|
|
61
|
+
type: "AUTH_STEP",
|
|
62
|
+
payload: { step, ...result, ms: Date.now() - t0 }
|
|
63
|
+
}).catch(() => {
|
|
175
64
|
});
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
await pg.goto(emailContent.magicLink, { waitUntil: 'domcontentloaded' });
|
|
194
|
-
await pg.waitForLoadState('networkidle').catch(() => { });
|
|
195
|
-
return true; // magic link navigation = login complete
|
|
196
|
-
}
|
|
197
|
-
code = emailContent?.otp ?? undefined;
|
|
198
|
-
if (code) {
|
|
199
|
-
logger.info('[crawler] Email OTP received');
|
|
200
|
-
}
|
|
201
|
-
else {
|
|
202
|
-
logger.warn('[crawler] Email OTP timed out after 60s');
|
|
203
|
-
}
|
|
204
|
-
}
|
|
205
|
-
if (!code) {
|
|
206
|
-
console.warn('[crawler] 2FA detected but no oneTimeCode provided');
|
|
207
|
-
return false;
|
|
65
|
+
}
|
|
66
|
+
throw e;
|
|
67
|
+
}
|
|
68
|
+
}
|
|
69
|
+
try {
|
|
70
|
+
const pg = await stagehand.resolvePage();
|
|
71
|
+
for (const sel of CAPTCHA_SELECTORS) {
|
|
72
|
+
const count = await pg.locator(sel).count().catch(() => 0);
|
|
73
|
+
if (count > 0) {
|
|
74
|
+
logger.warn({ selector: sel }, "[crawler] CAPTCHA detected on login page");
|
|
75
|
+
if (captchaOpts?.apiKey) {
|
|
76
|
+
logger.info({ provider: captchaOpts.provider ?? "2captcha" }, "[crawler] Attempting auto-solve");
|
|
77
|
+
try {
|
|
78
|
+
const result = await solveCaptcha(pg, startUrl, captchaOpts.apiKey, captchaOpts.provider ?? "2captcha");
|
|
79
|
+
if (result) {
|
|
80
|
+
logger.info({ provider: result.provider }, "[crawler] CAPTCHA solved \u2014 proceeding with login");
|
|
81
|
+
break;
|
|
208
82
|
}
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
const sel = 'input[autocomplete="one-time-code"], input[name*="otp" i], input[name*="code" i], input[name*="token" i], input[inputmode="numeric"], input[type="number"]';
|
|
218
|
-
await pg.locator(sel).first().fill(code);
|
|
219
|
-
return { description: '2fa_inject via DOM fallback' };
|
|
220
|
-
}
|
|
221
|
-
});
|
|
222
|
-
await logStep('2fa_submit', async () => {
|
|
223
|
-
try {
|
|
224
|
-
return await actWithMouseSim(stagehand, 'Click verify or confirm to submit the verification code');
|
|
225
|
-
}
|
|
226
|
-
catch (err) {
|
|
227
|
-
if (!isNoObjectGeneratedError(err))
|
|
228
|
-
throw err;
|
|
229
|
-
logger.warn('[crawler] 2fa_submit act() failed — falling back to DOM');
|
|
230
|
-
const sel = 'button[type="submit"], button:has-text("Verify"), button:has-text("Confirm"), button:has-text("Submit"), button:has-text("Continue")';
|
|
231
|
-
await pg.locator(sel).first().click();
|
|
232
|
-
return { description: '2fa_submit via DOM fallback' };
|
|
233
|
-
}
|
|
234
|
-
});
|
|
235
|
-
await pg.waitForLoadState('networkidle').catch(() => pg.waitForTimeout(2000));
|
|
83
|
+
} catch (solveErr) {
|
|
84
|
+
logger.warn({ err: solveErr }, "[crawler] CAPTCHA solve failed \u2014 falling back to manual error");
|
|
85
|
+
throw new Error(`CAPTCHA_DETECTED: Auto-solve failed (${String(solveErr)}). Use session capture instead.`);
|
|
86
|
+
}
|
|
87
|
+
} else {
|
|
88
|
+
throw new Error(
|
|
89
|
+
'CAPTCHA_DETECTED: Login page has a CAPTCHA. Use "Session Capture \u2192 Launch Browser" to log in manually, then capture the session. Alternatively, disable CAPTCHA for the ZeTa crawler IP.'
|
|
90
|
+
);
|
|
236
91
|
}
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
|
|
92
|
+
break;
|
|
93
|
+
}
|
|
94
|
+
}
|
|
95
|
+
const MAX_LOGIN_INSTRUCTIONS = 2e3;
|
|
96
|
+
if (loginInstructions && loginInstructions.length > MAX_LOGIN_INSTRUCTIONS) {
|
|
97
|
+
throw new Error(`loginInstructions exceeds ${MAX_LOGIN_INSTRUCTIONS} character limit`);
|
|
98
|
+
}
|
|
99
|
+
if (loginInstructions) {
|
|
100
|
+
await logStep("custom_recipe", async () => {
|
|
240
101
|
try {
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
return true;
|
|
247
|
-
}
|
|
102
|
+
return await actWithMouseSim(stagehand, loginInstructions);
|
|
103
|
+
} catch (err) {
|
|
104
|
+
if (!isNoObjectGeneratedError(err)) throw err;
|
|
105
|
+
logger.warn("[crawler] custom_recipe act() failed \u2014 Stagehand V3 elementId schema error, skipping to standard fill");
|
|
106
|
+
return { description: "custom_recipe skipped (schema error)" };
|
|
248
107
|
}
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
108
|
+
});
|
|
109
|
+
await pg.waitForLoadState("networkidle").catch(() => pg.waitForTimeout(1500));
|
|
110
|
+
}
|
|
111
|
+
await logStep("fill_username", async () => {
|
|
112
|
+
try {
|
|
113
|
+
return await actWithMouseSim(stagehand, `Type "${creds.username}" into the username or email input field`);
|
|
114
|
+
} catch (err) {
|
|
115
|
+
if (!isNoObjectGeneratedError(err)) throw err;
|
|
116
|
+
logger.warn("[crawler] fill_username act() failed \u2014 falling back to DOM");
|
|
117
|
+
const sel = 'input[type="email"], input[autocomplete="email"], input[autocomplete="username"], input[name*="email" i], input[name*="user" i], input[placeholder*="email" i], input[placeholder*="user" i], input[id*="email" i], input[id*="user" i]';
|
|
118
|
+
await pg.locator(sel).first().fill(creds.username);
|
|
119
|
+
return { description: "fill_username via DOM fallback" };
|
|
120
|
+
}
|
|
121
|
+
});
|
|
122
|
+
await logStep("fill_password", async () => {
|
|
123
|
+
const observations = await stagehand.observe("Find the password input field");
|
|
124
|
+
const selector = observations[0]?.selector || 'input[type="password"]';
|
|
125
|
+
try {
|
|
126
|
+
await pg.locator(selector).fill(creds.password);
|
|
127
|
+
} catch {
|
|
128
|
+
await pg.evaluate((pwd) => {
|
|
129
|
+
const input = document.querySelector('input[type="password"]');
|
|
130
|
+
if (!input) throw new Error("No password field found on page");
|
|
131
|
+
const setter = Object.getOwnPropertyDescriptor(window.HTMLInputElement.prototype, "value")?.set;
|
|
132
|
+
setter?.call(input, pwd);
|
|
133
|
+
input.dispatchEvent(new Event("input", { bubbles: true }));
|
|
134
|
+
input.dispatchEvent(new Event("change", { bubbles: true }));
|
|
135
|
+
}, creds.password);
|
|
136
|
+
}
|
|
137
|
+
});
|
|
138
|
+
await logStep("click_submit", async () => {
|
|
139
|
+
try {
|
|
140
|
+
return await actWithMouseSim(stagehand, "Click the sign in, log in, login, or continue button to submit credentials");
|
|
141
|
+
} catch (err) {
|
|
142
|
+
if (!isNoObjectGeneratedError(err)) throw err;
|
|
143
|
+
logger.warn("[crawler] click_submit act() failed \u2014 falling back to DOM");
|
|
144
|
+
const sel = 'button[type="submit"], input[type="submit"], button:has-text("Sign in"), button:has-text("Log in"), button:has-text("Login"), button:has-text("Continue"), button:has-text("Next"), button:has-text("Sign In"), [role="button"]:has-text("Sign in"), [role="button"]:has-text("Login")';
|
|
145
|
+
await pg.locator(sel).first().click();
|
|
146
|
+
await pg.waitForTimeout(500);
|
|
147
|
+
return { description: "click_submit via DOM fallback" };
|
|
148
|
+
}
|
|
149
|
+
});
|
|
150
|
+
await pg.waitForLoadState("networkidle").catch(() => pg.waitForTimeout(2e3));
|
|
151
|
+
const twoFA = await stagehand.extract(
|
|
152
|
+
"Is there a two-factor authentication, OTP, or verification code input visible right now?",
|
|
153
|
+
TwoFASchema
|
|
154
|
+
);
|
|
155
|
+
if (twoFA.has2FA) {
|
|
156
|
+
if (opts?.tenantId && opts?.projectId) {
|
|
157
|
+
AppBrain.saveCrawlEvidence(opts.tenantId, opts.projectId, {
|
|
158
|
+
jobId: opts?.jobId,
|
|
159
|
+
type: "AUTH_STEP",
|
|
160
|
+
payload: { step: "2fa_detect", success: true, detail: twoFA.fieldLabel ?? "2FA detected", ms: 0 }
|
|
161
|
+
}).catch(() => {
|
|
162
|
+
});
|
|
163
|
+
}
|
|
164
|
+
let code = creds.oneTimeCode ?? await creds.getOneTimeCode?.();
|
|
165
|
+
if (!code && emailOtpOpts?.apiKey && emailOtpOpts?.inboxId) {
|
|
166
|
+
const since = (/* @__PURE__ */ new Date()).toISOString();
|
|
167
|
+
logger.info("[crawler] 2FA detected \u2014 waiting for email OTP or magic link via MailSlurp");
|
|
168
|
+
const emailContent = await waitForEmailContent(
|
|
169
|
+
{ provider: "mailslurp", apiKey: emailOtpOpts.apiKey, inboxId: emailOtpOpts.inboxId },
|
|
170
|
+
since
|
|
171
|
+
);
|
|
172
|
+
if (emailContent?.magicLink) {
|
|
173
|
+
logger.info({ url: emailContent.magicLink }, "[crawler] Magic link received \u2014 navigating");
|
|
174
|
+
await pg.goto(emailContent.magicLink, { waitUntil: "domcontentloaded" });
|
|
175
|
+
await pg.waitForLoadState("networkidle").catch(() => {
|
|
176
|
+
});
|
|
177
|
+
return true;
|
|
255
178
|
}
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
|
|
179
|
+
code = emailContent?.otp ?? void 0;
|
|
180
|
+
if (code) {
|
|
181
|
+
logger.info("[crawler] Email OTP received");
|
|
182
|
+
} else {
|
|
183
|
+
logger.warn("[crawler] Email OTP timed out after 60s");
|
|
261
184
|
}
|
|
262
|
-
|
|
185
|
+
}
|
|
186
|
+
if (!code) {
|
|
187
|
+
console.warn("[crawler] 2FA detected but no oneTimeCode provided");
|
|
263
188
|
return false;
|
|
189
|
+
}
|
|
190
|
+
await logStep("2fa_inject", async () => {
|
|
191
|
+
try {
|
|
192
|
+
return await actWithMouseSim(stagehand, `Enter "${code}" into the ${twoFA.fieldLabel ?? "verification code"} field`);
|
|
193
|
+
} catch (err) {
|
|
194
|
+
if (!isNoObjectGeneratedError(err)) throw err;
|
|
195
|
+
logger.warn("[crawler] 2fa_inject act() failed \u2014 falling back to DOM");
|
|
196
|
+
const sel = 'input[autocomplete="one-time-code"], input[name*="otp" i], input[name*="code" i], input[name*="token" i], input[inputmode="numeric"], input[type="number"]';
|
|
197
|
+
await pg.locator(sel).first().fill(code);
|
|
198
|
+
return { description: "2fa_inject via DOM fallback" };
|
|
199
|
+
}
|
|
200
|
+
});
|
|
201
|
+
await logStep("2fa_submit", async () => {
|
|
202
|
+
try {
|
|
203
|
+
return await actWithMouseSim(stagehand, "Click verify or confirm to submit the verification code");
|
|
204
|
+
} catch (err) {
|
|
205
|
+
if (!isNoObjectGeneratedError(err)) throw err;
|
|
206
|
+
logger.warn("[crawler] 2fa_submit act() failed \u2014 falling back to DOM");
|
|
207
|
+
const sel = 'button[type="submit"], button:has-text("Verify"), button:has-text("Confirm"), button:has-text("Submit"), button:has-text("Continue")';
|
|
208
|
+
await pg.locator(sel).first().click();
|
|
209
|
+
return { description: "2fa_submit via DOM fallback" };
|
|
210
|
+
}
|
|
211
|
+
});
|
|
212
|
+
await pg.waitForLoadState("networkidle").catch(() => pg.waitForTimeout(2e3));
|
|
264
213
|
}
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
|
|
214
|
+
const currentUrl = pg.url();
|
|
215
|
+
const loginPathPattern = /\/(login|signin|sign-in|auth|authenticate|log-in|account\/login)(\?|$|\/)/i;
|
|
216
|
+
try {
|
|
217
|
+
const { pathname } = new URL(currentUrl);
|
|
218
|
+
const wasRedirected = currentUrl !== startUrl;
|
|
219
|
+
const stillOnLoginPath = loginPathPattern.test(pathname);
|
|
220
|
+
if (wasRedirected && !stillOnLoginPath) {
|
|
221
|
+
logger.info({ url: currentUrl }, "Login confirmed via URL change");
|
|
222
|
+
return true;
|
|
223
|
+
}
|
|
224
|
+
} catch {
|
|
225
|
+
}
|
|
226
|
+
const passwordCount = await pg.locator('input[type="password"]').count().catch(() => 1);
|
|
227
|
+
const bodyText = await pg.evaluate(() => document.body?.innerText?.trim()?.length ?? 0).catch(() => 0);
|
|
228
|
+
if (passwordCount === 0 && bodyText > 30) {
|
|
229
|
+
logger.info("Login confirmed: password field no longer visible");
|
|
230
|
+
return true;
|
|
231
|
+
}
|
|
232
|
+
logger.warn({ url: currentUrl }, "Login appears failed");
|
|
233
|
+
try {
|
|
234
|
+
const screenshotPath = `/tmp/zeta-auth-failure-${Date.now()}.jpg`;
|
|
235
|
+
await pg.screenshot({ path: screenshotPath, type: "jpeg", quality: 70, fullPage: false });
|
|
236
|
+
logger.warn({ screenshotPath }, "[crawler] Auth failure screenshot saved");
|
|
237
|
+
} catch {
|
|
271
238
|
}
|
|
239
|
+
return false;
|
|
240
|
+
} catch (err) {
|
|
241
|
+
const msg = String(err.message ?? err);
|
|
242
|
+
if (msg.includes("CAPTCHA_DETECTED")) throw err;
|
|
243
|
+
logger.error({ err }, "Login error");
|
|
244
|
+
return false;
|
|
245
|
+
}
|
|
272
246
|
}
|
|
247
|
+
export {
|
|
248
|
+
loginWithStagehand
|
|
249
|
+
};
|
|
250
|
+
//# sourceMappingURL=ai-login.js.map
|
|
@@ -1,68 +1,58 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
}
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
}
|
|
55
|
-
/** Stats for monitoring. */
|
|
56
|
-
stats() {
|
|
57
|
-
const now = Date.now();
|
|
58
|
-
const stale = this.pool.filter(s => now - s.createdAt >= this.maxAgeMs || s.requestCount >= this.maxRequests).length;
|
|
59
|
-
const checkedOut = this.pool.filter(s => !!s.checkedOutAt).length;
|
|
60
|
-
return { total: this.pool.length, available: this.pool.length - checkedOut - stale, checkedOut, stale };
|
|
61
|
-
}
|
|
62
|
-
/** Add a new session to the pool at runtime. */
|
|
63
|
-
addSession(role, storageState) {
|
|
64
|
-
const id = `session-${this.pool.length}-${role}`;
|
|
65
|
-
this.pool.push({ id, role, storageState, createdAt: Date.now(), checkedOutAt: null, requestCount: 0 });
|
|
66
|
-
return id;
|
|
67
|
-
}
|
|
1
|
+
class AuthTokenRotator {
|
|
2
|
+
pool;
|
|
3
|
+
maxAgeMs;
|
|
4
|
+
maxRequests;
|
|
5
|
+
constructor(opts) {
|
|
6
|
+
this.maxAgeMs = opts.maxAgeMs ?? 4 * 60 * 60 * 1e3;
|
|
7
|
+
this.maxRequests = opts.maxRequestsPerSession ?? 100;
|
|
8
|
+
this.pool = opts.sessions.map((s, i) => ({
|
|
9
|
+
id: `session-${i}-${s.role}`,
|
|
10
|
+
role: s.role,
|
|
11
|
+
storageState: s.storageState,
|
|
12
|
+
createdAt: Date.now(),
|
|
13
|
+
checkedOutAt: null,
|
|
14
|
+
requestCount: 0
|
|
15
|
+
}));
|
|
16
|
+
}
|
|
17
|
+
/** Get the next available session. Returns null if all are checked out. */
|
|
18
|
+
checkout() {
|
|
19
|
+
const now = Date.now();
|
|
20
|
+
const available = this.pool.filter(
|
|
21
|
+
(s) => !s.checkedOutAt && now - s.createdAt < this.maxAgeMs && s.requestCount < this.maxRequests
|
|
22
|
+
);
|
|
23
|
+
if (available.length === 0) return null;
|
|
24
|
+
available.sort((a, b) => a.requestCount - b.requestCount);
|
|
25
|
+
const session = available[0];
|
|
26
|
+
session.checkedOutAt = now;
|
|
27
|
+
return session;
|
|
28
|
+
}
|
|
29
|
+
/** Return a session to the pool after use. */
|
|
30
|
+
checkin(sessionId, updatedStorageState) {
|
|
31
|
+
const s = this.pool.find((e) => e.id === sessionId);
|
|
32
|
+
if (!s) return;
|
|
33
|
+
s.checkedOutAt = null;
|
|
34
|
+
s.requestCount++;
|
|
35
|
+
if (updatedStorageState) s.storageState = updatedStorageState;
|
|
36
|
+
}
|
|
37
|
+
/** Get all available sessions (for parallel crawl setup). */
|
|
38
|
+
getAll() {
|
|
39
|
+
return this.pool;
|
|
40
|
+
}
|
|
41
|
+
/** Stats for monitoring. */
|
|
42
|
+
stats() {
|
|
43
|
+
const now = Date.now();
|
|
44
|
+
const stale = this.pool.filter((s) => now - s.createdAt >= this.maxAgeMs || s.requestCount >= this.maxRequests).length;
|
|
45
|
+
const checkedOut = this.pool.filter((s) => !!s.checkedOutAt).length;
|
|
46
|
+
return { total: this.pool.length, available: this.pool.length - checkedOut - stale, checkedOut, stale };
|
|
47
|
+
}
|
|
48
|
+
/** Add a new session to the pool at runtime. */
|
|
49
|
+
addSession(role, storageState) {
|
|
50
|
+
const id = `session-${this.pool.length}-${role}`;
|
|
51
|
+
this.pool.push({ id, role, storageState, createdAt: Date.now(), checkedOutAt: null, requestCount: 0 });
|
|
52
|
+
return id;
|
|
53
|
+
}
|
|
68
54
|
}
|
|
55
|
+
export {
|
|
56
|
+
AuthTokenRotator
|
|
57
|
+
};
|
|
58
|
+
//# sourceMappingURL=auth-token-rotator.js.map
|