@detiq/crawler 0.1.7 → 0.1.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/ai-login.js +12 -1
- package/dist/ai-login.js.map +2 -2
- package/dist/crawler.js +9 -6
- package/dist/crawler.js.map +2 -2
- package/dist/http-engine.js +2 -1
- package/dist/http-engine.js.map +2 -2
- package/dist/job-queue.js +3 -2
- package/dist/job-queue.js.map +2 -2
- package/dist/skyvern.js +0 -4
- package/dist/skyvern.js.map +2 -2
- package/dist/test-executor.js +18 -6
- package/dist/test-executor.js.map +2 -2
- package/package.json +1 -1
- package/src/crawler.ts +10 -5
- package/src/crawler.ts.rej +12 -0
- package/src/http-engine.ts +2 -1
- package/src/job-queue.ts +3 -2
- package/src/test-executor.ts +20 -6
- package/types/crawler.d.ts +2 -0
- package/types/test-executor.d.ts +7 -0
package/dist/ai-login.js
CHANGED
|
@@ -99,7 +99,18 @@ async function loginWithStagehand(stagehand, creds, startUrl, captchaOpts, email
|
|
|
99
99
|
await logStep("fill_password", async () => {
|
|
100
100
|
const observations = await stagehand.observe("Find the password input field");
|
|
101
101
|
const selector = observations[0]?.selector || 'input[type="password"]';
|
|
102
|
-
|
|
102
|
+
try {
|
|
103
|
+
await pg.locator(selector).fill(creds.password);
|
|
104
|
+
} catch {
|
|
105
|
+
await pg.evaluate((pwd) => {
|
|
106
|
+
const input = document.querySelector('input[type="password"]');
|
|
107
|
+
if (!input) throw new Error("No password field found on page");
|
|
108
|
+
const setter = Object.getOwnPropertyDescriptor(window.HTMLInputElement.prototype, "value")?.set;
|
|
109
|
+
setter?.call(input, pwd);
|
|
110
|
+
input.dispatchEvent(new Event("input", { bubbles: true }));
|
|
111
|
+
input.dispatchEvent(new Event("change", { bubbles: true }));
|
|
112
|
+
}, creds.password);
|
|
113
|
+
}
|
|
103
114
|
});
|
|
104
115
|
await logStep("click_submit", () => actWithMouseSim(stagehand, "Click the sign in, log in, login, or continue button to submit credentials"));
|
|
105
116
|
await pg.waitForLoadState("networkidle").catch(() => pg.waitForTimeout(2e3));
|
package/dist/ai-login.js.map
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"version": 3,
|
|
3
3
|
"sources": ["../src/ai-login.ts"],
|
|
4
|
-
"sourcesContent": ["import { z } from 'zod';\nimport type { Stagehand } from '@browserbasehq/stagehand';\nimport { AppBrain } from '@detiq/app-brain';\nimport { solveCaptcha } from './captcha-solver.js';\nimport { waitForEmailContent } from './email-otp.js';\nimport { logger } from './logger.js';\nimport type { Credentials } from './auth.js';\nimport { simulateMouseMovement } from './stealth/human-behavior.js';\n\nconst CLICK_INTENT = /\\b(click|tap|press|submit|select|choose|toggle)\\b/i;\n\n/** Simulate human mouse movement before click-type actions, then call stagehand.act(). */\nasync function actWithMouseSim(stagehand: Stagehand, instruction: string): Promise<any> {\n if (CLICK_INTENT.test(instruction)) {\n try {\n const pg = await (stagehand as any).resolvePage();\n const viewport = pg.viewportSize() ?? { width: 1280, height: 720 };\n // Move mouse toward center of page with some randomness before Stagehand clicks\n const targetX = viewport.width * 0.3 + Math.random() * viewport.width * 0.4;\n const targetY = viewport.height * 0.2 + Math.random() * viewport.height * 0.5;\n await simulateMouseMovement(pg, targetX, targetY);\n } catch { /* non-fatal \u2014 don't block the action */ }\n }\n return stagehand.act(instruction);\n}\n\n/**\n * Recorded login step from the Chrome extension's step recorder.\n * {username} and {password} in `value` are substituted at replay time.\n */\nexport interface LoginStep {\n action: 'fill' | 'click' | 'wait';\n selector: string;\n value?: string;\n ms?: number;\n}\n\nconst TwoFASchema = z.object({\n has2FA: z.boolean(),\n fieldLabel: z.string().optional(),\n});\n\nconst CAPTCHA_SELECTORS = [\n 'iframe[src*=\"recaptcha\"]',\n 'iframe[src*=\"hcaptcha\"]',\n 'iframe[src*=\"challenges.cloudflare.com\"]',\n '.cf-turnstile',\n '[data-sitekey]',\n '#captcha',\n '.captcha',\n '[id*=\"captcha\"]',\n '[class*=\"captcha\"]',\n];\n\n/**\n * AI-guided login using Stagehand act/extract.\n *\n * @param loginInstructions - Optional natural language description of the login flow.\n * When provided, an initial stagehand.act() is called with these instructions to\n * navigate the UI (e.g. \"click Sign In in the nav, fill email, click Continue\").\n * The standard 3-step credential fill follows immediately after.\n * @param opts - Optional tenant/project/job IDs for writing AUTH_STEP CrawlEvidence.\n */\nexport async function loginWithStagehand(\n stagehand: Stagehand,\n creds: Credentials,\n startUrl: string,\n captchaOpts?: { apiKey?: string; provider?: '2captcha' | 'capsolver' },\n emailOtpOpts?: { apiKey?: string; inboxId?: string },\n loginInstructions?: string,\n opts?: { tenantId?: string; projectId?: string; jobId?: string },\n): Promise<boolean> {\n async function logStep(step: string, fn: () => Promise<any>): Promise<{ success: boolean; detail: string }> {\n const t0 = Date.now();\n try {\n const r = await fn();\n const result = { success: true, detail: r?.description ?? r?.message ?? step };\n if (opts?.tenantId && opts?.projectId) {\n AppBrain.saveCrawlEvidence(opts.tenantId, opts.projectId, {\n jobId: opts?.jobId,\n type: 'AUTH_STEP',\n payload: { step, ...result, ms: Date.now() - t0 },\n }).catch(() => {});\n }\n return result;\n } catch (e: any) {\n const result = { success: false, detail: e.message ?? 'failed' };\n if (opts?.tenantId && opts?.projectId) {\n AppBrain.saveCrawlEvidence(opts.tenantId, opts.projectId, {\n jobId: opts?.jobId,\n type: 'AUTH_STEP',\n payload: { step, ...result, ms: Date.now() - t0 },\n }).catch(() => {});\n }\n throw e;\n }\n }\n\n try {\n const pg = await (stagehand as any).resolvePage();\n\n // CAPTCHA detection\n for (const sel of CAPTCHA_SELECTORS) {\n const count = await pg.locator(sel).count().catch(() => 0);\n if (count > 0) {\n logger.warn({ selector: sel }, '[crawler] CAPTCHA detected on login page');\n if (captchaOpts?.apiKey) {\n logger.info({ provider: captchaOpts.provider ?? '2captcha' }, '[crawler] Attempting auto-solve');\n try {\n const result = await solveCaptcha(pg, startUrl, captchaOpts.apiKey, captchaOpts.provider ?? '2captcha');\n if (result) {\n logger.info({ provider: result.provider }, '[crawler] CAPTCHA solved \u2014 proceeding with login');\n break;\n }\n } catch (solveErr) {\n logger.warn({ err: solveErr }, '[crawler] CAPTCHA solve failed \u2014 falling back to manual error');\n throw new Error(`CAPTCHA_DETECTED: Auto-solve failed (${String(solveErr)}). Use session capture instead.`);\n }\n } else {\n throw new Error(\n 'CAPTCHA_DETECTED: Login page has a CAPTCHA. Use \"Session Capture \u2192 Launch Browser\" to log in manually, then capture the session. Alternatively, disable CAPTCHA for the ZeTa crawler IP.'\n );\n }\n break;\n }\n }\n\n // If caller provided natural language login instructions, use them to navigate to the login form\n const MAX_LOGIN_INSTRUCTIONS = 2000;\n if (loginInstructions && loginInstructions.length > MAX_LOGIN_INSTRUCTIONS) {\n throw new Error(`loginInstructions exceeds ${MAX_LOGIN_INSTRUCTIONS} character limit`);\n }\n if (loginInstructions) {\n await logStep('custom_recipe', () => actWithMouseSim(stagehand, loginInstructions));\n await pg.waitForLoadState('networkidle').catch(() => pg.waitForTimeout(1500));\n }\n\n await logStep('fill_username', () => actWithMouseSim(stagehand, `Type \"${creds.username}\" into the username or email input field`));\n // Don't send password to LLM \u2014 use direct Playwright fill to avoid\n // leaking credentials into the Stagehand/LLM provider's request logs.\n await logStep('fill_password', async () => {\n const observations = await stagehand.observe('Find the password input field');\n const selector = observations[0]?.selector || 'input[type=\"password\"]';\n await pg.locator(selector).fill(creds.password);\n });\n await logStep('click_submit', () => actWithMouseSim(stagehand, 'Click the sign in, log in, login, or continue button to submit credentials'));\n await pg.waitForLoadState('networkidle').catch(() => pg.waitForTimeout(2000));\n\n const twoFA = await stagehand.extract(\n 'Is there a two-factor authentication, OTP, or verification code input visible right now?',\n TwoFASchema as any\n );\n\n if (twoFA.has2FA) {\n if (opts?.tenantId && opts?.projectId) {\n AppBrain.saveCrawlEvidence(opts.tenantId, opts.projectId, {\n jobId: opts?.jobId,\n type: 'AUTH_STEP',\n payload: { step: '2fa_detect', success: true, detail: twoFA.fieldLabel ?? '2FA detected', ms: 0 },\n }).catch(() => {});\n }\n\n let code = creds.oneTimeCode ?? await creds.getOneTimeCode?.();\n\n if (!code && emailOtpOpts?.apiKey && emailOtpOpts?.inboxId) {\n const since = new Date().toISOString();\n logger.info('[crawler] 2FA detected \u2014 waiting for email OTP or magic link via MailSlurp');\n const emailContent = await waitForEmailContent(\n { provider: 'mailslurp', apiKey: emailOtpOpts.apiKey, inboxId: emailOtpOpts.inboxId },\n since\n );\n if (emailContent?.magicLink) {\n logger.info({ url: emailContent.magicLink }, '[crawler] Magic link received \u2014 navigating');\n await pg.goto(emailContent.magicLink, { waitUntil: 'domcontentloaded' });\n await pg.waitForLoadState('networkidle').catch(() => {});\n return true; // magic link navigation = login complete\n }\n code = emailContent?.otp ?? undefined;\n if (code) {\n logger.info('[crawler] Email OTP received');\n } else {\n logger.warn('[crawler] Email OTP timed out after 60s');\n }\n }\n\n if (!code) {\n console.warn('[crawler] 2FA detected but no oneTimeCode provided');\n return false;\n }\n await logStep('2fa_inject', () => actWithMouseSim(stagehand, `Enter \"${code}\" into the ${twoFA.fieldLabel ?? 'verification code'} field`));\n await logStep('2fa_submit', () => actWithMouseSim(stagehand, 'Click verify or confirm to submit the verification code'));\n await pg.waitForLoadState('networkidle').catch(() => pg.waitForTimeout(2000));\n }\n\n // URL-based success check\n const currentUrl = pg.url();\n const loginPathPattern = /\\/(login|signin|sign-in|auth|authenticate|log-in|account\\/login)(\\?|$|\\/)/i;\n try {\n const { pathname } = new URL(currentUrl);\n const wasRedirected = currentUrl !== startUrl;\n const stillOnLoginPath = loginPathPattern.test(pathname);\n if (wasRedirected && !stillOnLoginPath) {\n logger.info({ url: currentUrl }, 'Login confirmed via URL change');\n return true;\n }\n } catch { /* invalid URL \u2014 fall through */ }\n\n const passwordCount = await pg.locator('input[type=\"password\"]').count().catch(() => 1);\n const bodyText = await pg.evaluate(() => (document as any).body?.innerText?.trim()?.length ?? 0).catch(() => 0);\n if (passwordCount === 0 && bodyText > 30) {\n logger.info('Login confirmed: password field no longer visible');\n return true;\n }\n\n logger.warn({ url: currentUrl }, 'Login appears failed');\n try {\n const screenshotPath = `/tmp/zeta-auth-failure-${Date.now()}.jpg`;\n await pg.screenshot({ path: screenshotPath, type: 'jpeg', quality: 70, fullPage: false });\n logger.warn({ screenshotPath }, '[crawler] Auth failure screenshot saved');\n } catch { /* non-fatal */ }\n return false;\n } catch (err) {\n const msg = String((err as Error).message ?? err);\n if (msg.includes('CAPTCHA_DETECTED')) throw err;\n logger.error({ err }, 'Login error');\n return false;\n }\n}\n"],
|
|
5
|
-
"mappings": "AAAA,SAAS,SAAS;AAElB,SAAS,gBAAgB;AACzB,SAAS,oBAAoB;AAC7B,SAAS,2BAA2B;AACpC,SAAS,cAAc;AAEvB,SAAS,6BAA6B;AAEtC,MAAM,eAAe;AAGrB,eAAe,gBAAgB,WAAsB,aAAmC;AACtF,MAAI,aAAa,KAAK,WAAW,GAAG;AAClC,QAAI;AACF,YAAM,KAAK,MAAO,UAAkB,YAAY;AAChD,YAAM,WAAW,GAAG,aAAa,KAAK,EAAE,OAAO,MAAM,QAAQ,IAAI;AAEjE,YAAM,UAAU,SAAS,QAAQ,MAAM,KAAK,OAAO,IAAI,SAAS,QAAQ;AACxE,YAAM,UAAU,SAAS,SAAS,MAAM,KAAK,OAAO,IAAI,SAAS,SAAS;AAC1E,YAAM,sBAAsB,IAAI,SAAS,OAAO;AAAA,IAClD,QAAQ;AAAA,IAA2C;AAAA,EACrD;AACA,SAAO,UAAU,IAAI,WAAW;AAClC;AAaA,MAAM,cAAc,EAAE,OAAO;AAAA,EAC3B,QAAQ,EAAE,QAAQ;AAAA,EAClB,YAAY,EAAE,OAAO,EAAE,SAAS;AAClC,CAAC;AAED,MAAM,oBAAoB;AAAA,EACxB;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AACF;AAWA,eAAsB,mBACpB,WACA,OACA,UACA,aACA,cACA,mBACA,MACkB;AAClB,iBAAe,QAAQ,MAAc,IAAuE;AAC1G,UAAM,KAAK,KAAK,IAAI;AACpB,QAAI;AACF,YAAM,IAAI,MAAM,GAAG;AACnB,YAAM,SAAS,EAAE,SAAS,MAAM,QAAQ,GAAG,eAAe,GAAG,WAAW,KAAK;AAC7E,UAAI,MAAM,YAAY,MAAM,WAAW;AACrC,iBAAS,kBAAkB,KAAK,UAAU,KAAK,WAAW;AAAA,UACxD,OAAO,MAAM;AAAA,UACb,MAAM;AAAA,UACN,SAAS,EAAE,MAAM,GAAG,QAAQ,IAAI,KAAK,IAAI,IAAI,GAAG;AAAA,QAClD,CAAC,EAAE,MAAM,MAAM;AAAA,QAAC,CAAC;AAAA,MACnB;AACA,aAAO;AAAA,IACT,SAAS,GAAQ;AACf,YAAM,SAAS,EAAE,SAAS,OAAO,QAAQ,EAAE,WAAW,SAAS;AAC/D,UAAI,MAAM,YAAY,MAAM,WAAW;AACrC,iBAAS,kBAAkB,KAAK,UAAU,KAAK,WAAW;AAAA,UACxD,OAAO,MAAM;AAAA,UACb,MAAM;AAAA,UACN,SAAS,EAAE,MAAM,GAAG,QAAQ,IAAI,KAAK,IAAI,IAAI,GAAG;AAAA,QAClD,CAAC,EAAE,MAAM,MAAM;AAAA,QAAC,CAAC;AAAA,MACnB;AACA,YAAM;AAAA,IACR;AAAA,EACF;AAEA,MAAI;AACF,UAAM,KAAK,MAAO,UAAkB,YAAY;AAGhD,eAAW,OAAO,mBAAmB;AACnC,YAAM,QAAQ,MAAM,GAAG,QAAQ,GAAG,EAAE,MAAM,EAAE,MAAM,MAAM,CAAC;AACzD,UAAI,QAAQ,GAAG;AACb,eAAO,KAAK,EAAE,UAAU,IAAI,GAAG,0CAA0C;AACzE,YAAI,aAAa,QAAQ;AACvB,iBAAO,KAAK,EAAE,UAAU,YAAY,YAAY,WAAW,GAAG,iCAAiC;AAC/F,cAAI;AACF,kBAAM,SAAS,MAAM,aAAa,IAAI,UAAU,YAAY,QAAQ,YAAY,YAAY,UAAU;AACtG,gBAAI,QAAQ;AACV,qBAAO,KAAK,EAAE,UAAU,OAAO,SAAS,GAAG,uDAAkD;AAC7F;AAAA,YACF;AAAA,UACF,SAAS,UAAU;AACjB,mBAAO,KAAK,EAAE,KAAK,SAAS,GAAG,oEAA+D;AAC9F,kBAAM,IAAI,MAAM,wCAAwC,OAAO,QAAQ,CAAC,iCAAiC;AAAA,UAC3G;AAAA,QACF,OAAO;AACL,gBAAM,IAAI;AAAA,YACR;AAAA,UACF;AAAA,QACF;AACA;AAAA,MACF;AAAA,IACF;AAGA,UAAM,yBAAyB;AAC/B,QAAI,qBAAqB,kBAAkB,SAAS,wBAAwB;AAC1E,YAAM,IAAI,MAAM,6BAA6B,sBAAsB,kBAAkB;AAAA,IACvF;AACA,QAAI,mBAAmB;AACrB,YAAM,QAAQ,iBAAiB,MAAM,gBAAgB,WAAW,iBAAiB,CAAC;AAClF,YAAM,GAAG,iBAAiB,aAAa,EAAE,MAAM,MAAM,GAAG,eAAe,IAAI,CAAC;AAAA,IAC9E;AAEA,UAAM,QAAQ,iBAAiB,MAAM,gBAAgB,WAAW,SAAS,MAAM,QAAQ,0CAA0C,CAAC;AAGlI,UAAM,QAAQ,iBAAiB,YAAY;AACzC,YAAM,eAAe,MAAM,UAAU,QAAQ,+BAA+B;AAC5E,YAAM,WAAW,aAAa,CAAC,GAAG,YAAY;AAC9C,
|
|
4
|
+
"sourcesContent": ["import { z } from 'zod';\nimport type { Stagehand } from '@browserbasehq/stagehand';\nimport { AppBrain } from '@detiq/app-brain';\nimport { solveCaptcha } from './captcha-solver.js';\nimport { waitForEmailContent } from './email-otp.js';\nimport { logger } from './logger.js';\nimport type { Credentials } from './auth.js';\nimport { simulateMouseMovement } from './stealth/human-behavior.js';\n\nconst CLICK_INTENT = /\\b(click|tap|press|submit|select|choose|toggle)\\b/i;\n\n/** Simulate human mouse movement before click-type actions, then call stagehand.act(). */\nasync function actWithMouseSim(stagehand: Stagehand, instruction: string): Promise<any> {\n if (CLICK_INTENT.test(instruction)) {\n try {\n const pg = await (stagehand as any).resolvePage();\n const viewport = pg.viewportSize() ?? { width: 1280, height: 720 };\n // Move mouse toward center of page with some randomness before Stagehand clicks\n const targetX = viewport.width * 0.3 + Math.random() * viewport.width * 0.4;\n const targetY = viewport.height * 0.2 + Math.random() * viewport.height * 0.5;\n await simulateMouseMovement(pg, targetX, targetY);\n } catch { /* non-fatal \u2014 don't block the action */ }\n }\n return stagehand.act(instruction);\n}\n\n/**\n * Recorded login step from the Chrome extension's step recorder.\n * {username} and {password} in `value` are substituted at replay time.\n */\nexport interface LoginStep {\n action: 'fill' | 'click' | 'wait';\n selector: string;\n value?: string;\n ms?: number;\n}\n\nconst TwoFASchema = z.object({\n has2FA: z.boolean(),\n fieldLabel: z.string().optional(),\n});\n\nconst CAPTCHA_SELECTORS = [\n 'iframe[src*=\"recaptcha\"]',\n 'iframe[src*=\"hcaptcha\"]',\n 'iframe[src*=\"challenges.cloudflare.com\"]',\n '.cf-turnstile',\n '[data-sitekey]',\n '#captcha',\n '.captcha',\n '[id*=\"captcha\"]',\n '[class*=\"captcha\"]',\n];\n\n/**\n * AI-guided login using Stagehand act/extract.\n *\n * @param loginInstructions - Optional natural language description of the login flow.\n * When provided, an initial stagehand.act() is called with these instructions to\n * navigate the UI (e.g. \"click Sign In in the nav, fill email, click Continue\").\n * The standard 3-step credential fill follows immediately after.\n * @param opts - Optional tenant/project/job IDs for writing AUTH_STEP CrawlEvidence.\n */\nexport async function loginWithStagehand(\n stagehand: Stagehand,\n creds: Credentials,\n startUrl: string,\n captchaOpts?: { apiKey?: string; provider?: '2captcha' | 'capsolver' },\n emailOtpOpts?: { apiKey?: string; inboxId?: string },\n loginInstructions?: string,\n opts?: { tenantId?: string; projectId?: string; jobId?: string },\n): Promise<boolean> {\n async function logStep(step: string, fn: () => Promise<any>): Promise<{ success: boolean; detail: string }> {\n const t0 = Date.now();\n try {\n const r = await fn();\n const result = { success: true, detail: r?.description ?? r?.message ?? step };\n if (opts?.tenantId && opts?.projectId) {\n AppBrain.saveCrawlEvidence(opts.tenantId, opts.projectId, {\n jobId: opts?.jobId,\n type: 'AUTH_STEP',\n payload: { step, ...result, ms: Date.now() - t0 },\n }).catch(() => {});\n }\n return result;\n } catch (e: any) {\n const result = { success: false, detail: e.message ?? 'failed' };\n if (opts?.tenantId && opts?.projectId) {\n AppBrain.saveCrawlEvidence(opts.tenantId, opts.projectId, {\n jobId: opts?.jobId,\n type: 'AUTH_STEP',\n payload: { step, ...result, ms: Date.now() - t0 },\n }).catch(() => {});\n }\n throw e;\n }\n }\n\n try {\n const pg = await (stagehand as any).resolvePage();\n\n // CAPTCHA detection\n for (const sel of CAPTCHA_SELECTORS) {\n const count = await pg.locator(sel).count().catch(() => 0);\n if (count > 0) {\n logger.warn({ selector: sel }, '[crawler] CAPTCHA detected on login page');\n if (captchaOpts?.apiKey) {\n logger.info({ provider: captchaOpts.provider ?? '2captcha' }, '[crawler] Attempting auto-solve');\n try {\n const result = await solveCaptcha(pg, startUrl, captchaOpts.apiKey, captchaOpts.provider ?? '2captcha');\n if (result) {\n logger.info({ provider: result.provider }, '[crawler] CAPTCHA solved \u2014 proceeding with login');\n break;\n }\n } catch (solveErr) {\n logger.warn({ err: solveErr }, '[crawler] CAPTCHA solve failed \u2014 falling back to manual error');\n throw new Error(`CAPTCHA_DETECTED: Auto-solve failed (${String(solveErr)}). Use session capture instead.`);\n }\n } else {\n throw new Error(\n 'CAPTCHA_DETECTED: Login page has a CAPTCHA. Use \"Session Capture \u2192 Launch Browser\" to log in manually, then capture the session. Alternatively, disable CAPTCHA for the ZeTa crawler IP.'\n );\n }\n break;\n }\n }\n\n // If caller provided natural language login instructions, use them to navigate to the login form\n const MAX_LOGIN_INSTRUCTIONS = 2000;\n if (loginInstructions && loginInstructions.length > MAX_LOGIN_INSTRUCTIONS) {\n throw new Error(`loginInstructions exceeds ${MAX_LOGIN_INSTRUCTIONS} character limit`);\n }\n if (loginInstructions) {\n await logStep('custom_recipe', () => actWithMouseSim(stagehand, loginInstructions));\n await pg.waitForLoadState('networkidle').catch(() => pg.waitForTimeout(1500));\n }\n\n await logStep('fill_username', () => actWithMouseSim(stagehand, `Type \"${creds.username}\" into the username or email input field`));\n // Don't send password to LLM \u2014 use direct Playwright fill to avoid\n // leaking credentials into the Stagehand/LLM provider's request logs.\n await logStep('fill_password', async () => {\n const observations = await stagehand.observe('Find the password input field');\n const selector = observations[0]?.selector || 'input[type=\"password\"]';\n try {\n await pg.locator(selector).fill(creds.password);\n } catch {\n // Stagehand locator failed (e.g. form re-rendered, proxy can't locate element).\n // Fall back to direct DOM fill via evaluate so the password never touches the LLM.\n await pg.evaluate((pwd: string) => {\n const input = document.querySelector('input[type=\"password\"]') as HTMLInputElement | null;\n if (!input) throw new Error('No password field found on page');\n const setter = Object.getOwnPropertyDescriptor(window.HTMLInputElement.prototype, 'value')?.set;\n setter?.call(input, pwd);\n input.dispatchEvent(new Event('input', { bubbles: true }));\n input.dispatchEvent(new Event('change', { bubbles: true }));\n }, creds.password);\n }\n });\n await logStep('click_submit', () => actWithMouseSim(stagehand, 'Click the sign in, log in, login, or continue button to submit credentials'));\n await pg.waitForLoadState('networkidle').catch(() => pg.waitForTimeout(2000));\n\n const twoFA = await stagehand.extract(\n 'Is there a two-factor authentication, OTP, or verification code input visible right now?',\n TwoFASchema as any\n );\n\n if (twoFA.has2FA) {\n if (opts?.tenantId && opts?.projectId) {\n AppBrain.saveCrawlEvidence(opts.tenantId, opts.projectId, {\n jobId: opts?.jobId,\n type: 'AUTH_STEP',\n payload: { step: '2fa_detect', success: true, detail: twoFA.fieldLabel ?? '2FA detected', ms: 0 },\n }).catch(() => {});\n }\n\n let code = creds.oneTimeCode ?? await creds.getOneTimeCode?.();\n\n if (!code && emailOtpOpts?.apiKey && emailOtpOpts?.inboxId) {\n const since = new Date().toISOString();\n logger.info('[crawler] 2FA detected \u2014 waiting for email OTP or magic link via MailSlurp');\n const emailContent = await waitForEmailContent(\n { provider: 'mailslurp', apiKey: emailOtpOpts.apiKey, inboxId: emailOtpOpts.inboxId },\n since\n );\n if (emailContent?.magicLink) {\n logger.info({ url: emailContent.magicLink }, '[crawler] Magic link received \u2014 navigating');\n await pg.goto(emailContent.magicLink, { waitUntil: 'domcontentloaded' });\n await pg.waitForLoadState('networkidle').catch(() => {});\n return true; // magic link navigation = login complete\n }\n code = emailContent?.otp ?? undefined;\n if (code) {\n logger.info('[crawler] Email OTP received');\n } else {\n logger.warn('[crawler] Email OTP timed out after 60s');\n }\n }\n\n if (!code) {\n console.warn('[crawler] 2FA detected but no oneTimeCode provided');\n return false;\n }\n await logStep('2fa_inject', () => actWithMouseSim(stagehand, `Enter \"${code}\" into the ${twoFA.fieldLabel ?? 'verification code'} field`));\n await logStep('2fa_submit', () => actWithMouseSim(stagehand, 'Click verify or confirm to submit the verification code'));\n await pg.waitForLoadState('networkidle').catch(() => pg.waitForTimeout(2000));\n }\n\n // URL-based success check\n const currentUrl = pg.url();\n const loginPathPattern = /\\/(login|signin|sign-in|auth|authenticate|log-in|account\\/login)(\\?|$|\\/)/i;\n try {\n const { pathname } = new URL(currentUrl);\n const wasRedirected = currentUrl !== startUrl;\n const stillOnLoginPath = loginPathPattern.test(pathname);\n if (wasRedirected && !stillOnLoginPath) {\n logger.info({ url: currentUrl }, 'Login confirmed via URL change');\n return true;\n }\n } catch { /* invalid URL \u2014 fall through */ }\n\n const passwordCount = await pg.locator('input[type=\"password\"]').count().catch(() => 1);\n const bodyText = await pg.evaluate(() => (document as any).body?.innerText?.trim()?.length ?? 0).catch(() => 0);\n if (passwordCount === 0 && bodyText > 30) {\n logger.info('Login confirmed: password field no longer visible');\n return true;\n }\n\n logger.warn({ url: currentUrl }, 'Login appears failed');\n try {\n const screenshotPath = `/tmp/zeta-auth-failure-${Date.now()}.jpg`;\n await pg.screenshot({ path: screenshotPath, type: 'jpeg', quality: 70, fullPage: false });\n logger.warn({ screenshotPath }, '[crawler] Auth failure screenshot saved');\n } catch { /* non-fatal */ }\n return false;\n } catch (err) {\n const msg = String((err as Error).message ?? err);\n if (msg.includes('CAPTCHA_DETECTED')) throw err;\n logger.error({ err }, 'Login error');\n return false;\n }\n}\n"],
|
|
5
|
+
"mappings": "AAAA,SAAS,SAAS;AAElB,SAAS,gBAAgB;AACzB,SAAS,oBAAoB;AAC7B,SAAS,2BAA2B;AACpC,SAAS,cAAc;AAEvB,SAAS,6BAA6B;AAEtC,MAAM,eAAe;AAGrB,eAAe,gBAAgB,WAAsB,aAAmC;AACtF,MAAI,aAAa,KAAK,WAAW,GAAG;AAClC,QAAI;AACF,YAAM,KAAK,MAAO,UAAkB,YAAY;AAChD,YAAM,WAAW,GAAG,aAAa,KAAK,EAAE,OAAO,MAAM,QAAQ,IAAI;AAEjE,YAAM,UAAU,SAAS,QAAQ,MAAM,KAAK,OAAO,IAAI,SAAS,QAAQ;AACxE,YAAM,UAAU,SAAS,SAAS,MAAM,KAAK,OAAO,IAAI,SAAS,SAAS;AAC1E,YAAM,sBAAsB,IAAI,SAAS,OAAO;AAAA,IAClD,QAAQ;AAAA,IAA2C;AAAA,EACrD;AACA,SAAO,UAAU,IAAI,WAAW;AAClC;AAaA,MAAM,cAAc,EAAE,OAAO;AAAA,EAC3B,QAAQ,EAAE,QAAQ;AAAA,EAClB,YAAY,EAAE,OAAO,EAAE,SAAS;AAClC,CAAC;AAED,MAAM,oBAAoB;AAAA,EACxB;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AACF;AAWA,eAAsB,mBACpB,WACA,OACA,UACA,aACA,cACA,mBACA,MACkB;AAClB,iBAAe,QAAQ,MAAc,IAAuE;AAC1G,UAAM,KAAK,KAAK,IAAI;AACpB,QAAI;AACF,YAAM,IAAI,MAAM,GAAG;AACnB,YAAM,SAAS,EAAE,SAAS,MAAM,QAAQ,GAAG,eAAe,GAAG,WAAW,KAAK;AAC7E,UAAI,MAAM,YAAY,MAAM,WAAW;AACrC,iBAAS,kBAAkB,KAAK,UAAU,KAAK,WAAW;AAAA,UACxD,OAAO,MAAM;AAAA,UACb,MAAM;AAAA,UACN,SAAS,EAAE,MAAM,GAAG,QAAQ,IAAI,KAAK,IAAI,IAAI,GAAG;AAAA,QAClD,CAAC,EAAE,MAAM,MAAM;AAAA,QAAC,CAAC;AAAA,MACnB;AACA,aAAO;AAAA,IACT,SAAS,GAAQ;AACf,YAAM,SAAS,EAAE,SAAS,OAAO,QAAQ,EAAE,WAAW,SAAS;AAC/D,UAAI,MAAM,YAAY,MAAM,WAAW;AACrC,iBAAS,kBAAkB,KAAK,UAAU,KAAK,WAAW;AAAA,UACxD,OAAO,MAAM;AAAA,UACb,MAAM;AAAA,UACN,SAAS,EAAE,MAAM,GAAG,QAAQ,IAAI,KAAK,IAAI,IAAI,GAAG;AAAA,QAClD,CAAC,EAAE,MAAM,MAAM;AAAA,QAAC,CAAC;AAAA,MACnB;AACA,YAAM;AAAA,IACR;AAAA,EACF;AAEA,MAAI;AACF,UAAM,KAAK,MAAO,UAAkB,YAAY;AAGhD,eAAW,OAAO,mBAAmB;AACnC,YAAM,QAAQ,MAAM,GAAG,QAAQ,GAAG,EAAE,MAAM,EAAE,MAAM,MAAM,CAAC;AACzD,UAAI,QAAQ,GAAG;AACb,eAAO,KAAK,EAAE,UAAU,IAAI,GAAG,0CAA0C;AACzE,YAAI,aAAa,QAAQ;AACvB,iBAAO,KAAK,EAAE,UAAU,YAAY,YAAY,WAAW,GAAG,iCAAiC;AAC/F,cAAI;AACF,kBAAM,SAAS,MAAM,aAAa,IAAI,UAAU,YAAY,QAAQ,YAAY,YAAY,UAAU;AACtG,gBAAI,QAAQ;AACV,qBAAO,KAAK,EAAE,UAAU,OAAO,SAAS,GAAG,uDAAkD;AAC7F;AAAA,YACF;AAAA,UACF,SAAS,UAAU;AACjB,mBAAO,KAAK,EAAE,KAAK,SAAS,GAAG,oEAA+D;AAC9F,kBAAM,IAAI,MAAM,wCAAwC,OAAO,QAAQ,CAAC,iCAAiC;AAAA,UAC3G;AAAA,QACF,OAAO;AACL,gBAAM,IAAI;AAAA,YACR;AAAA,UACF;AAAA,QACF;AACA;AAAA,MACF;AAAA,IACF;AAGA,UAAM,yBAAyB;AAC/B,QAAI,qBAAqB,kBAAkB,SAAS,wBAAwB;AAC1E,YAAM,IAAI,MAAM,6BAA6B,sBAAsB,kBAAkB;AAAA,IACvF;AACA,QAAI,mBAAmB;AACrB,YAAM,QAAQ,iBAAiB,MAAM,gBAAgB,WAAW,iBAAiB,CAAC;AAClF,YAAM,GAAG,iBAAiB,aAAa,EAAE,MAAM,MAAM,GAAG,eAAe,IAAI,CAAC;AAAA,IAC9E;AAEA,UAAM,QAAQ,iBAAiB,MAAM,gBAAgB,WAAW,SAAS,MAAM,QAAQ,0CAA0C,CAAC;AAGlI,UAAM,QAAQ,iBAAiB,YAAY;AACzC,YAAM,eAAe,MAAM,UAAU,QAAQ,+BAA+B;AAC5E,YAAM,WAAW,aAAa,CAAC,GAAG,YAAY;AAC9C,UAAI;AACF,cAAM,GAAG,QAAQ,QAAQ,EAAE,KAAK,MAAM,QAAQ;AAAA,MAChD,QAAQ;AAGN,cAAM,GAAG,SAAS,CAAC,QAAgB;AACjC,gBAAM,QAAQ,SAAS,cAAc,wBAAwB;AAC7D,cAAI,CAAC,MAAO,OAAM,IAAI,MAAM,iCAAiC;AAC7D,gBAAM,SAAS,OAAO,yBAAyB,OAAO,iBAAiB,WAAW,OAAO,GAAG;AAC5F,kBAAQ,KAAK,OAAO,GAAG;AACvB,gBAAM,cAAc,IAAI,MAAM,SAAS,EAAE,SAAS,KAAK,CAAC,CAAC;AACzD,gBAAM,cAAc,IAAI,MAAM,UAAU,EAAE,SAAS,KAAK,CAAC,CAAC;AAAA,QAC5D,GAAG,MAAM,QAAQ;AAAA,MACnB;AAAA,IACF,CAAC;AACD,UAAM,QAAQ,gBAAgB,MAAM,gBAAgB,WAAW,4EAA4E,CAAC;AAC5I,UAAM,GAAG,iBAAiB,aAAa,EAAE,MAAM,MAAM,GAAG,eAAe,GAAI,CAAC;AAE5E,UAAM,QAAQ,MAAM,UAAU;AAAA,MAC5B;AAAA,MACA;AAAA,IACF;AAEA,QAAI,MAAM,QAAQ;AAChB,UAAI,MAAM,YAAY,MAAM,WAAW;AACrC,iBAAS,kBAAkB,KAAK,UAAU,KAAK,WAAW;AAAA,UACxD,OAAO,MAAM;AAAA,UACb,MAAM;AAAA,UACN,SAAS,EAAE,MAAM,cAAc,SAAS,MAAM,QAAQ,MAAM,cAAc,gBAAgB,IAAI,EAAE;AAAA,QAClG,CAAC,EAAE,MAAM,MAAM;AAAA,QAAC,CAAC;AAAA,MACnB;AAEA,UAAI,OAAO,MAAM,eAAe,MAAM,MAAM,iBAAiB;AAE7D,UAAI,CAAC,QAAQ,cAAc,UAAU,cAAc,SAAS;AAC1D,cAAM,SAAQ,oBAAI,KAAK,GAAE,YAAY;AACrC,eAAO,KAAK,iFAA4E;AACxF,cAAM,eAAe,MAAM;AAAA,UACzB,EAAE,UAAU,aAAa,QAAQ,aAAa,QAAQ,SAAS,aAAa,QAAQ;AAAA,UACpF;AAAA,QACF;AACA,YAAI,cAAc,WAAW;AAC3B,iBAAO,KAAK,EAAE,KAAK,aAAa,UAAU,GAAG,iDAA4C;AACzF,gBAAM,GAAG,KAAK,aAAa,WAAW,EAAE,WAAW,mBAAmB,CAAC;AACvE,gBAAM,GAAG,iBAAiB,aAAa,EAAE,MAAM,MAAM;AAAA,UAAC,CAAC;AACvD,iBAAO;AAAA,QACT;AACA,eAAO,cAAc,OAAO;AAC5B,YAAI,MAAM;AACR,iBAAO,KAAK,8BAA8B;AAAA,QAC5C,OAAO;AACL,iBAAO,KAAK,yCAAyC;AAAA,QACvD;AAAA,MACF;AAEA,UAAI,CAAC,MAAM;AACT,gBAAQ,KAAK,oDAAoD;AACjE,eAAO;AAAA,MACT;AACA,YAAM,QAAQ,cAAc,MAAM,gBAAgB,WAAW,UAAU,IAAI,cAAc,MAAM,cAAc,mBAAmB,QAAQ,CAAC;AACzI,YAAM,QAAQ,cAAc,MAAM,gBAAgB,WAAW,yDAAyD,CAAC;AACvH,YAAM,GAAG,iBAAiB,aAAa,EAAE,MAAM,MAAM,GAAG,eAAe,GAAI,CAAC;AAAA,IAC9E;AAGA,UAAM,aAAa,GAAG,IAAI;AAC1B,UAAM,mBAAmB;AACzB,QAAI;AACF,YAAM,EAAE,SAAS,IAAI,IAAI,IAAI,UAAU;AACvC,YAAM,gBAAgB,eAAe;AACrC,YAAM,mBAAmB,iBAAiB,KAAK,QAAQ;AACvD,UAAI,iBAAiB,CAAC,kBAAkB;AACtC,eAAO,KAAK,EAAE,KAAK,WAAW,GAAG,gCAAgC;AACjE,eAAO;AAAA,MACT;AAAA,IACF,QAAQ;AAAA,IAAmC;AAE3C,UAAM,gBAAgB,MAAM,GAAG,QAAQ,wBAAwB,EAAE,MAAM,EAAE,MAAM,MAAM,CAAC;AACtF,UAAM,WAAW,MAAM,GAAG,SAAS,MAAO,SAAiB,MAAM,WAAW,KAAK,GAAG,UAAU,CAAC,EAAE,MAAM,MAAM,CAAC;AAC9G,QAAI,kBAAkB,KAAK,WAAW,IAAI;AACxC,aAAO,KAAK,mDAAmD;AAC/D,aAAO;AAAA,IACT;AAEA,WAAO,KAAK,EAAE,KAAK,WAAW,GAAG,sBAAsB;AACvD,QAAI;AACF,YAAM,iBAAiB,0BAA0B,KAAK,IAAI,CAAC;AAC3D,YAAM,GAAG,WAAW,EAAE,MAAM,gBAAgB,MAAM,QAAQ,SAAS,IAAI,UAAU,MAAM,CAAC;AACxF,aAAO,KAAK,EAAE,eAAe,GAAG,yCAAyC;AAAA,IAC3E,QAAQ;AAAA,IAAkB;AAC1B,WAAO;AAAA,EACT,SAAS,KAAK;AACZ,UAAM,MAAM,OAAQ,IAAc,WAAW,GAAG;AAChD,QAAI,IAAI,SAAS,kBAAkB,EAAG,OAAM;AAC5C,WAAO,MAAM,EAAE,IAAI,GAAG,aAAa;AACnC,WAAO;AAAA,EACT;AACF;",
|
|
6
6
|
"names": []
|
|
7
7
|
}
|
package/dist/crawler.js
CHANGED
|
@@ -736,7 +736,8 @@ async function crawlProject(opts, onProgress) {
|
|
|
736
736
|
}, state.origins);
|
|
737
737
|
}
|
|
738
738
|
await onProgress?.(12, "Restoring session");
|
|
739
|
-
await rawPage.
|
|
739
|
+
await rawPage.goto(opts.appUrl, { waitUntil: "domcontentloaded" }).catch(() => {
|
|
740
|
+
});
|
|
740
741
|
await waitForStableDOM(page);
|
|
741
742
|
if (page.url() === opts.appUrl || page.url() === opts.appUrl.replace(/\/$/, "")) {
|
|
742
743
|
await waitForURLChange(page, opts.appUrl).catch(() => {
|
|
@@ -1732,8 +1733,11 @@ async function crawlProject(opts, onProgress) {
|
|
|
1732
1733
|
}
|
|
1733
1734
|
})()}`);
|
|
1734
1735
|
const [, elements, nextLinks, atElements] = await Promise.all([
|
|
1735
|
-
page.screenshot({ path: shotPath, fullPage: true }).then(() => AppBrain.saveScreenshot(opts.tenantId, opts.projectId, screen.id, shotPath, screen.version, "DESKTOP")),
|
|
1736
|
-
extractElements(stagehand, page, elementLLM ?? void 0).catch(() =>
|
|
1736
|
+
page.screenshot({ path: shotPath, fullPage: true }).then(() => AppBrain.saveScreenshot(opts.tenantId, opts.projectId, screen.id, shotPath, screen.version, "DESKTOP")).catch((err) => console.error("[crawler] page.screenshot failed:", err)),
|
|
1737
|
+
extractElements(stagehand, page, elementLLM ?? void 0).catch((err) => {
|
|
1738
|
+
console.error("[crawler] extractElements failed:", err);
|
|
1739
|
+
return [];
|
|
1740
|
+
}),
|
|
1737
1741
|
// Always run AI link discovery (even when cache exists) — cache is merged in, not used as a replacement.
|
|
1738
1742
|
// Skipping AI when cache exists was a bug: stale cache from broken crawls permanently blocked discovery.
|
|
1739
1743
|
discoverLinks(stagehand, origin, page, linkLLM ?? void 0, aiSkipCount > 0).then((freshLinks) => {
|
|
@@ -2155,7 +2159,7 @@ ${snippet}`,
|
|
|
2155
2159
|
fcp: rawPerf?.fcp,
|
|
2156
2160
|
cls: rawPerf?.cls,
|
|
2157
2161
|
ttfb: rawPerf?.ttfb,
|
|
2158
|
-
viewport: "DESKTOP",
|
|
2162
|
+
viewport: opts.viewport ?? "DESKTOP",
|
|
2159
2163
|
...lhScores ? {
|
|
2160
2164
|
lighthousePerf: lhScores.perf,
|
|
2161
2165
|
lighthouseA11y: lhScores.a11y,
|
|
@@ -2555,8 +2559,7 @@ ${snippet}`,
|
|
|
2555
2559
|
logger.info({ url: sessionCheckUrl }, "[P5] Session alive \u2014 proceeding with viewport pass");
|
|
2556
2560
|
}
|
|
2557
2561
|
} catch (err) {
|
|
2558
|
-
logger.warn({ err: String(err) }, "[P5] Session health check failed \u2014
|
|
2559
|
-
sessionAliveForViewports = false;
|
|
2562
|
+
logger.warn({ err: String(err) }, "[P5] Session health check navigation failed \u2014 proceeding with viewport pass anyway");
|
|
2560
2563
|
}
|
|
2561
2564
|
const extraProfiles = (opts.deviceProfiles ?? []).filter((p) => p !== "DESKTOP");
|
|
2562
2565
|
if (sessionAliveForViewports && extraProfiles.length > 0 && discovered.length > 0) {
|