@detiq/crawler 0.1.3 → 0.1.5

package/Dockerfile

@@ -40,6 +40,10 @@ RUN npx prisma generate --schema packages/database/prisma/schema.prisma && \
 # Screenshot cache dir
 RUN mkdir -p /app/.screenshots-cache
 
+RUN addgroup --system appgroup && adduser --system --ingroup appgroup appuser
+RUN chown -R appuser:appgroup /app
+USER appuser
+
 EXPOSE 4001
 # index.js is a side-effect-free library barrel (also imported by @detiq/api) — it holds
 # no listener/timer, so running it directly starts the process, finishes its imports,

package/dist/ai-login.js

@@ -1,5 +1,5 @@
 import { z } from "zod";
-import { AppBrain } from "@qip/app-brain";
+import { AppBrain } from "@detiq/app-brain";
 import { solveCaptcha } from "./captcha-solver.js";
 import { waitForEmailContent } from "./email-otp.js";
 import { logger } from "./logger.js";
@@ -87,12 +87,20 @@ async function loginWithStagehand(stagehand, creds, startUrl, captchaOpts, email
         break;
       }
     }
+    const MAX_LOGIN_INSTRUCTIONS = 2e3;
+    if (loginInstructions && loginInstructions.length > MAX_LOGIN_INSTRUCTIONS) {
+      throw new Error(`loginInstructions exceeds ${MAX_LOGIN_INSTRUCTIONS} character limit`);
+    }
     if (loginInstructions) {
       await logStep("custom_recipe", () => actWithMouseSim(stagehand, loginInstructions));
       await pg.waitForLoadState("networkidle").catch(() => pg.waitForTimeout(1500));
     }
     await logStep("fill_username", () => actWithMouseSim(stagehand, `Type "${creds.username}" into the username or email input field`));
-    await logStep("fill_password", () => actWithMouseSim(stagehand, `Type "${creds.password}" into the password input field`));
+    await logStep("fill_password", async () => {
+      const observations = await stagehand.observe("Find the password input field");
+      const selector = observations[0]?.selector || 'input[type="password"]';
+      await pg.locator(selector).fill(creds.password);
+    });
     await logStep("click_submit", () => actWithMouseSim(stagehand, "Click the sign in, log in, login, or continue button to submit credentials"));
     await pg.waitForLoadState("networkidle").catch(() => pg.waitForTimeout(2e3));
     const twoFA = await stagehand.extract(

package/dist/ai-login.js.map

@@ -1,7 +1,7 @@
 {
   "version": 3,
   "sources": ["../src/ai-login.ts"],
-  "sourcesContent": ["import { z } from 'zod';\nimport type { Stagehand } from '@browserbasehq/stagehand';\nimport { AppBrain } from '@qip/app-brain';\nimport { solveCaptcha } from './captcha-solver.js';\nimport { waitForEmailContent } from './email-otp.js';\nimport { logger } from './logger.js';\nimport type { Credentials } from './auth.js';\nimport { simulateMouseMovement } from './stealth/human-behavior.js';\n\nconst CLICK_INTENT = /\\b(click|tap|press|submit|select|choose|toggle)\\b/i;\n\n/** Simulate human mouse movement before click-type actions, then call stagehand.act(). */\nasync function actWithMouseSim(stagehand: Stagehand, instruction: string): Promise<any> {\n  if (CLICK_INTENT.test(instruction)) {\n    try {\n      const pg = await (stagehand as any).resolvePage();\n      const viewport = pg.viewportSize() ?? { width: 1280, height: 720 };\n      // Move mouse toward center of page with some randomness before Stagehand clicks\n      const targetX = viewport.width * 0.3 + Math.random() * viewport.width * 0.4;\n      const targetY = viewport.height * 0.2 + Math.random() * viewport.height * 0.5;\n      await simulateMouseMovement(pg, targetX, targetY);\n    } catch { /* non-fatal \u2014 don't block the action */ }\n  }\n  return stagehand.act(instruction);\n}\n\n/**\n * Recorded login step from the Chrome extension's step recorder.\n * {username} and {password} in `value` are substituted at replay time.\n */\nexport interface LoginStep {\n  action: 'fill' | 'click' | 'wait';\n  selector: string;\n  value?: string;\n  ms?: number;\n}\n\nconst TwoFASchema = z.object({\n  has2FA: z.boolean(),\n  fieldLabel: z.string().optional(),\n});\n\nconst CAPTCHA_SELECTORS = [\n  'iframe[src*=\"recaptcha\"]',\n  'iframe[src*=\"hcaptcha\"]',\n  'iframe[src*=\"challenges.cloudflare.com\"]',\n  '.cf-turnstile',\n  '[data-sitekey]',\n  '#captcha',\n  '.captcha',\n  '[id*=\"captcha\"]',\n  '[class*=\"captcha\"]',\n];\n\n/**\n * AI-guided login using Stagehand act/extract.\n *\n * @param loginInstructions - Optional natural language description of the login flow.\n *   When provided, an initial stagehand.act() is called with these instructions to\n *   navigate the UI (e.g. \"click Sign In in the nav, fill email, click Continue\").\n *   The standard 3-step credential fill follows immediately after.\n * @param opts - Optional tenant/project/job IDs for writing AUTH_STEP CrawlEvidence.\n */\nexport async function loginWithStagehand(\n  stagehand: Stagehand,\n  creds: Credentials,\n  startUrl: string,\n  captchaOpts?: { apiKey?: string; provider?: '2captcha' | 'capsolver' },\n  emailOtpOpts?: { apiKey?: string; inboxId?: string },\n  loginInstructions?: string,\n  opts?: { tenantId?: string; projectId?: string; jobId?: string },\n): Promise<boolean> {\n  async function logStep(step: string, fn: () => Promise<any>): Promise<{ success: boolean; detail: string }> {\n    const t0 = Date.now();\n    try {\n      const r = await fn();\n      const result = { success: true, detail: r?.description ?? r?.message ?? step };\n      if (opts?.tenantId && opts?.projectId) {\n        AppBrain.saveCrawlEvidence(opts.tenantId, opts.projectId, {\n          jobId: opts?.jobId,\n          type: 'AUTH_STEP',\n          payload: { step, ...result, ms: Date.now() - t0 },\n        }).catch(() => {});\n      }\n      return result;\n    } catch (e: any) {\n      const result = { success: false, detail: e.message ?? 'failed' };\n      if (opts?.tenantId && opts?.projectId) {\n        AppBrain.saveCrawlEvidence(opts.tenantId, opts.projectId, {\n          jobId: opts?.jobId,\n          type: 'AUTH_STEP',\n          payload: { step, ...result, ms: Date.now() - t0 },\n        }).catch(() => {});\n      }\n      throw e;\n    }\n  }\n\n  try {\n    const pg = await (stagehand as any).resolvePage();\n\n    // CAPTCHA detection\n    for (const sel of CAPTCHA_SELECTORS) {\n      const count = await pg.locator(sel).count().catch(() => 0);\n      if (count > 0) {\n        logger.warn({ selector: sel }, '[crawler] CAPTCHA detected on login page');\n        if (captchaOpts?.apiKey) {\n          logger.info({ provider: captchaOpts.provider ?? '2captcha' }, '[crawler] Attempting auto-solve');\n          try {\n            const result = await solveCaptcha(pg, startUrl, captchaOpts.apiKey, captchaOpts.provider ?? '2captcha');\n            if (result) {\n              logger.info({ provider: result.provider }, '[crawler] CAPTCHA solved \u2014 proceeding with login');\n              break;\n            }\n          } catch (solveErr) {\n            logger.warn({ err: solveErr }, '[crawler] CAPTCHA solve failed \u2014 falling back to manual error');\n            throw new Error(`CAPTCHA_DETECTED: Auto-solve failed (${String(solveErr)}). Use session capture instead.`);\n          }\n        } else {\n          throw new Error(\n            'CAPTCHA_DETECTED: Login page has a CAPTCHA. Use \"Session Capture \u2192 Launch Browser\" to log in manually, then capture the session. Alternatively, disable CAPTCHA for the ZeTa crawler IP.'\n          );\n        }\n        break;\n      }\n    }\n\n    // If caller provided natural language login instructions, use them to navigate to the login form\n    if (loginInstructions) {\n      await logStep('custom_recipe', () => actWithMouseSim(stagehand, loginInstructions));\n      await pg.waitForLoadState('networkidle').catch(() => pg.waitForTimeout(1500));\n    }\n\n    await logStep('fill_username', () => actWithMouseSim(stagehand, `Type \"${creds.username}\" into the username or email input field`));\n    await logStep('fill_password', () => actWithMouseSim(stagehand, `Type \"${creds.password}\" into the password input field`));\n    await logStep('click_submit', () => actWithMouseSim(stagehand, 'Click the sign in, log in, login, or continue button to submit credentials'));\n    await pg.waitForLoadState('networkidle').catch(() => pg.waitForTimeout(2000));\n\n    const twoFA = await stagehand.extract(\n      'Is there a two-factor authentication, OTP, or verification code input visible right now?',\n      TwoFASchema as any\n    );\n\n    if (twoFA.has2FA) {\n      if (opts?.tenantId && opts?.projectId) {\n        AppBrain.saveCrawlEvidence(opts.tenantId, opts.projectId, {\n          jobId: opts?.jobId,\n          type: 'AUTH_STEP',\n          payload: { step: '2fa_detect', success: true, detail: twoFA.fieldLabel ?? '2FA detected', ms: 0 },\n        }).catch(() => {});\n      }\n\n      let code = creds.oneTimeCode ?? await creds.getOneTimeCode?.();\n\n      if (!code && emailOtpOpts?.apiKey && emailOtpOpts?.inboxId) {\n        const since = new Date().toISOString();\n        logger.info('[crawler] 2FA detected \u2014 waiting for email OTP or magic link via MailSlurp');\n        const emailContent = await waitForEmailContent(\n          { provider: 'mailslurp', apiKey: emailOtpOpts.apiKey, inboxId: emailOtpOpts.inboxId },\n          since\n        );\n        if (emailContent?.magicLink) {\n          logger.info({ url: emailContent.magicLink }, '[crawler] Magic link received \u2014 navigating');\n          await pg.goto(emailContent.magicLink, { waitUntil: 'domcontentloaded' });\n          await pg.waitForLoadState('networkidle').catch(() => {});\n          return true; // magic link navigation = login complete\n        }\n        code = emailContent?.otp ?? undefined;\n        if (code) {\n          logger.info('[crawler] Email OTP received');\n        } else {\n          logger.warn('[crawler] Email OTP timed out after 60s');\n        }\n      }\n\n      if (!code) {\n        console.warn('[crawler] 2FA detected but no oneTimeCode provided');\n        return false;\n      }\n      await logStep('2fa_inject', () => actWithMouseSim(stagehand, `Enter \"${code}\" into the ${twoFA.fieldLabel ?? 'verification code'} field`));\n      await logStep('2fa_submit', () => actWithMouseSim(stagehand, 'Click verify or confirm to submit the verification code'));\n      await pg.waitForLoadState('networkidle').catch(() => pg.waitForTimeout(2000));\n    }\n\n    // URL-based success check\n    const currentUrl = pg.url();\n    const loginPathPattern = /\\/(login|signin|sign-in|auth|authenticate|log-in|account\\/login)(\\?|$|\\/)/i;\n    try {\n      const { pathname } = new URL(currentUrl);\n      const wasRedirected = currentUrl !== startUrl;\n      const stillOnLoginPath = loginPathPattern.test(pathname);\n      if (wasRedirected && !stillOnLoginPath) {\n        logger.info({ url: currentUrl }, 'Login confirmed via URL change');\n        return true;\n      }\n    } catch { /* invalid URL \u2014 fall through */ }\n\n    const passwordCount = await pg.locator('input[type=\"password\"]').count().catch(() => 1);\n    const bodyText = await pg.evaluate(() => (document as any).body?.innerText?.trim()?.length ?? 0).catch(() => 0);\n    if (passwordCount === 0 && bodyText > 30) {\n      logger.info('Login confirmed: password field no longer visible');\n      return true;\n    }\n\n    logger.warn({ url: currentUrl }, 'Login appears failed');\n    try {\n      const screenshotPath = `/tmp/zeta-auth-failure-${Date.now()}.jpg`;\n      await pg.screenshot({ path: screenshotPath, type: 'jpeg', quality: 70, fullPage: false });\n      logger.warn({ screenshotPath }, '[crawler] Auth failure screenshot saved');\n    } catch { /* non-fatal */ }\n    return false;\n  } catch (err) {\n    const msg = String((err as Error).message ?? err);\n    if (msg.includes('CAPTCHA_DETECTED')) throw err;\n    logger.error({ err }, 'Login error');\n    return false;\n  }\n}\n"],
-  "mappings": "AAAA,SAAS,SAAS;AAElB,SAAS,gBAAgB;AACzB,SAAS,oBAAoB;AAC7B,SAAS,2BAA2B;AACpC,SAAS,cAAc;AAEvB,SAAS,6BAA6B;AAEtC,MAAM,eAAe;AAGrB,eAAe,gBAAgB,WAAsB,aAAmC;AACtF,MAAI,aAAa,KAAK,WAAW,GAAG;AAClC,QAAI;AACF,YAAM,KAAK,MAAO,UAAkB,YAAY;AAChD,YAAM,WAAW,GAAG,aAAa,KAAK,EAAE,OAAO,MAAM,QAAQ,IAAI;AAEjE,YAAM,UAAU,SAAS,QAAQ,MAAM,KAAK,OAAO,IAAI,SAAS,QAAQ;AACxE,YAAM,UAAU,SAAS,SAAS,MAAM,KAAK,OAAO,IAAI,SAAS,SAAS;AAC1E,YAAM,sBAAsB,IAAI,SAAS,OAAO;AAAA,IAClD,QAAQ;AAAA,IAA2C;AAAA,EACrD;AACA,SAAO,UAAU,IAAI,WAAW;AAClC;AAaA,MAAM,cAAc,EAAE,OAAO;AAAA,EAC3B,QAAQ,EAAE,QAAQ;AAAA,EAClB,YAAY,EAAE,OAAO,EAAE,SAAS;AAClC,CAAC;AAED,MAAM,oBAAoB;AAAA,EACxB;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AACF;AAWA,eAAsB,mBACpB,WACA,OACA,UACA,aACA,cACA,mBACA,MACkB;AAClB,iBAAe,QAAQ,MAAc,IAAuE;AAC1G,UAAM,KAAK,KAAK,IAAI;AACpB,QAAI;AACF,YAAM,IAAI,MAAM,GAAG;AACnB,YAAM,SAAS,EAAE,SAAS,MAAM,QAAQ,GAAG,eAAe,GAAG,WAAW,KAAK;AAC7E,UAAI,MAAM,YAAY,MAAM,WAAW;AACrC,iBAAS,kBAAkB,KAAK,UAAU,KAAK,WAAW;AAAA,UACxD,OAAO,MAAM;AAAA,UACb,MAAM;AAAA,UACN,SAAS,EAAE,MAAM,GAAG,QAAQ,IAAI,KAAK,IAAI,IAAI,GAAG;AAAA,QAClD,CAAC,EAAE,MAAM,MAAM;AAAA,QAAC,CAAC;AAAA,MACnB;AACA,aAAO;AAAA,IACT,SAAS,GAAQ;AACf,YAAM,SAAS,EAAE,SAAS,OAAO,QAAQ,EAAE,WAAW,SAAS;AAC/D,UAAI,MAAM,YAAY,MAAM,WAAW;AACrC,iBAAS,kBAAkB,KAAK,UAAU,KAAK,WAAW;AAAA,UACxD,OAAO,MAAM;AAAA,UACb,MAAM;AAAA,UACN,SAAS,EAAE,MAAM,GAAG,QAAQ,IAAI,KAAK,IAAI,IAAI,GAAG;AAAA,QAClD,CAAC,EAAE,MAAM,MAAM;AAAA,QAAC,CAAC;AAAA,MACnB;AACA,YAAM;AAAA,IACR;AAAA,EACF;AAEA,MAAI;AACF,UAAM,KAAK,MAAO,UAAkB,YAAY;AAGhD,eAAW,OAAO,mBAAmB;AACnC,YAAM,QAAQ,MAAM,GAAG,QAAQ,GAAG,EAAE,MAAM,EAAE,MAAM,MAAM,CAAC;AACzD,UAAI,QAAQ,GAAG;AACb,eAAO,KAAK,EAAE,UAAU,IAAI,GAAG,0CAA0C;AACzE,YAAI,aAAa,QAAQ;AACvB,iBAAO,KAAK,EAAE,UAAU,YAAY,YAAY,WAAW,GAAG,iCAAiC;AAC/F,cAAI;AACF,kBAAM,SAAS,MAAM,aAAa,IAAI,UAAU,YAAY,QAAQ,YAAY,YAAY,UAAU;AACtG,gBAAI,QAAQ;AACV,qBAAO,KAAK,EAAE,UAAU,OAAO,SAAS,GAAG,uDAAkD;AAC7F;AAAA,YACF;AAAA,UACF,SAAS,UAAU;AACjB,mBAAO,KAAK,EAAE,KAAK,SAAS,GAAG,oEAA+D;AAC9F,kBAAM,IAAI,MAAM,wCAAwC,OAAO,QAAQ,CAAC,iCAAiC;AAAA,UAC3G;AAAA,QACF,OAAO;AACL,gBAAM,IAAI;AAAA,YACR;AAAA,UACF;AAAA,QACF;AACA;AAAA,MACF;AAAA,IACF;AAGA,QAAI,mBAAmB;AACrB,YAAM,QAAQ,iBAAiB,MAAM,gBAAgB,WAAW,iBAAiB,CAAC;AAClF,YAAM,GAAG,iBAAiB,aAAa,EAAE,MAAM,MAAM,GAAG,eAAe,IAAI,CAAC;AAAA,IAC9E;AAEA,UAAM,QAAQ,iBAAiB,MAAM,gBAAgB,WAAW,SAAS,MAAM,QAAQ,0CAA0C,CAAC;AAClI,UAAM,QAAQ,iBAAiB,MAAM,gBAAgB,WAAW,SAAS,MAAM,QAAQ,iCAAiC,CAAC;AACzH,UAAM,QAAQ,gBAAgB,MAAM,gBAAgB,WAAW,4EAA4E,CAAC;AAC5I,UAAM,GAAG,iBAAiB,aAAa,EAAE,MAAM,MAAM,GAAG,eAAe,GAAI,CAAC;AAE5E,UAAM,QAAQ,MAAM,UAAU;AAAA,MAC5B;AAAA,MACA;AAAA,IACF;AAEA,QAAI,MAAM,QAAQ;AAChB,UAAI,MAAM,YAAY,MAAM,WAAW;AACrC,iBAAS,kBAAkB,KAAK,UAAU,KAAK,WAAW;AAAA,UACxD,OAAO,MAAM;AAAA,UACb,MAAM;AAAA,UACN,SAAS,EAAE,MAAM,cAAc,SAAS,MAAM,QAAQ,MAAM,cAAc,gBAAgB,IAAI,EAAE;AAAA,QAClG,CAAC,EAAE,MAAM,MAAM;AAAA,QAAC,CAAC;AAAA,MACnB;AAEA,UAAI,OAAO,MAAM,eAAe,MAAM,MAAM,iBAAiB;AAE7D,UAAI,CAAC,QAAQ,cAAc,UAAU,cAAc,SAAS;AAC1D,cAAM,SAAQ,oBAAI,KAAK,GAAE,YAAY;AACrC,eAAO,KAAK,iFAA4E;AACxF,cAAM,eAAe,MAAM;AAAA,UACzB,EAAE,UAAU,aAAa,QAAQ,aAAa,QAAQ,SAAS,aAAa,QAAQ;AAAA,UACpF;AAAA,QACF;AACA,YAAI,cAAc,WAAW;AAC3B,iBAAO,KAAK,EAAE,KAAK,aAAa,UAAU,GAAG,iDAA4C;AACzF,gBAAM,GAAG,KAAK,aAAa,WAAW,EAAE,WAAW,mBAAmB,CAAC;AACvE,gBAAM,GAAG,iBAAiB,aAAa,EAAE,MAAM,MAAM;AAAA,UAAC,CAAC;AACvD,iBAAO;AAAA,QACT;AACA,eAAO,cAAc,OAAO;AAC5B,YAAI,MAAM;AACR,iBAAO,KAAK,8BAA8B;AAAA,QAC5C,OAAO;AACL,iBAAO,KAAK,yCAAyC;AAAA,QACvD;AAAA,MACF;AAEA,UAAI,CAAC,MAAM;AACT,gBAAQ,KAAK,oDAAoD;AACjE,eAAO;AAAA,MACT;AACA,YAAM,QAAQ,cAAc,MAAM,gBAAgB,WAAW,UAAU,IAAI,cAAc,MAAM,cAAc,mBAAmB,QAAQ,CAAC;AACzI,YAAM,QAAQ,cAAc,MAAM,gBAAgB,WAAW,yDAAyD,CAAC;AACvH,YAAM,GAAG,iBAAiB,aAAa,EAAE,MAAM,MAAM,GAAG,eAAe,GAAI,CAAC;AAAA,IAC9E;AAGA,UAAM,aAAa,GAAG,IAAI;AAC1B,UAAM,mBAAmB;AACzB,QAAI;AACF,YAAM,EAAE,SAAS,IAAI,IAAI,IAAI,UAAU;AACvC,YAAM,gBAAgB,eAAe;AACrC,YAAM,mBAAmB,iBAAiB,KAAK,QAAQ;AACvD,UAAI,iBAAiB,CAAC,kBAAkB;AACtC,eAAO,KAAK,EAAE,KAAK,WAAW,GAAG,gCAAgC;AACjE,eAAO;AAAA,MACT;AAAA,IACF,QAAQ;AAAA,IAAmC;AAE3C,UAAM,gBAAgB,MAAM,GAAG,QAAQ,wBAAwB,EAAE,MAAM,EAAE,MAAM,MAAM,CAAC;AACtF,UAAM,WAAW,MAAM,GAAG,SAAS,MAAO,SAAiB,MAAM,WAAW,KAAK,GAAG,UAAU,CAAC,EAAE,MAAM,MAAM,CAAC;AAC9G,QAAI,kBAAkB,KAAK,WAAW,IAAI;AACxC,aAAO,KAAK,mDAAmD;AAC/D,aAAO;AAAA,IACT;AAEA,WAAO,KAAK,EAAE,KAAK,WAAW,GAAG,sBAAsB;AACvD,QAAI;AACF,YAAM,iBAAiB,0BAA0B,KAAK,IAAI,CAAC;AAC3D,YAAM,GAAG,WAAW,EAAE,MAAM,gBAAgB,MAAM,QAAQ,SAAS,IAAI,UAAU,MAAM,CAAC;AACxF,aAAO,KAAK,EAAE,eAAe,GAAG,yCAAyC;AAAA,IAC3E,QAAQ;AAAA,IAAkB;AAC1B,WAAO;AAAA,EACT,SAAS,KAAK;AACZ,UAAM,MAAM,OAAQ,IAAc,WAAW,GAAG;AAChD,QAAI,IAAI,SAAS,kBAAkB,EAAG,OAAM;AAC5C,WAAO,MAAM,EAAE,IAAI,GAAG,aAAa;AACnC,WAAO;AAAA,EACT;AACF;",
+  "sourcesContent": ["import { z } from 'zod';\nimport type { Stagehand } from '@browserbasehq/stagehand';\nimport { AppBrain } from '@detiq/app-brain';\nimport { solveCaptcha } from './captcha-solver.js';\nimport { waitForEmailContent } from './email-otp.js';\nimport { logger } from './logger.js';\nimport type { Credentials } from './auth.js';\nimport { simulateMouseMovement } from './stealth/human-behavior.js';\n\nconst CLICK_INTENT = /\\b(click|tap|press|submit|select|choose|toggle)\\b/i;\n\n/** Simulate human mouse movement before click-type actions, then call stagehand.act(). */\nasync function actWithMouseSim(stagehand: Stagehand, instruction: string): Promise<any> {\n  if (CLICK_INTENT.test(instruction)) {\n    try {\n      const pg = await (stagehand as any).resolvePage();\n      const viewport = pg.viewportSize() ?? { width: 1280, height: 720 };\n      // Move mouse toward center of page with some randomness before Stagehand clicks\n      const targetX = viewport.width * 0.3 + Math.random() * viewport.width * 0.4;\n      const targetY = viewport.height * 0.2 + Math.random() * viewport.height * 0.5;\n      await simulateMouseMovement(pg, targetX, targetY);\n    } catch { /* non-fatal \u2014 don't block the action */ }\n  }\n  return stagehand.act(instruction);\n}\n\n/**\n * Recorded login step from the Chrome extension's step recorder.\n * {username} and {password} in `value` are substituted at replay time.\n */\nexport interface LoginStep {\n  action: 'fill' | 'click' | 'wait';\n  selector: string;\n  value?: string;\n  ms?: number;\n}\n\nconst TwoFASchema = z.object({\n  has2FA: z.boolean(),\n  fieldLabel: z.string().optional(),\n});\n\nconst CAPTCHA_SELECTORS = [\n  'iframe[src*=\"recaptcha\"]',\n  'iframe[src*=\"hcaptcha\"]',\n  'iframe[src*=\"challenges.cloudflare.com\"]',\n  '.cf-turnstile',\n  '[data-sitekey]',\n  '#captcha',\n  '.captcha',\n  '[id*=\"captcha\"]',\n  '[class*=\"captcha\"]',\n];\n\n/**\n * AI-guided login using Stagehand act/extract.\n *\n * @param loginInstructions - Optional natural language description of the login flow.\n *   When provided, an initial stagehand.act() is called with these instructions to\n *   navigate the UI (e.g. \"click Sign In in the nav, fill email, click Continue\").\n *   The standard 3-step credential fill follows immediately after.\n * @param opts - Optional tenant/project/job IDs for writing AUTH_STEP CrawlEvidence.\n */\nexport async function loginWithStagehand(\n  stagehand: Stagehand,\n  creds: Credentials,\n  startUrl: string,\n  captchaOpts?: { apiKey?: string; provider?: '2captcha' | 'capsolver' },\n  emailOtpOpts?: { apiKey?: string; inboxId?: string },\n  loginInstructions?: string,\n  opts?: { tenantId?: string; projectId?: string; jobId?: string },\n): Promise<boolean> {\n  async function logStep(step: string, fn: () => Promise<any>): Promise<{ success: boolean; detail: string }> {\n    const t0 = Date.now();\n    try {\n      const r = await fn();\n      const result = { success: true, detail: r?.description ?? r?.message ?? step };\n      if (opts?.tenantId && opts?.projectId) {\n        AppBrain.saveCrawlEvidence(opts.tenantId, opts.projectId, {\n          jobId: opts?.jobId,\n          type: 'AUTH_STEP',\n          payload: { step, ...result, ms: Date.now() - t0 },\n        }).catch(() => {});\n      }\n      return result;\n    } catch (e: any) {\n      const result = { success: false, detail: e.message ?? 'failed' };\n      if (opts?.tenantId && opts?.projectId) {\n        AppBrain.saveCrawlEvidence(opts.tenantId, opts.projectId, {\n          jobId: opts?.jobId,\n          type: 'AUTH_STEP',\n          payload: { step, ...result, ms: Date.now() - t0 },\n        }).catch(() => {});\n      }\n      throw e;\n    }\n  }\n\n  try {\n    const pg = await (stagehand as any).resolvePage();\n\n    // CAPTCHA detection\n    for (const sel of CAPTCHA_SELECTORS) {\n      const count = await pg.locator(sel).count().catch(() => 0);\n      if (count > 0) {\n        logger.warn({ selector: sel }, '[crawler] CAPTCHA detected on login page');\n        if (captchaOpts?.apiKey) {\n          logger.info({ provider: captchaOpts.provider ?? '2captcha' }, '[crawler] Attempting auto-solve');\n          try {\n            const result = await solveCaptcha(pg, startUrl, captchaOpts.apiKey, captchaOpts.provider ?? '2captcha');\n            if (result) {\n              logger.info({ provider: result.provider }, '[crawler] CAPTCHA solved \u2014 proceeding with login');\n              break;\n            }\n          } catch (solveErr) {\n            logger.warn({ err: solveErr }, '[crawler] CAPTCHA solve failed \u2014 falling back to manual error');\n            throw new Error(`CAPTCHA_DETECTED: Auto-solve failed (${String(solveErr)}). Use session capture instead.`);\n          }\n        } else {\n          throw new Error(\n            'CAPTCHA_DETECTED: Login page has a CAPTCHA. Use \"Session Capture \u2192 Launch Browser\" to log in manually, then capture the session. Alternatively, disable CAPTCHA for the ZeTa crawler IP.'\n          );\n        }\n        break;\n      }\n    }\n\n    // If caller provided natural language login instructions, use them to navigate to the login form\n    const MAX_LOGIN_INSTRUCTIONS = 2000;\n    if (loginInstructions && loginInstructions.length > MAX_LOGIN_INSTRUCTIONS) {\n      throw new Error(`loginInstructions exceeds ${MAX_LOGIN_INSTRUCTIONS} character limit`);\n    }\n    if (loginInstructions) {\n      await logStep('custom_recipe', () => actWithMouseSim(stagehand, loginInstructions));\n      await pg.waitForLoadState('networkidle').catch(() => pg.waitForTimeout(1500));\n    }\n\n    await logStep('fill_username', () => actWithMouseSim(stagehand, `Type \"${creds.username}\" into the username or email input field`));\n    // Don't send password to LLM \u2014 use direct Playwright fill to avoid\n    // leaking credentials into the Stagehand/LLM provider's request logs.\n    await logStep('fill_password', async () => {\n      const observations = await stagehand.observe('Find the password input field');\n      const selector = observations[0]?.selector || 'input[type=\"password\"]';\n      await pg.locator(selector).fill(creds.password);\n    });\n    await logStep('click_submit', () => actWithMouseSim(stagehand, 'Click the sign in, log in, login, or continue button to submit credentials'));\n    await pg.waitForLoadState('networkidle').catch(() => pg.waitForTimeout(2000));\n\n    const twoFA = await stagehand.extract(\n      'Is there a two-factor authentication, OTP, or verification code input visible right now?',\n      TwoFASchema as any\n    );\n\n    if (twoFA.has2FA) {\n      if (opts?.tenantId && opts?.projectId) {\n        AppBrain.saveCrawlEvidence(opts.tenantId, opts.projectId, {\n          jobId: opts?.jobId,\n          type: 'AUTH_STEP',\n          payload: { step: '2fa_detect', success: true, detail: twoFA.fieldLabel ?? '2FA detected', ms: 0 },\n        }).catch(() => {});\n      }\n\n      let code = creds.oneTimeCode ?? await creds.getOneTimeCode?.();\n\n      if (!code && emailOtpOpts?.apiKey && emailOtpOpts?.inboxId) {\n        const since = new Date().toISOString();\n        logger.info('[crawler] 2FA detected \u2014 waiting for email OTP or magic link via MailSlurp');\n        const emailContent = await waitForEmailContent(\n          { provider: 'mailslurp', apiKey: emailOtpOpts.apiKey, inboxId: emailOtpOpts.inboxId },\n          since\n        );\n        if (emailContent?.magicLink) {\n          logger.info({ url: emailContent.magicLink }, '[crawler] Magic link received \u2014 navigating');\n          await pg.goto(emailContent.magicLink, { waitUntil: 'domcontentloaded' });\n          await pg.waitForLoadState('networkidle').catch(() => {});\n          return true; // magic link navigation = login complete\n        }\n        code = emailContent?.otp ?? undefined;\n        if (code) {\n          logger.info('[crawler] Email OTP received');\n        } else {\n          logger.warn('[crawler] Email OTP timed out after 60s');\n        }\n      }\n\n      if (!code) {\n        console.warn('[crawler] 2FA detected but no oneTimeCode provided');\n        return false;\n      }\n      await logStep('2fa_inject', () => actWithMouseSim(stagehand, `Enter \"${code}\" into the ${twoFA.fieldLabel ?? 'verification code'} field`));\n      await logStep('2fa_submit', () => actWithMouseSim(stagehand, 'Click verify or confirm to submit the verification code'));\n      await pg.waitForLoadState('networkidle').catch(() => pg.waitForTimeout(2000));\n    }\n\n    // URL-based success check\n    const currentUrl = pg.url();\n    const loginPathPattern = /\\/(login|signin|sign-in|auth|authenticate|log-in|account\\/login)(\\?|$|\\/)/i;\n    try {\n      const { pathname } = new URL(currentUrl);\n      const wasRedirected = currentUrl !== startUrl;\n      const stillOnLoginPath = loginPathPattern.test(pathname);\n      if (wasRedirected && !stillOnLoginPath) {\n        logger.info({ url: currentUrl }, 'Login confirmed via URL change');\n        return true;\n      }\n    } catch { /* invalid URL \u2014 fall through */ }\n\n    const passwordCount = await pg.locator('input[type=\"password\"]').count().catch(() => 1);\n    const bodyText = await pg.evaluate(() => (document as any).body?.innerText?.trim()?.length ?? 0).catch(() => 0);\n    if (passwordCount === 0 && bodyText > 30) {\n      logger.info('Login confirmed: password field no longer visible');\n      return true;\n    }\n\n    logger.warn({ url: currentUrl }, 'Login appears failed');\n    try {\n      const screenshotPath = `/tmp/zeta-auth-failure-${Date.now()}.jpg`;\n      await pg.screenshot({ path: screenshotPath, type: 'jpeg', quality: 70, fullPage: false });\n      logger.warn({ screenshotPath }, '[crawler] Auth failure screenshot saved');\n    } catch { /* non-fatal */ }\n    return false;\n  } catch (err) {\n    const msg = String((err as Error).message ?? err);\n    if (msg.includes('CAPTCHA_DETECTED')) throw err;\n    logger.error({ err }, 'Login error');\n    return false;\n  }\n}\n"],
+  "mappings": "AAAA,SAAS,SAAS;AAElB,SAAS,gBAAgB;AACzB,SAAS,oBAAoB;AAC7B,SAAS,2BAA2B;AACpC,SAAS,cAAc;AAEvB,SAAS,6BAA6B;AAEtC,MAAM,eAAe;AAGrB,eAAe,gBAAgB,WAAsB,aAAmC;AACtF,MAAI,aAAa,KAAK,WAAW,GAAG;AAClC,QAAI;AACF,YAAM,KAAK,MAAO,UAAkB,YAAY;AAChD,YAAM,WAAW,GAAG,aAAa,KAAK,EAAE,OAAO,MAAM,QAAQ,IAAI;AAEjE,YAAM,UAAU,SAAS,QAAQ,MAAM,KAAK,OAAO,IAAI,SAAS,QAAQ;AACxE,YAAM,UAAU,SAAS,SAAS,MAAM,KAAK,OAAO,IAAI,SAAS,SAAS;AAC1E,YAAM,sBAAsB,IAAI,SAAS,OAAO;AAAA,IAClD,QAAQ;AAAA,IAA2C;AAAA,EACrD;AACA,SAAO,UAAU,IAAI,WAAW;AAClC;AAaA,MAAM,cAAc,EAAE,OAAO;AAAA,EAC3B,QAAQ,EAAE,QAAQ;AAAA,EAClB,YAAY,EAAE,OAAO,EAAE,SAAS;AAClC,CAAC;AAED,MAAM,oBAAoB;AAAA,EACxB;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AACF;AAWA,eAAsB,mBACpB,WACA,OACA,UACA,aACA,cACA,mBACA,MACkB;AAClB,iBAAe,QAAQ,MAAc,IAAuE;AAC1G,UAAM,KAAK,KAAK,IAAI;AACpB,QAAI;AACF,YAAM,IAAI,MAAM,GAAG;AACnB,YAAM,SAAS,EAAE,SAAS,MAAM,QAAQ,GAAG,eAAe,GAAG,WAAW,KAAK;AAC7E,UAAI,MAAM,YAAY,MAAM,WAAW;AACrC,iBAAS,kBAAkB,KAAK,UAAU,KAAK,WAAW;AAAA,UACxD,OAAO,MAAM;AAAA,UACb,MAAM;AAAA,UACN,SAAS,EAAE,MAAM,GAAG,QAAQ,IAAI,KAAK,IAAI,IAAI,GAAG;AAAA,QAClD,CAAC,EAAE,MAAM,MAAM;AAAA,QAAC,CAAC;AAAA,MACnB;AACA,aAAO;AAAA,IACT,SAAS,GAAQ;AACf,YAAM,SAAS,EAAE,SAAS,OAAO,QAAQ,EAAE,WAAW,SAAS;AAC/D,UAAI,MAAM,YAAY,MAAM,WAAW;AACrC,iBAAS,kBAAkB,KAAK,UAAU,KAAK,WAAW;AAAA,UACxD,OAAO,MAAM;AAAA,UACb,MAAM;AAAA,UACN,SAAS,EAAE,MAAM,GAAG,QAAQ,IAAI,KAAK,IAAI,IAAI,GAAG;AAAA,QAClD,CAAC,EAAE,MAAM,MAAM;AAAA,QAAC,CAAC;AAAA,MACnB;AACA,YAAM;AAAA,IACR;AAAA,EACF;AAEA,MAAI;AACF,UAAM,KAAK,MAAO,UAAkB,YAAY;AAGhD,eAAW,OAAO,mBAAmB;AACnC,YAAM,QAAQ,MAAM,GAAG,QAAQ,GAAG,EAAE,MAAM,EAAE,MAAM,MAAM,CAAC;AACzD,UAAI,QAAQ,GAAG;AACb,eAAO,KAAK,EAAE,UAAU,IAAI,GAAG,0CAA0C;AACzE,YAAI,aAAa,QAAQ;AACvB,iBAAO,KAAK,EAAE,UAAU,YAAY,YAAY,WAAW,GAAG,iCAAiC;AAC/F,cAAI;AACF,kBAAM,SAAS,MAAM,aAAa,IAAI,UAAU,YAAY,QAAQ,YAAY,YAAY,UAAU;AACtG,gBAAI,QAAQ;AACV,qBAAO,KAAK,EAAE,UAAU,OAAO,SAAS,GAAG,uDAAkD;AAC7F;AAAA,YACF;AAAA,UACF,SAAS,UAAU;AACjB,mBAAO,KAAK,EAAE,KAAK,SAAS,GAAG,oEAA+D;AAC9F,kBAAM,IAAI,MAAM,wCAAwC,OAAO,QAAQ,CAAC,iCAAiC;AAAA,UAC3G;AAAA,QACF,OAAO;AACL,gBAAM,IAAI;AAAA,YACR;AAAA,UACF;AAAA,QACF;AACA;AAAA,MACF;AAAA,IACF;AAGA,UAAM,yBAAyB;AAC/B,QAAI,qBAAqB,kBAAkB,SAAS,wBAAwB;AAC1E,YAAM,IAAI,MAAM,6BAA6B,sBAAsB,kBAAkB;AAAA,IACvF;AACA,QAAI,mBAAmB;AACrB,YAAM,QAAQ,iBAAiB,MAAM,gBAAgB,WAAW,iBAAiB,CAAC;AAClF,YAAM,GAAG,iBAAiB,aAAa,EAAE,MAAM,MAAM,GAAG,eAAe,IAAI,CAAC;AAAA,IAC9E;AAEA,UAAM,QAAQ,iBAAiB,MAAM,gBAAgB,WAAW,SAAS,MAAM,QAAQ,0CAA0C,CAAC;AAGlI,UAAM,QAAQ,iBAAiB,YAAY;AACzC,YAAM,eAAe,MAAM,UAAU,QAAQ,+BAA+B;AAC5E,YAAM,WAAW,aAAa,CAAC,GAAG,YAAY;AAC9C,YAAM,GAAG,QAAQ,QAAQ,EAAE,KAAK,MAAM,QAAQ;AAAA,IAChD,CAAC;AACD,UAAM,QAAQ,gBAAgB,MAAM,gBAAgB,WAAW,4EAA4E,CAAC;AAC5I,UAAM,GAAG,iBAAiB,aAAa,EAAE,MAAM,MAAM,GAAG,eAAe,GAAI,CAAC;AAE5E,UAAM,QAAQ,MAAM,UAAU;AAAA,MAC5B;AAAA,MACA;AAAA,IACF;AAEA,QAAI,MAAM,QAAQ;AAChB,UAAI,MAAM,YAAY,MAAM,WAAW;AACrC,iBAAS,kBAAkB,KAAK,UAAU,KAAK,WAAW;AAAA,UACxD,OAAO,MAAM;AAAA,UACb,MAAM;AAAA,UACN,SAAS,EAAE,MAAM,cAAc,SAAS,MAAM,QAAQ,MAAM,cAAc,gBAAgB,IAAI,EAAE;AAAA,QAClG,CAAC,EAAE,MAAM,MAAM;AAAA,QAAC,CAAC;AAAA,MACnB;AAEA,UAAI,OAAO,MAAM,eAAe,MAAM,MAAM,iBAAiB;AAE7D,UAAI,CAAC,QAAQ,cAAc,UAAU,cAAc,SAAS;AAC1D,cAAM,SAAQ,oBAAI,KAAK,GAAE,YAAY;AACrC,eAAO,KAAK,iFAA4E;AACxF,cAAM,eAAe,MAAM;AAAA,UACzB,EAAE,UAAU,aAAa,QAAQ,aAAa,QAAQ,SAAS,aAAa,QAAQ;AAAA,UACpF;AAAA,QACF;AACA,YAAI,cAAc,WAAW;AAC3B,iBAAO,KAAK,EAAE,KAAK,aAAa,UAAU,GAAG,iDAA4C;AACzF,gBAAM,GAAG,KAAK,aAAa,WAAW,EAAE,WAAW,mBAAmB,CAAC;AACvE,gBAAM,GAAG,iBAAiB,aAAa,EAAE,MAAM,MAAM;AAAA,UAAC,CAAC;AACvD,iBAAO;AAAA,QACT;AACA,eAAO,cAAc,OAAO;AAC5B,YAAI,MAAM;AACR,iBAAO,KAAK,8BAA8B;AAAA,QAC5C,OAAO;AACL,iBAAO,KAAK,yCAAyC;AAAA,QACvD;AAAA,MACF;AAEA,UAAI,CAAC,MAAM;AACT,gBAAQ,KAAK,oDAAoD;AACjE,eAAO;AAAA,MACT;AACA,YAAM,QAAQ,cAAc,MAAM,gBAAgB,WAAW,UAAU,IAAI,cAAc,MAAM,cAAc,mBAAmB,QAAQ,CAAC;AACzI,YAAM,QAAQ,cAAc,MAAM,gBAAgB,WAAW,yDAAyD,CAAC;AACvH,YAAM,GAAG,iBAAiB,aAAa,EAAE,MAAM,MAAM,GAAG,eAAe,GAAI,CAAC;AAAA,IAC9E;AAGA,UAAM,aAAa,GAAG,IAAI;AAC1B,UAAM,mBAAmB;AACzB,QAAI;AACF,YAAM,EAAE,SAAS,IAAI,IAAI,IAAI,UAAU;AACvC,YAAM,gBAAgB,eAAe;AACrC,YAAM,mBAAmB,iBAAiB,KAAK,QAAQ;AACvD,UAAI,iBAAiB,CAAC,kBAAkB;AACtC,eAAO,KAAK,EAAE,KAAK,WAAW,GAAG,gCAAgC;AACjE,eAAO;AAAA,MACT;AAAA,IACF,QAAQ;AAAA,IAAmC;AAE3C,UAAM,gBAAgB,MAAM,GAAG,QAAQ,wBAAwB,EAAE,MAAM,EAAE,MAAM,MAAM,CAAC;AACtF,UAAM,WAAW,MAAM,GAAG,SAAS,MAAO,SAAiB,MAAM,WAAW,KAAK,GAAG,UAAU,CAAC,EAAE,MAAM,MAAM,CAAC;AAC9G,QAAI,kBAAkB,KAAK,WAAW,IAAI;AACxC,aAAO,KAAK,mDAAmD;AAC/D,aAAO;AAAA,IACT;AAEA,WAAO,KAAK,EAAE,KAAK,WAAW,GAAG,sBAAsB;AACvD,QAAI;AACF,YAAM,iBAAiB,0BAA0B,KAAK,IAAI,CAAC;AAC3D,YAAM,GAAG,WAAW,EAAE,MAAM,gBAAgB,MAAM,QAAQ,SAAS,IAAI,UAAU,MAAM,CAAC;AACxF,aAAO,KAAK,EAAE,eAAe,GAAG,yCAAyC;AAAA,IAC3E,QAAQ;AAAA,IAAkB;AAC1B,WAAO;AAAA,EACT,SAAS,KAAK;AACZ,UAAM,MAAM,OAAQ,IAAc,WAAW,GAAG;AAChD,QAAI,IAAI,SAAS,kBAAkB,EAAG,OAAM;AAC5C,WAAO,MAAM,EAAE,IAAI,GAAG,aAAa;AACnC,WAAO;AAAA,EACT;AACF;",
   "names": []
 }

package/dist/crawler.js

@@ -9,9 +9,9 @@ stealthChromium.use(StealthPlugin());
 import { z } from "zod";
 import fs from "node:fs/promises";
 import path from "node:path";
-import { AppBrain, recordLLMCall } from "@qip/app-brain";
-import { aiVisualCompare, askTracked, estimateCost, getTaskAgentLLMConfig } from "@qip/agents";
-import { config } from "@qip/core";
+import { AppBrain, recordLLMCall } from "@detiq/app-brain";
+import { aiVisualCompare, askTracked, estimateCost, getTaskAgentLLMConfig } from "@detiq/agents";
+import { config } from "@detiq/core";
 import { pHashVerdict, mergeAiVisualVerdict } from "./visual-diff-verdict.js";
 import { skyvernFallback, skyvernNavigate } from "./skyvern.js";
 import { loginWithStagehand } from "./ai-login.js";
@@ -1002,6 +1002,7 @@ async function crawlProject(opts, onProgress) {
     }
     let checkpointSuccessCount = 0;
     let maxSeenTotal = 0;
+    let lastEmittedPct = 20;
     const startUrl = page.url();
     const resumeUrls = opts.resumeUrls ?? [];
     const gqlIntrospectionPromise = (async () => {
@@ -1045,8 +1046,9 @@ async function crawlProject(opts, onProgress) {
           const u = s.url;
           if (!u) return false;
           try {
-            const { origin: origin2, pathname } = new URL(u);
-            if (origin2 !== new URL(startUrl).origin) return false;
+            const { pathname } = new URL(u);
+            const stripWww = (h) => h.replace(/^www\./i, "").toLowerCase();
+            if (stripWww(new URL(u).hostname) !== stripWww(new URL(startUrl).hostname)) return false;
             if (isGeneratedDynamicPath(pathname)) return false;
             return true;
           } catch {
@@ -1937,7 +1939,7 @@ ${snippet}`,
                 if (changeType !== "NONE") {
                   let newScreenshotCloudUrl;
                   try {
-                    const { getTenantStorageConfig, uploadScreenshotToTenantStorage, buildArtifactKey } = await import("@qip/app-brain");
+                    const { getTenantStorageConfig, uploadScreenshotToTenantStorage, buildArtifactKey } = await import("@detiq/app-brain");
                     const storageCfg = await getTenantStorageConfig(opts.tenantId).catch(() => null);
                     if (storageCfg) {
                       const key = buildArtifactKey(opts.tenantId, opts.projectId, "visual-diffs", `${screen.id}-v${screen.version}.png`, storageCfg.prefix);
@@ -2157,8 +2159,10 @@ ${snippet}`,
         }
         discovered.push({ url, screenId: screen.id, elements: elements.length, elementFacts: elements, screenshotPath: shotPath, version: screen.version });
         maxSeenTotal = Math.max(maxSeenTotal, visited.size + queue.length);
-        const screenPct = Math.min(60, visited.size / Math.max(1, maxSeenTotal) * 60);
-        await onProgress?.(20 + Math.round(screenPct), `Crawling screens (${visited.size} mapped, ${queue.length} pending)`);
+        const rawPct = 20 + Math.round(Math.min(60, visited.size / Math.max(1, maxSeenTotal) * 60));
+        const clampedPct = Math.max(lastEmittedPct, rawPct);
+        lastEmittedPct = clampedPct;
+        await onProgress?.(clampedPct, `Crawling screens (${visited.size} mapped, ${queue.length} pending)`);
         logger.info({ name, elements: elements.length }, "Screen crawled");
         if (++checkpointSuccessCount % 10 === 0) {
           await opts.onCheckpoint?.(