@detiq/crawler 0.2.7 → 0.2.10

package/package.json

@@ -1,8 +1,11 @@
 {
   "name": "@detiq/crawler",
-  "version": "0.2.7",
+  "version": "0.2.10",
   "private": false,
-  "publishConfig": { "access": "public", "registry": "https://registry.npmjs.org/" },
+  "publishConfig": {
+    "access": "public",
+    "registry": "https://registry.npmjs.org/"
+  },
   "type": "module",
   "main": "dist/index.js",
   "types": "types/index.d.ts",

package/src/ai-login.ts

@@ -9,6 +9,14 @@ import { simulateMouseMovement } from './stealth/human-behavior.js';
 
 const CLICK_INTENT = /\b(click|tap|press|submit|select|choose|toggle)\b/i;
 
+/** Stagehand V3 enforces elementId as /^\d+-\d+$/ — LLM sometimes returns bare integers.
+ *  Catch the resulting AI_NoObjectGeneratedError so callers can fall back to DOM. */
+function isNoObjectGeneratedError(err: any): boolean {
+  const name = String(err?.name ?? '');
+  const msg = String(err?.message ?? '');
+  return name.includes('NoObjectGenerated') || msg.includes('NoObjectGenerated') || msg.includes('No object generated');
+}
+
 /** Simulate human mouse movement before click-type actions, then call stagehand.act(). */
 async function actWithMouseSim(stagehand: Stagehand, instruction: string): Promise<any> {
   if (CLICK_INTENT.test(instruction)) {
@@ -131,11 +139,29 @@ export async function loginWithStagehand(
       throw new Error(`loginInstructions exceeds ${MAX_LOGIN_INSTRUCTIONS} character limit`);
     }
     if (loginInstructions) {
-      await logStep('custom_recipe', () => actWithMouseSim(stagehand, loginInstructions));
+      await logStep('custom_recipe', async () => {
+        try {
+          return await actWithMouseSim(stagehand, loginInstructions);
+        } catch (err: any) {
+          if (!isNoObjectGeneratedError(err)) throw err;
+          logger.warn('[crawler] custom_recipe act() failed — Stagehand V3 elementId schema error, skipping to standard fill');
+          return { description: 'custom_recipe skipped (schema error)' };
+        }
+      });
       await pg.waitForLoadState('networkidle').catch(() => pg.waitForTimeout(1500));
     }
 
-    await logStep('fill_username', () => actWithMouseSim(stagehand, `Type "${creds.username}" into the username or email input field`));
+    await logStep('fill_username', async () => {
+      try {
+        return await actWithMouseSim(stagehand, `Type "${creds.username}" into the username or email input field`);
+      } catch (err: any) {
+        if (!isNoObjectGeneratedError(err)) throw err;
+        logger.warn('[crawler] fill_username act() failed — falling back to DOM');
+        const sel = 'input[type="email"], input[autocomplete="email"], input[autocomplete="username"], input[name*="email" i], input[name*="user" i], input[placeholder*="email" i], input[placeholder*="user" i], input[id*="email" i], input[id*="user" i]';
+        await pg.locator(sel).first().fill(creds.username);
+        return { description: 'fill_username via DOM fallback' };
+      }
+    });
     // Don't send password to LLM — use direct Playwright fill to avoid
     // leaking credentials into the Stagehand/LLM provider's request logs.
     await logStep('fill_password', async () => {
@@ -156,7 +182,18 @@ export async function loginWithStagehand(
         }, creds.password);
       }
     });
-    await logStep('click_submit', () => actWithMouseSim(stagehand, 'Click the sign in, log in, login, or continue button to submit credentials'));
+    await logStep('click_submit', async () => {
+      try {
+        return await actWithMouseSim(stagehand, 'Click the sign in, log in, login, or continue button to submit credentials');
+      } catch (err: any) {
+        if (!isNoObjectGeneratedError(err)) throw err;
+        logger.warn('[crawler] click_submit act() failed — falling back to DOM');
+        const sel = 'button[type="submit"], input[type="submit"], button:has-text("Sign in"), button:has-text("Log in"), button:has-text("Login"), button:has-text("Continue"), button:has-text("Next"), button:has-text("Sign In"), [role="button"]:has-text("Sign in"), [role="button"]:has-text("Login")';
+        await pg.locator(sel).first().click();
+        await pg.waitForTimeout(500);
+        return { description: 'click_submit via DOM fallback' };
+      }
+    });
     await pg.waitForLoadState('networkidle').catch(() => pg.waitForTimeout(2000));
 
     const twoFA = await stagehand.extract(
@@ -200,8 +237,28 @@ export async function loginWithStagehand(
         console.warn('[crawler] 2FA detected but no oneTimeCode provided');
         return false;
       }
-      await logStep('2fa_inject', () => actWithMouseSim(stagehand, `Enter "${code}" into the ${twoFA.fieldLabel ?? 'verification code'} field`));
-      await logStep('2fa_submit', () => actWithMouseSim(stagehand, 'Click verify or confirm to submit the verification code'));
+      await logStep('2fa_inject', async () => {
+        try {
+          return await actWithMouseSim(stagehand, `Enter "${code}" into the ${twoFA.fieldLabel ?? 'verification code'} field`);
+        } catch (err: any) {
+          if (!isNoObjectGeneratedError(err)) throw err;
+          logger.warn('[crawler] 2fa_inject act() failed — falling back to DOM');
+          const sel = 'input[autocomplete="one-time-code"], input[name*="otp" i], input[name*="code" i], input[name*="token" i], input[inputmode="numeric"], input[type="number"]';
+          await pg.locator(sel).first().fill(code as string);
+          return { description: '2fa_inject via DOM fallback' };
+        }
+      });
+      await logStep('2fa_submit', async () => {
+        try {
+          return await actWithMouseSim(stagehand, 'Click verify or confirm to submit the verification code');
+        } catch (err: any) {
+          if (!isNoObjectGeneratedError(err)) throw err;
+          logger.warn('[crawler] 2fa_submit act() failed — falling back to DOM');
+          const sel = 'button[type="submit"], button:has-text("Verify"), button:has-text("Confirm"), button:has-text("Submit"), button:has-text("Continue")';
+          await pg.locator(sel).first().click();
+          return { description: '2fa_submit via DOM fallback' };
+        }
+      });
       await pg.waitForLoadState('networkidle').catch(() => pg.waitForTimeout(2000));
     }
 

package/src/crawler.ts

@@ -1802,7 +1802,7 @@ export async function crawlProject(opts: {
                 type: 'AUTH_FAILURE',
                 payload: { url, landed: landedUrl, reason: 'Re-authentication failed. Session is unrecoverable. Re-capture the session in project settings and retry the crawl.' },
               }).catch(() => { });
-              break;
+              throw new Error('[AUTH_FAILURE] Re-authentication failed. Session is unrecoverable. Re-capture the session in project settings and retry the crawl.');
             }
           } else {
             logger.warn('[crawler] Session expired mid-crawl, no credentials to recover — stopping BFS');
@@ -1811,7 +1811,7 @@ export async function crawlProject(opts: {
               type: 'AUTH_FAILURE',
               payload: { url, landed: landedUrl, reason: 'Session expired and no credentials are configured. Add credentials in project settings and re-capture the session.' },
             }).catch(() => { });
-            break;
+            throw new Error('[AUTH_FAILURE] Session expired and no credentials are configured. Add credentials in project settings and re-capture the session.');
           }
         }
         // If weWantedThisPage (intentional /login, /session-expired, /404 navigation) —

package/src/job-queue.ts

@@ -383,7 +383,10 @@ export async function enqueueJob(type: string, config: any): Promise<string> {
   if (type === 'CRAWL' && process.env.JOB_QUEUE_BACKEND === 'sqs') {
     const { sendJobToSqs } = await import('./sqs-queue.js');
     await sendJobToSqs(job.id, tenantId, config);
-  } else {
+  } else if (process.env.DISABLE_WORKER !== 'true') {
+    // Skip inline dispatch when DISABLE_WORKER=true — the dedicated crawler process
+    // will pick this up via its poll loop. Inline dispatch in the API container
+    // would launch Playwright browsers and crash the API.
     setImmediate(() => claimSpecificJob(job.id).then((claimed) => claimed && dispatchClaimedJob(claimed)));
   }