@detiq/crawler 1.1.1 → 1.1.5

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@detiq/crawler",
-  "version": "1.1.1",
+  "version": "1.1.5",
   "private": false,
   "publishConfig": {
     "access": "public",

package/src/crawler.ts

@@ -1638,6 +1638,92 @@ export async function crawlProject(opts: {
 
     let pendingSessionRestore: (() => Promise<void>) | null = null;
 
+    // F10: Parallel lite-worker browser tabs — pull from shared queue while main loop runs.
+    // Each worker uses a fresh Playwright page from the same browser context so it shares
+    // cookies/session with the main page. DOM-only extraction (no Stagehand AI).
+    // Default 5 = 3-5× crawl throughput on a single browser context. Set CRAWL_PAGE_PARALLELISM=1 to disable.
+    const CRAWL_PAGE_PARALLELISM = Math.max(1, parseInt(process.env.CRAWL_PAGE_PARALLELISM ?? '5', 10));
+
+    async function runLiteWorker(workerPage: any): Promise<void> {
+      while (queue.length > 0 && visited.size < maxScreens && visited.size < segmentSize) {
+        if (opts.abortSignal?.aborted) break;
+        if (++totalAttemptsThisSegment > maxAttempts) break;
+
+        const url = queue.shift();
+        if (!url) break;
+
+        const canonUrl = canonicalize(url);
+        if (visited.has(canonUrl)) continue;
+
+        try {
+          const { pathname } = new URL(url);
+          if (SKIP_PATH.test(pathname) || isGeneratedDynamicPath(pathname)) continue;
+        } catch { continue; }
+
+        // Claim URL before first await — prevents duplicate processing across parallel workers
+        visited.set(canonUrl, '_pending_');
+
+        try {
+          const navErr = await workerPage.goto(url, { waitUntil: 'domcontentloaded', timeout: 15_000 }).then(() => null as null, (e: any) => e);
+          if (navErr && /timeout/i.test(String((navErr as any)?.message ?? navErr))) {
+            await workerPage.goto(url, { waitUntil: 'domcontentloaded', timeout: 30_000 }).catch(() => {});
+          }
+
+          const pathParts = new URL(url).pathname.split('/').filter(Boolean);
+          const name = pathParts.map((p: string) => p.charAt(0).toUpperCase() + p.slice(1).replace(/[-_]/g, ' ')).join(' — ') || 'Home';
+
+          const screen = await AppBrain.saveScreen(opts.tenantId, opts.projectId, name, url, opts.jobId, discoveryMethodFor(url));
+          visited.set(canonUrl, screen.id);
+
+          const shotPath = path.join(opts.screenshotDir ?? '/tmp', `${screen.id}-desktop-v${screen.version}.png`);
+
+          const [elements] = await Promise.all([
+            domFallbackElements(workerPage).catch(() => [] as Awaited<ReturnType<typeof domFallbackElements>>),
+            workerPage.screenshot({ path: shotPath, fullPage: true })
+              .then(async () => {
+                const cloudUrl = await uploadToCloud(shotPath, `${opts.tenantId}/${opts.projectId}/screenshots/${path.basename(shotPath)}`);
+                return (AppBrain.saveScreenshot as any)(opts.tenantId, opts.projectId, screen.id, shotPath, screen.version, 'DESKTOP', cloudUrl);
+              })
+              .catch(() => {}),
+          ]);
+
+          const nextLinks: string[] = await (workerPage.evaluate as any)((originStr: string) => {
+            return Array.from(document.querySelectorAll('a[href]'))
+              .map((a: any) => {
+                try { return new URL(a.getAttribute('href')!, window.location.href).href; }
+                catch { return ''; }
+              })
+              .filter((h: string) => h && new URL(h).origin === originStr && !h.includes('#'))
+              .slice(0, 200);
+          }, origin).catch(() => []);
+
+          for (const link of nextLinks) {
+            const cLink = canonicalize(link);
+            if (!visited.has(cLink)) {
+              queue.push(link);
+              edges.push({ fromUrl: url, toUrl: link });
+            }
+          }
+
+          discovered.push({ url, screenId: screen.id, elements: elements.length, elementFacts: elements, screenshotPath: shotPath, version: screen.version });
+        } catch (err: any) {
+          logger.warn({ url, err: err?.message ?? String(err) }, '[lite-worker] URL processing failed');
+          discovered.push({ url, error: err?.message ?? String(err) });
+        }
+      }
+    }
+
+    const liteWorkerPromises: Promise<void>[] = [];
+    if (CRAWL_PAGE_PARALLELISM > 1 && playwrightBrowser) {
+      const pwCtx = playwrightBrowser.contexts()[0];
+      for (let i = 1; i < CRAWL_PAGE_PARALLELISM; i++) {
+        const wp = await pwCtx.newPage();
+        await wp.addInitScript(() => { Object.defineProperty(navigator, 'webdriver', { get: () => undefined }); }).catch(() => {});
+        wp.setDefaultTimeout(45_000);
+        liteWorkerPromises.push(runLiteWorker(wp).finally(() => wp.close().catch(() => {})));
+      }
+    }
+
     while (queue.length > 0 && visited.size < maxScreens && visited.size < segmentSize) {
       if (pendingSessionRestore) {
         await pendingSessionRestore();
@@ -1767,9 +1853,17 @@ export async function crawlProject(opts: {
             }
           }
 
-          await page.goto(url, { waitUntil: 'domcontentloaded', timeout: 30_000 }).catch((navErr: any) => {
-            logger.warn({ url, err: navErr?.message ?? String(navErr) }, '[crawler] navigation failed — skipping');
-          });
+          // Try fast 15 s first; only escalate to 30 s on timeout so happy-path pages don't wait
+          const navErr15 = await page.goto(url, { waitUntil: 'domcontentloaded', timeout: 15_000 }).then(() => null, (e: any) => e);
+          if (navErr15) {
+            if (/timeout/i.test(String(navErr15?.message ?? navErr15))) {
+              await page.goto(url, { waitUntil: 'domcontentloaded', timeout: 30_000 }).catch((navErr: any) => {
+                logger.warn({ url, err: navErr?.message ?? String(navErr) }, '[crawler] navigation failed — skipping');
+              });
+            } else {
+              logger.warn({ url, err: navErr15?.message ?? String(navErr15) }, '[crawler] navigation failed — skipping');
+            }
+          }
           rawPage.off('response', onRedirectResponse);
           if (redirectHops.length > 0) {
             redirectRegistry.register({
@@ -2350,7 +2444,8 @@ export async function crawlProject(opts: {
           }),
           // Always run AI link discovery (even when cache exists) — cache is merged in, not used as a replacement.
           // Skipping AI when cache exists was a bug: stale cache from broken crawls permanently blocked discovery.
-          discoverLinks(stagehand, origin, page, linkLLM ?? undefined, aiSkipCount > 0)
+          // Skip AI on simple pages (< 5 interactive elements) — DOM-only is sufficient and saves ~12s per page.
+          discoverLinks(stagehand, origin, page, linkLLM ?? undefined, aiSkipCount > 0 || preDomCount < 5)
             .then((freshLinks) => {
               if (aiSkipCount > 0) aiSkipCount--;
               aiErrorStreak = 0;
@@ -2909,6 +3004,9 @@ export async function crawlProject(opts: {
       }
     }
 
+    // Wait for any lite workers still processing URLs from the shared queue
+    if (liteWorkerPromises.length > 0) await Promise.all(liteWorkerPromises);
+
     // Discovery mode: return collected URLs and skip all post-crawl analysis phases.
     // Use visited.values() (original URLs) not .keys() (canonical forms with /{id} placeholders).
     if (opts.discoveryMode) {
@@ -5052,7 +5150,7 @@ async function hasMeaningfulAuthOrPublicState(page: any, requestedUrl: string):
     }
   })();
 
-  const domLooksRelevant = await page.evaluate(() => {
+  const domState = await page.evaluate(() => {
     const text = (document.body?.innerText ?? '').toLowerCase();
     const inputs = Array.from(document.querySelectorAll('input')) as HTMLInputElement[];
     const hasPassword = inputs.some((i) => i.type === 'password');
@@ -5062,9 +5160,16 @@ async function hasMeaningfulAuthOrPublicState(page: any, requestedUrl: string):
     });
     const hasAuthCopy = /(sign in|log in|login|sign up|create account|register|forgot password|reset password|verify email|one-time|otp|continue with email)/i.test(text);
     const hasPublicNav = /(pricing|contact|features|documentation|docs|about|privacy|terms)/i.test(text);
-    return hasPassword || hasIdentityInput || hasAuthCopy || hasPublicNav;
-  }).catch(() => false);
+    // Soft-404: page says "not found" / "404" with no real auth form
+    const isSoft404 = !hasPassword && !hasIdentityInput &&
+      /(page not found|404|this page (could not|doesn'?t) (be found|exist)|we couldn'?t find|no longer exists|been moved|doesn'?t exist)/i.test(text);
+    return { hasPassword, hasIdentityInput, hasAuthCopy, hasPublicNav, isSoft404 };
+  }).catch(() => ({ hasPassword: false, hasIdentityInput: false, hasAuthCopy: false, hasPublicNav: false, isSoft404: false }));
+
+  // Soft-404 with no auth form — path match alone is not enough
+  if (domState.isSoft404) return false;
 
+  const domLooksRelevant = domState.hasPassword || domState.hasIdentityInput || domState.hasAuthCopy || domState.hasPublicNav;
   return pathLooksRelevant || domLooksRelevant;
 }
 

package/src/job-queue.ts

@@ -191,7 +191,7 @@ const runningControllers = new Map<string, AbortController>();
 const WORKER_ID = `${os.hostname()}:${process.pid}`;
 const STALE_JOB_TIMEOUT_MINUTES = Number(process.env.JOB_STALE_TIMEOUT_MINUTES) || 30;
 const WORKER_POLL_INTERVAL_MS = Number(process.env.JOB_WORKER_POLL_INTERVAL_MS) || 15_000;
-const WORKER_CONCURRENCY = Number(process.env.JOB_WORKER_CONCURRENCY) || 3;
+const WORKER_CONCURRENCY = Number(process.env.JOB_WORKER_CONCURRENCY) || 8;
 const JOB_HEARTBEAT_INTERVAL_MS = Number(process.env.JOB_HEARTBEAT_INTERVAL_MS) || 15_000;
 // How many headless browsers a single RUN_TEST_SUITE job launches at once —
 // distinct from WORKER_CONCURRENCY (how many *jobs* the poll loop claims at
@@ -199,7 +199,7 @@ const JOB_HEARTBEAT_INTERVAL_MS = Number(process.env.JOB_HEARTBEAT_INTERVAL_MS)
 // primary scale-out path for running more tests at once is still horizontal
 // (more worker containers / SQS), not raising this past what one container
 // can safely hold — see startJobWorkerLoop's doc comment.
-const TEST_EXEC_CONCURRENCY = Number(process.env.TEST_EXEC_CONCURRENCY) || 3;
+const TEST_EXEC_CONCURRENCY = Number(process.env.TEST_EXEC_CONCURRENCY) || 5;
 
 type ClaimedJob = { id: string; tenantId: string; projectId: string; type: string; config: any };
 
@@ -1689,6 +1689,7 @@ async function processDiscoverJob(jobId: string, config: any, opts?: ProcessJobO
     const maxDiscoverUrls: number = config.maxDiscoverUrls ?? 2000;
     const proxyUrl: string | undefined = config.proxyUrl ?? crawlCfg.proxyUrl;
     const screenshotDir = os.tmpdir();
+    const discoverStorageConfig = await getTenantStorageConfig(tenantId).catch(() => null);
 
     await onProgress(3, 'Seeding from sitemap/robots.txt…');
 
@@ -1714,6 +1715,7 @@ async function processDiscoverJob(jobId: string, config: any, opts?: ProcessJobO
       maxScreens: maxDiscoverUrls,
       proxyUrl,
       useFirefox: crawlCfg.useFirefox,
+      storageConfig: discoverStorageConfig,
       abortSignal: controller.signal,
       jobId,
       onUrlsDiscovered: async (urls: string[]) => {
@@ -1794,6 +1796,7 @@ async function processDiscoverJob(jobId: string, config: any, opts?: ProcessJobO
         proxyUrl,
         chromeProfilePath: crawlCfg.chromeProfilePath,
         useFirefox: crawlCfg.useFirefox,
+        storageConfig: discoverStorageConfig,
         captchaSolverApiKey: crawlCfg.captchaSolverApiKey,
         captchaSolverProvider: crawlCfg.captchaSolverProvider,
         mailslurpApiKey: crawlCfg.mailslurpApiKey,

package/src/sitemap.ts

@@ -111,9 +111,11 @@ export async function fetchSitemapUrls(sitemapUrl: string, origin: string, depth
   const { pageUrls, subSitemapUrls } = extractUrlsFromParsedSitemap(parsed, origin);
   const collected = [...pageUrls];
 
-  for (const subUrl of subSitemapUrls) {
-    if (collected.length >= MAX_URLS) break;
-    collected.push(...(await fetchSitemapUrls(subUrl, origin, depth + 1, dispatcher)));
+  if (subSitemapUrls.length > 0 && collected.length < MAX_URLS) {
+    const subResults = await Promise.all(
+      subSitemapUrls.slice(0, 10).map(subUrl => fetchSitemapUrls(subUrl, origin, depth + 1, dispatcher))
+    );
+    for (const sub of subResults) collected.push(...sub);
   }
 
   return [...new Set(collected)].slice(0, MAX_URLS);