@govtechsg/oobee 0.10.97 → 0.11.0

package/dist/utils.js

@@ -346,15 +346,34 @@ export const cleanUp = async (randomToken, isError = false) => {
         catch (error) {
             consoleLogger.warn(`Unable to force remove userDataDirectory: ${error.message}`);
         }
-        // Also remove _pool* sibling directories created by browser pool re-launches
-        try {
-            const poolDirs = globSync(`${constants.userDataDirectory}_pool*`);
+        // Also remove _pool* sibling directories created by browser pool re-launches.
+        // On Windows, Chrome writes files (optimization_guide_model_store,
+        // segmentation_platform, first_party_sets.db, etc.) during its shutdown
+        // sequence and may still hold file locks. Retry up to 10 times at 5s intervals.
+        const removePoolDirs = () => {
+            // On Windows, glob interprets backslashes as escape characters, not path
+            // separators. Normalize to forward slashes so the pattern actually matches.
+            const globPattern = `${constants.userDataDirectory}_pool*`.replace(/\\/g, '/');
+            const poolDirs = globSync(globPattern);
             for (const dir of poolDirs) {
                 fs.rmSync(dir, { recursive: true, force: true });
             }
-        }
-        catch (error) {
-            consoleLogger.warn(`Unable to remove pool directories: ${error.message}`);
+            return poolDirs.length;
+        };
+        for (let attempt = 0; attempt < 10; attempt++) {
+            try {
+                if (removePoolDirs() === 0)
+                    break; // nothing left to clean
+                break; // rmSync succeeded for all dirs
+            }
+            catch {
+                if (attempt < 9) {
+                    await new Promise(r => setTimeout(r, 5000));
+                }
+                else {
+                    consoleLogger.warn('Unable to remove pool directories after 10 attempts');
+                }
+            }
         }
     }
     if (process.env.TMPDIR && fs.existsSync('/.dockerenv'))
@@ -374,7 +393,7 @@ export const cleanUp = async (randomToken, isError = false) => {
             const crawleeDir = path.join(storagePath, 'crawlee');
             const dataset = await Dataset.open(crawleeDir);
             await dataset.drop();
-            const requestQueue = await RequestQueue.open(crawleeDir);
+            const requestQueue = await RequestQueue.open(`${crawleeDir}_rq`);
             await requestQueue.drop();
         }
         catch (error) {
@@ -386,6 +405,12 @@ export const cleanUp = async (randomToken, isError = false) => {
         catch (error) {
             consoleLogger.warn(`Unable to force remove crawlee folder: ${error.message}`);
         }
+        try {
+            fs.rmSync(path.join(storagePath, 'crawlee_rq'), { recursive: true, force: true });
+        }
+        catch (error) {
+            consoleLogger.warn(`Unable to force remove crawlee_rq folder: ${error.message}`);
+        }
         try {
             fs.rmSync(path.join(storagePath, 'pdfs'), { recursive: true, force: true });
         }
@@ -808,9 +833,11 @@ export const zipResults = async (zipName, resultsPath) => {
     if (!stats.isDirectory()) {
         throw new Error(`resultsPath is not a directory: ${resultsPath}`);
     }
-    async function addFolderToZip(folderPath, zipFolder) {
+    async function addFolderToZip(folderPath, zipFolder, isRoot = false) {
         const items = await fs.readdir(folderPath);
         for (const item of items) {
+            if (isRoot && item.startsWith('crawlee'))
+                continue;
             const fullPath = path.join(folderPath, item);
             const stats = await fs.stat(fullPath);
             if (stats.isDirectory()) {
@@ -824,7 +851,7 @@ export const zipResults = async (zipName, resultsPath) => {
         }
     }
     const zip = new JSZip();
-    await addFolderToZip(resultsPath, zip);
+    await addFolderToZip(resultsPath, zip, true);
     const zipStream = zip.generateNodeStream({
         type: 'nodebuffer',
         streamFiles: true,

package/oobee-client-scanner.js

@@ -3,7 +3,7 @@
  * DO NOT EDIT MANUALLY. Re-generate with: node dist/generateOobeeClientScanner.js
  *
  * Embedded at generation time:
- *   App version : 0.10.97
+ *   App version : 0.11.0
  *   Sentry DSN  : (from OOBEE_SENTRY_DSN env var or constants.ts default)
  *   Sentry SDK  : @sentry/browser 10.58.0 (loaded from CDN at runtime)
  *
@@ -33593,7 +33593,11 @@
     function isElementTooSmall(element) {
         // Get the bounding rectangle of the element
         const rect = element.getBoundingClientRect();
-        // Check if the element has a valid width or height
+        // If either dimension is 0, the element is non-perceivable
+        if (rect.width === 0 || rect.height === 0) {
+            return true;
+        }
+        // Check if the element has a valid width and height
         if (rect.width > 0 || rect.height > 0) {
             return false; // Element is not too small
         }
@@ -34883,7 +34887,7 @@
   // ── Sentry browser telemetry (Sentry JS SDK, loaded from CDN) ────────────
   
   var _oobeeSentryDsn          = "https://3b8c7ee46b06f33815a1301b6713ebc3@o4509047624761344.ingest.us.sentry.io/4509327783559168";
-  var _oobeeAppVersion         = "0.10.97";
+  var _oobeeAppVersion         = "0.11.0";
   var _oobeeSentryVersion      = "10.58.0";
   var _oobeeSentryInitialized  = false;
   var _oobeeSentryLoadPromise  = null;

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@govtechsg/oobee",
   "main": "dist/npmIndex.js",
-  "version": "0.10.97",
+  "version": "0.11.0",
   "type": "module",
   "author": "Government Technology Agency <info@tech.gov.sg>",
   "bin": {

package/src/combine.ts

@@ -96,6 +96,13 @@ const combineRun = async (details: Data, deviceToScan: string) => {
       consoleLogger.info(`Suppressed Playwright post-close connection error: ${err.message}`);
       return;
     }
+    // Suppress EPERM errors from Crawlee's async lock-file operations that fire
+    // after a crawl phase finishes. On Windows, the sitemap phase's async lock
+    // cleanup can race with the domain phase starting in the same directory.
+    if (err.message?.includes('EPERM') && err.message?.includes('.json.lock')) {
+      consoleLogger.info(`Suppressed Crawlee lock-file EPERM (stale async cleanup): ${err.message}`);
+      return;
+    }
     throw err;
   };
   process.on('uncaughtException', psTreeHandler);
@@ -186,6 +193,7 @@ const combineRun = async (details: Data, deviceToScan: string) => {
         strategy,
         userUrl: url,
         scanDuration,
+        ruleset,
       });
       urlsCrawledObj = sitemapResult.urlsCrawled;
       durationExceeded = sitemapResult.durationExceeded;
@@ -236,6 +244,7 @@ const combineRun = async (details: Data, deviceToScan: string) => {
         extraHTTPHeaders,
         safeMode,
         scanDuration,
+        ruleset,
       );
       urlsCrawledObj = intelligentResult.urlsCrawled;
       durationExceeded = intelligentResult.durationExceeded;

package/src/constants/common.ts

@@ -2157,6 +2157,15 @@ export const getPlaywrightLaunchOptions = (browser?: string): LaunchOptions => {
       !(browser === BrowserTypes.CHROME && arg === '--edge-skip-compat-layer-relaunch'),
   );
 
+  // Cap browser disk cache to 10MB per instance to prevent storage bloat
+  // during long crawls with multiple pool rotations
+  finalArgs.push('--disk-cache-size=10485760');
+
+  // Prevent Windows from throttling background Chromium processes
+  if (os.platform() === 'win32') {
+    finalArgs.push('--disable-features=UseEcoQoSForBackgroundProcess');
+  }
+
   // Headless flags (unchanged)
   if (process.env.CRAWLEE_HEADLESS === '1') {
     if (!finalArgs.includes('--mute-audio')) finalArgs.push('--mute-audio');

package/src/constants/constants.ts

@@ -30,10 +30,27 @@ export const blackListedFileExtensions = [
   'svg',
   'gif',
   'woff',
+  'woff2',
   'zip',
   'webp',
   'json',
   'xml',
+  'ico',
+  'bmp',
+  'tiff',
+  'tif',
+  'avi',
+  'mov',
+  'wmv',
+  'flv',
+  'ogg',
+  'wav',
+  'doc',
+  'docx',
+  'xls',
+  'xlsx',
+  'ppt',
+  'pptx',
 ];
 
 export const getIntermediateScreenshotsPath = (datasetsPath: string): string =>

package/src/crawlers/commonCrawlerFunc.ts

@@ -1135,10 +1135,9 @@ export const createCrawleeSubFolders = async (
   randomToken: string,
 ): Promise<{ dataset: Dataset; requestQueue: RequestQueue }> => {
 
-  const crawleeDir = path.join(getStoragePath(randomToken),"crawlee");
-
-  const dataset = await Dataset.open(crawleeDir);
-  const requestQueue = await RequestQueue.open(crawleeDir);
+  const baseDir = path.join(getStoragePath(randomToken), "crawlee");
+  const dataset = await Dataset.open(baseDir);
+  const requestQueue = await RequestQueue.open(`${baseDir}_rq`);
   return { dataset, requestQueue };
 };
 
@@ -1215,53 +1214,90 @@ export const postNavigationHooks = [
 
 export const getPreLaunchHook = (userDataDirectory: string) => {
   let launchCount = 0;
+  let previousPoolDir: string | null = null;
 
   return async (_pageId: string, launchContext: any) => {
     const fsp = await import('fs/promises').then(m => m.default);
     launchCount += 1;
 
-    // First launch uses the base directory; subsequent launches get a unique
-    // directory so that lingering file handles from a retired browser don't
-    // cause Chrome exit code 21 on Windows.
-    const effectiveDir =
-      launchCount === 1
-        ? userDataDirectory
-        : `${userDataDirectory}_pool${launchCount}`;
+    // Clean up the previous pool directory. When preLaunchHooks fires, the
+    // previous browser has been retired and is finishing its last pages.
+    // Schedule cleanup with enough delay for Chrome to fully exit —
+    // closeInactiveBrowserAfterSecs (30s) + Windows file-lock grace.
+    if (previousPoolDir) {
+      const dirToClean = previousPoolDir;
+      const isWin = process.platform === 'win32';
+      const delay = isWin ? 40000 : 35000;
+      setTimeout(async () => {
+        for (let attempt = 0; attempt < 3; attempt++) {
+          try {
+            await fsp.rm(dirToClean, { recursive: true, force: true });
+            return;
+          } catch {
+            await new Promise(r => setTimeout(r, 5000));
+          }
+        }
+      }, delay);
+    }
+
+    // Every browser gets its own directory. The base userDataDirectory is
+    // treated as a read-only cookie source (the pristine clone of the user's
+    // profile) and is never used directly by a running browser.
+    const effectiveDir = `${userDataDirectory}_pool${launchCount}`;
 
     await fsp.mkdir(effectiveDir, { recursive: true });
 
-    // For pool re-launches, best-effort clone profile data from base directory
-    // so authenticated sessions are preserved across browser pool retirements.
-    if (launchCount > 1) {
-      const skipDirs = new Set([
-        'Singleton', 'lockfile', 'LOCK',
-        'Cache', 'Code Cache', 'GPUCache', 'DawnGraphiteCache', 'DawnWebGPUCache',
-        'Service Worker', 'ScriptCache', 'ShaderCache', 'GrShaderCache',
-        'component_crx_cache', 'optimization_guide_model_store',
-        'BrowserMetrics', 'Crashpad', 'FileTypePolicies',
-      ]);
-      try {
-        const copyRecursive = async (src: string, dest: string) => {
-          const stat = await fsp.stat(src).catch(() => null);
-          if (!stat) return;
-          if (stat.isDirectory()) {
-            await fsp.mkdir(dest, { recursive: true }).catch(() => {});
-            const entries = await fsp.readdir(src).catch(() => []);
-            await Promise.all(
-              entries
-                .filter(entry => !entry.startsWith('Singleton') && !skipDirs.has(entry))
-                .map(entry =>
-                  copyRecursive(path.join(src, entry), path.join(dest, entry)).catch(() => {}),
-                ),
-            );
-          } else {
-            await fsp.copyFile(src, dest).catch(() => {});
-          }
-        };
-        await copyRecursive(userDataDirectory, effectiveDir).catch(() => {});
-      } catch {
-        // Silent fallback: use empty profile if clone fails
+    // Copy auth-relevant files from the pristine base directory so
+    // authenticated sessions are preserved across pool rotations.
+    try {
+      const localStateSrc = path.join(userDataDirectory, 'Local State');
+      if (await fsp.stat(localStateSrc).catch(() => null)) {
+        await fsp.copyFile(localStateSrc, path.join(effectiveDir, 'Local State')).catch(() => {});
       }
+
+      const entries = await fsp.readdir(userDataDirectory, { withFileTypes: true }).catch(() => []);
+      const profileDirs = (entries as any[]).filter(
+        (e: any) => e.isDirectory() && /^(Default|Profile \d+)$/i.test(e.name),
+      );
+
+      for (const profile of profileDirs) {
+        const srcProfile = path.join(userDataDirectory, profile.name);
+        const destProfile = path.join(effectiveDir, profile.name);
+        await fsp.mkdir(destProfile, { recursive: true }).catch(() => {});
+
+        // Cookies (macOS layout: <Profile>/Cookies)
+        const cookiesSrc = path.join(srcProfile, 'Cookies');
+        if (await fsp.stat(cookiesSrc).catch(() => null)) {
+          await fsp.copyFile(cookiesSrc, path.join(destProfile, 'Cookies')).catch(() => {});
+        }
+
+        // Cookies (Windows layout: <Profile>/Network/Cookies)
+        const networkCookiesSrc = path.join(srcProfile, 'Network', 'Cookies');
+        if (await fsp.stat(networkCookiesSrc).catch(() => null)) {
+          const destNetwork = path.join(destProfile, 'Network');
+          await fsp.mkdir(destNetwork, { recursive: true }).catch(() => {});
+          await fsp.copyFile(networkCookiesSrc, path.join(destNetwork, 'Cookies')).catch(() => {});
+        }
+
+        // Local Storage (auth tokens, session data)
+        const localStorageSrc = path.join(srcProfile, 'Local Storage');
+        const localStorageStat = await fsp.stat(localStorageSrc).catch(() => null);
+        if (localStorageStat && localStorageStat.isDirectory()) {
+          const destLocalStorage = path.join(destProfile, 'Local Storage');
+          await fsp.mkdir(destLocalStorage, { recursive: true }).catch(() => {});
+          const lsFiles = await fsp.readdir(localStorageSrc).catch(() => []);
+          await Promise.all(
+            lsFiles.map(f =>
+              fsp.copyFile(
+                path.join(localStorageSrc, f),
+                path.join(destLocalStorage, f),
+              ).catch(() => {}),
+            ),
+          );
+        }
+      }
+    } catch {
+      // Silent fallback: use empty profile if clone fails
     }
 
     // Clean any stale lock files that may block browser launches on Windows
@@ -1275,11 +1311,38 @@ export const getPreLaunchHook = (userDataDirectory: string) => {
     ];
     await Promise.all(lockFiles.map(f => fsp.rm(f, { force: true }).catch(() => {})));
 
+    previousPoolDir = effectiveDir;
     // eslint-disable-next-line no-param-reassign
     launchContext.userDataDir = effectiveDir;
   };
 };
 
+export const getPostPageCloseHook = (userDataDirectory: string) => {
+  return async (_pageId: string, browserController: any) => {
+    if (browserController.activePages === 0) {
+      const dir = browserController.launchContext?.userDataDir;
+      if (dir && dir !== userDataDirectory && dir.includes('_pool')) {
+        const fsp = await import('fs/promises').then(m => m.default);
+        // Chrome on Windows writes optimization/segmentation data during shutdown;
+        // wait longer (5s) for the process to fully release file handles.
+        const isWin = process.platform === 'win32';
+        const initialDelay = isWin ? 5000 : 2000;
+        setTimeout(async () => {
+          for (let attempt = 0; attempt < 3; attempt++) {
+            try {
+              await fsp.rm(dir, { recursive: true, force: true });
+              return;
+            } catch {
+              // Retry after a short back-off (Windows file locks)
+              await new Promise(r => setTimeout(r, 2000));
+            }
+          }
+        }, initialDelay);
+      }
+    }
+  };
+};
+
 export const failedRequestHandler = async ({ request }: { request: Request }) => {
   guiInfoLog(guiInfoStatusTypes.ERROR, { numScanned: 0, urlScanned: request.url });
   log.error(`Failed Request - ${request.url}: ${request.errorMessages}`);

package/src/crawlers/crawlDomain.ts

@@ -5,6 +5,7 @@ import type { PlaywrightCrawlingContext, RequestOptions } from 'crawlee';
 import {
   createCrawleeSubFolders,
   getPreLaunchHook,
+  getPostPageCloseHook,
   preNavigationHooks,
   runAxeScript,
   isUrlPdf,
@@ -382,8 +383,11 @@ const crawlDomain = async ({
   };
 
   let isAbortingScanNow = false;
+  const remainingBudget = fromCrawlIntelligentSitemap
+    ? Math.max(0, maxRequestsPerCrawl - urlsCrawledFromIntelligent.scanned.length)
+    : maxRequestsPerCrawl;
   const rateController = new CrawlRateController(
-    maxRequestsPerCrawl,
+    remainingBudget,
     specifiedMaxConcurrency || constants.maxConcurrency,
   );
 
@@ -398,6 +402,8 @@ const crawlDomain = async ({
       retryOnBlocked: true,
       browserPoolOptions: {
         useFingerprints: false,
+        retireBrowserAfterPageCount: 500,
+        closeInactiveBrowserAfterSecs: 30,
         preLaunchHooks: [
           getPreLaunchHook(userDataDirectory),
           async (_pageId, launchContext) => {
@@ -413,55 +419,73 @@ const crawlDomain = async ({
             };
           },
         ],
+        postPageCloseHooks: [getPostPageCloseHook(userDataDirectory)],
       },
       requestQueue,
       maxRequestRetries: 3,
       maxSessionRotations: 1,
       preNavigationHooks: [
         ...preNavigationHooks(extraHTTPHeaders),
+        async ({ request }) => {
+          const url = request.url.toLowerCase();
+          try {
+            const pathname = new URL(url).pathname;
+            const ext = pathname.split('.').pop();
+            if (ext && blackListedFileExtensions.includes(ext)) {
+              request.skipNavigation = true;
+            }
+          } catch {}
+        },
       ],
       postNavigationHooks: [
         async crawlingContext => {
           const { page, request } = crawlingContext;
 
-          await page.evaluate(() => {
-            return new Promise(resolve => {
-              let timeout;
-              let mutationCount = 0;
-              const MAX_MUTATIONS = 500; // stop if things never quiet down
-              const OBSERVER_TIMEOUT = 5000; // hard cap on total wait
-
-              const observer = new MutationObserver(() => {
-                clearTimeout(timeout);
-
-                mutationCount += 1;
-                if (mutationCount > MAX_MUTATIONS) {
-                  observer.disconnect();
-                  resolve('Too many mutations, exiting.');
-                  return;
-                }
+          try {
+            await page.evaluate(() => {
+              return new Promise(resolve => {
+                let timeout;
+                let mutationCount = 0;
+                const MAX_MUTATIONS = 500; // stop if things never quiet down
+                const OBSERVER_TIMEOUT = 5000; // hard cap on total wait
+
+                const observer = new MutationObserver(() => {
+                  clearTimeout(timeout);
+
+                  mutationCount += 1;
+                  if (mutationCount > MAX_MUTATIONS) {
+                    observer.disconnect();
+                    resolve('Too many mutations, exiting.');
+                    return;
+                  }
+
+                  // restart quiet‑period timer
+                  timeout = setTimeout(() => {
+                    observer.disconnect();
+                    resolve('DOM stabilized.');
+                  }, 1000);
+                });
 
-                // restart quiet‑period timer
+                // overall timeout in case the page never settles
                 timeout = setTimeout(() => {
                   observer.disconnect();
-                  resolve('DOM stabilized.');
-                }, 1000);
+                  resolve('Observer timeout reached.');
+                }, OBSERVER_TIMEOUT);
+
+                const root = document.documentElement || document.body || document;
+                if (!root || typeof observer.observe !== 'function') {
+                  resolve('No root node to observe.');
+                } else {
+                  observer.observe(root, { childList: true, subtree: true });
+                }
               });
-
-              // overall timeout in case the page never settles
-              timeout = setTimeout(() => {
-                observer.disconnect();
-                resolve('Observer timeout reached.');
-              }, OBSERVER_TIMEOUT);
-
-              const root = document.documentElement || document.body || document;
-              if (!root || typeof observer.observe !== 'function') {
-                resolve('No root node to observe.');
-              } else {
-                observer.observe(root, { childList: true, subtree: true });
-              }
             });
-          });
+          } catch (err) {
+            if (err.message?.includes('was destroyed')) {
+              return;
+            }
+            throw err;
+          }
 
           let finalUrl = page.url();
           const requestLabelUrl = request.label;
@@ -543,21 +567,17 @@ const crawlDomain = async ({
             (request.skipNavigation && actualUrl === 'about:blank')
           ) {
             if (!isScanPdfs) {
-              // Don't inform the user it is skipped since web crawler is best-effort.
-              /*
-            guiInfoLog(guiInfoStatusTypes.SKIPPED, {
-              numScanned: urlsCrawled.scanned.length,
-              urlScanned: request.url,
-            });
-            urlsCrawled.userExcluded.push({
-              url: request.url,
-              pageTitle: request.url,
-              actualUrl: request.url, // because about:blank is not useful
-              metadata: STATUS_CODE_METADATA[1],
-              httpStatusCode: 0,
-            });
-            */
-
+              guiInfoLog(guiInfoStatusTypes.SKIPPED, {
+                numScanned: urlsCrawled.scanned.length,
+                urlScanned: request.url,
+              });
+              urlsCrawled.userExcluded.push({
+                url: request.url,
+                pageTitle: request.url,
+                actualUrl: request.url,
+                metadata: STATUS_CODE_METADATA[1],
+                httpStatusCode: 1,
+              });
               return;
             }
             const { pdfFileName, url: downloadedPdfUrl } = handlePdfDownload(
@@ -573,20 +593,17 @@ const crawlDomain = async ({
           }
 
           if (isBlacklistedFileExtensions(actualUrl, blackListedFileExtensions)) {
-            // Don't inform the user it is skipped since web crawler is best-effort.
-            /*
-          guiInfoLog(guiInfoStatusTypes.SKIPPED, {
-            numScanned: urlsCrawled.scanned.length,
-            urlScanned: request.url,
-          });
-          urlsCrawled.userExcluded.push({
-            url: request.url,
-            pageTitle: request.url,
-            actualUrl, // because about:blank is not useful
-            metadata: STATUS_CODE_METADATA[1],
-            httpStatusCode: 0,
-          });
-          */
+            guiInfoLog(guiInfoStatusTypes.SKIPPED, {
+              numScanned: urlsCrawled.scanned.length,
+              urlScanned: request.url,
+            });
+            urlsCrawled.userExcluded.push({
+              url: request.url,
+              pageTitle: request.url,
+              actualUrl,
+              metadata: STATUS_CODE_METADATA[1],
+              httpStatusCode: 1,
+            });
             return;
           }
 
@@ -788,7 +805,59 @@ const crawlDomain = async ({
           return;
         }
 
+        // Handle download-triggered navigation errors: Playwright throws
+        // "Download is starting" when page.goto() hits a file download URL.
+        // Crawlee retries 3 times (all fail) then lands here.
+        const isDownloadError = request.errorMessages?.some(
+          (msg: string) => msg.includes('Download is starting'),
+        );
+        if (isDownloadError) {
+          if (isScanPdfs) {
+            // Re-enqueue with skipNavigation so the requestHandler's PDF download path handles it
+            try {
+              await requestQueue.addRequest({
+                url: request.url,
+                skipNavigation: true,
+                label: request.url,
+                uniqueKey: `download_${request.url}`,
+              });
+            } catch {}
+          } else {
+            guiInfoLog(guiInfoStatusTypes.SKIPPED, {
+              numScanned: urlsCrawled.scanned.length,
+              urlScanned: request.url,
+            });
+            urlsCrawled.userExcluded.push({
+              url: request.url,
+              pageTitle: request.url,
+              actualUrl: request.url,
+              metadata: STATUS_CODE_METADATA[1],
+              httpStatusCode: 1,
+            });
+          }
+          return;
+        }
+
         const status = response?.status();
+
+        // Re-enqueue rate-limited (403) URLs once for a retry after concurrency recovers.
+        // Without this, URLs that fail during a rate-limit burst are permanently lost
+        // even though the site is accessible at lower concurrency.
+        // Don't call onFailure here — the re-enqueued request will get a fresh attempt.
+        // If it fails again (rateLimitRetried=true), it falls through to the normal
+        // onFailure + circuit breaker path below.
+        if (status === 403 && !request.userData?.rateLimitRetried) {
+          try {
+            await requestQueue.addRequest({
+              url: request.url,
+              label: request.url,
+              uniqueKey: `ratelimit_${request.url}`,
+              userData: { rateLimitRetried: true },
+            });
+          } catch {}
+          return;
+        }
+
         if (rateController.onFailure(status, crawler.autoscaledPool)) {
           consoleLogger.info(
             `Aborting crawl: consecutive HTTP failures threshold reached (site may be rate-limiting). Successfully scanned ${urlsCrawled.scanned.length} pages.`,
@@ -832,7 +901,9 @@ const crawlDomain = async ({
 
   // Additional passes: keep re-visiting scanned seed-hostname pages for
   // click-discovery until no new pages are found or limits are reached.
-  if (!safeMode && !isAbortingScanNow && !durationExceeded) {
+  // Skip when called from intelligent sitemap — the domain phase is only meant
+  // to discover new pages via <a> links, not re-click 3000+ already-scanned pages.
+  if (!safeMode && !isAbortingScanNow && !durationExceeded && !fromCrawlIntelligentSitemap) {
     const seedHostname = new URL(url).hostname;
     const clickPassVisited = new Set<string>();
     let prevScannedCount: number;