@govtechsg/oobee 0.10.98 → 0.11.1

package/dist/utils.js

@@ -346,15 +346,34 @@ export const cleanUp = async (randomToken, isError = false) => {
         catch (error) {
             consoleLogger.warn(`Unable to force remove userDataDirectory: ${error.message}`);
         }
-        // Also remove _pool* sibling directories created by browser pool re-launches
-        try {
-            const poolDirs = globSync(`${constants.userDataDirectory}_pool*`);
+        // Also remove _pool* sibling directories created by browser pool re-launches.
+        // On Windows, Chrome writes files (optimization_guide_model_store,
+        // segmentation_platform, first_party_sets.db, etc.) during its shutdown
+        // sequence and may still hold file locks. Retry up to 10 times at 5s intervals.
+        const removePoolDirs = () => {
+            // On Windows, glob interprets backslashes as escape characters, not path
+            // separators. Normalize to forward slashes so the pattern actually matches.
+            const globPattern = `${constants.userDataDirectory}_pool*`.replace(/\\/g, '/');
+            const poolDirs = globSync(globPattern);
             for (const dir of poolDirs) {
                 fs.rmSync(dir, { recursive: true, force: true });
             }
-        }
-        catch (error) {
-            consoleLogger.warn(`Unable to remove pool directories: ${error.message}`);
+            return poolDirs.length;
+        };
+        for (let attempt = 0; attempt < 10; attempt++) {
+            try {
+                if (removePoolDirs() === 0)
+                    break; // nothing left to clean
+                break; // rmSync succeeded for all dirs
+            }
+            catch {
+                if (attempt < 9) {
+                    await new Promise(r => setTimeout(r, 5000));
+                }
+                else {
+                    consoleLogger.warn('Unable to remove pool directories after 10 attempts');
+                }
+            }
         }
     }
     if (process.env.TMPDIR && fs.existsSync('/.dockerenv'))
@@ -374,7 +393,7 @@ export const cleanUp = async (randomToken, isError = false) => {
             const crawleeDir = path.join(storagePath, 'crawlee');
             const dataset = await Dataset.open(crawleeDir);
             await dataset.drop();
-            const requestQueue = await RequestQueue.open(crawleeDir);
+            const requestQueue = await RequestQueue.open(`${crawleeDir}_rq`);
             await requestQueue.drop();
         }
         catch (error) {
@@ -386,6 +405,12 @@ export const cleanUp = async (randomToken, isError = false) => {
         catch (error) {
             consoleLogger.warn(`Unable to force remove crawlee folder: ${error.message}`);
         }
+        try {
+            fs.rmSync(path.join(storagePath, 'crawlee_rq'), { recursive: true, force: true });
+        }
+        catch (error) {
+            consoleLogger.warn(`Unable to force remove crawlee_rq folder: ${error.message}`);
+        }
         try {
             fs.rmSync(path.join(storagePath, 'pdfs'), { recursive: true, force: true });
         }
@@ -808,9 +833,11 @@ export const zipResults = async (zipName, resultsPath) => {
     if (!stats.isDirectory()) {
         throw new Error(`resultsPath is not a directory: ${resultsPath}`);
     }
-    async function addFolderToZip(folderPath, zipFolder) {
+    async function addFolderToZip(folderPath, zipFolder, isRoot = false) {
         const items = await fs.readdir(folderPath);
         for (const item of items) {
+            if (isRoot && item.startsWith('crawlee'))
+                continue;
             const fullPath = path.join(folderPath, item);
             const stats = await fs.stat(fullPath);
             if (stats.isDirectory()) {
@@ -824,7 +851,7 @@ export const zipResults = async (zipName, resultsPath) => {
         }
     }
     const zip = new JSZip();
-    await addFolderToZip(resultsPath, zip);
+    await addFolderToZip(resultsPath, zip, true);
     const zipStream = zip.generateNodeStream({
         type: 'nodebuffer',
         streamFiles: true,

package/oobee-client-scanner.js

@@ -3,7 +3,7 @@
  * DO NOT EDIT MANUALLY. Re-generate with: node dist/generateOobeeClientScanner.js
  *
  * Embedded at generation time:
- *   App version : 0.10.98
+ *   App version : 0.11.1
  *   Sentry DSN  : (from OOBEE_SENTRY_DSN env var or constants.ts default)
  *   Sentry SDK  : @sentry/browser 10.58.0 (loaded from CDN at runtime)
  *
@@ -34887,7 +34887,7 @@
   // ── Sentry browser telemetry (Sentry JS SDK, loaded from CDN) ────────────
   
   var _oobeeSentryDsn          = "https://3b8c7ee46b06f33815a1301b6713ebc3@o4509047624761344.ingest.us.sentry.io/4509327783559168";
-  var _oobeeAppVersion         = "0.10.98";
+  var _oobeeAppVersion         = "0.11.1";
   var _oobeeSentryVersion      = "10.58.0";
   var _oobeeSentryInitialized  = false;
   var _oobeeSentryLoadPromise  = null;

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@govtechsg/oobee",
   "main": "dist/npmIndex.js",
-  "version": "0.10.98",
+  "version": "0.11.1",
   "type": "module",
   "author": "Government Technology Agency <info@tech.gov.sg>",
   "bin": {

package/src/combine.ts

@@ -18,6 +18,7 @@ import {
   getS3UploadPrefix,
   uploadFolderToS3,
 } from './services/s3Uploader.js';
+import { writeManifest, resetCaptureEntries, isPageCaptureEnabled } from './crawlers/pageCapture.js';
 
 // Class exports
 export class ViewportSettingsClass {
@@ -75,6 +76,10 @@ const combineRun = async (details: Data, deviceToScan: string) => {
   process.env.CRAWLEE_STORAGE_DIR = randomToken;
   constants.sitemapFetchedLinks = null;
 
+  if (isPageCaptureEnabled() && !process.env.OOBEE_SCAN_PRODUCT) {
+    process.env.OOBEE_SCAN_PRODUCT = 'U&A';
+  }
+
   if (process.env.CRAWLEE_SYSTEM_INFO_V2 === undefined) {
     // Set the environment variable to enable system info v2
     // Resolves issue with when wmic is not installed on Windows
@@ -96,6 +101,13 @@ const combineRun = async (details: Data, deviceToScan: string) => {
       consoleLogger.info(`Suppressed Playwright post-close connection error: ${err.message}`);
       return;
     }
+    // Suppress EPERM errors from Crawlee's async lock-file operations that fire
+    // after a crawl phase finishes. On Windows, the sitemap phase's async lock
+    // cleanup can race with the domain phase starting in the same directory.
+    if (err.message?.includes('EPERM') && err.message?.includes('.json.lock')) {
+      consoleLogger.info(`Suppressed Crawlee lock-file EPERM (stale async cleanup): ${err.message}`);
+      return;
+    }
     throw err;
   };
   process.on('uncaughtException', psTreeHandler);
@@ -278,6 +290,8 @@ const combineRun = async (details: Data, deviceToScan: string) => {
   if (scanDetails.urlsCrawled) {
     if (scanDetails.urlsCrawled.scanned.length > 0) {
       await createAndUpdateResultsFolders(randomToken);
+      await writeManifest(randomToken);
+      resetCaptureEntries();
       const pagesNotScanned = [
         ...urlsCrawledObj.error,
         ...urlsCrawledObj.invalid,

package/src/constants/common.ts

@@ -2157,6 +2157,15 @@ export const getPlaywrightLaunchOptions = (browser?: string): LaunchOptions => {
       !(browser === BrowserTypes.CHROME && arg === '--edge-skip-compat-layer-relaunch'),
   );
 
+  // Cap browser disk cache to 10MB per instance to prevent storage bloat
+  // during long crawls with multiple pool rotations
+  finalArgs.push('--disk-cache-size=10485760');
+
+  // Prevent Windows from throttling background Chromium processes
+  if (os.platform() === 'win32') {
+    finalArgs.push('--disable-features=UseEcoQoSForBackgroundProcess');
+  }
+
   // Headless flags (unchanged)
   if (process.env.CRAWLEE_HEADLESS === '1') {
     if (!finalArgs.includes('--mute-audio')) finalArgs.push('--mute-audio');

package/src/constants/constants.ts

@@ -30,10 +30,27 @@ export const blackListedFileExtensions = [
   'svg',
   'gif',
   'woff',
+  'woff2',
   'zip',
   'webp',
   'json',
   'xml',
+  'ico',
+  'bmp',
+  'tiff',
+  'tif',
+  'avi',
+  'mov',
+  'wmv',
+  'flv',
+  'ogg',
+  'wav',
+  'doc',
+  'docx',
+  'xls',
+  'xlsx',
+  'ppt',
+  'pptx',
 ];
 
 export const getIntermediateScreenshotsPath = (datasetsPath: string): string =>

package/src/crawlers/commonCrawlerFunc.ts

@@ -1135,10 +1135,9 @@ export const createCrawleeSubFolders = async (
   randomToken: string,
 ): Promise<{ dataset: Dataset; requestQueue: RequestQueue }> => {
 
-  const crawleeDir = path.join(getStoragePath(randomToken),"crawlee");
-
-  const dataset = await Dataset.open(crawleeDir);
-  const requestQueue = await RequestQueue.open(crawleeDir);
+  const baseDir = path.join(getStoragePath(randomToken), "crawlee");
+  const dataset = await Dataset.open(baseDir);
+  const requestQueue = await RequestQueue.open(`${baseDir}_rq`);
   return { dataset, requestQueue };
 };
 
@@ -1215,77 +1214,90 @@ export const postNavigationHooks = [
 
 export const getPreLaunchHook = (userDataDirectory: string) => {
   let launchCount = 0;
+  let previousPoolDir: string | null = null;
 
   return async (_pageId: string, launchContext: any) => {
     const fsp = await import('fs/promises').then(m => m.default);
     launchCount += 1;
 
-    // First launch uses the base directory; subsequent launches get a unique
-    // directory so that lingering file handles from a retired browser don't
-    // cause Chrome exit code 21 on Windows.
-    const effectiveDir =
-      launchCount === 1
-        ? userDataDirectory
-        : `${userDataDirectory}_pool${launchCount}`;
+    // Clean up the previous pool directory. When preLaunchHooks fires, the
+    // previous browser has been retired and is finishing its last pages.
+    // Schedule cleanup with enough delay for Chrome to fully exit —
+    // closeInactiveBrowserAfterSecs (30s) + Windows file-lock grace.
+    if (previousPoolDir) {
+      const dirToClean = previousPoolDir;
+      const isWin = process.platform === 'win32';
+      const delay = isWin ? 40000 : 35000;
+      setTimeout(async () => {
+        for (let attempt = 0; attempt < 3; attempt++) {
+          try {
+            await fsp.rm(dirToClean, { recursive: true, force: true });
+            return;
+          } catch {
+            await new Promise(r => setTimeout(r, 5000));
+          }
+        }
+      }, delay);
+    }
+
+    // Every browser gets its own directory. The base userDataDirectory is
+    // treated as a read-only cookie source (the pristine clone of the user's
+    // profile) and is never used directly by a running browser.
+    const effectiveDir = `${userDataDirectory}_pool${launchCount}`;
 
     await fsp.mkdir(effectiveDir, { recursive: true });
 
-    // For pool re-launches, copy only auth-relevant files from the base
-    // directory so authenticated sessions are preserved without cloning the
-    // entire profile (which grows with caches, IndexedDB, etc. during the crawl).
-    if (launchCount > 1) {
-      try {
-        // Copy top-level Local State (cookie encryption keys on Windows)
-        const localStateSrc = path.join(userDataDirectory, 'Local State');
-        if (await fsp.stat(localStateSrc).catch(() => null)) {
-          await fsp.copyFile(localStateSrc, path.join(effectiveDir, 'Local State')).catch(() => {});
-        }
+    // Copy auth-relevant files from the pristine base directory so
+    // authenticated sessions are preserved across pool rotations.
+    try {
+      const localStateSrc = path.join(userDataDirectory, 'Local State');
+      if (await fsp.stat(localStateSrc).catch(() => null)) {
+        await fsp.copyFile(localStateSrc, path.join(effectiveDir, 'Local State')).catch(() => {});
+      }
 
-        // Find profile directories (Default, Profile 1, Profile 2, etc.)
-        const entries = await fsp.readdir(userDataDirectory, { withFileTypes: true }).catch(() => []);
-        const profileDirs = (entries as any[]).filter(
-          (e: any) => e.isDirectory() && /^(Default|Profile \d+)$/i.test(e.name),
-        );
-
-        for (const profile of profileDirs) {
-          const srcProfile = path.join(userDataDirectory, profile.name);
-          const destProfile = path.join(effectiveDir, profile.name);
-          await fsp.mkdir(destProfile, { recursive: true }).catch(() => {});
-
-          // Cookies (macOS layout: <Profile>/Cookies)
-          const cookiesSrc = path.join(srcProfile, 'Cookies');
-          if (await fsp.stat(cookiesSrc).catch(() => null)) {
-            await fsp.copyFile(cookiesSrc, path.join(destProfile, 'Cookies')).catch(() => {});
-          }
+      const entries = await fsp.readdir(userDataDirectory, { withFileTypes: true }).catch(() => []);
+      const profileDirs = (entries as any[]).filter(
+        (e: any) => e.isDirectory() && /^(Default|Profile \d+)$/i.test(e.name),
+      );
 
-          // Cookies (Windows layout: <Profile>/Network/Cookies)
-          const networkCookiesSrc = path.join(srcProfile, 'Network', 'Cookies');
-          if (await fsp.stat(networkCookiesSrc).catch(() => null)) {
-            const destNetwork = path.join(destProfile, 'Network');
-            await fsp.mkdir(destNetwork, { recursive: true }).catch(() => {});
-            await fsp.copyFile(networkCookiesSrc, path.join(destNetwork, 'Cookies')).catch(() => {});
-          }
+      for (const profile of profileDirs) {
+        const srcProfile = path.join(userDataDirectory, profile.name);
+        const destProfile = path.join(effectiveDir, profile.name);
+        await fsp.mkdir(destProfile, { recursive: true }).catch(() => {});
 
-          // Local Storage (auth tokens, session data)
-          const localStorageSrc = path.join(srcProfile, 'Local Storage');
-          const localStorageStat = await fsp.stat(localStorageSrc).catch(() => null);
-          if (localStorageStat && localStorageStat.isDirectory()) {
-            const destLocalStorage = path.join(destProfile, 'Local Storage');
-            await fsp.mkdir(destLocalStorage, { recursive: true }).catch(() => {});
-            const lsFiles = await fsp.readdir(localStorageSrc).catch(() => []);
-            await Promise.all(
-              lsFiles.map(f =>
-                fsp.copyFile(
-                  path.join(localStorageSrc, f),
-                  path.join(destLocalStorage, f),
-                ).catch(() => {}),
-              ),
-            );
-          }
+        // Cookies (macOS layout: <Profile>/Cookies)
+        const cookiesSrc = path.join(srcProfile, 'Cookies');
+        if (await fsp.stat(cookiesSrc).catch(() => null)) {
+          await fsp.copyFile(cookiesSrc, path.join(destProfile, 'Cookies')).catch(() => {});
+        }
+
+        // Cookies (Windows layout: <Profile>/Network/Cookies)
+        const networkCookiesSrc = path.join(srcProfile, 'Network', 'Cookies');
+        if (await fsp.stat(networkCookiesSrc).catch(() => null)) {
+          const destNetwork = path.join(destProfile, 'Network');
+          await fsp.mkdir(destNetwork, { recursive: true }).catch(() => {});
+          await fsp.copyFile(networkCookiesSrc, path.join(destNetwork, 'Cookies')).catch(() => {});
+        }
+
+        // Local Storage (auth tokens, session data)
+        const localStorageSrc = path.join(srcProfile, 'Local Storage');
+        const localStorageStat = await fsp.stat(localStorageSrc).catch(() => null);
+        if (localStorageStat && localStorageStat.isDirectory()) {
+          const destLocalStorage = path.join(destProfile, 'Local Storage');
+          await fsp.mkdir(destLocalStorage, { recursive: true }).catch(() => {});
+          const lsFiles = await fsp.readdir(localStorageSrc).catch(() => []);
+          await Promise.all(
+            lsFiles.map(f =>
+              fsp.copyFile(
+                path.join(localStorageSrc, f),
+                path.join(destLocalStorage, f),
+              ).catch(() => {}),
+            ),
+          );
         }
-      } catch {
-        // Silent fallback: use empty profile if clone fails
       }
+    } catch {
+      // Silent fallback: use empty profile if clone fails
     }
 
     // Clean any stale lock files that may block browser launches on Windows
@@ -1299,11 +1311,38 @@ export const getPreLaunchHook = (userDataDirectory: string) => {
     ];
     await Promise.all(lockFiles.map(f => fsp.rm(f, { force: true }).catch(() => {})));
 
+    previousPoolDir = effectiveDir;
     // eslint-disable-next-line no-param-reassign
     launchContext.userDataDir = effectiveDir;
   };
 };
 
+export const getPostPageCloseHook = (userDataDirectory: string) => {
+  return async (_pageId: string, browserController: any) => {
+    if (browserController.activePages === 0) {
+      const dir = browserController.launchContext?.userDataDir;
+      if (dir && dir !== userDataDirectory && dir.includes('_pool')) {
+        const fsp = await import('fs/promises').then(m => m.default);
+        // Chrome on Windows writes optimization/segmentation data during shutdown;
+        // wait longer (5s) for the process to fully release file handles.
+        const isWin = process.platform === 'win32';
+        const initialDelay = isWin ? 5000 : 2000;
+        setTimeout(async () => {
+          for (let attempt = 0; attempt < 3; attempt++) {
+            try {
+              await fsp.rm(dir, { recursive: true, force: true });
+              return;
+            } catch {
+              // Retry after a short back-off (Windows file locks)
+              await new Promise(r => setTimeout(r, 2000));
+            }
+          }
+        }, initialDelay);
+      }
+    }
+  };
+};
+
 export const failedRequestHandler = async ({ request }: { request: Request }) => {
   guiInfoLog(guiInfoStatusTypes.ERROR, { numScanned: 0, urlScanned: request.url });
   log.error(`Failed Request - ${request.url}: ${request.errorMessages}`);

package/src/crawlers/crawlDomain.ts

@@ -5,6 +5,7 @@ import type { PlaywrightCrawlingContext, RequestOptions } from 'crawlee';
 import {
   createCrawleeSubFolders,
   getPreLaunchHook,
+  getPostPageCloseHook,
   preNavigationHooks,
   runAxeScript,
   isUrlPdf,
@@ -40,6 +41,7 @@ import {
 } from './pdfScanFunc.js';
 import { consoleLogger, guiInfoLog } from '../logs.js';
 import { ViewportSettingsClass } from '../combine.js';
+import { capturePageData } from './pageCapture.js';
 
 const isBlacklisted = (url: string, blacklistedPatterns: string[]) => {
   if (!blacklistedPatterns) {
@@ -398,7 +400,7 @@ const crawlDomain = async ({
         launcher: constants.launcher,
         launchOptions: getPlaywrightLaunchOptions(browser),
       },
-      retryOnBlocked: true,
+      retryOnBlocked: false,
       browserPoolOptions: {
         useFingerprints: false,
         retireBrowserAfterPageCount: 500,
@@ -418,12 +420,22 @@ const crawlDomain = async ({
             };
           },
         ],
+        postPageCloseHooks: [getPostPageCloseHook(userDataDirectory)],
       },
       requestQueue,
       maxRequestRetries: 3,
-      maxSessionRotations: 1,
       preNavigationHooks: [
         ...preNavigationHooks(extraHTTPHeaders),
+        async ({ request }) => {
+          const url = request.url.toLowerCase();
+          try {
+            const pathname = new URL(url).pathname;
+            const ext = pathname.split('.').pop();
+            if (ext && blackListedFileExtensions.includes(ext)) {
+              request.skipNavigation = true;
+            }
+          } catch {}
+        },
       ],
       postNavigationHooks: [
         async crawlingContext => {
@@ -555,21 +567,17 @@ const crawlDomain = async ({
             (request.skipNavigation && actualUrl === 'about:blank')
           ) {
             if (!isScanPdfs) {
-              // Don't inform the user it is skipped since web crawler is best-effort.
-              /*
-            guiInfoLog(guiInfoStatusTypes.SKIPPED, {
-              numScanned: urlsCrawled.scanned.length,
-              urlScanned: request.url,
-            });
-            urlsCrawled.userExcluded.push({
-              url: request.url,
-              pageTitle: request.url,
-              actualUrl: request.url, // because about:blank is not useful
-              metadata: STATUS_CODE_METADATA[1],
-              httpStatusCode: 0,
-            });
-            */
-
+              guiInfoLog(guiInfoStatusTypes.SKIPPED, {
+                numScanned: urlsCrawled.scanned.length,
+                urlScanned: request.url,
+              });
+              urlsCrawled.userExcluded.push({
+                url: request.url,
+                pageTitle: request.url,
+                actualUrl: request.url,
+                metadata: STATUS_CODE_METADATA[1],
+                httpStatusCode: 1,
+              });
               return;
             }
             const { pdfFileName, url: downloadedPdfUrl } = handlePdfDownload(
@@ -585,20 +593,17 @@ const crawlDomain = async ({
           }
 
           if (isBlacklistedFileExtensions(actualUrl, blackListedFileExtensions)) {
-            // Don't inform the user it is skipped since web crawler is best-effort.
-            /*
-          guiInfoLog(guiInfoStatusTypes.SKIPPED, {
-            numScanned: urlsCrawled.scanned.length,
-            urlScanned: request.url,
-          });
-          urlsCrawled.userExcluded.push({
-            url: request.url,
-            pageTitle: request.url,
-            actualUrl, // because about:blank is not useful
-            metadata: STATUS_CODE_METADATA[1],
-            httpStatusCode: 0,
-          });
-          */
+            guiInfoLog(guiInfoStatusTypes.SKIPPED, {
+              numScanned: urlsCrawled.scanned.length,
+              urlScanned: request.url,
+            });
+            urlsCrawled.userExcluded.push({
+              url: request.url,
+              pageTitle: request.url,
+              actualUrl,
+              metadata: STATUS_CODE_METADATA[1],
+              httpStatusCode: 1,
+            });
             return;
           }
 
@@ -639,6 +644,21 @@ const crawlDomain = async ({
             }
 
             const responseStatus = response?.status();
+            if (responseStatus === 403) {
+              rateController.onFailure(responseStatus, activeCrawler.autoscaledPool);
+              guiInfoLog(guiInfoStatusTypes.SKIPPED, {
+                numScanned: urlsCrawled.scanned.length,
+                urlScanned: request.url,
+              });
+              urlsCrawled.userExcluded.push({
+                url: request.url,
+                pageTitle: request.url,
+                actualUrl,
+                metadata: STATUS_CODE_METADATA[403] || STATUS_CODE_METADATA[599],
+                httpStatusCode: 403,
+              });
+              return;
+            }
             if (responseStatus && responseStatus >= 300) {
               guiInfoLog(guiInfoStatusTypes.SKIPPED, {
                 numScanned: urlsCrawled.scanned.length,
@@ -656,6 +676,8 @@ const crawlDomain = async ({
 
             const results = await runAxeScript({ includeScreenshots, page, randomToken, ruleset });
 
+            await capturePageData(page, actualUrl, randomToken);
+
             // Detect JS redirects that fire during/after axe scan.
             // Listen for navigation, then give a brief window for pending redirects to complete.
             try {
@@ -800,7 +822,56 @@ const crawlDomain = async ({
           return;
         }
 
+        // Handle download-triggered navigation errors: Playwright throws
+        // "Download is starting" when page.goto() hits a file download URL.
+        // Crawlee retries 3 times (all fail) then lands here.
+        const isDownloadError = request.errorMessages?.some(
+          (msg: string) => msg.includes('Download is starting'),
+        );
+        if (isDownloadError) {
+          if (isScanPdfs) {
+            // Re-enqueue with skipNavigation so the requestHandler's PDF download path handles it
+            try {
+              await requestQueue.addRequest({
+                url: request.url,
+                skipNavigation: true,
+                label: request.url,
+                uniqueKey: `download_${request.url}`,
+              });
+            } catch {}
+          } else {
+            guiInfoLog(guiInfoStatusTypes.SKIPPED, {
+              numScanned: urlsCrawled.scanned.length,
+              urlScanned: request.url,
+            });
+            urlsCrawled.userExcluded.push({
+              url: request.url,
+              pageTitle: request.url,
+              actualUrl: request.url,
+              metadata: STATUS_CODE_METADATA[1],
+              httpStatusCode: 1,
+            });
+          }
+          return;
+        }
+
         const status = response?.status();
+
+        // Re-enqueue rate-limited (403) URLs once for a retry after concurrency recovers.
+        // Call onFailure to reduce concurrency immediately on rate-limit detection.
+        if (status === 403 && !request.userData?.rateLimitRetried) {
+          rateController.onFailure(status, crawler.autoscaledPool);
+          try {
+            await requestQueue.addRequest({
+              url: request.url,
+              label: request.url,
+              uniqueKey: `ratelimit_${request.url}`,
+              userData: { rateLimitRetried: true },
+            });
+          } catch {}
+          return;
+        }
+
         if (rateController.onFailure(status, crawler.autoscaledPool)) {
           consoleLogger.info(
             `Aborting crawl: consecutive HTTP failures threshold reached (site may be rate-limiting). Successfully scanned ${urlsCrawled.scanned.length} pages.`,
@@ -844,7 +915,9 @@ const crawlDomain = async ({
 
   // Additional passes: keep re-visiting scanned seed-hostname pages for
   // click-discovery until no new pages are found or limits are reached.
-  if (!safeMode && !isAbortingScanNow && !durationExceeded) {
+  // Skip when called from intelligent sitemap — the domain phase is only meant
+  // to discover new pages via <a> links, not re-click 3000+ already-scanned pages.
+  if (!safeMode && !isAbortingScanNow && !durationExceeded && !fromCrawlIntelligentSitemap) {
     const seedHostname = new URL(url).hostname;
     const clickPassVisited = new Set<string>();
     let prevScannedCount: number;

package/src/crawlers/crawlLocalFile.ts

@@ -21,6 +21,7 @@ import { runPdfScan, mapPdfScanResults, doPdfScreenshots } from './pdfScanFunc.j
 import { guiInfoLog } from '../logs.js';
 import crawlSitemap from './crawlSitemap.js';
 import { getPdfStoragePath, getStoragePath, register } from '../utils.js';
+import { capturePageData } from './pageCapture.js';
 
 export const crawlLocalFile = async ({
   url,
@@ -186,6 +187,8 @@ export const crawlLocalFile = async ({
 
     const actualUrl = page.url() || request.loadedUrl || url;
 
+    await capturePageData(page, actualUrl, randomToken);
+
     guiInfoLog(guiInfoStatusTypes.SCANNED, {
       numScanned: urlsCrawled.scanned.length,
       urlScanned: url,