@govtechsg/oobee 0.10.97 → 0.10.98

package/AGENTS.md

@@ -299,6 +299,14 @@ When making changes, validate these areas which have well-established edge cases
 - Without the circuit breaker, a rate-limited crawl with thousands of enqueued URLs would run indefinitely, never hit the success threshold, and never generate a report.
 - When enqueuing all sitemap URLs (which we do for accurate `totalLinksFetchedFromSitemaps` reporting), always ensure either a scan duration (`-d`) or the circuit breaker is in place as a safety net.
 
+### Scan Consistency Between crawlDomain and crawlSitemap
+- Both crawlers must produce equivalent axe scan results for the same page. Any difference in how the page is observed/stabilized before `runAxeScript()` will cause inconsistent accessibility findings between scan types.
+- **postNavigationHook DOM observer must be identical**: Both crawlers use a MutationObserver in `postNavigationHooks` to wait for DOM stabilization before proceeding to the request handler. The observer must call `observer.observe(root, { childList: true, subtree: true })` — without this call, the hook degrades to a fixed 5-second timeout (the `OBSERVER_TIMEOUT` fallback) and the DOM may be in a different state when scanning begins.
+- **`runAxeScript()` has its own secondary DOM observer** (in `commonCrawlerFunc.ts`) that additionally watches `attributes: true`. This is a second stabilization gate shared by all crawlers — it ensures attribute animations settle before axe runs.
+- **`waitForPageLoaded(page, 10000)` in the requestHandler** is the third stabilization check (waits for `load` event or 10s timeout). All crawlers call this at the top of their request handler.
+- **Parameters passed to `runAxeScript()` must match**: both must pass `ruleset` (controls `DISABLE_OOBEE` / `ENABLE_WCAG_AAA`). Any new parameter added to `runAxeScript()` must be propagated to all crawlers via `combine.ts`.
+- **Error handling in postNavigationHook**: wrap `page.evaluate()` in try/catch to handle pages destroyed during the DOM observer (navigation, timeout, crash). Without this, the error propagates and may affect Crawlee's retry logic differently between crawlers.
+
 ### Axe & Custom Checks
 - When axe reports color-contrast violations but cannot determine the actual colors, skip augmenting the message with contrast context (avoids crashes on null/undefined color values).
 - Violation messages are enriched with live DOM context (element text, computed styles, dimensions) via `page.evaluate()` during scan. Handle cases where elements are no longer in DOM at evaluation time.

package/dist/combine.js

@@ -117,6 +117,7 @@ const combineRun = async (details, deviceToScan) => {
                 strategy,
                 userUrl: url,
                 scanDuration,
+                ruleset,
             });
             urlsCrawledObj = sitemapResult.urlsCrawled;
             durationExceeded = sitemapResult.durationExceeded;
@@ -149,7 +150,7 @@ const combineRun = async (details, deviceToScan) => {
             }
             break;
         case ScannerTypes.INTELLIGENT:
-            const intelligentResult = await crawlIntelligentSitemap(url, randomToken, host, viewportSettings, maxRequestsPerCrawl, browser, userDataDirectory, strategy, specifiedMaxConcurrency, fileTypes, blacklistedPatterns, includeScreenshots, followRobots, extraHTTPHeaders, safeMode, scanDuration);
+            const intelligentResult = await crawlIntelligentSitemap(url, randomToken, host, viewportSettings, maxRequestsPerCrawl, browser, userDataDirectory, strategy, specifiedMaxConcurrency, fileTypes, blacklistedPatterns, includeScreenshots, followRobots, extraHTTPHeaders, safeMode, scanDuration, ruleset);
             urlsCrawledObj = intelligentResult.urlsCrawled;
             durationExceeded = intelligentResult.durationExceeded;
             break;

package/dist/crawlers/commonCrawlerFunc.js

@@ -974,33 +974,45 @@ export const getPreLaunchHook = (userDataDirectory) => {
             ? userDataDirectory
             : `${userDataDirectory}_pool${launchCount}`;
         await fsp.mkdir(effectiveDir, { recursive: true });
-        // For pool re-launches, best-effort clone profile data from base directory
-        // so authenticated sessions are preserved across browser pool retirements.
+        // For pool re-launches, copy only auth-relevant files from the base
+        // directory so authenticated sessions are preserved without cloning the
+        // entire profile (which grows with caches, IndexedDB, etc. during the crawl).
         if (launchCount > 1) {
-            const skipDirs = new Set([
-                'Singleton', 'lockfile', 'LOCK',
-                'Cache', 'Code Cache', 'GPUCache', 'DawnGraphiteCache', 'DawnWebGPUCache',
-                'Service Worker', 'ScriptCache', 'ShaderCache', 'GrShaderCache',
-                'component_crx_cache', 'optimization_guide_model_store',
-                'BrowserMetrics', 'Crashpad', 'FileTypePolicies',
-            ]);
             try {
-                const copyRecursive = async (src, dest) => {
-                    const stat = await fsp.stat(src).catch(() => null);
-                    if (!stat)
-                        return;
-                    if (stat.isDirectory()) {
-                        await fsp.mkdir(dest, { recursive: true }).catch(() => { });
-                        const entries = await fsp.readdir(src).catch(() => []);
-                        await Promise.all(entries
-                            .filter(entry => !entry.startsWith('Singleton') && !skipDirs.has(entry))
-                            .map(entry => copyRecursive(path.join(src, entry), path.join(dest, entry)).catch(() => { })));
+                // Copy top-level Local State (cookie encryption keys on Windows)
+                const localStateSrc = path.join(userDataDirectory, 'Local State');
+                if (await fsp.stat(localStateSrc).catch(() => null)) {
+                    await fsp.copyFile(localStateSrc, path.join(effectiveDir, 'Local State')).catch(() => { });
+                }
+                // Find profile directories (Default, Profile 1, Profile 2, etc.)
+                const entries = await fsp.readdir(userDataDirectory, { withFileTypes: true }).catch(() => []);
+                const profileDirs = entries.filter((e) => e.isDirectory() && /^(Default|Profile \d+)$/i.test(e.name));
+                for (const profile of profileDirs) {
+                    const srcProfile = path.join(userDataDirectory, profile.name);
+                    const destProfile = path.join(effectiveDir, profile.name);
+                    await fsp.mkdir(destProfile, { recursive: true }).catch(() => { });
+                    // Cookies (macOS layout: <Profile>/Cookies)
+                    const cookiesSrc = path.join(srcProfile, 'Cookies');
+                    if (await fsp.stat(cookiesSrc).catch(() => null)) {
+                        await fsp.copyFile(cookiesSrc, path.join(destProfile, 'Cookies')).catch(() => { });
                     }
-                    else {
-                        await fsp.copyFile(src, dest).catch(() => { });
+                    // Cookies (Windows layout: <Profile>/Network/Cookies)
+                    const networkCookiesSrc = path.join(srcProfile, 'Network', 'Cookies');
+                    if (await fsp.stat(networkCookiesSrc).catch(() => null)) {
+                        const destNetwork = path.join(destProfile, 'Network');
+                        await fsp.mkdir(destNetwork, { recursive: true }).catch(() => { });
+                        await fsp.copyFile(networkCookiesSrc, path.join(destNetwork, 'Cookies')).catch(() => { });
                     }
-                };
-                await copyRecursive(userDataDirectory, effectiveDir).catch(() => { });
+                    // Local Storage (auth tokens, session data)
+                    const localStorageSrc = path.join(srcProfile, 'Local Storage');
+                    const localStorageStat = await fsp.stat(localStorageSrc).catch(() => null);
+                    if (localStorageStat && localStorageStat.isDirectory()) {
+                        const destLocalStorage = path.join(destProfile, 'Local Storage');
+                        await fsp.mkdir(destLocalStorage, { recursive: true }).catch(() => { });
+                        const lsFiles = await fsp.readdir(localStorageSrc).catch(() => []);
+                        await Promise.all(lsFiles.map(f => fsp.copyFile(path.join(localStorageSrc, f), path.join(destLocalStorage, f)).catch(() => { })));
+                    }
+                }
             }
             catch {
                 // Silent fallback: use empty profile if clone fails

package/dist/crawlers/crawlDomain.js

@@ -274,7 +274,10 @@ const crawlDomain = async ({ url, randomToken, host: _host, viewportSettings, ma
         }
     };
     let isAbortingScanNow = false;
-    const rateController = new CrawlRateController(maxRequestsPerCrawl, specifiedMaxConcurrency || constants.maxConcurrency);
+    const remainingBudget = fromCrawlIntelligentSitemap
+        ? Math.max(0, maxRequestsPerCrawl - urlsCrawledFromIntelligent.scanned.length)
+        : maxRequestsPerCrawl;
+    const rateController = new CrawlRateController(remainingBudget, specifiedMaxConcurrency || constants.maxConcurrency);
     const { nonAuthHeaders, httpCredentials } = splitAuthHeaders(extraHTTPHeaders);
     const crawler = register(new crawlee.PlaywrightCrawler({
         launchContext: {
@@ -284,6 +287,8 @@ const crawlDomain = async ({ url, randomToken, host: _host, viewportSettings, ma
         retryOnBlocked: true,
         browserPoolOptions: {
             useFingerprints: false,
+            retireBrowserAfterPageCount: 500,
+            closeInactiveBrowserAfterSecs: 30,
             preLaunchHooks: [
                 getPreLaunchHook(userDataDirectory),
                 async (_pageId, launchContext) => {
@@ -309,40 +314,48 @@ const crawlDomain = async ({ url, randomToken, host: _host, viewportSettings, ma
         postNavigationHooks: [
             async (crawlingContext) => {
                 const { page, request } = crawlingContext;
-                await page.evaluate(() => {
-                    return new Promise(resolve => {
-                        let timeout;
-                        let mutationCount = 0;
-                        const MAX_MUTATIONS = 500; // stop if things never quiet down
-                        const OBSERVER_TIMEOUT = 5000; // hard cap on total wait
-                        const observer = new MutationObserver(() => {
-                            clearTimeout(timeout);
-                            mutationCount += 1;
-                            if (mutationCount > MAX_MUTATIONS) {
-                                observer.disconnect();
-                                resolve('Too many mutations, exiting.');
-                                return;
-                            }
-                            // restart quiet‑period timer
+                try {
+                    await page.evaluate(() => {
+                        return new Promise(resolve => {
+                            let timeout;
+                            let mutationCount = 0;
+                            const MAX_MUTATIONS = 500; // stop if things never quiet down
+                            const OBSERVER_TIMEOUT = 5000; // hard cap on total wait
+                            const observer = new MutationObserver(() => {
+                                clearTimeout(timeout);
+                                mutationCount += 1;
+                                if (mutationCount > MAX_MUTATIONS) {
+                                    observer.disconnect();
+                                    resolve('Too many mutations, exiting.');
+                                    return;
+                                }
+                                // restart quiet‑period timer
+                                timeout = setTimeout(() => {
+                                    observer.disconnect();
+                                    resolve('DOM stabilized.');
+                                }, 1000);
+                            });
+                            // overall timeout in case the page never settles
                             timeout = setTimeout(() => {
                                 observer.disconnect();
-                                resolve('DOM stabilized.');
-                            }, 1000);
+                                resolve('Observer timeout reached.');
+                            }, OBSERVER_TIMEOUT);
+                            const root = document.documentElement || document.body || document;
+                            if (!root || typeof observer.observe !== 'function') {
+                                resolve('No root node to observe.');
+                            }
+                            else {
+                                observer.observe(root, { childList: true, subtree: true });
+                            }
                         });
-                        // overall timeout in case the page never settles
-                        timeout = setTimeout(() => {
-                            observer.disconnect();
-                            resolve('Observer timeout reached.');
-                        }, OBSERVER_TIMEOUT);
-                        const root = document.documentElement || document.body || document;
-                        if (!root || typeof observer.observe !== 'function') {
-                            resolve('No root node to observe.');
-                        }
-                        else {
-                            observer.observe(root, { childList: true, subtree: true });
-                        }
                     });
-                });
+                }
+                catch (err) {
+                    if (err.message?.includes('was destroyed')) {
+                        return;
+                    }
+                    throw err;
+                }
                 let finalUrl = page.url();
                 const requestLabelUrl = request.label;
                 // to handle scenario where the redirected link is not within the scanning website

package/dist/crawlers/crawlIntelligentSitemap.js

@@ -5,7 +5,7 @@ import crawlDomain from './crawlDomain.js';
 import crawlSitemap from './crawlSitemap.js';
 import { getPlaywrightLaunchOptions, getSitemapsFromRobotsTxt, initModifiedUserAgent } from '../constants/common.js';
 import { register } from '../utils.js';
-const crawlIntelligentSitemap = async (url, randomToken, host, viewportSettings, maxRequestsPerCrawl, browser, userDataDirectory, strategy, specifiedMaxConcurrency, fileTypes, blacklistedPatterns, includeScreenshots, followRobots, extraHTTPHeaders, safeMode, scanDuration) => {
+const crawlIntelligentSitemap = async (url, randomToken, host, viewportSettings, maxRequestsPerCrawl, browser, userDataDirectory, strategy, specifiedMaxConcurrency, fileTypes, blacklistedPatterns, includeScreenshots, followRobots, extraHTTPHeaders, safeMode, scanDuration, ruleset = []) => {
     const startTime = Date.now(); // Track start time
     let urlsCrawledFinal;
     const urlsCrawled = { ...constants.urlsCrawledObj };
@@ -153,6 +153,7 @@ const crawlIntelligentSitemap = async (url, randomToken, host, viewportSettings,
             urlsCrawledFromIntelligent: urlsCrawled,
             crawledFromLocalFile: false,
             scanDuration: scanDuration > 0 ? remainingDuration : 0,
+            ruleset,
         });
     }
     const elapsed = Date.now() - startTime;
@@ -180,6 +181,7 @@ const crawlIntelligentSitemap = async (url, randomToken, host, viewportSettings,
             datasetFromIntelligent: dataset,
             urlsCrawledFromIntelligent: urlsCrawled,
             scanDuration: remainingScanDuration,
+            ruleset,
         });
     }
     else if (!hasDurationRemaining) {

package/dist/crawlers/crawlLocalFile.js

@@ -69,6 +69,7 @@ export const crawlLocalFile = async ({ url, randomToken, host, viewportSettings,
             datasetFromIntelligent,
             urlsCrawledFromIntelligent,
             crawledFromLocalFile: true,
+            ruleset,
         });
         urlsCrawled = { ...urlsCrawled, ...updatedUrlsCrawled };
         return urlsCrawled;

package/dist/crawlers/crawlRateController.js

@@ -26,12 +26,9 @@ export class CrawlRateController {
         }
     }
     onFailure(httpStatus, pool) {
-        if (typeof httpStatus !== 'number' || httpStatus < 400) {
-            return false;
-        }
         this.consecutiveSuccesses = 0;
         this.consecutiveFailures++;
-        if (pool && pool.maxConcurrency > 1) {
+        if (typeof httpStatus === 'number' && httpStatus >= 400 && pool && pool.maxConcurrency > 1) {
             pool.maxConcurrency = Math.max(1, Math.floor(pool.maxConcurrency / 2));
             consoleLogger.info(`Rate limited (HTTP ${httpStatus}) — reducing concurrency to ${pool.maxConcurrency}`);
         }

package/dist/crawlers/crawlSitemap.js

@@ -6,13 +6,16 @@ import { getLinksFromSitemap, getPlaywrightLaunchOptions, isSkippedUrl, waitForP
 import { areLinksEqual, isFollowStrategy, isWhitelistedContentType, normUrl, register } from '../utils.js';
 import { handlePdfDownload, runPdfScan, mapPdfScanResults, doPdfScreenshots, } from './pdfScanFunc.js';
 import { consoleLogger, guiInfoLog } from '../logs.js';
-const crawlSitemap = async ({ sitemapUrl, randomToken, host, viewportSettings, maxRequestsPerCrawl, browser, userDataDirectory, specifiedMaxConcurrency, fileTypes, blacklistedPatterns, includeScreenshots, extraHTTPHeaders, strategy = EnqueueStrategy.All, userUrl = '', scanDuration = 0, fromCrawlIntelligentSitemap = false, userUrlInputFromIntelligent = null, datasetFromIntelligent = null, urlsCrawledFromIntelligent = null, crawledFromLocalFile = false, }) => {
+const crawlSitemap = async ({ sitemapUrl, randomToken, host, viewportSettings, maxRequestsPerCrawl, browser, userDataDirectory, specifiedMaxConcurrency, fileTypes, blacklistedPatterns, includeScreenshots, extraHTTPHeaders, strategy = EnqueueStrategy.All, userUrl = '', scanDuration = 0, fromCrawlIntelligentSitemap = false, userUrlInputFromIntelligent = null, datasetFromIntelligent = null, urlsCrawledFromIntelligent = null, crawledFromLocalFile = false, ruleset = [], }) => {
     const crawlStartTime = Date.now();
     let dataset;
     let urlsCrawled;
     let durationExceeded = false;
     let isAbortingScan = false;
-    const rateController = new CrawlRateController(maxRequestsPerCrawl, specifiedMaxConcurrency || constants.maxConcurrency);
+    const remainingBudget = fromCrawlIntelligentSitemap
+        ? Math.max(0, maxRequestsPerCrawl - urlsCrawledFromIntelligent.scanned.length)
+        : maxRequestsPerCrawl;
+    const rateController = new CrawlRateController(remainingBudget, specifiedMaxConcurrency || constants.maxConcurrency);
     const initialNoSuccessFailureAbortThreshold = Math.max(5, Math.min(maxRequestsPerCrawl, 25));
     if (fromCrawlIntelligentSitemap) {
         dataset = datasetFromIntelligent;
@@ -46,6 +49,8 @@ const crawlSitemap = async ({ sitemapUrl, randomToken, host, viewportSettings, m
         retryOnBlocked: true,
         browserPoolOptions: {
             useFingerprints: false,
+            retireBrowserAfterPageCount: 500,
+            closeInactiveBrowserAfterSecs: 30,
             preLaunchHooks: [
                 getPreLaunchHook(userDataDirectory),
                 async (_pageId, launchContext) => {
@@ -97,6 +102,9 @@ const crawlSitemap = async ({ sitemapUrl, randomToken, host, viewportSettings, m
                             if (!root || typeof observer.observe !== 'function') {
                                 resolve('No root node to observe.');
                             }
+                            else {
+                                observer.observe(root, { childList: true, subtree: true });
+                            }
                         });
                     });
                 }
@@ -209,7 +217,7 @@ const crawlSitemap = async ({ sitemapUrl, randomToken, host, viewportSettings, m
                         });
                         return;
                     }
-                    const results = await runAxeScript({ includeScreenshots, page, randomToken });
+                    const results = await runAxeScript({ includeScreenshots, page, randomToken, ruleset });
                     // Detect JS redirects that fire during/after axe scan.
                     // Listen for navigation, then give a brief window for pending redirects to complete.
                     try {

package/dist/crawlers/custom/flagUnlabelledClickableElements.js

@@ -382,7 +382,11 @@ export async function flagUnlabelledClickableElements() {
     function isElementTooSmall(element) {
         // Get the bounding rectangle of the element
         const rect = element.getBoundingClientRect();
-        // Check if the element has a valid width or height
+        // If either dimension is 0, the element is non-perceivable
+        if (rect.width === 0 || rect.height === 0) {
+            return true;
+        }
+        // Check if the element has a valid width and height
         if (rect.width > 0 || rect.height > 0) {
             return false; // Element is not too small
         }

package/oobee-client-scanner.js

@@ -3,7 +3,7 @@
  * DO NOT EDIT MANUALLY. Re-generate with: node dist/generateOobeeClientScanner.js
  *
  * Embedded at generation time:
- *   App version : 0.10.97
+ *   App version : 0.10.98
  *   Sentry DSN  : (from OOBEE_SENTRY_DSN env var or constants.ts default)
  *   Sentry SDK  : @sentry/browser 10.58.0 (loaded from CDN at runtime)
  *
@@ -33593,7 +33593,11 @@
     function isElementTooSmall(element) {
         // Get the bounding rectangle of the element
         const rect = element.getBoundingClientRect();
-        // Check if the element has a valid width or height
+        // If either dimension is 0, the element is non-perceivable
+        if (rect.width === 0 || rect.height === 0) {
+            return true;
+        }
+        // Check if the element has a valid width and height
         if (rect.width > 0 || rect.height > 0) {
             return false; // Element is not too small
         }
@@ -34883,7 +34887,7 @@
   // ── Sentry browser telemetry (Sentry JS SDK, loaded from CDN) ────────────
   
   var _oobeeSentryDsn          = "https://3b8c7ee46b06f33815a1301b6713ebc3@o4509047624761344.ingest.us.sentry.io/4509327783559168";
-  var _oobeeAppVersion         = "0.10.97";
+  var _oobeeAppVersion         = "0.10.98";
   var _oobeeSentryVersion      = "10.58.0";
   var _oobeeSentryInitialized  = false;
   var _oobeeSentryLoadPromise  = null;

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@govtechsg/oobee",
   "main": "dist/npmIndex.js",
-  "version": "0.10.97",
+  "version": "0.10.98",
   "type": "module",
   "author": "Government Technology Agency <info@tech.gov.sg>",
   "bin": {

package/src/combine.ts

@@ -186,6 +186,7 @@ const combineRun = async (details: Data, deviceToScan: string) => {
         strategy,
         userUrl: url,
         scanDuration,
+        ruleset,
       });
       urlsCrawledObj = sitemapResult.urlsCrawled;
       durationExceeded = sitemapResult.durationExceeded;
@@ -236,6 +237,7 @@ const combineRun = async (details: Data, deviceToScan: string) => {
         extraHTTPHeaders,
         safeMode,
         scanDuration,
+        ruleset,
       );
       urlsCrawledObj = intelligentResult.urlsCrawled;
       durationExceeded = intelligentResult.durationExceeded;

package/src/crawlers/commonCrawlerFunc.ts

@@ -1230,35 +1230,59 @@ export const getPreLaunchHook = (userDataDirectory: string) => {
 
     await fsp.mkdir(effectiveDir, { recursive: true });
 
-    // For pool re-launches, best-effort clone profile data from base directory
-    // so authenticated sessions are preserved across browser pool retirements.
+    // For pool re-launches, copy only auth-relevant files from the base
+    // directory so authenticated sessions are preserved without cloning the
+    // entire profile (which grows with caches, IndexedDB, etc. during the crawl).
     if (launchCount > 1) {
-      const skipDirs = new Set([
-        'Singleton', 'lockfile', 'LOCK',
-        'Cache', 'Code Cache', 'GPUCache', 'DawnGraphiteCache', 'DawnWebGPUCache',
-        'Service Worker', 'ScriptCache', 'ShaderCache', 'GrShaderCache',
-        'component_crx_cache', 'optimization_guide_model_store',
-        'BrowserMetrics', 'Crashpad', 'FileTypePolicies',
-      ]);
       try {
-        const copyRecursive = async (src: string, dest: string) => {
-          const stat = await fsp.stat(src).catch(() => null);
-          if (!stat) return;
-          if (stat.isDirectory()) {
-            await fsp.mkdir(dest, { recursive: true }).catch(() => {});
-            const entries = await fsp.readdir(src).catch(() => []);
+        // Copy top-level Local State (cookie encryption keys on Windows)
+        const localStateSrc = path.join(userDataDirectory, 'Local State');
+        if (await fsp.stat(localStateSrc).catch(() => null)) {
+          await fsp.copyFile(localStateSrc, path.join(effectiveDir, 'Local State')).catch(() => {});
+        }
+
+        // Find profile directories (Default, Profile 1, Profile 2, etc.)
+        const entries = await fsp.readdir(userDataDirectory, { withFileTypes: true }).catch(() => []);
+        const profileDirs = (entries as any[]).filter(
+          (e: any) => e.isDirectory() && /^(Default|Profile \d+)$/i.test(e.name),
+        );
+
+        for (const profile of profileDirs) {
+          const srcProfile = path.join(userDataDirectory, profile.name);
+          const destProfile = path.join(effectiveDir, profile.name);
+          await fsp.mkdir(destProfile, { recursive: true }).catch(() => {});
+
+          // Cookies (macOS layout: <Profile>/Cookies)
+          const cookiesSrc = path.join(srcProfile, 'Cookies');
+          if (await fsp.stat(cookiesSrc).catch(() => null)) {
+            await fsp.copyFile(cookiesSrc, path.join(destProfile, 'Cookies')).catch(() => {});
+          }
+
+          // Cookies (Windows layout: <Profile>/Network/Cookies)
+          const networkCookiesSrc = path.join(srcProfile, 'Network', 'Cookies');
+          if (await fsp.stat(networkCookiesSrc).catch(() => null)) {
+            const destNetwork = path.join(destProfile, 'Network');
+            await fsp.mkdir(destNetwork, { recursive: true }).catch(() => {});
+            await fsp.copyFile(networkCookiesSrc, path.join(destNetwork, 'Cookies')).catch(() => {});
+          }
+
+          // Local Storage (auth tokens, session data)
+          const localStorageSrc = path.join(srcProfile, 'Local Storage');
+          const localStorageStat = await fsp.stat(localStorageSrc).catch(() => null);
+          if (localStorageStat && localStorageStat.isDirectory()) {
+            const destLocalStorage = path.join(destProfile, 'Local Storage');
+            await fsp.mkdir(destLocalStorage, { recursive: true }).catch(() => {});
+            const lsFiles = await fsp.readdir(localStorageSrc).catch(() => []);
             await Promise.all(
-              entries
-                .filter(entry => !entry.startsWith('Singleton') && !skipDirs.has(entry))
-                .map(entry =>
-                  copyRecursive(path.join(src, entry), path.join(dest, entry)).catch(() => {}),
-                ),
+              lsFiles.map(f =>
+                fsp.copyFile(
+                  path.join(localStorageSrc, f),
+                  path.join(destLocalStorage, f),
+                ).catch(() => {}),
+              ),
             );
-          } else {
-            await fsp.copyFile(src, dest).catch(() => {});
           }
-        };
-        await copyRecursive(userDataDirectory, effectiveDir).catch(() => {});
+        }
       } catch {
         // Silent fallback: use empty profile if clone fails
       }

package/src/crawlers/crawlDomain.ts

@@ -382,8 +382,11 @@ const crawlDomain = async ({
   };
 
   let isAbortingScanNow = false;
+  const remainingBudget = fromCrawlIntelligentSitemap
+    ? Math.max(0, maxRequestsPerCrawl - urlsCrawledFromIntelligent.scanned.length)
+    : maxRequestsPerCrawl;
   const rateController = new CrawlRateController(
-    maxRequestsPerCrawl,
+    remainingBudget,
     specifiedMaxConcurrency || constants.maxConcurrency,
   );
 
@@ -398,6 +401,8 @@ const crawlDomain = async ({
       retryOnBlocked: true,
       browserPoolOptions: {
         useFingerprints: false,
+        retireBrowserAfterPageCount: 500,
+        closeInactiveBrowserAfterSecs: 30,
         preLaunchHooks: [
           getPreLaunchHook(userDataDirectory),
           async (_pageId, launchContext) => {
@@ -424,44 +429,51 @@ const crawlDomain = async ({
         async crawlingContext => {
           const { page, request } = crawlingContext;
 
-          await page.evaluate(() => {
-            return new Promise(resolve => {
-              let timeout;
-              let mutationCount = 0;
-              const MAX_MUTATIONS = 500; // stop if things never quiet down
-              const OBSERVER_TIMEOUT = 5000; // hard cap on total wait
-
-              const observer = new MutationObserver(() => {
-                clearTimeout(timeout);
-
-                mutationCount += 1;
-                if (mutationCount > MAX_MUTATIONS) {
-                  observer.disconnect();
-                  resolve('Too many mutations, exiting.');
-                  return;
-                }
+          try {
+            await page.evaluate(() => {
+              return new Promise(resolve => {
+                let timeout;
+                let mutationCount = 0;
+                const MAX_MUTATIONS = 500; // stop if things never quiet down
+                const OBSERVER_TIMEOUT = 5000; // hard cap on total wait
+
+                const observer = new MutationObserver(() => {
+                  clearTimeout(timeout);
+
+                  mutationCount += 1;
+                  if (mutationCount > MAX_MUTATIONS) {
+                    observer.disconnect();
+                    resolve('Too many mutations, exiting.');
+                    return;
+                  }
+
+                  // restart quiet‑period timer
+                  timeout = setTimeout(() => {
+                    observer.disconnect();
+                    resolve('DOM stabilized.');
+                  }, 1000);
+                });
 
-                // restart quiet‑period timer
+                // overall timeout in case the page never settles
                 timeout = setTimeout(() => {
                   observer.disconnect();
-                  resolve('DOM stabilized.');
-                }, 1000);
+                  resolve('Observer timeout reached.');
+                }, OBSERVER_TIMEOUT);
+
+                const root = document.documentElement || document.body || document;
+                if (!root || typeof observer.observe !== 'function') {
+                  resolve('No root node to observe.');
+                } else {
+                  observer.observe(root, { childList: true, subtree: true });
+                }
               });
-
-              // overall timeout in case the page never settles
-              timeout = setTimeout(() => {
-                observer.disconnect();
-                resolve('Observer timeout reached.');
-              }, OBSERVER_TIMEOUT);
-
-              const root = document.documentElement || document.body || document;
-              if (!root || typeof observer.observe !== 'function') {
-                resolve('No root node to observe.');
-              } else {
-                observer.observe(root, { childList: true, subtree: true });
-              }
             });
-          });
+          } catch (err) {
+            if (err.message?.includes('was destroyed')) {
+              return;
+            }
+            throw err;
+          }
 
           let finalUrl = page.url();
           const requestLabelUrl = request.label;

package/src/crawlers/crawlIntelligentSitemap.ts

@@ -2,7 +2,7 @@ import fs from 'fs';
 import { chromium, Page } from 'playwright';
 import { EnqueueStrategy } from 'crawlee';
 import { createCrawleeSubFolders, splitAuthHeaders, addAuthRouteHandler } from './commonCrawlerFunc.js';
-import constants, { FileTypes, guiInfoStatusTypes, sitemapPaths } from '../constants/constants.js';
+import constants, { FileTypes, guiInfoStatusTypes, RuleFlags, sitemapPaths } from '../constants/constants.js';
 import { consoleLogger, guiInfoLog } from '../logs.js';
 import crawlDomain from './crawlDomain.js';
 import crawlSitemap from './crawlSitemap.js';
@@ -27,6 +27,7 @@ const crawlIntelligentSitemap = async (
   extraHTTPHeaders: Record<string, string>,
   safeMode: boolean,
   scanDuration: number,
+  ruleset: RuleFlags[] = [],
 ) => {
   const startTime = Date.now(); // Track start time
 
@@ -191,6 +192,7 @@ const crawlIntelligentSitemap = async (
       urlsCrawledFromIntelligent: urlsCrawled,
       crawledFromLocalFile: false,
       scanDuration: scanDuration > 0 ? remainingDuration : 0,
+      ruleset,
     });
   }
 
@@ -222,6 +224,7 @@ const crawlIntelligentSitemap = async (
       datasetFromIntelligent: dataset,
       urlsCrawledFromIntelligent: urlsCrawled,
       scanDuration: remainingScanDuration,
+      ruleset,
     });
   } else if (!hasDurationRemaining) {
     consoleLogger.info(

package/src/crawlers/crawlLocalFile.ts

@@ -127,6 +127,7 @@ export const crawlLocalFile = async ({
       datasetFromIntelligent,
       urlsCrawledFromIntelligent,
       crawledFromLocalFile: true,
+      ruleset,
     });
 
     urlsCrawled = { ...urlsCrawled, ...updatedUrlsCrawled };

package/src/crawlers/crawlRateController.ts

@@ -36,14 +36,10 @@ export class CrawlRateController {
   }
 
   onFailure(httpStatus: number | undefined, pool?: { maxConcurrency: number }): boolean {
-    if (typeof httpStatus !== 'number' || httpStatus < 400) {
-      return false;
-    }
-
     this.consecutiveSuccesses = 0;
     this.consecutiveFailures++;
 
-    if (pool && pool.maxConcurrency > 1) {
+    if (typeof httpStatus === 'number' && httpStatus >= 400 && pool && pool.maxConcurrency > 1) {
       pool.maxConcurrency = Math.max(1, Math.floor(pool.maxConcurrency / 2));
       consoleLogger.info(
         `Rate limited (HTTP ${httpStatus}) — reducing concurrency to ${pool.maxConcurrency}`,

package/src/crawlers/crawlSitemap.ts

@@ -16,6 +16,7 @@ import constants, {
   UrlsCrawled,
   disallowedListOfPatterns,
   FileTypes,
+  RuleFlags,
 } from '../constants/constants.js';
 import {
   getLinksFromSitemap,
@@ -55,6 +56,7 @@ const crawlSitemap = async ({
   datasetFromIntelligent = null,
   urlsCrawledFromIntelligent = null,
   crawledFromLocalFile = false,
+  ruleset = [],
 }: {
   sitemapUrl: string;
   randomToken: string;
@@ -76,14 +78,18 @@ const crawlSitemap = async ({
   datasetFromIntelligent?: Dataset;
   urlsCrawledFromIntelligent?: UrlsCrawled;
   crawledFromLocalFile?: boolean;
+  ruleset?: RuleFlags[];
 }) => {
   const crawlStartTime = Date.now();
   let dataset: crawlee.Dataset;
   let urlsCrawled: UrlsCrawled;
   let durationExceeded = false;
   let isAbortingScan = false;
+  const remainingBudget = fromCrawlIntelligentSitemap
+    ? Math.max(0, maxRequestsPerCrawl - urlsCrawledFromIntelligent.scanned.length)
+    : maxRequestsPerCrawl;
   const rateController = new CrawlRateController(
-    maxRequestsPerCrawl,
+    remainingBudget,
     specifiedMaxConcurrency || constants.maxConcurrency,
   );
   const initialNoSuccessFailureAbortThreshold = Math.max(5, Math.min(maxRequestsPerCrawl, 25));
@@ -136,6 +142,8 @@ const crawlSitemap = async ({
       retryOnBlocked: true,
       browserPoolOptions: {
         useFingerprints: false,
+        retireBrowserAfterPageCount: 500,
+        closeInactiveBrowserAfterSecs: 30,
         preLaunchHooks: [
           getPreLaunchHook(userDataDirectory),
           async (_pageId, launchContext) => {
@@ -191,6 +199,8 @@ const crawlSitemap = async ({
                 const root = document.documentElement || document.body || document;
                 if (!root || typeof observer.observe !== 'function') {
                   resolve('No root node to observe.');
+                } else {
+                  observer.observe(root, { childList: true, subtree: true });
                 }
               });
             });
@@ -334,7 +344,7 @@ const crawlSitemap = async ({
               return;
             }
 
-            const results = await runAxeScript({ includeScreenshots, page, randomToken });
+            const results = await runAxeScript({ includeScreenshots, page, randomToken, ruleset });
 
             // Detect JS redirects that fire during/after axe scan.
             // Listen for navigation, then give a brief window for pending redirects to complete.

package/src/crawlers/custom/flagUnlabelledClickableElements.ts

@@ -457,8 +457,13 @@ function hasPointerCursor(node: Node): boolean {
   function isElementTooSmall(element: Element) {
     // Get the bounding rectangle of the element
     const rect = element.getBoundingClientRect();
-    
-    // Check if the element has a valid width or height
+
+    // If either dimension is 0, the element is non-perceivable
+    if (rect.width === 0 || rect.height === 0) {
+        return true;
+    }
+
+    // Check if the element has a valid width and height
     if (rect.width > 0 || rect.height > 0) {
         return false; // Element is not too small
     }