@govtechsg/oobee 0.10.97 → 0.11.0

package/dist/crawlers/crawlDomain.js

@@ -1,6 +1,6 @@
 import crawlee from 'crawlee';
 import { CrawlRateController } from './crawlRateController.js';
-import { createCrawleeSubFolders, getPreLaunchHook, preNavigationHooks, runAxeScript, isUrlPdf, shouldSkipClickDueToDisallowedHref, shouldSkipDueToUnsupportedContent, splitAuthHeaders, } from './commonCrawlerFunc.js';
+import { createCrawleeSubFolders, getPreLaunchHook, getPostPageCloseHook, preNavigationHooks, runAxeScript, isUrlPdf, shouldSkipClickDueToDisallowedHref, shouldSkipDueToUnsupportedContent, splitAuthHeaders, } from './commonCrawlerFunc.js';
 import constants, { blackListedFileExtensions, guiInfoStatusTypes, cssQuerySelectors, STATUS_CODE_METADATA, disallowedListOfPatterns, disallowedSelectorPatterns, FileTypes, } from '../constants/constants.js';
 import { getPlaywrightLaunchOptions, isBlacklistedFileExtensions, isSkippedUrl, isDisallowedInRobotsTxt, getUrlsFromRobotsTxt, waitForPageLoaded, } from '../constants/common.js';
 import { areLinksEqual, isFollowStrategy, isSameHostname, normUrl, register } from '../utils.js';
@@ -274,7 +274,10 @@ const crawlDomain = async ({ url, randomToken, host: _host, viewportSettings, ma
         }
     };
     let isAbortingScanNow = false;
-    const rateController = new CrawlRateController(maxRequestsPerCrawl, specifiedMaxConcurrency || constants.maxConcurrency);
+    const remainingBudget = fromCrawlIntelligentSitemap
+        ? Math.max(0, maxRequestsPerCrawl - urlsCrawledFromIntelligent.scanned.length)
+        : maxRequestsPerCrawl;
+    const rateController = new CrawlRateController(remainingBudget, specifiedMaxConcurrency || constants.maxConcurrency);
     const { nonAuthHeaders, httpCredentials } = splitAuthHeaders(extraHTTPHeaders);
     const crawler = register(new crawlee.PlaywrightCrawler({
         launchContext: {
@@ -284,6 +287,8 @@ const crawlDomain = async ({ url, randomToken, host: _host, viewportSettings, ma
         retryOnBlocked: true,
         browserPoolOptions: {
             useFingerprints: false,
+            retireBrowserAfterPageCount: 500,
+            closeInactiveBrowserAfterSecs: 30,
             preLaunchHooks: [
                 getPreLaunchHook(userDataDirectory),
                 async (_pageId, launchContext) => {
@@ -299,50 +304,70 @@ const crawlDomain = async ({ url, randomToken, host: _host, viewportSettings, ma
                     };
                 },
             ],
+            postPageCloseHooks: [getPostPageCloseHook(userDataDirectory)],
         },
         requestQueue,
         maxRequestRetries: 3,
         maxSessionRotations: 1,
         preNavigationHooks: [
             ...preNavigationHooks(extraHTTPHeaders),
+            async ({ request }) => {
+                const url = request.url.toLowerCase();
+                try {
+                    const pathname = new URL(url).pathname;
+                    const ext = pathname.split('.').pop();
+                    if (ext && blackListedFileExtensions.includes(ext)) {
+                        request.skipNavigation = true;
+                    }
+                }
+                catch { }
+            },
         ],
         postNavigationHooks: [
             async (crawlingContext) => {
                 const { page, request } = crawlingContext;
-                await page.evaluate(() => {
-                    return new Promise(resolve => {
-                        let timeout;
-                        let mutationCount = 0;
-                        const MAX_MUTATIONS = 500; // stop if things never quiet down
-                        const OBSERVER_TIMEOUT = 5000; // hard cap on total wait
-                        const observer = new MutationObserver(() => {
-                            clearTimeout(timeout);
-                            mutationCount += 1;
-                            if (mutationCount > MAX_MUTATIONS) {
-                                observer.disconnect();
-                                resolve('Too many mutations, exiting.');
-                                return;
-                            }
-                            // restart quiet‑period timer
+                try {
+                    await page.evaluate(() => {
+                        return new Promise(resolve => {
+                            let timeout;
+                            let mutationCount = 0;
+                            const MAX_MUTATIONS = 500; // stop if things never quiet down
+                            const OBSERVER_TIMEOUT = 5000; // hard cap on total wait
+                            const observer = new MutationObserver(() => {
+                                clearTimeout(timeout);
+                                mutationCount += 1;
+                                if (mutationCount > MAX_MUTATIONS) {
+                                    observer.disconnect();
+                                    resolve('Too many mutations, exiting.');
+                                    return;
+                                }
+                                // restart quiet‑period timer
+                                timeout = setTimeout(() => {
+                                    observer.disconnect();
+                                    resolve('DOM stabilized.');
+                                }, 1000);
+                            });
+                            // overall timeout in case the page never settles
                             timeout = setTimeout(() => {
                                 observer.disconnect();
-                                resolve('DOM stabilized.');
-                            }, 1000);
+                                resolve('Observer timeout reached.');
+                            }, OBSERVER_TIMEOUT);
+                            const root = document.documentElement || document.body || document;
+                            if (!root || typeof observer.observe !== 'function') {
+                                resolve('No root node to observe.');
+                            }
+                            else {
+                                observer.observe(root, { childList: true, subtree: true });
+                            }
                         });
-                        // overall timeout in case the page never settles
-                        timeout = setTimeout(() => {
-                            observer.disconnect();
-                            resolve('Observer timeout reached.');
-                        }, OBSERVER_TIMEOUT);
-                        const root = document.documentElement || document.body || document;
-                        if (!root || typeof observer.observe !== 'function') {
-                            resolve('No root node to observe.');
-                        }
-                        else {
-                            observer.observe(root, { childList: true, subtree: true });
-                        }
                     });
-                });
+                }
+                catch (err) {
+                    if (err.message?.includes('was destroyed')) {
+                        return;
+                    }
+                    throw err;
+                }
                 let finalUrl = page.url();
                 const requestLabelUrl = request.label;
                 // to handle scenario where the redirected link is not within the scanning website
@@ -402,20 +427,17 @@ const crawlDomain = async ({ url, randomToken, host: _host, viewportSettings, ma
                 if (shouldSkipDueToUnsupportedContent(response, request.url) ||
                     (request.skipNavigation && actualUrl === 'about:blank')) {
                     if (!isScanPdfs) {
-                        // Don't inform the user it is skipped since web crawler is best-effort.
-                        /*
-                      guiInfoLog(guiInfoStatusTypes.SKIPPED, {
-                        numScanned: urlsCrawled.scanned.length,
-                        urlScanned: request.url,
-                      });
-                      urlsCrawled.userExcluded.push({
-                        url: request.url,
-                        pageTitle: request.url,
-                        actualUrl: request.url, // because about:blank is not useful
-                        metadata: STATUS_CODE_METADATA[1],
-                        httpStatusCode: 0,
-                      });
-                      */
+                        guiInfoLog(guiInfoStatusTypes.SKIPPED, {
+                            numScanned: urlsCrawled.scanned.length,
+                            urlScanned: request.url,
+                        });
+                        urlsCrawled.userExcluded.push({
+                            url: request.url,
+                            pageTitle: request.url,
+                            actualUrl: request.url,
+                            metadata: STATUS_CODE_METADATA[1],
+                            httpStatusCode: 1,
+                        });
                         return;
                     }
                     const { pdfFileName, url: downloadedPdfUrl } = handlePdfDownload(randomToken, pdfDownloads, request, sendRequest, urlsCrawled);
@@ -423,20 +445,17 @@ const crawlDomain = async ({ url, randomToken, host: _host, viewportSettings, ma
                     return;
                 }
                 if (isBlacklistedFileExtensions(actualUrl, blackListedFileExtensions)) {
-                    // Don't inform the user it is skipped since web crawler is best-effort.
-                    /*
-                  guiInfoLog(guiInfoStatusTypes.SKIPPED, {
-                    numScanned: urlsCrawled.scanned.length,
-                    urlScanned: request.url,
-                  });
-                  urlsCrawled.userExcluded.push({
-                    url: request.url,
-                    pageTitle: request.url,
-                    actualUrl, // because about:blank is not useful
-                    metadata: STATUS_CODE_METADATA[1],
-                    httpStatusCode: 0,
-                  });
-                  */
+                    guiInfoLog(guiInfoStatusTypes.SKIPPED, {
+                        numScanned: urlsCrawled.scanned.length,
+                        urlScanned: request.url,
+                    });
+                    urlsCrawled.userExcluded.push({
+                        url: request.url,
+                        pageTitle: request.url,
+                        actualUrl,
+                        metadata: STATUS_CODE_METADATA[1],
+                        httpStatusCode: 1,
+                    });
                     return;
                 }
                 if (!isFollowStrategy(url, actualUrl, strategy) &&
@@ -621,7 +640,57 @@ const crawlDomain = async ({ url, randomToken, host: _host, viewportSettings, ma
             if (isAbortingScanNow) {
                 return;
             }
+            // Handle download-triggered navigation errors: Playwright throws
+            // "Download is starting" when page.goto() hits a file download URL.
+            // Crawlee retries 3 times (all fail) then lands here.
+            const isDownloadError = request.errorMessages?.some((msg) => msg.includes('Download is starting'));
+            if (isDownloadError) {
+                if (isScanPdfs) {
+                    // Re-enqueue with skipNavigation so the requestHandler's PDF download path handles it
+                    try {
+                        await requestQueue.addRequest({
+                            url: request.url,
+                            skipNavigation: true,
+                            label: request.url,
+                            uniqueKey: `download_${request.url}`,
+                        });
+                    }
+                    catch { }
+                }
+                else {
+                    guiInfoLog(guiInfoStatusTypes.SKIPPED, {
+                        numScanned: urlsCrawled.scanned.length,
+                        urlScanned: request.url,
+                    });
+                    urlsCrawled.userExcluded.push({
+                        url: request.url,
+                        pageTitle: request.url,
+                        actualUrl: request.url,
+                        metadata: STATUS_CODE_METADATA[1],
+                        httpStatusCode: 1,
+                    });
+                }
+                return;
+            }
             const status = response?.status();
+            // Re-enqueue rate-limited (403) URLs once for a retry after concurrency recovers.
+            // Without this, URLs that fail during a rate-limit burst are permanently lost
+            // even though the site is accessible at lower concurrency.
+            // Don't call onFailure here — the re-enqueued request will get a fresh attempt.
+            // If it fails again (rateLimitRetried=true), it falls through to the normal
+            // onFailure + circuit breaker path below.
+            if (status === 403 && !request.userData?.rateLimitRetried) {
+                try {
+                    await requestQueue.addRequest({
+                        url: request.url,
+                        label: request.url,
+                        uniqueKey: `ratelimit_${request.url}`,
+                        userData: { rateLimitRetried: true },
+                    });
+                }
+                catch { }
+                return;
+            }
             if (rateController.onFailure(status, crawler.autoscaledPool)) {
                 consoleLogger.info(`Aborting crawl: consecutive HTTP failures threshold reached (site may be rate-limiting). Successfully scanned ${urlsCrawled.scanned.length} pages.`);
                 isAbortingScanNow = true;
@@ -656,7 +725,9 @@ const crawlDomain = async ({ url, randomToken, host: _host, viewportSettings, ma
     await crawler.run();
     // Additional passes: keep re-visiting scanned seed-hostname pages for
     // click-discovery until no new pages are found or limits are reached.
-    if (!safeMode && !isAbortingScanNow && !durationExceeded) {
+    // Skip when called from intelligent sitemap — the domain phase is only meant
+    // to discover new pages via <a> links, not re-click 3000+ already-scanned pages.
+    if (!safeMode && !isAbortingScanNow && !durationExceeded && !fromCrawlIntelligentSitemap) {
         const seedHostname = new URL(url).hostname;
         const clickPassVisited = new Set();
         let prevScannedCount;

package/dist/crawlers/crawlIntelligentSitemap.js

@@ -5,7 +5,7 @@ import crawlDomain from './crawlDomain.js';
 import crawlSitemap from './crawlSitemap.js';
 import { getPlaywrightLaunchOptions, getSitemapsFromRobotsTxt, initModifiedUserAgent } from '../constants/common.js';
 import { register } from '../utils.js';
-const crawlIntelligentSitemap = async (url, randomToken, host, viewportSettings, maxRequestsPerCrawl, browser, userDataDirectory, strategy, specifiedMaxConcurrency, fileTypes, blacklistedPatterns, includeScreenshots, followRobots, extraHTTPHeaders, safeMode, scanDuration) => {
+const crawlIntelligentSitemap = async (url, randomToken, host, viewportSettings, maxRequestsPerCrawl, browser, userDataDirectory, strategy, specifiedMaxConcurrency, fileTypes, blacklistedPatterns, includeScreenshots, followRobots, extraHTTPHeaders, safeMode, scanDuration, ruleset = []) => {
     const startTime = Date.now(); // Track start time
     let urlsCrawledFinal;
     const urlsCrawled = { ...constants.urlsCrawledObj };
@@ -153,6 +153,7 @@ const crawlIntelligentSitemap = async (url, randomToken, host, viewportSettings,
             urlsCrawledFromIntelligent: urlsCrawled,
             crawledFromLocalFile: false,
             scanDuration: scanDuration > 0 ? remainingDuration : 0,
+            ruleset,
         });
     }
     const elapsed = Date.now() - startTime;
@@ -180,6 +181,7 @@ const crawlIntelligentSitemap = async (url, randomToken, host, viewportSettings,
             datasetFromIntelligent: dataset,
             urlsCrawledFromIntelligent: urlsCrawled,
             scanDuration: remainingScanDuration,
+            ruleset,
         });
     }
     else if (!hasDurationRemaining) {

package/dist/crawlers/crawlLocalFile.js

@@ -69,6 +69,7 @@ export const crawlLocalFile = async ({ url, randomToken, host, viewportSettings,
             datasetFromIntelligent,
             urlsCrawledFromIntelligent,
             crawledFromLocalFile: true,
+            ruleset,
         });
         urlsCrawled = { ...urlsCrawled, ...updatedUrlsCrawled };
         return urlsCrawled;

package/dist/crawlers/crawlRateController.js

@@ -26,12 +26,9 @@ export class CrawlRateController {
         }
     }
     onFailure(httpStatus, pool) {
-        if (typeof httpStatus !== 'number' || httpStatus < 400) {
-            return false;
-        }
         this.consecutiveSuccesses = 0;
         this.consecutiveFailures++;
-        if (pool && pool.maxConcurrency > 1) {
+        if (typeof httpStatus === 'number' && httpStatus >= 400 && pool && pool.maxConcurrency > 1) {
             pool.maxConcurrency = Math.max(1, Math.floor(pool.maxConcurrency / 2));
             consoleLogger.info(`Rate limited (HTTP ${httpStatus}) — reducing concurrency to ${pool.maxConcurrency}`);
         }

package/dist/crawlers/crawlSitemap.js

@@ -1,18 +1,21 @@
 import crawlee, { EnqueueStrategy, RequestList } from 'crawlee';
 import { CrawlRateController } from './crawlRateController.js';
-import { createCrawleeSubFolders, getPreLaunchHook, preNavigationHooks, runAxeScript, splitAuthHeaders, } from './commonCrawlerFunc.js';
-import constants, { STATUS_CODE_METADATA, guiInfoStatusTypes, disallowedListOfPatterns, FileTypes, } from '../constants/constants.js';
-import { getLinksFromSitemap, getPlaywrightLaunchOptions, isSkippedUrl, waitForPageLoaded, isFilePath, } from '../constants/common.js';
+import { createCrawleeSubFolders, getPreLaunchHook, getPostPageCloseHook, preNavigationHooks, runAxeScript, isUrlPdf, splitAuthHeaders, } from './commonCrawlerFunc.js';
+import constants, { STATUS_CODE_METADATA, guiInfoStatusTypes, blackListedFileExtensions, disallowedListOfPatterns, disallowedSelectorPatterns, FileTypes, } from '../constants/constants.js';
+import { getLinksFromSitemap, getPlaywrightLaunchOptions, isDisallowedInRobotsTxt, isSkippedUrl, waitForPageLoaded, isFilePath, } from '../constants/common.js';
 import { areLinksEqual, isFollowStrategy, isWhitelistedContentType, normUrl, register } from '../utils.js';
 import { handlePdfDownload, runPdfScan, mapPdfScanResults, doPdfScreenshots, } from './pdfScanFunc.js';
 import { consoleLogger, guiInfoLog } from '../logs.js';
-const crawlSitemap = async ({ sitemapUrl, randomToken, host, viewportSettings, maxRequestsPerCrawl, browser, userDataDirectory, specifiedMaxConcurrency, fileTypes, blacklistedPatterns, includeScreenshots, extraHTTPHeaders, strategy = EnqueueStrategy.All, userUrl = '', scanDuration = 0, fromCrawlIntelligentSitemap = false, userUrlInputFromIntelligent = null, datasetFromIntelligent = null, urlsCrawledFromIntelligent = null, crawledFromLocalFile = false, }) => {
+const crawlSitemap = async ({ sitemapUrl, randomToken, host, viewportSettings, maxRequestsPerCrawl, browser, userDataDirectory, specifiedMaxConcurrency, fileTypes, blacklistedPatterns, includeScreenshots, extraHTTPHeaders, strategy = EnqueueStrategy.All, userUrl = '', scanDuration = 0, fromCrawlIntelligentSitemap = false, userUrlInputFromIntelligent = null, datasetFromIntelligent = null, urlsCrawledFromIntelligent = null, crawledFromLocalFile = false, ruleset = [], }) => {
     const crawlStartTime = Date.now();
     let dataset;
     let urlsCrawled;
     let durationExceeded = false;
     let isAbortingScan = false;
-    const rateController = new CrawlRateController(maxRequestsPerCrawl, specifiedMaxConcurrency || constants.maxConcurrency);
+    const remainingBudget = fromCrawlIntelligentSitemap
+        ? Math.max(0, maxRequestsPerCrawl - urlsCrawledFromIntelligent.scanned.length)
+        : maxRequestsPerCrawl;
+    const rateController = new CrawlRateController(remainingBudget, specifiedMaxConcurrency || constants.maxConcurrency);
     const initialNoSuccessFailureAbortThreshold = Math.max(5, Math.min(maxRequestsPerCrawl, 25));
     if (fromCrawlIntelligentSitemap) {
         dataset = datasetFromIntelligent;
@@ -38,6 +41,11 @@ const crawlSitemap = async ({ sitemapUrl, randomToken, host, viewportSettings, m
     const requestList = await RequestList.open({
         sources: linksFromSitemap,
     });
+    // Always create a request queue alongside the request list. An empty queue
+    // has zero impact on crawl behavior (Crawlee processes RequestList first).
+    // Having it available enables: download re-enqueue for PDF scanning,
+    // 403 rate-limit retry, and enqueueLinks for intelligent sitemap discovery.
+    const { requestQueue } = await createCrawleeSubFolders(randomToken);
     const crawler = register(new crawlee.PlaywrightCrawler({
         launchContext: {
             launcher: constants.launcher,
@@ -46,6 +54,8 @@ const crawlSitemap = async ({ sitemapUrl, randomToken, host, viewportSettings, m
         retryOnBlocked: true,
         browserPoolOptions: {
             useFingerprints: false,
+            retireBrowserAfterPageCount: 500,
+            closeInactiveBrowserAfterSecs: 30,
             preLaunchHooks: [
                 getPreLaunchHook(userDataDirectory),
                 async (_pageId, launchContext) => {
@@ -60,8 +70,10 @@ const crawlSitemap = async ({ sitemapUrl, randomToken, host, viewportSettings, m
                     };
                 },
             ],
+            postPageCloseHooks: [getPostPageCloseHook(userDataDirectory)],
         },
         requestList,
+        requestQueue,
         maxRequestRetries: 3,
         maxSessionRotations: 1,
         postNavigationHooks: [
@@ -97,6 +109,9 @@ const crawlSitemap = async ({ sitemapUrl, randomToken, host, viewportSettings, m
                             if (!root || typeof observer.observe !== 'function') {
                                 resolve('No root node to observe.');
                             }
+                            else {
+                                observer.observe(root, { childList: true, subtree: true });
+                            }
                         });
                     });
                 }
@@ -117,14 +132,22 @@ const crawlSitemap = async ({ sitemapUrl, randomToken, host, viewportSettings, m
                 if (isNotSupportedDocument) {
                     request.skipNavigation = true;
                     request.userData.isNotSupportedDocument = true;
-                    // Log for verification (optional, but not required for correctness)
-                    // console.log(`[SKIP] Not supported: ${request.url}`);
                     return;
                 }
+                try {
+                    const pathname = new URL(url).pathname;
+                    const ext = pathname.split('.').pop();
+                    if (ext && blackListedFileExtensions.includes(ext)) {
+                        request.skipNavigation = true;
+                        request.userData.isNotSupportedDocument = true;
+                        return;
+                    }
+                }
+                catch { }
             },
         ],
         requestHandlerTimeoutSecs: 90,
-        requestHandler: async ({ page, request, response, sendRequest }) => {
+        requestHandler: async ({ page, request, response, sendRequest, enqueueLinks }) => {
             // Log documents that are not supported
             if (request.userData?.isNotSupportedDocument) {
                 guiInfoLog(guiInfoStatusTypes.SKIPPED, {
@@ -209,7 +232,7 @@ const crawlSitemap = async ({ sitemapUrl, randomToken, host, viewportSettings, m
                         });
                         return;
                     }
-                    const results = await runAxeScript({ includeScreenshots, page, randomToken });
+                    const results = await runAxeScript({ includeScreenshots, page, randomToken, ruleset });
                     // Detect JS redirects that fire during/after axe scan.
                     // Listen for navigation, then give a brief window for pending redirects to complete.
                     try {
@@ -260,6 +283,34 @@ const crawlSitemap = async ({ sitemapUrl, randomToken, host, viewportSettings, m
                         results.url = request.url;
                         results.actualUrl = actualUrl;
                         await dataset.pushData(results);
+                        // Discover <a> links from this page for the intelligent sitemap flow.
+                        // This eliminates the need for a separate crawlDomain supplement phase
+                        // that would re-visit all these pages just to extract links.
+                        if (fromCrawlIntelligentSitemap) {
+                            try {
+                                await enqueueLinks({
+                                    selector: `a:not(${disallowedSelectorPatterns})`,
+                                    strategy,
+                                    requestQueue,
+                                    transformRequestFunction: (req) => {
+                                        try {
+                                            req.url = req.url.replace(/(?<=&|\?)utm_.*?(&|$)/gim, '');
+                                        }
+                                        catch { }
+                                        if (isDisallowedInRobotsTxt(req.url))
+                                            return null;
+                                        if (isUrlPdf(req.url)) {
+                                            req.skipNavigation = true;
+                                        }
+                                        req.label = req.url;
+                                        return req;
+                                    },
+                                });
+                            }
+                            catch {
+                                // Best-effort link discovery; don't fail the scan
+                            }
+                        }
                     }
                 }
                 else {
@@ -268,18 +319,34 @@ const crawlSitemap = async ({ sitemapUrl, randomToken, host, viewportSettings, m
                         urlScanned: request.url,
                     });
                     if (isScanHtml) {
-                        // carry through the HTTP status metadata
-                        const status = response?.status();
-                        const metadata = typeof status === 'number'
-                            ? STATUS_CODE_METADATA[status] || STATUS_CODE_METADATA[599]
-                            : STATUS_CODE_METADATA[2];
-                        urlsCrawled.invalid.push({
-                            actualUrl,
-                            url: request.url,
-                            pageTitle: request.url,
-                            metadata,
-                            httpStatusCode: typeof status === 'number' ? status : 0,
-                        });
+                        // Non-HTML content types (images, PDFs, binary files) and URLs
+                        // that redirect to non-HTML resources should be classified as
+                        // unsupported documents, not generic page errors.
+                        const isNonHtmlContent = contentType &&
+                            !contentType.startsWith('text/html') &&
+                            !contentType.includes('html');
+                        if (isNonHtmlContent && status !== 0) {
+                            urlsCrawled.userExcluded.push({
+                                actualUrl,
+                                url: request.url,
+                                pageTitle: request.url,
+                                metadata: STATUS_CODE_METADATA[1],
+                                httpStatusCode: 1,
+                            });
+                        }
+                        else {
+                            const httpStatus = response?.status();
+                            const metadata = typeof httpStatus === 'number'
+                                ? STATUS_CODE_METADATA[httpStatus] || STATUS_CODE_METADATA[599]
+                                : STATUS_CODE_METADATA[2];
+                            urlsCrawled.invalid.push({
+                                actualUrl,
+                                url: request.url,
+                                pageTitle: request.url,
+                                metadata,
+                                httpStatusCode: typeof httpStatus === 'number' ? httpStatus : 0,
+                            });
+                        }
                     }
                 }
             }
@@ -294,7 +361,53 @@ const crawlSitemap = async ({ sitemapUrl, randomToken, host, viewportSettings, m
             if (isAbortingScan) {
                 return;
             }
+            // Handle download-triggered navigation errors: Playwright throws
+            // "Download is starting" when page.goto() hits a file download URL.
+            const isDownloadError = request.errorMessages?.some((msg) => msg.includes('Download is starting'));
+            if (isDownloadError) {
+                if (isScanPdfs) {
+                    // Re-enqueue with skipNavigation so the requestHandler's PDF download path handles it
+                    try {
+                        await requestQueue.addRequest({
+                            url: request.url,
+                            skipNavigation: true,
+                            label: request.url,
+                            uniqueKey: `download_${request.url}`,
+                        });
+                    }
+                    catch { }
+                }
+                else {
+                    guiInfoLog(guiInfoStatusTypes.SKIPPED, {
+                        numScanned: urlsCrawled.scanned.length,
+                        urlScanned: request.url,
+                    });
+                    urlsCrawled.userExcluded.push({
+                        url: request.url,
+                        pageTitle: request.url,
+                        actualUrl: request.url,
+                        metadata: STATUS_CODE_METADATA[1],
+                        httpStatusCode: 1,
+                    });
+                }
+                return;
+            }
             const status = response?.status();
+            // Re-enqueue rate-limited (403) URLs once for a retry after concurrency recovers.
+            // Don't call onFailure here — the re-enqueued request gets a fresh attempt.
+            // If it fails again (rateLimitRetried=true), it falls through to the normal path.
+            if (status === 403 && !request.userData?.rateLimitRetried) {
+                try {
+                    await requestQueue.addRequest({
+                        url: request.url,
+                        label: request.url,
+                        uniqueKey: `ratelimit_${request.url}`,
+                        userData: { rateLimitRetried: true },
+                    });
+                }
+                catch { }
+                return;
+            }
             if (rateController.onFailure(status, crawler.autoscaledPool)) {
                 consoleLogger.info(`Aborting crawl: consecutive HTTP failures threshold reached (site may be rate-limiting). Successfully scanned ${urlsCrawled.scanned.length} pages.`);
                 isAbortingScan = true;

package/dist/crawlers/custom/flagUnlabelledClickableElements.js

@@ -382,7 +382,11 @@ export async function flagUnlabelledClickableElements() {
     function isElementTooSmall(element) {
         // Get the bounding rectangle of the element
         const rect = element.getBoundingClientRect();
-        // Check if the element has a valid width or height
+        // If either dimension is 0, the element is non-perceivable
+        if (rect.width === 0 || rect.height === 0) {
+            return true;
+        }
+        // Check if the element has a valid width and height
         if (rect.width > 0 || rect.height > 0) {
             return false; // Element is not too small
         }

package/dist/mergeAxeResults.js

@@ -831,7 +831,7 @@ generateJsonFiles = false, preferredBrowser) => {
         consoleLogger.info(`Dataset drop: ${error.message}`);
     }
     try {
-        const requestQueue = await RequestQueue.open(crawleeDir);
+        const requestQueue = await RequestQueue.open(`${crawleeDir}_rq`);
         await requestQueue.drop();
     }
     catch (error) {
@@ -841,6 +841,7 @@ generateJsonFiles = false, preferredBrowser) => {
     const crawleePath = path.join(storagePath, 'crawlee');
     try {
         await fs.promises.rm(crawleePath, { recursive: true, force: true });
+        await fs.promises.rm(`${crawleePath}_rq`, { recursive: true, force: true });
     }
     catch {
         // Best-effort; storage was already dropped via API