@govtechsg/oobee 0.10.85 → 0.10.87

package/dist/crawlers/commonCrawlerFunc.js

@@ -116,7 +116,7 @@ export const filterAxeResults = (results, pageTitle, customFlowDetails) => {
             return;
         const conformance = tags.filter(tag => tag.startsWith('wcag') || tag === 'best-practice');
         nodes.forEach(node => {
-            const { html } = node;
+            const { html, target } = node;
             if (!(rule in passed.rules)) {
                 passed.rules[rule] = {
                     description,
@@ -128,7 +128,8 @@ export const filterAxeResults = (results, pageTitle, customFlowDetails) => {
                 };
             }
             const finalHtml = truncateHtml(html);
-            passed.rules[rule].items.push({ html: finalHtml, screenshotPath: '', message: '', xpath: '' });
+            const xpath = target.length === 1 && typeof target[0] === 'string' ? target[0] : undefined;
+            passed.rules[rule].items.push({ html: finalHtml, screenshotPath: '', message: '', xpath: xpath || '' });
             passed.totalItems += 1;
             passed.rules[rule].totalItems += 1;
             totalItems += 1;

package/dist/crawlers/crawlDomain.js

@@ -4,7 +4,7 @@ import fsp from 'fs/promises';
 import { createCrawleeSubFolders, runAxeScript, isUrlPdf, shouldSkipClickDueToDisallowedHref, shouldSkipDueToUnsupportedContent, } from './commonCrawlerFunc.js';
 import constants, { blackListedFileExtensions, guiInfoStatusTypes, cssQuerySelectors, STATUS_CODE_METADATA, disallowedListOfPatterns, disallowedSelectorPatterns, FileTypes, } from '../constants/constants.js';
 import { getPlaywrightLaunchOptions, isBlacklistedFileExtensions, isSkippedUrl, isDisallowedInRobotsTxt, getUrlsFromRobotsTxt, waitForPageLoaded, } from '../constants/common.js';
-import { areLinksEqual, isFollowStrategy, register } from '../utils.js';
+import { areLinksEqual, isFollowStrategy, normUrl, register } from '../utils.js';
 import { handlePdfDownload, runPdfScan, mapPdfScanResults, doPdfScreenshots, } from './pdfScanFunc.js';
 import { consoleLogger, guiInfoLog } from '../logs.js';
 const isBlacklisted = (url, blacklistedPatterns) => {
@@ -37,8 +37,8 @@ const crawlDomain = async ({ url, randomToken, host: _host, viewportSettings, ma
     const pdfDownloads = [];
     const uuidToPdfMapping = {};
     const queuedUrlSet = new Set();
-    const scannedUrlSet = new Set(urlsCrawled.scanned.map(item => item.url));
-    const scannedResolvedUrlSet = new Set(urlsCrawled.scanned.map(item => item.actualUrl || item.url));
+    const scannedUrlSet = new Set(urlsCrawled.scanned.map(item => normUrl(item.url)));
+    const scannedResolvedUrlSet = new Set(urlsCrawled.scanned.map(item => normUrl(item.actualUrl || item.url)));
     const isScanHtml = [FileTypes.All, FileTypes.HtmlOnly].includes(fileTypes);
     const isScanPdfs = [FileTypes.All, FileTypes.PdfOnly].includes(fileTypes);
     const { maxConcurrency } = constants;
@@ -70,11 +70,12 @@ const crawlDomain = async ({ url, randomToken, host: _host, viewportSettings, ma
         const initialPageUrl = workingPage.url().toString();
         const selectedElementsString = cssQuerySelectors.join(', ');
         const isExcluded = (newPageUrl) => {
-            const isAlreadyScanned = urlsCrawled.scanned.some(item => item.url === newPageUrl);
+            const isAlreadyScanned = scannedUrlSet.has(normUrl(newPageUrl));
             const isBlacklistedUrl = isBlacklisted(newPageUrl, blacklistedPatterns);
             const isNotFollowStrategy = !isFollowStrategy(newPageUrl, initialPageUrl, strategy);
             const isNotSupportedDocument = disallowedListOfPatterns.some(pattern => newPageUrl.toLowerCase().startsWith(pattern));
-            return isNotSupportedDocument || isAlreadyScanned || isBlacklistedUrl || isNotFollowStrategy;
+            const isRobotsDisallowed = isDisallowedInRobotsTxt(newPageUrl);
+            return isNotSupportedDocument || isAlreadyScanned || isBlacklistedUrl || isNotFollowStrategy || isRobotsDisallowed;
         };
         const setPageListeners = (pageListener) => {
             // event listener to handle new page popups upon button click
@@ -235,7 +236,7 @@ const crawlDomain = async ({ url, randomToken, host: _host, viewportSettings, ma
                     catch (e) {
                         consoleLogger.error(e);
                     }
-                    if (scannedUrlSet.has(req.url)) {
+                    if (scannedUrlSet.has(normUrl(req.url))) {
                         req.skipNavigation = true;
                     }
                     if (isDisallowedInRobotsTxt(req.url))
@@ -358,7 +359,7 @@ const crawlDomain = async ({ url, randomToken, host: _host, viewportSettings, ma
                     finalUrl = requestLabelUrl;
                 }
                 const isRedirected = !areLinksEqual(finalUrl, requestLabelUrl);
-                if (isRedirected) {
+                if (isRedirected && !isDisallowedInRobotsTxt(finalUrl)) {
                     await enqueueUniqueRequest({ url: finalUrl, label: finalUrl });
                 }
                 else {
@@ -399,7 +400,7 @@ const crawlDomain = async ({ url, randomToken, host: _host, viewportSettings, ma
                     return;
                 }
                 // if URL has already been scanned
-                if (scannedUrlSet.has(request.url)) {
+                if (scannedUrlSet.has(normUrl(request.url))) {
                     await enqueueProcess(page, enqueueLinks, browserContext);
                     return;
                 }
@@ -493,8 +494,32 @@ const crawlDomain = async ({ url, randomToken, host: _host, viewportSettings, ma
                         return;
                     }
                     const results = await runAxeScript({ includeScreenshots, page, randomToken, ruleset });
+                    // Detect JS redirects that fire during/after axe scan.
+                    // Listen for navigation, then give a brief window for pending redirects to complete.
+                    try {
+                        let navigatedToUrl = null;
+                        const onFrameNavigated = (frame) => {
+                            if (frame === page.mainFrame()) {
+                                navigatedToUrl = frame.url();
+                            }
+                        };
+                        page.on('framenavigated', onFrameNavigated);
+                        await page.waitForTimeout(1000);
+                        page.off('framenavigated', onFrameNavigated);
+                        const postScanUrl = navigatedToUrl || page.url();
+                        if (postScanUrl && postScanUrl !== 'about:blank' && !isFollowStrategy(postScanUrl, request.url, 'same-hostname')) {
+                            urlsCrawled.notScannedRedirects.push({
+                                fromUrl: request.url,
+                                toUrl: postScanUrl,
+                            });
+                            return;
+                        }
+                    }
+                    catch (_) {
+                        // Page/context was destroyed during navigation — handled by outer catch
+                    }
                     if (isRedirected) {
-                        const isLoadedUrlInCrawledUrls = scannedResolvedUrlSet.has(actualUrl);
+                        const isLoadedUrlInCrawledUrls = scannedResolvedUrlSet.has(normUrl(actualUrl));
                         if (isLoadedUrlInCrawledUrls) {
                             urlsCrawled.notScannedRedirects.push({
                                 fromUrl: request.url,
@@ -513,8 +538,8 @@ const crawlDomain = async ({ url, randomToken, host: _host, viewportSettings, ma
                                 pageTitle: results.pageTitle,
                                 actualUrl, // i.e. actualUrl
                             });
-                            scannedUrlSet.add(request.url);
-                            scannedResolvedUrlSet.add(actualUrl);
+                            scannedUrlSet.add(normUrl(request.url));
+                            scannedResolvedUrlSet.add(normUrl(actualUrl));
                             urlsCrawled.scannedRedirects.push({
                                 fromUrl: request.url,
                                 toUrl: actualUrl, // i.e. actualUrl
@@ -535,8 +560,8 @@ const crawlDomain = async ({ url, randomToken, host: _host, viewportSettings, ma
                             actualUrl: request.url,
                             pageTitle: results.pageTitle,
                         });
-                        scannedUrlSet.add(request.url);
-                        scannedResolvedUrlSet.add(request.url);
+                        scannedUrlSet.add(normUrl(request.url));
+                        scannedResolvedUrlSet.add(normUrl(request.url));
                         await dataset.pushData(results);
                     }
                 }

package/dist/crawlers/crawlIntelligentSitemap.js

@@ -3,7 +3,7 @@ import constants, { guiInfoStatusTypes, sitemapPaths } from '../constants/consta
 import { consoleLogger, guiInfoLog } from '../logs.js';
 import crawlDomain from './crawlDomain.js';
 import crawlSitemap from './crawlSitemap.js';
-import { getPlaywrightLaunchOptions } from '../constants/common.js';
+import { getPlaywrightLaunchOptions, getSitemapsFromRobotsTxt } from '../constants/common.js';
 import { register } from '../utils.js';
 const crawlIntelligentSitemap = async (url, randomToken, host, viewportSettings, maxRequestsPerCrawl, browser, userDataDirectory, strategy, specifiedMaxConcurrency, fileTypes, blacklistedPatterns, includeScreenshots, followRobots, extraHTTPHeaders, safeMode, scanDuration) => {
     const startTime = Date.now(); // Track start time
@@ -66,12 +66,30 @@ const crawlIntelligentSitemap = async (url, randomToken, host, viewportSettings,
             return false;
         }
     };
+    // Discover sitemaps from robots.txt first (supports multiple Sitemap: directives)
+    let sitemapUrls = [];
     try {
-        sitemapUrl = await findSitemap(url, userDataDirectory, extraHTTPHeaders);
+        sitemapUrls = await getSitemapsFromRobotsTxt(url, browser, userDataDirectory, extraHTTPHeaders);
+        if (sitemapUrls.length > 0) {
+            console.log(`Found ${sitemapUrls.length} sitemap(s) in robots.txt: ${sitemapUrls.join(', ')}`);
+            sitemapExist = true;
+        }
     }
     catch (error) {
         consoleLogger.error(error);
     }
+    // Fall back to hardcoded path probing if robots.txt had no sitemaps
+    if (!sitemapExist) {
+        try {
+            sitemapUrl = await findSitemap(url, userDataDirectory, extraHTTPHeaders);
+            if (sitemapExist) {
+                sitemapUrls = [sitemapUrl];
+            }
+        }
+        catch (error) {
+            consoleLogger.error(error);
+        }
+    }
     if (!sitemapExist) {
         console.log('Unable to find sitemap. Commencing website crawl instead.');
         return await crawlDomain({
@@ -90,34 +108,48 @@ const crawlIntelligentSitemap = async (url, randomToken, host, viewportSettings,
             followRobots,
             extraHTTPHeaders,
             safeMode,
-            scanDuration, // Use full duration since no sitemap
+            scanDuration,
+        });
+    }
+    // Process all discovered sitemaps sequentially, sharing dataset and urlsCrawled
+    for (const currentSitemapUrl of sitemapUrls) {
+        if (urlsCrawled.scanned.length >= maxRequestsPerCrawl)
+            break;
+        const elapsed = Date.now() - startTime;
+        const remainingDuration = scanDuration > 0 ? Math.max(scanDuration - elapsed / 1000, 0) : scanDuration;
+        if (scanDuration > 0 && remainingDuration <= 0) {
+            durationExceeded = true;
+            break;
+        }
+        console.log(`Processing sitemap: ${currentSitemapUrl}`);
+        urlsCrawledFinal = await crawlSitemap({
+            sitemapUrl: currentSitemapUrl,
+            randomToken,
+            host,
+            viewportSettings,
+            maxRequestsPerCrawl,
+            browser,
+            userDataDirectory,
+            specifiedMaxConcurrency,
+            fileTypes,
+            blacklistedPatterns,
+            includeScreenshots,
+            extraHTTPHeaders,
+            strategy,
+            userUrl: url,
+            fromCrawlIntelligentSitemap,
+            userUrlInputFromIntelligent: url,
+            datasetFromIntelligent: dataset,
+            urlsCrawledFromIntelligent: urlsCrawled,
+            crawledFromLocalFile: false,
+            scanDuration: scanDuration > 0 ? remainingDuration : 0,
         });
     }
-    console.log(`Sitemap found at ${sitemapUrl}`);
-    urlsCrawledFinal = await crawlSitemap({
-        sitemapUrl,
-        randomToken,
-        host,
-        viewportSettings,
-        maxRequestsPerCrawl,
-        browser,
-        userDataDirectory,
-        specifiedMaxConcurrency,
-        fileTypes,
-        blacklistedPatterns,
-        includeScreenshots,
-        extraHTTPHeaders,
-        fromCrawlIntelligentSitemap,
-        userUrlInputFromIntelligent: url,
-        datasetFromIntelligent: dataset,
-        urlsCrawledFromIntelligent: urlsCrawled,
-        crawledFromLocalFile: false,
-        scanDuration,
-    });
     const elapsed = Date.now() - startTime;
-    const remainingScanDuration = Math.max(scanDuration - elapsed / 1000, 0); // in seconds
-    if (urlsCrawledFinal.scanned.length < maxRequestsPerCrawl && remainingScanDuration > 0) {
-        console.log(`Continuing crawl from root website. Remaining scan time: ${remainingScanDuration.toFixed(1)}s`);
+    const remainingScanDuration = scanDuration > 0 ? Math.max(scanDuration - elapsed / 1000, 0) : 0;
+    const hasDurationRemaining = scanDuration === 0 || remainingScanDuration > 0;
+    if (urlsCrawled.scanned.length < maxRequestsPerCrawl && hasDurationRemaining) {
+        console.log(`Continuing crawl from root website.${scanDuration > 0 ? ` Remaining scan time: ${remainingScanDuration.toFixed(1)}s` : ''}`);
         urlsCrawledFinal = await crawlDomain({
             url,
             randomToken,
@@ -136,15 +168,15 @@ const crawlIntelligentSitemap = async (url, randomToken, host, viewportSettings,
             safeMode,
             fromCrawlIntelligentSitemap,
             datasetFromIntelligent: dataset,
-            urlsCrawledFromIntelligent: urlsCrawledFinal,
+            urlsCrawledFromIntelligent: urlsCrawled,
             scanDuration: remainingScanDuration,
         });
     }
-    else if (remainingScanDuration <= 0) {
+    else if (!hasDurationRemaining) {
         console.log(`Crawl duration exceeded before more pages could be found (limit: ${scanDuration}s).`);
         durationExceeded = true;
     }
     guiInfoLog(guiInfoStatusTypes.COMPLETED, {});
-    return { urlsCrawled: urlsCrawledFinal, durationExceeded };
+    return { urlsCrawled, durationExceeded };
 };
 export default crawlIntelligentSitemap;

package/dist/crawlers/crawlSitemap.js

@@ -1,17 +1,18 @@
-import crawlee, { RequestList } from 'crawlee';
+import crawlee, { EnqueueStrategy, RequestList } from 'crawlee';
 import * as path from 'path';
 import fsp from 'fs/promises';
 import { createCrawleeSubFolders, preNavigationHooks, runAxeScript, } from './commonCrawlerFunc.js';
 import constants, { STATUS_CODE_METADATA, guiInfoStatusTypes, disallowedListOfPatterns, FileTypes, } from '../constants/constants.js';
 import { getLinksFromSitemap, getPlaywrightLaunchOptions, isSkippedUrl, waitForPageLoaded, isFilePath, } from '../constants/common.js';
-import { areLinksEqual, isWhitelistedContentType, register } from '../utils.js';
+import { areLinksEqual, isFollowStrategy, isWhitelistedContentType, normUrl, register } from '../utils.js';
 import { handlePdfDownload, runPdfScan, mapPdfScanResults, doPdfScreenshots, } from './pdfScanFunc.js';
 import { guiInfoLog } from '../logs.js';
-const crawlSitemap = async ({ sitemapUrl, randomToken, host, viewportSettings, maxRequestsPerCrawl, browser, userDataDirectory, specifiedMaxConcurrency, fileTypes, blacklistedPatterns, includeScreenshots, extraHTTPHeaders, scanDuration = 0, fromCrawlIntelligentSitemap = false, userUrlInputFromIntelligent = null, datasetFromIntelligent = null, urlsCrawledFromIntelligent = null, crawledFromLocalFile = false, }) => {
+const crawlSitemap = async ({ sitemapUrl, randomToken, host, viewportSettings, maxRequestsPerCrawl, browser, userDataDirectory, specifiedMaxConcurrency, fileTypes, blacklistedPatterns, includeScreenshots, extraHTTPHeaders, strategy = EnqueueStrategy.All, userUrl = '', scanDuration = 0, fromCrawlIntelligentSitemap = false, userUrlInputFromIntelligent = null, datasetFromIntelligent = null, urlsCrawledFromIntelligent = null, crawledFromLocalFile = false, }) => {
     const crawlStartTime = Date.now();
     let dataset;
     let urlsCrawled;
     let durationExceeded = false;
+    let isAbortingScan = false;
     if (fromCrawlIntelligentSitemap) {
         dataset = datasetFromIntelligent;
         urlsCrawled = urlsCrawledFromIntelligent;
@@ -24,7 +25,7 @@ const crawlSitemap = async ({ sitemapUrl, randomToken, host, viewportSettings, m
         console.log('Local file crawling not supported for sitemap. Please provide a valid URL.');
         return;
     }
-    const linksFromSitemap = await getLinksFromSitemap(sitemapUrl, maxRequestsPerCrawl, browser, userDataDirectory, userUrlInputFromIntelligent, fromCrawlIntelligentSitemap, extraHTTPHeaders);
+    const linksFromSitemap = await getLinksFromSitemap(sitemapUrl, maxRequestsPerCrawl, browser, userDataDirectory, userUrlInputFromIntelligent, fromCrawlIntelligentSitemap, extraHTTPHeaders, strategy, userUrl || sitemapUrl);
     sitemapUrl = encodeURI(sitemapUrl);
     const pdfDownloads = [];
     const uuidToPdfMapping = {};
@@ -144,106 +145,162 @@ const crawlSitemap = async ({ sitemapUrl, randomToken, host, viewportSettings, m
                 });
                 return;
             }
-            await waitForPageLoaded(page, 10000);
-            const actualUrl = page.url() || request.loadedUrl || request.url;
-            const hasExceededDuration = scanDuration > 0 && Date.now() - crawlStartTime > scanDuration * 1000;
-            if (urlsCrawled.scanned.length >= maxRequestsPerCrawl || hasExceededDuration) {
-                if (hasExceededDuration) {
-                    console.log(`Crawl duration of ${scanDuration}s exceeded. Aborting sitemap crawl.`);
-                    durationExceeded = true;
-                }
-                crawler.autoscaledPool.abort(); // stops new requests
-                return;
-            }
-            if (request.skipNavigation && actualUrl === 'about:blank') {
-                if (isScanPdfs) {
-                    // pushes download promise into pdfDownloads
-                    const { pdfFileName, url } = handlePdfDownload(randomToken, pdfDownloads, request, sendRequest, urlsCrawled);
-                    uuidToPdfMapping[pdfFileName] = url;
+            try {
+                await waitForPageLoaded(page, 10000);
+                const actualUrl = page.url() || request.loadedUrl || request.url;
+                const hasExceededDuration = scanDuration > 0 && Date.now() - crawlStartTime > scanDuration * 1000;
+                if (urlsCrawled.scanned.length >= maxRequestsPerCrawl || hasExceededDuration) {
+                    isAbortingScan = true;
+                    if (hasExceededDuration) {
+                        console.log(`Crawl duration of ${scanDuration}s exceeded. Aborting sitemap crawl.`);
+                        durationExceeded = true;
+                    }
+                    crawler.autoscaledPool.abort(); // stops new requests
                     return;
                 }
-                guiInfoLog(guiInfoStatusTypes.SKIPPED, {
-                    numScanned: urlsCrawled.scanned.length,
-                    urlScanned: request.url,
-                });
-                urlsCrawled.userExcluded.push({
-                    url: request.url,
-                    pageTitle: request.url,
-                    actualUrl: request.url, // because about:blank is not useful
-                    metadata: STATUS_CODE_METADATA[1],
-                    httpStatusCode: 1,
-                });
-                return;
-            }
-            const contentType = response?.headers?.()['content-type'] || '';
-            const status = response ? response.status() : 0;
-            if (isScanHtml && status < 300 && isWhitelistedContentType(contentType)) {
-                const isRedirected = !areLinksEqual(page.url(), request.url);
-                const isLoadedUrlInCrawledUrls = urlsCrawled.scanned.some(item => (item.actualUrl || item.url) === page.url());
-                if (isRedirected && isLoadedUrlInCrawledUrls) {
-                    urlsCrawled.notScannedRedirects.push({
-                        fromUrl: request.url,
-                        toUrl: actualUrl, // i.e. actualUrl
+                if (request.skipNavigation && actualUrl === 'about:blank') {
+                    if (isScanPdfs) {
+                        // pushes download promise into pdfDownloads
+                        const { pdfFileName, url } = handlePdfDownload(randomToken, pdfDownloads, request, sendRequest, urlsCrawled);
+                        uuidToPdfMapping[pdfFileName] = url;
+                        return;
+                    }
+                    guiInfoLog(guiInfoStatusTypes.SKIPPED, {
+                        numScanned: urlsCrawled.scanned.length,
+                        urlScanned: request.url,
                     });
-                    return;
-                }
-                // This logic is different from crawlDomain, as it also checks if the pae is redirected before checking if it is excluded using exclusions.txt
-                if (isRedirected && blacklistedPatterns && isSkippedUrl(actualUrl, blacklistedPatterns)) {
                     urlsCrawled.userExcluded.push({
                         url: request.url,
                         pageTitle: request.url,
-                        actualUrl,
-                        metadata: STATUS_CODE_METADATA[0],
-                        httpStatusCode: 0,
+                        actualUrl: request.url, // because about:blank is not useful
+                        metadata: STATUS_CODE_METADATA[1],
+                        httpStatusCode: 1,
                     });
+                    return;
+                }
+                const contentType = response?.headers?.()['content-type'] || '';
+                const status = response ? response.status() : 0;
+                if (isScanHtml && status < 300 && isWhitelistedContentType(contentType)) {
+                    const isRedirected = !areLinksEqual(page.url(), request.url);
+                    const isLoadedUrlInCrawledUrls = urlsCrawled.scanned.some(item => normUrl(item.actualUrl || item.url) === normUrl(page.url()));
+                    if (isRedirected && isLoadedUrlInCrawledUrls) {
+                        urlsCrawled.notScannedRedirects.push({
+                            fromUrl: request.url,
+                            toUrl: actualUrl, // i.e. actualUrl
+                        });
+                        return;
+                    }
+                    // This logic is different from crawlDomain, as it also checks if the pae is redirected before checking if it is excluded using exclusions.txt
+                    if (isRedirected && blacklistedPatterns && isSkippedUrl(actualUrl, blacklistedPatterns)) {
+                        urlsCrawled.userExcluded.push({
+                            url: request.url,
+                            pageTitle: request.url,
+                            actualUrl,
+                            metadata: STATUS_CODE_METADATA[0],
+                            httpStatusCode: 0,
+                        });
+                        guiInfoLog(guiInfoStatusTypes.SKIPPED, {
+                            numScanned: urlsCrawled.scanned.length,
+                            urlScanned: request.url,
+                        });
+                        return;
+                    }
+                    if (isRedirected && !isFollowStrategy(actualUrl, request.url, 'same-hostname')) {
+                        urlsCrawled.notScannedRedirects.push({
+                            fromUrl: request.url,
+                            toUrl: actualUrl,
+                        });
+                        guiInfoLog(guiInfoStatusTypes.SKIPPED, {
+                            numScanned: urlsCrawled.scanned.length,
+                            urlScanned: request.url,
+                        });
+                        return;
+                    }
+                    const results = await runAxeScript({ includeScreenshots, page, randomToken });
+                    // Detect JS redirects that fire during/after axe scan.
+                    // Listen for navigation, then give a brief window for pending redirects to complete.
+                    try {
+                        let navigatedToUrl = null;
+                        const onFrameNavigated = (frame) => {
+                            if (frame === page.mainFrame()) {
+                                navigatedToUrl = frame.url();
+                            }
+                        };
+                        page.on('framenavigated', onFrameNavigated);
+                        await page.waitForTimeout(1000);
+                        page.off('framenavigated', onFrameNavigated);
+                        const postScanUrl = navigatedToUrl || page.url();
+                        if (postScanUrl && postScanUrl !== 'about:blank' && !isFollowStrategy(postScanUrl, request.url, 'same-hostname')) {
+                            urlsCrawled.notScannedRedirects.push({
+                                fromUrl: request.url,
+                                toUrl: postScanUrl,
+                            });
+                            guiInfoLog(guiInfoStatusTypes.SKIPPED, {
+                                numScanned: urlsCrawled.scanned.length,
+                                urlScanned: request.url,
+                            });
+                            return;
+                        }
+                    }
+                    catch (_) {
+                        // Page/context was destroyed during navigation — handled by outer catch
+                    }
+                    guiInfoLog(guiInfoStatusTypes.SCANNED, {
+                        numScanned: urlsCrawled.scanned.length,
+                        urlScanned: request.url,
+                    });
+                    urlsCrawled.scanned.push({
+                        url: request.url,
+                        pageTitle: results.pageTitle,
+                        actualUrl, // i.e. actualUrl
+                    });
+                    urlsCrawled.scannedRedirects.push({
+                        fromUrl: request.url,
+                        toUrl: actualUrl,
+                    });
+                    results.url = request.url;
+                    results.actualUrl = actualUrl;
+                    await dataset.pushData(results);
+                }
+                else {
                     guiInfoLog(guiInfoStatusTypes.SKIPPED, {
                         numScanned: urlsCrawled.scanned.length,
                         urlScanned: request.url,
                     });
-                    return;
+                    if (isScanHtml) {
+                        // carry through the HTTP status metadata
+                        const status = response?.status();
+                        const metadata = typeof status === 'number'
+                            ? STATUS_CODE_METADATA[status] || STATUS_CODE_METADATA[599]
+                            : STATUS_CODE_METADATA[2];
+                        urlsCrawled.invalid.push({
+                            actualUrl,
+                            url: request.url,
+                            pageTitle: request.url,
+                            metadata,
+                            httpStatusCode: typeof status === 'number' ? status : 0,
+                        });
+                    }
                 }
-                const results = await runAxeScript({ includeScreenshots, page, randomToken });
-                guiInfoLog(guiInfoStatusTypes.SCANNED, {
-                    numScanned: urlsCrawled.scanned.length,
-                    urlScanned: request.url,
-                });
-                urlsCrawled.scanned.push({
-                    url: request.url,
-                    pageTitle: results.pageTitle,
-                    actualUrl, // i.e. actualUrl
-                });
-                urlsCrawled.scannedRedirects.push({
-                    fromUrl: request.url,
-                    toUrl: actualUrl,
-                });
-                results.url = request.url;
-                results.actualUrl = actualUrl;
-                await dataset.pushData(results);
             }
-            else {
-                guiInfoLog(guiInfoStatusTypes.SKIPPED, {
-                    numScanned: urlsCrawled.scanned.length,
-                    urlScanned: request.url,
-                });
-                if (isScanHtml) {
-                    // carry through the HTTP status metadata
-                    const status = response?.status();
-                    const metadata = typeof status === 'number'
-                        ? STATUS_CODE_METADATA[status] || STATUS_CODE_METADATA[599]
-                        : STATUS_CODE_METADATA[2];
-                    urlsCrawled.invalid.push({
-                        actualUrl,
+            catch (e) {
+                if (!isAbortingScan) {
+                    guiInfoLog(guiInfoStatusTypes.ERROR, {
+                        numScanned: urlsCrawled.scanned.length,
+                        urlScanned: request.url,
+                    });
+                    urlsCrawled.error.push({
                         url: request.url,
                         pageTitle: request.url,
-                        metadata,
-                        httpStatusCode: typeof status === 'number' ? status : 0,
+                        actualUrl: request.url,
+                        metadata: STATUS_CODE_METADATA[2],
+                        httpStatusCode: 0,
                     });
                 }
             }
         },
         failedRequestHandler: async ({ request, response, error }) => {
-            // check if scanned pages have reached limit due to multi-instances of handler running
-            if (urlsCrawled.scanned.length >= maxRequestsPerCrawl) {
+            if (isAbortingScan) {
                 return;
             }
             guiInfoLog(guiInfoStatusTypes.ERROR, {