@govtechsg/oobee 0.10.85 → 0.10.87

package/src/crawlers/commonCrawlerFunc.ts

@@ -196,7 +196,7 @@ export const filterAxeResults = (
     const conformance = tags.filter(tag => tag.startsWith('wcag') || tag === 'best-practice');
 
     nodes.forEach(node => {
-      const { html } = node;
+      const { html, target } = node;
       if (!(rule in passed.rules)) {
         passed.rules[rule] = {
           description,
@@ -207,9 +207,10 @@ export const filterAxeResults = (
           items: [],
         };
       }
-      
+
       const finalHtml = truncateHtml(html);
-      passed.rules[rule].items.push({ html: finalHtml, screenshotPath: '', message: '', xpath: '' });
+      const xpath = target.length === 1 && typeof target[0] === 'string' ? target[0] : undefined;
+      passed.rules[rule].items.push({ html: finalHtml, screenshotPath: '', message: '', xpath: xpath || '' });
 
       passed.totalItems += 1;
       passed.rules[rule].totalItems += 1;

package/src/crawlers/crawlDomain.ts

@@ -29,7 +29,7 @@ import {
   getUrlsFromRobotsTxt,
   waitForPageLoaded,
 } from '../constants/common.js';
-import { areLinksEqual, isFollowStrategy, register } from '../utils.js';
+import { areLinksEqual, isFollowStrategy, normUrl, register } from '../utils.js';
 import {
   handlePdfDownload,
   runPdfScan,
@@ -116,9 +116,9 @@ const crawlDomain = async ({
   const pdfDownloads: Promise<void>[] = [];
   const uuidToPdfMapping: Record<string, string> = {};
   const queuedUrlSet = new Set<string>();
-  const scannedUrlSet = new Set<string>(urlsCrawled.scanned.map(item => item.url));
+  const scannedUrlSet = new Set<string>(urlsCrawled.scanned.map(item => normUrl(item.url)));
   const scannedResolvedUrlSet = new Set<string>(
-    urlsCrawled.scanned.map(item => item.actualUrl || item.url),
+    urlsCrawled.scanned.map(item => normUrl(item.actualUrl || item.url)),
   );
   const isScanHtml = [FileTypes.All, FileTypes.HtmlOnly].includes(fileTypes as FileTypes);
   const isScanPdfs = [FileTypes.All, FileTypes.PdfOnly].includes(fileTypes as FileTypes);
@@ -166,13 +166,14 @@ const crawlDomain = async ({
     const selectedElementsString = cssQuerySelectors.join(', ');
 
     const isExcluded = (newPageUrl: string): boolean => {
-      const isAlreadyScanned: boolean = urlsCrawled.scanned.some(item => item.url === newPageUrl);
+      const isAlreadyScanned: boolean = scannedUrlSet.has(normUrl(newPageUrl));
       const isBlacklistedUrl: boolean = isBlacklisted(newPageUrl, blacklistedPatterns);
       const isNotFollowStrategy: boolean = !isFollowStrategy(newPageUrl, initialPageUrl, strategy);
       const isNotSupportedDocument: boolean = disallowedListOfPatterns.some(pattern =>
         newPageUrl.toLowerCase().startsWith(pattern),
       );
-      return isNotSupportedDocument || isAlreadyScanned || isBlacklistedUrl || isNotFollowStrategy;
+      const isRobotsDisallowed: boolean = isDisallowedInRobotsTxt(newPageUrl);
+      return isNotSupportedDocument || isAlreadyScanned || isBlacklistedUrl || isNotFollowStrategy || isRobotsDisallowed;
     };
     const setPageListeners = (pageListener: Page): void => {
       // event listener to handle new page popups upon button click
@@ -341,7 +342,7 @@ const crawlDomain = async ({
           } catch (e) {
             consoleLogger.error(e);
           }
-          if (scannedUrlSet.has(req.url)) {
+          if (scannedUrlSet.has(normUrl(req.url))) {
             req.skipNavigation = true;
           }
           if (isDisallowedInRobotsTxt(req.url)) return null;
@@ -481,7 +482,7 @@ const crawlDomain = async ({
           }
 
           const isRedirected = !areLinksEqual(finalUrl, requestLabelUrl);
-          if (isRedirected) {
+          if (isRedirected && !isDisallowedInRobotsTxt(finalUrl)) {
             await enqueueUniqueRequest({ url: finalUrl, label: finalUrl });
           } else {
             request.skipNavigation = false;
@@ -537,7 +538,7 @@ const crawlDomain = async ({
           }
 
           // if URL has already been scanned
-          if (scannedUrlSet.has(request.url)) {
+          if (scannedUrlSet.has(normUrl(request.url))) {
             await enqueueProcess(page, enqueueLinks, browserContext);
             return;
           }
@@ -654,8 +655,33 @@ const crawlDomain = async ({
 
             const results = await runAxeScript({ includeScreenshots, page, randomToken, ruleset });
 
+            // Detect JS redirects that fire during/after axe scan.
+            // Listen for navigation, then give a brief window for pending redirects to complete.
+            try {
+              let navigatedToUrl: string | null = null;
+              const onFrameNavigated = (frame: Frame) => {
+                if (frame === page.mainFrame()) {
+                  navigatedToUrl = frame.url();
+                }
+              };
+              page.on('framenavigated', onFrameNavigated);
+              await page.waitForTimeout(1000);
+              page.off('framenavigated', onFrameNavigated);
+
+              const postScanUrl = navigatedToUrl || page.url();
+              if (postScanUrl && postScanUrl !== 'about:blank' && !isFollowStrategy(postScanUrl, request.url, 'same-hostname')) {
+                urlsCrawled.notScannedRedirects.push({
+                  fromUrl: request.url,
+                  toUrl: postScanUrl,
+                });
+                return;
+              }
+            } catch (_) {
+              // Page/context was destroyed during navigation — handled by outer catch
+            }
+
             if (isRedirected) {
-              const isLoadedUrlInCrawledUrls = scannedResolvedUrlSet.has(actualUrl);
+              const isLoadedUrlInCrawledUrls = scannedResolvedUrlSet.has(normUrl(actualUrl));
 
               if (isLoadedUrlInCrawledUrls) {
                 urlsCrawled.notScannedRedirects.push({
@@ -677,8 +703,8 @@ const crawlDomain = async ({
                   pageTitle: results.pageTitle,
                   actualUrl, // i.e. actualUrl
                 });
-                scannedUrlSet.add(request.url);
-                scannedResolvedUrlSet.add(actualUrl);
+                scannedUrlSet.add(normUrl(request.url));
+                scannedResolvedUrlSet.add(normUrl(actualUrl));
 
                 urlsCrawled.scannedRedirects.push({
                   fromUrl: request.url,
@@ -700,8 +726,8 @@ const crawlDomain = async ({
                 actualUrl: request.url,
                 pageTitle: results.pageTitle,
               });
-              scannedUrlSet.add(request.url);
-              scannedResolvedUrlSet.add(request.url);
+              scannedUrlSet.add(normUrl(request.url));
+              scannedResolvedUrlSet.add(normUrl(request.url));
               await dataset.pushData(results);
             }
           } else {

package/src/crawlers/crawlIntelligentSitemap.ts

@@ -7,7 +7,7 @@ import { consoleLogger, guiInfoLog } from '../logs.js';
 import crawlDomain from './crawlDomain.js';
 import crawlSitemap from './crawlSitemap.js';
 import { ViewportSettingsClass } from '../combine.js';
-import { getPlaywrightLaunchOptions } from '../constants/common.js';
+import { getPlaywrightLaunchOptions, getSitemapsFromRobotsTxt } from '../constants/common.js';
 import { register } from '../utils.js';
 
 const crawlIntelligentSitemap = async (
@@ -100,12 +100,30 @@ const crawlIntelligentSitemap = async (
     }
   };
 
+  // Discover sitemaps from robots.txt first (supports multiple Sitemap: directives)
+  let sitemapUrls: string[] = [];
   try {
-    sitemapUrl = await findSitemap(url, userDataDirectory, extraHTTPHeaders);
+    sitemapUrls = await getSitemapsFromRobotsTxt(url, browser, userDataDirectory, extraHTTPHeaders);
+    if (sitemapUrls.length > 0) {
+      console.log(`Found ${sitemapUrls.length} sitemap(s) in robots.txt: ${sitemapUrls.join(', ')}`);
+      sitemapExist = true;
+    }
   } catch (error) {
     consoleLogger.error(error);
   }
 
+  // Fall back to hardcoded path probing if robots.txt had no sitemaps
+  if (!sitemapExist) {
+    try {
+      sitemapUrl = await findSitemap(url, userDataDirectory, extraHTTPHeaders);
+      if (sitemapExist) {
+        sitemapUrls = [sitemapUrl];
+      }
+    } catch (error) {
+      consoleLogger.error(error);
+    }
+  }
+
   if (!sitemapExist) {
     console.log('Unable to find sitemap. Commencing website crawl instead.');
     return await crawlDomain({
@@ -124,38 +142,53 @@ const crawlIntelligentSitemap = async (
       followRobots,
       extraHTTPHeaders,
       safeMode,
-      scanDuration, // Use full duration since no sitemap
+      scanDuration,
     });
   }
 
-  console.log(`Sitemap found at ${sitemapUrl}`);
-  urlsCrawledFinal = await crawlSitemap({
-    sitemapUrl,
-    randomToken,
-    host,
-    viewportSettings,
-    maxRequestsPerCrawl,
-    browser,
-    userDataDirectory,
-    specifiedMaxConcurrency,
-    fileTypes,
-    blacklistedPatterns,
-    includeScreenshots,
-    extraHTTPHeaders,
-    fromCrawlIntelligentSitemap,
-    userUrlInputFromIntelligent: url,
-    datasetFromIntelligent: dataset,
-    urlsCrawledFromIntelligent: urlsCrawled,
-    crawledFromLocalFile: false,
-    scanDuration,
-  });
+  // Process all discovered sitemaps sequentially, sharing dataset and urlsCrawled
+  for (const currentSitemapUrl of sitemapUrls) {
+    if (urlsCrawled.scanned.length >= maxRequestsPerCrawl) break;
+
+    const elapsed = Date.now() - startTime;
+    const remainingDuration = scanDuration > 0 ? Math.max(scanDuration - elapsed / 1000, 0) : scanDuration;
+    if (scanDuration > 0 && remainingDuration <= 0) {
+      durationExceeded = true;
+      break;
+    }
+
+    console.log(`Processing sitemap: ${currentSitemapUrl}`);
+    urlsCrawledFinal = await crawlSitemap({
+      sitemapUrl: currentSitemapUrl,
+      randomToken,
+      host,
+      viewportSettings,
+      maxRequestsPerCrawl,
+      browser,
+      userDataDirectory,
+      specifiedMaxConcurrency,
+      fileTypes,
+      blacklistedPatterns,
+      includeScreenshots,
+      extraHTTPHeaders,
+      strategy,
+      userUrl: url,
+      fromCrawlIntelligentSitemap,
+      userUrlInputFromIntelligent: url,
+      datasetFromIntelligent: dataset,
+      urlsCrawledFromIntelligent: urlsCrawled,
+      crawledFromLocalFile: false,
+      scanDuration: scanDuration > 0 ? remainingDuration : 0,
+    });
+  }
 
   const elapsed = Date.now() - startTime;
-  const remainingScanDuration = Math.max(scanDuration - elapsed / 1000, 0); // in seconds
+  const remainingScanDuration = scanDuration > 0 ? Math.max(scanDuration - elapsed / 1000, 0) : 0;
+  const hasDurationRemaining = scanDuration === 0 || remainingScanDuration > 0;
 
-  if (urlsCrawledFinal.scanned.length < maxRequestsPerCrawl && remainingScanDuration > 0) {
+  if (urlsCrawled.scanned.length < maxRequestsPerCrawl && hasDurationRemaining) {
     console.log(
-      `Continuing crawl from root website. Remaining scan time: ${remainingScanDuration.toFixed(1)}s`,
+      `Continuing crawl from root website.${scanDuration > 0 ? ` Remaining scan time: ${remainingScanDuration.toFixed(1)}s` : ''}`,
     );
     urlsCrawledFinal = await crawlDomain({
       url,
@@ -175,10 +208,10 @@ const crawlIntelligentSitemap = async (
       safeMode,
       fromCrawlIntelligentSitemap,
       datasetFromIntelligent: dataset,
-      urlsCrawledFromIntelligent: urlsCrawledFinal,
+      urlsCrawledFromIntelligent: urlsCrawled,
       scanDuration: remainingScanDuration,
     });
-  } else if (remainingScanDuration <= 0) {
+  } else if (!hasDurationRemaining) {
     console.log(
       `Crawl duration exceeded before more pages could be found (limit: ${scanDuration}s).`,
     );
@@ -186,7 +219,7 @@ const crawlIntelligentSitemap = async (
   }
 
   guiInfoLog(guiInfoStatusTypes.COMPLETED, {});
-  return { urlsCrawled: urlsCrawledFinal, durationExceeded };
+  return { urlsCrawled, durationExceeded };
 };
 
 export default crawlIntelligentSitemap;

package/src/crawlers/crawlSitemap.ts

@@ -1,4 +1,4 @@
-import crawlee, { LaunchContext, Request, RequestList, Dataset } from 'crawlee';
+import crawlee, { EnqueueStrategy, LaunchContext, Request, RequestList, Dataset } from 'crawlee';
 import fs from 'fs';
 import * as path from 'path';
 import fsp from 'fs/promises';
@@ -23,7 +23,7 @@ import {
   waitForPageLoaded,
   isFilePath,
 } from '../constants/common.js';
-import { areLinksEqual, isWhitelistedContentType, register } from '../utils.js';
+import { areLinksEqual, isFollowStrategy, isWhitelistedContentType, normUrl, register } from '../utils.js';
 import {
   handlePdfDownload,
   runPdfScan,
@@ -46,6 +46,8 @@ const crawlSitemap = async ({
   blacklistedPatterns,
   includeScreenshots,
   extraHTTPHeaders,
+  strategy = EnqueueStrategy.All,
+  userUrl = '',
   scanDuration = 0,
   fromCrawlIntelligentSitemap = false,
   userUrlInputFromIntelligent = null,
@@ -65,6 +67,8 @@ const crawlSitemap = async ({
   blacklistedPatterns: string[];
   includeScreenshots: boolean;
   extraHTTPHeaders: Record<string, string>;
+  strategy?: EnqueueStrategy;
+  userUrl?: string;
   scanDuration?: number;
   fromCrawlIntelligentSitemap?: boolean;
   userUrlInputFromIntelligent?: string;
@@ -76,6 +80,7 @@ const crawlSitemap = async ({
   let dataset: crawlee.Dataset;
   let urlsCrawled: UrlsCrawled;
   let durationExceeded = false;
+  let isAbortingScan = false;
 
   if (fromCrawlIntelligentSitemap) {
     dataset = datasetFromIntelligent;
@@ -98,6 +103,8 @@ const crawlSitemap = async ({
     userUrlInputFromIntelligent,
     fromCrawlIntelligentSitemap,
     extraHTTPHeaders,
+    strategy,
+    userUrl || sitemapUrl,
   );
 
   sitemapUrl = encodeURI(sitemapUrl);
@@ -244,135 +251,193 @@ const crawlSitemap = async ({
           return;
         }
 
-        await waitForPageLoaded(page, 10000);
+        try {
+          await waitForPageLoaded(page, 10000);
 
-        const actualUrl = page.url() || request.loadedUrl || request.url;
+          const actualUrl = page.url() || request.loadedUrl || request.url;
 
-        const hasExceededDuration =
-          scanDuration > 0 && Date.now() - crawlStartTime > scanDuration * 1000;
+          const hasExceededDuration =
+            scanDuration > 0 && Date.now() - crawlStartTime > scanDuration * 1000;
 
-        if (urlsCrawled.scanned.length >= maxRequestsPerCrawl || hasExceededDuration) {
-          if (hasExceededDuration) {
-            console.log(`Crawl duration of ${scanDuration}s exceeded. Aborting sitemap crawl.`);
-            durationExceeded = true;
+          if (urlsCrawled.scanned.length >= maxRequestsPerCrawl || hasExceededDuration) {
+            isAbortingScan = true;
+            if (hasExceededDuration) {
+              console.log(`Crawl duration of ${scanDuration}s exceeded. Aborting sitemap crawl.`);
+              durationExceeded = true;
+            }
+            crawler.autoscaledPool.abort(); // stops new requests
+            return;
           }
-          crawler.autoscaledPool.abort(); // stops new requests
-          return;
-        }
 
-        if (request.skipNavigation && actualUrl === 'about:blank') {
-          if (isScanPdfs) {
-            // pushes download promise into pdfDownloads
-            const { pdfFileName, url } = handlePdfDownload(
-              randomToken,
-              pdfDownloads,
-              request,
-              sendRequest,
-              urlsCrawled,
-            );
+          if (request.skipNavigation && actualUrl === 'about:blank') {
+            if (isScanPdfs) {
+              // pushes download promise into pdfDownloads
+              const { pdfFileName, url } = handlePdfDownload(
+                randomToken,
+                pdfDownloads,
+                request,
+                sendRequest,
+                urlsCrawled,
+              );
+
+              uuidToPdfMapping[pdfFileName] = url;
+              return;
+            }
+
+            guiInfoLog(guiInfoStatusTypes.SKIPPED, {
+              numScanned: urlsCrawled.scanned.length,
+              urlScanned: request.url,
+            });
+            urlsCrawled.userExcluded.push({
+              url: request.url,
+              pageTitle: request.url,
+              actualUrl: request.url, // because about:blank is not useful
+              metadata: STATUS_CODE_METADATA[1],
+              httpStatusCode: 1,
+            });
 
-            uuidToPdfMapping[pdfFileName] = url;
             return;
           }
 
-          guiInfoLog(guiInfoStatusTypes.SKIPPED, {
-            numScanned: urlsCrawled.scanned.length,
-            urlScanned: request.url,
-          });
-          urlsCrawled.userExcluded.push({
-            url: request.url,
-            pageTitle: request.url,
-            actualUrl: request.url, // because about:blank is not useful
-            metadata: STATUS_CODE_METADATA[1],
-            httpStatusCode: 1,
-          });
+          const contentType = response?.headers?.()['content-type'] || '';
+          const status = response ? response.status() : 0;
 
-          return;
-        }
+          if (isScanHtml && status < 300 && isWhitelistedContentType(contentType)) {
+            const isRedirected = !areLinksEqual(page.url(), request.url);
+            const isLoadedUrlInCrawledUrls = urlsCrawled.scanned.some(
+              item => normUrl(item.actualUrl || item.url) === normUrl(page.url()),
+            );
 
-        const contentType = response?.headers?.()['content-type'] || '';
-        const status = response ? response.status() : 0;
+            if (isRedirected && isLoadedUrlInCrawledUrls) {
+              urlsCrawled.notScannedRedirects.push({
+                fromUrl: request.url,
+                toUrl: actualUrl, // i.e. actualUrl
+              });
+              return;
+            }
 
-        if (isScanHtml && status < 300 && isWhitelistedContentType(contentType)) {
-          const isRedirected = !areLinksEqual(page.url(), request.url);
-          const isLoadedUrlInCrawledUrls = urlsCrawled.scanned.some(
-            item => (item.actualUrl || item.url) === page.url(),
-          );
+            // This logic is different from crawlDomain, as it also checks if the pae is redirected before checking if it is excluded using exclusions.txt
+            if (isRedirected && blacklistedPatterns && isSkippedUrl(actualUrl, blacklistedPatterns)) {
+              urlsCrawled.userExcluded.push({
+                url: request.url,
+                pageTitle: request.url,
+                actualUrl,
+                metadata: STATUS_CODE_METADATA[0],
+                httpStatusCode: 0,
+              });
 
-          if (isRedirected && isLoadedUrlInCrawledUrls) {
-            urlsCrawled.notScannedRedirects.push({
-              fromUrl: request.url,
-              toUrl: actualUrl, // i.e. actualUrl
-            });
-            return;
-          }
+              guiInfoLog(guiInfoStatusTypes.SKIPPED, {
+                numScanned: urlsCrawled.scanned.length,
+                urlScanned: request.url,
+              });
+              return;
+            }
 
-          // This logic is different from crawlDomain, as it also checks if the pae is redirected before checking if it is excluded using exclusions.txt
-          if (isRedirected && blacklistedPatterns && isSkippedUrl(actualUrl, blacklistedPatterns)) {
-            urlsCrawled.userExcluded.push({
-              url: request.url,
-              pageTitle: request.url,
-              actualUrl,
-              metadata: STATUS_CODE_METADATA[0],
-              httpStatusCode: 0,
-            });
+            if (isRedirected && !isFollowStrategy(actualUrl, request.url, 'same-hostname')) {
+              urlsCrawled.notScannedRedirects.push({
+                fromUrl: request.url,
+                toUrl: actualUrl,
+              });
+              guiInfoLog(guiInfoStatusTypes.SKIPPED, {
+                numScanned: urlsCrawled.scanned.length,
+                urlScanned: request.url,
+              });
+              return;
+            }
 
-            guiInfoLog(guiInfoStatusTypes.SKIPPED, {
+            const results = await runAxeScript({ includeScreenshots, page, randomToken });
+
+            // Detect JS redirects that fire during/after axe scan.
+            // Listen for navigation, then give a brief window for pending redirects to complete.
+            try {
+              let navigatedToUrl: string | null = null;
+              const onFrameNavigated = (frame: any) => {
+                if (frame === page.mainFrame()) {
+                  navigatedToUrl = frame.url();
+                }
+              };
+              page.on('framenavigated', onFrameNavigated);
+              await page.waitForTimeout(1000);
+              page.off('framenavigated', onFrameNavigated);
+
+              const postScanUrl = navigatedToUrl || page.url();
+              if (postScanUrl && postScanUrl !== 'about:blank' && !isFollowStrategy(postScanUrl, request.url, 'same-hostname')) {
+                urlsCrawled.notScannedRedirects.push({
+                  fromUrl: request.url,
+                  toUrl: postScanUrl,
+                });
+                guiInfoLog(guiInfoStatusTypes.SKIPPED, {
+                  numScanned: urlsCrawled.scanned.length,
+                  urlScanned: request.url,
+                });
+                return;
+              }
+            } catch (_) {
+              // Page/context was destroyed during navigation — handled by outer catch
+            }
+
+            guiInfoLog(guiInfoStatusTypes.SCANNED, {
               numScanned: urlsCrawled.scanned.length,
               urlScanned: request.url,
             });
-            return;
-          }
-
-          const results = await runAxeScript({ includeScreenshots, page, randomToken });
-
-          guiInfoLog(guiInfoStatusTypes.SCANNED, {
-            numScanned: urlsCrawled.scanned.length,
-            urlScanned: request.url,
-          });
 
-          urlsCrawled.scanned.push({
-            url: request.url,
-            pageTitle: results.pageTitle,
-            actualUrl, // i.e. actualUrl
-          });
+            urlsCrawled.scanned.push({
+              url: request.url,
+              pageTitle: results.pageTitle,
+              actualUrl, // i.e. actualUrl
+            });
 
-          urlsCrawled.scannedRedirects.push({
-            fromUrl: request.url,
-            toUrl: actualUrl,
-          });
+            urlsCrawled.scannedRedirects.push({
+              fromUrl: request.url,
+              toUrl: actualUrl,
+            });
 
-          results.url = request.url;
-          results.actualUrl = actualUrl;
+            results.url = request.url;
+            results.actualUrl = actualUrl;
 
-          await dataset.pushData(results);
-        } else {
-          guiInfoLog(guiInfoStatusTypes.SKIPPED, {
-            numScanned: urlsCrawled.scanned.length,
-            urlScanned: request.url,
-          });
+            await dataset.pushData(results);
+          } else {
+            guiInfoLog(guiInfoStatusTypes.SKIPPED, {
+              numScanned: urlsCrawled.scanned.length,
+              urlScanned: request.url,
+            });
 
-          if (isScanHtml) {
-            // carry through the HTTP status metadata
-            const status = response?.status();
-            const metadata =
-              typeof status === 'number'
-                ? STATUS_CODE_METADATA[status] || STATUS_CODE_METADATA[599]
-                : STATUS_CODE_METADATA[2];
+            if (isScanHtml) {
+              // carry through the HTTP status metadata
+              const status = response?.status();
+              const metadata =
+                typeof status === 'number'
+                  ? STATUS_CODE_METADATA[status] || STATUS_CODE_METADATA[599]
+                  : STATUS_CODE_METADATA[2];
+
+              urlsCrawled.invalid.push({
+                actualUrl,
+                url: request.url,
+                pageTitle: request.url,
+                metadata,
+                httpStatusCode: typeof status === 'number' ? status : 0,
+              });
+            }
+          }
+        } catch (e) {
+          if (!isAbortingScan) {
+            guiInfoLog(guiInfoStatusTypes.ERROR, {
+              numScanned: urlsCrawled.scanned.length,
+              urlScanned: request.url,
+            });
 
-            urlsCrawled.invalid.push({
-              actualUrl,
+            urlsCrawled.error.push({
               url: request.url,
               pageTitle: request.url,
-              metadata,
-              httpStatusCode: typeof status === 'number' ? status : 0,
+              actualUrl: request.url,
+              metadata: STATUS_CODE_METADATA[2],
+              httpStatusCode: 0,
             });
           }
         }
       },
       failedRequestHandler: async ({ request, response, error }) => {
-        // check if scanned pages have reached limit due to multi-instances of handler running
-        if (urlsCrawled.scanned.length >= maxRequestsPerCrawl) {
+        if (isAbortingScan) {
           return;
         }