@govtechsg/oobee 0.10.84 → 0.10.86

package/src/crawlers/commonCrawlerFunc.ts

@@ -196,7 +196,7 @@ export const filterAxeResults = (
     const conformance = tags.filter(tag => tag.startsWith('wcag') || tag === 'best-practice');
 
     nodes.forEach(node => {
-      const { html } = node;
+      const { html, target } = node;
       if (!(rule in passed.rules)) {
         passed.rules[rule] = {
           description,
@@ -207,9 +207,10 @@ export const filterAxeResults = (
           items: [],
         };
       }
-      
+
       const finalHtml = truncateHtml(html);
-      passed.rules[rule].items.push({ html: finalHtml, screenshotPath: '', message: '', xpath: '' });
+      const xpath = target.length === 1 && typeof target[0] === 'string' ? target[0] : undefined;
+      passed.rules[rule].items.push({ html: finalHtml, screenshotPath: '', message: '', xpath: xpath || '' });
 
       passed.totalItems += 1;
       passed.rules[rule].totalItems += 1;

package/src/crawlers/crawlSitemap.ts

@@ -76,6 +76,7 @@ const crawlSitemap = async ({
   let dataset: crawlee.Dataset;
   let urlsCrawled: UrlsCrawled;
   let durationExceeded = false;
+  let isAbortingScan = false;
 
   if (fromCrawlIntelligentSitemap) {
     dataset = datasetFromIntelligent;
@@ -244,135 +245,152 @@ const crawlSitemap = async ({
           return;
         }
 
-        await waitForPageLoaded(page, 10000);
+        try {
+          await waitForPageLoaded(page, 10000);
 
-        const actualUrl = page.url() || request.loadedUrl || request.url;
+          const actualUrl = page.url() || request.loadedUrl || request.url;
 
-        const hasExceededDuration =
-          scanDuration > 0 && Date.now() - crawlStartTime > scanDuration * 1000;
+          const hasExceededDuration =
+            scanDuration > 0 && Date.now() - crawlStartTime > scanDuration * 1000;
 
-        if (urlsCrawled.scanned.length >= maxRequestsPerCrawl || hasExceededDuration) {
-          if (hasExceededDuration) {
-            console.log(`Crawl duration of ${scanDuration}s exceeded. Aborting sitemap crawl.`);
-            durationExceeded = true;
+          if (urlsCrawled.scanned.length >= maxRequestsPerCrawl || hasExceededDuration) {
+            isAbortingScan = true;
+            if (hasExceededDuration) {
+              console.log(`Crawl duration of ${scanDuration}s exceeded. Aborting sitemap crawl.`);
+              durationExceeded = true;
+            }
+            crawler.autoscaledPool.abort(); // stops new requests
+            return;
           }
-          crawler.autoscaledPool.abort(); // stops new requests
-          return;
-        }
 
-        if (request.skipNavigation && actualUrl === 'about:blank') {
-          if (isScanPdfs) {
-            // pushes download promise into pdfDownloads
-            const { pdfFileName, url } = handlePdfDownload(
-              randomToken,
-              pdfDownloads,
-              request,
-              sendRequest,
-              urlsCrawled,
-            );
+          if (request.skipNavigation && actualUrl === 'about:blank') {
+            if (isScanPdfs) {
+              // pushes download promise into pdfDownloads
+              const { pdfFileName, url } = handlePdfDownload(
+                randomToken,
+                pdfDownloads,
+                request,
+                sendRequest,
+                urlsCrawled,
+              );
+
+              uuidToPdfMapping[pdfFileName] = url;
+              return;
+            }
+
+            guiInfoLog(guiInfoStatusTypes.SKIPPED, {
+              numScanned: urlsCrawled.scanned.length,
+              urlScanned: request.url,
+            });
+            urlsCrawled.userExcluded.push({
+              url: request.url,
+              pageTitle: request.url,
+              actualUrl: request.url, // because about:blank is not useful
+              metadata: STATUS_CODE_METADATA[1],
+              httpStatusCode: 1,
+            });
 
-            uuidToPdfMapping[pdfFileName] = url;
             return;
           }
 
-          guiInfoLog(guiInfoStatusTypes.SKIPPED, {
-            numScanned: urlsCrawled.scanned.length,
-            urlScanned: request.url,
-          });
-          urlsCrawled.userExcluded.push({
-            url: request.url,
-            pageTitle: request.url,
-            actualUrl: request.url, // because about:blank is not useful
-            metadata: STATUS_CODE_METADATA[1],
-            httpStatusCode: 1,
-          });
+          const contentType = response?.headers?.()['content-type'] || '';
+          const status = response ? response.status() : 0;
 
-          return;
-        }
+          if (isScanHtml && status < 300 && isWhitelistedContentType(contentType)) {
+            const isRedirected = !areLinksEqual(page.url(), request.url);
+            const isLoadedUrlInCrawledUrls = urlsCrawled.scanned.some(
+              item => (item.actualUrl || item.url) === page.url(),
+            );
 
-        const contentType = response?.headers?.()['content-type'] || '';
-        const status = response ? response.status() : 0;
+            if (isRedirected && isLoadedUrlInCrawledUrls) {
+              urlsCrawled.notScannedRedirects.push({
+                fromUrl: request.url,
+                toUrl: actualUrl, // i.e. actualUrl
+              });
+              return;
+            }
 
-        if (isScanHtml && status < 300 && isWhitelistedContentType(contentType)) {
-          const isRedirected = !areLinksEqual(page.url(), request.url);
-          const isLoadedUrlInCrawledUrls = urlsCrawled.scanned.some(
-            item => (item.actualUrl || item.url) === page.url(),
-          );
+            // This logic is different from crawlDomain, as it also checks if the pae is redirected before checking if it is excluded using exclusions.txt
+            if (isRedirected && blacklistedPatterns && isSkippedUrl(actualUrl, blacklistedPatterns)) {
+              urlsCrawled.userExcluded.push({
+                url: request.url,
+                pageTitle: request.url,
+                actualUrl,
+                metadata: STATUS_CODE_METADATA[0],
+                httpStatusCode: 0,
+              });
 
-          if (isRedirected && isLoadedUrlInCrawledUrls) {
-            urlsCrawled.notScannedRedirects.push({
-              fromUrl: request.url,
-              toUrl: actualUrl, // i.e. actualUrl
-            });
-            return;
-          }
+              guiInfoLog(guiInfoStatusTypes.SKIPPED, {
+                numScanned: urlsCrawled.scanned.length,
+                urlScanned: request.url,
+              });
+              return;
+            }
 
-          // This logic is different from crawlDomain, as it also checks if the pae is redirected before checking if it is excluded using exclusions.txt
-          if (isRedirected && blacklistedPatterns && isSkippedUrl(actualUrl, blacklistedPatterns)) {
-            urlsCrawled.userExcluded.push({
-              url: request.url,
-              pageTitle: request.url,
-              actualUrl,
-              metadata: STATUS_CODE_METADATA[0],
-              httpStatusCode: 0,
-            });
+            const results = await runAxeScript({ includeScreenshots, page, randomToken });
 
-            guiInfoLog(guiInfoStatusTypes.SKIPPED, {
+            guiInfoLog(guiInfoStatusTypes.SCANNED, {
               numScanned: urlsCrawled.scanned.length,
               urlScanned: request.url,
             });
-            return;
-          }
 
-          const results = await runAxeScript({ includeScreenshots, page, randomToken });
-
-          guiInfoLog(guiInfoStatusTypes.SCANNED, {
-            numScanned: urlsCrawled.scanned.length,
-            urlScanned: request.url,
-          });
-
-          urlsCrawled.scanned.push({
-            url: request.url,
-            pageTitle: results.pageTitle,
-            actualUrl, // i.e. actualUrl
-          });
+            urlsCrawled.scanned.push({
+              url: request.url,
+              pageTitle: results.pageTitle,
+              actualUrl, // i.e. actualUrl
+            });
 
-          urlsCrawled.scannedRedirects.push({
-            fromUrl: request.url,
-            toUrl: actualUrl,
-          });
+            urlsCrawled.scannedRedirects.push({
+              fromUrl: request.url,
+              toUrl: actualUrl,
+            });
 
-          results.url = request.url;
-          results.actualUrl = actualUrl;
+            results.url = request.url;
+            results.actualUrl = actualUrl;
 
-          await dataset.pushData(results);
-        } else {
-          guiInfoLog(guiInfoStatusTypes.SKIPPED, {
-            numScanned: urlsCrawled.scanned.length,
-            urlScanned: request.url,
-          });
+            await dataset.pushData(results);
+          } else {
+            guiInfoLog(guiInfoStatusTypes.SKIPPED, {
+              numScanned: urlsCrawled.scanned.length,
+              urlScanned: request.url,
+            });
 
-          if (isScanHtml) {
-            // carry through the HTTP status metadata
-            const status = response?.status();
-            const metadata =
-              typeof status === 'number'
-                ? STATUS_CODE_METADATA[status] || STATUS_CODE_METADATA[599]
-                : STATUS_CODE_METADATA[2];
+            if (isScanHtml) {
+              // carry through the HTTP status metadata
+              const status = response?.status();
+              const metadata =
+                typeof status === 'number'
+                  ? STATUS_CODE_METADATA[status] || STATUS_CODE_METADATA[599]
+                  : STATUS_CODE_METADATA[2];
+
+              urlsCrawled.invalid.push({
+                actualUrl,
+                url: request.url,
+                pageTitle: request.url,
+                metadata,
+                httpStatusCode: typeof status === 'number' ? status : 0,
+              });
+            }
+          }
+        } catch (e) {
+          if (!isAbortingScan) {
+            guiInfoLog(guiInfoStatusTypes.ERROR, {
+              numScanned: urlsCrawled.scanned.length,
+              urlScanned: request.url,
+            });
 
-            urlsCrawled.invalid.push({
-              actualUrl,
+            urlsCrawled.error.push({
               url: request.url,
               pageTitle: request.url,
-              metadata,
-              httpStatusCode: typeof status === 'number' ? status : 0,
+              actualUrl: request.url,
+              metadata: STATUS_CODE_METADATA[2],
+              httpStatusCode: 0,
             });
           }
         }
       },
       failedRequestHandler: async ({ request, response, error }) => {
-        // check if scanned pages have reached limit due to multi-instances of handler running
-        if (urlsCrawled.scanned.length >= maxRequestsPerCrawl) {
+        if (isAbortingScan) {
           return;
         }