@govtechsg/oobee 0.10.84 → 0.10.86

package/dist/crawlers/commonCrawlerFunc.js

@@ -116,7 +116,7 @@ export const filterAxeResults = (results, pageTitle, customFlowDetails) => {
             return;
         const conformance = tags.filter(tag => tag.startsWith('wcag') || tag === 'best-practice');
         nodes.forEach(node => {
-            const { html } = node;
+            const { html, target } = node;
             if (!(rule in passed.rules)) {
                 passed.rules[rule] = {
                     description,
@@ -128,7 +128,8 @@ export const filterAxeResults = (results, pageTitle, customFlowDetails) => {
                 };
             }
             const finalHtml = truncateHtml(html);
-            passed.rules[rule].items.push({ html: finalHtml, screenshotPath: '', message: '', xpath: '' });
+            const xpath = target.length === 1 && typeof target[0] === 'string' ? target[0] : undefined;
+            passed.rules[rule].items.push({ html: finalHtml, screenshotPath: '', message: '', xpath: xpath || '' });
             passed.totalItems += 1;
             passed.rules[rule].totalItems += 1;
             totalItems += 1;

package/dist/crawlers/crawlSitemap.js

@@ -12,6 +12,7 @@ const crawlSitemap = async ({ sitemapUrl, randomToken, host, viewportSettings, m
     let dataset;
     let urlsCrawled;
     let durationExceeded = false;
+    let isAbortingScan = false;
     if (fromCrawlIntelligentSitemap) {
         dataset = datasetFromIntelligent;
         urlsCrawled = urlsCrawledFromIntelligent;
@@ -144,106 +145,123 @@ const crawlSitemap = async ({ sitemapUrl, randomToken, host, viewportSettings, m
                 });
                 return;
             }
-            await waitForPageLoaded(page, 10000);
-            const actualUrl = page.url() || request.loadedUrl || request.url;
-            const hasExceededDuration = scanDuration > 0 && Date.now() - crawlStartTime > scanDuration * 1000;
-            if (urlsCrawled.scanned.length >= maxRequestsPerCrawl || hasExceededDuration) {
-                if (hasExceededDuration) {
-                    console.log(`Crawl duration of ${scanDuration}s exceeded. Aborting sitemap crawl.`);
-                    durationExceeded = true;
-                }
-                crawler.autoscaledPool.abort(); // stops new requests
-                return;
-            }
-            if (request.skipNavigation && actualUrl === 'about:blank') {
-                if (isScanPdfs) {
-                    // pushes download promise into pdfDownloads
-                    const { pdfFileName, url } = handlePdfDownload(randomToken, pdfDownloads, request, sendRequest, urlsCrawled);
-                    uuidToPdfMapping[pdfFileName] = url;
+            try {
+                await waitForPageLoaded(page, 10000);
+                const actualUrl = page.url() || request.loadedUrl || request.url;
+                const hasExceededDuration = scanDuration > 0 && Date.now() - crawlStartTime > scanDuration * 1000;
+                if (urlsCrawled.scanned.length >= maxRequestsPerCrawl || hasExceededDuration) {
+                    isAbortingScan = true;
+                    if (hasExceededDuration) {
+                        console.log(`Crawl duration of ${scanDuration}s exceeded. Aborting sitemap crawl.`);
+                        durationExceeded = true;
+                    }
+                    crawler.autoscaledPool.abort(); // stops new requests
                     return;
                 }
-                guiInfoLog(guiInfoStatusTypes.SKIPPED, {
-                    numScanned: urlsCrawled.scanned.length,
-                    urlScanned: request.url,
-                });
-                urlsCrawled.userExcluded.push({
-                    url: request.url,
-                    pageTitle: request.url,
-                    actualUrl: request.url, // because about:blank is not useful
-                    metadata: STATUS_CODE_METADATA[1],
-                    httpStatusCode: 1,
-                });
-                return;
-            }
-            const contentType = response?.headers?.()['content-type'] || '';
-            const status = response ? response.status() : 0;
-            if (isScanHtml && status < 300 && isWhitelistedContentType(contentType)) {
-                const isRedirected = !areLinksEqual(page.url(), request.url);
-                const isLoadedUrlInCrawledUrls = urlsCrawled.scanned.some(item => (item.actualUrl || item.url) === page.url());
-                if (isRedirected && isLoadedUrlInCrawledUrls) {
-                    urlsCrawled.notScannedRedirects.push({
-                        fromUrl: request.url,
-                        toUrl: actualUrl, // i.e. actualUrl
+                if (request.skipNavigation && actualUrl === 'about:blank') {
+                    if (isScanPdfs) {
+                        // pushes download promise into pdfDownloads
+                        const { pdfFileName, url } = handlePdfDownload(randomToken, pdfDownloads, request, sendRequest, urlsCrawled);
+                        uuidToPdfMapping[pdfFileName] = url;
+                        return;
+                    }
+                    guiInfoLog(guiInfoStatusTypes.SKIPPED, {
+                        numScanned: urlsCrawled.scanned.length,
+                        urlScanned: request.url,
                     });
-                    return;
-                }
-                // This logic is different from crawlDomain, as it also checks if the pae is redirected before checking if it is excluded using exclusions.txt
-                if (isRedirected && blacklistedPatterns && isSkippedUrl(actualUrl, blacklistedPatterns)) {
                     urlsCrawled.userExcluded.push({
                         url: request.url,
                         pageTitle: request.url,
-                        actualUrl,
-                        metadata: STATUS_CODE_METADATA[0],
-                        httpStatusCode: 0,
+                        actualUrl: request.url, // because about:blank is not useful
+                        metadata: STATUS_CODE_METADATA[1],
+                        httpStatusCode: 1,
                     });
+                    return;
+                }
+                const contentType = response?.headers?.()['content-type'] || '';
+                const status = response ? response.status() : 0;
+                if (isScanHtml && status < 300 && isWhitelistedContentType(contentType)) {
+                    const isRedirected = !areLinksEqual(page.url(), request.url);
+                    const isLoadedUrlInCrawledUrls = urlsCrawled.scanned.some(item => (item.actualUrl || item.url) === page.url());
+                    if (isRedirected && isLoadedUrlInCrawledUrls) {
+                        urlsCrawled.notScannedRedirects.push({
+                            fromUrl: request.url,
+                            toUrl: actualUrl, // i.e. actualUrl
+                        });
+                        return;
+                    }
+                    // This logic is different from crawlDomain, as it also checks if the pae is redirected before checking if it is excluded using exclusions.txt
+                    if (isRedirected && blacklistedPatterns && isSkippedUrl(actualUrl, blacklistedPatterns)) {
+                        urlsCrawled.userExcluded.push({
+                            url: request.url,
+                            pageTitle: request.url,
+                            actualUrl,
+                            metadata: STATUS_CODE_METADATA[0],
+                            httpStatusCode: 0,
+                        });
+                        guiInfoLog(guiInfoStatusTypes.SKIPPED, {
+                            numScanned: urlsCrawled.scanned.length,
+                            urlScanned: request.url,
+                        });
+                        return;
+                    }
+                    const results = await runAxeScript({ includeScreenshots, page, randomToken });
+                    guiInfoLog(guiInfoStatusTypes.SCANNED, {
+                        numScanned: urlsCrawled.scanned.length,
+                        urlScanned: request.url,
+                    });
+                    urlsCrawled.scanned.push({
+                        url: request.url,
+                        pageTitle: results.pageTitle,
+                        actualUrl, // i.e. actualUrl
+                    });
+                    urlsCrawled.scannedRedirects.push({
+                        fromUrl: request.url,
+                        toUrl: actualUrl,
+                    });
+                    results.url = request.url;
+                    results.actualUrl = actualUrl;
+                    await dataset.pushData(results);
+                }
+                else {
                     guiInfoLog(guiInfoStatusTypes.SKIPPED, {
                         numScanned: urlsCrawled.scanned.length,
                         urlScanned: request.url,
                     });
-                    return;
+                    if (isScanHtml) {
+                        // carry through the HTTP status metadata
+                        const status = response?.status();
+                        const metadata = typeof status === 'number'
+                            ? STATUS_CODE_METADATA[status] || STATUS_CODE_METADATA[599]
+                            : STATUS_CODE_METADATA[2];
+                        urlsCrawled.invalid.push({
+                            actualUrl,
+                            url: request.url,
+                            pageTitle: request.url,
+                            metadata,
+                            httpStatusCode: typeof status === 'number' ? status : 0,
+                        });
+                    }
                 }
-                const results = await runAxeScript({ includeScreenshots, page, randomToken });
-                guiInfoLog(guiInfoStatusTypes.SCANNED, {
-                    numScanned: urlsCrawled.scanned.length,
-                    urlScanned: request.url,
-                });
-                urlsCrawled.scanned.push({
-                    url: request.url,
-                    pageTitle: results.pageTitle,
-                    actualUrl, // i.e. actualUrl
-                });
-                urlsCrawled.scannedRedirects.push({
-                    fromUrl: request.url,
-                    toUrl: actualUrl,
-                });
-                results.url = request.url;
-                results.actualUrl = actualUrl;
-                await dataset.pushData(results);
             }
-            else {
-                guiInfoLog(guiInfoStatusTypes.SKIPPED, {
-                    numScanned: urlsCrawled.scanned.length,
-                    urlScanned: request.url,
-                });
-                if (isScanHtml) {
-                    // carry through the HTTP status metadata
-                    const status = response?.status();
-                    const metadata = typeof status === 'number'
-                        ? STATUS_CODE_METADATA[status] || STATUS_CODE_METADATA[599]
-                        : STATUS_CODE_METADATA[2];
-                    urlsCrawled.invalid.push({
-                        actualUrl,
+            catch (e) {
+                if (!isAbortingScan) {
+                    guiInfoLog(guiInfoStatusTypes.ERROR, {
+                        numScanned: urlsCrawled.scanned.length,
+                        urlScanned: request.url,
+                    });
+                    urlsCrawled.error.push({
                         url: request.url,
                         pageTitle: request.url,
-                        metadata,
-                        httpStatusCode: typeof status === 'number' ? status : 0,
+                        actualUrl: request.url,
+                        metadata: STATUS_CODE_METADATA[2],
+                        httpStatusCode: 0,
                     });
                 }
             }
         },
         failedRequestHandler: async ({ request, response, error }) => {
-            // check if scanned pages have reached limit due to multi-instances of handler running
-            if (urlsCrawled.scanned.length >= maxRequestsPerCrawl) {
+            if (isAbortingScan) {
                 return;
             }
             guiInfoLog(guiInfoStatusTypes.ERROR, {