@govtechsg/oobee 0.10.29 → 0.10.33

package/exclusions.txt

@@ -1,2 +1,3 @@
 \.*login.singpass.gov.sg\.*
-\.*auth.singpass.gov.sg\.*
+\.*auth.singpass.gov.sg\.*
+\.*form.gov.sg\.*

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@govtechsg/oobee",
   "main": "dist/npmIndex.js",
-  "version": "0.10.29",
+  "version": "0.10.33",
   "type": "module",
   "author": "Government Technology Agency <info@tech.gov.sg>",
   "dependencies": {

package/src/combine.ts

@@ -210,6 +210,7 @@ const combineRun = async (details: Data, deviceToScan: string) => {
         ...urlsCrawledObj.error,
         ...urlsCrawledObj.invalid,
         ...urlsCrawledObj.forbidden,
+        ...urlsCrawledObj.userExcluded,
       ];
       const basicFormHTMLSnippet = await generateArtifacts(
         randomToken,
@@ -240,6 +241,8 @@ const combineRun = async (details: Data, deviceToScan: string) => {
         pagesNotScanned.length,
         metadata,
       );
+    } else {
+      printMessage([`No pages were scanned.`], alertMessageOptions);
     }
   } else {
     printMessage([`No pages were scanned.`], alertMessageOptions);

package/src/constants/common.ts

@@ -1819,13 +1819,72 @@ export const urlWithoutAuth = (url: string): string => {
 };
 
 export const waitForPageLoaded = async (page, timeout = 10000) => {
+  const OBSERVER_TIMEOUT = timeout; // Ensure observer timeout does not exceed the main timeout
+
   return Promise.race([
-    page.waitForLoadState('load'),
-    page.waitForLoadState('networkidle'),
-    new Promise(resolve => setTimeout(resolve, timeout)),
+    page.waitForLoadState('load'), // Ensure page load completes
+    page.waitForLoadState('networkidle'), // Wait for network requests to settle
+    new Promise(resolve => setTimeout(resolve, timeout)), // Hard timeout as a fallback
+    page.evaluate((OBSERVER_TIMEOUT) => {
+      return new Promise((resolve) => {
+        // Skip mutation check for PDFs
+        if (document.contentType === 'application/pdf') {
+          resolve('Skipping DOM mutation check for PDF.');
+          return;
+        }
+
+        let timeout;
+        let mutationCount = 0;
+        const MAX_MUTATIONS = 250; // Limit max mutations
+        const mutationHash = {};
+
+        const observer = new MutationObserver(mutationsList => {
+          clearTimeout(timeout);
+
+          mutationCount++;
+          if (mutationCount > MAX_MUTATIONS) {
+            observer.disconnect();
+            resolve('Too many mutations detected, exiting.');
+            return;
+          }
+
+          mutationsList.forEach(mutation => {
+            if (mutation.target instanceof Element) {
+              Array.from(mutation.target.attributes).forEach(attr => {
+                const mutationKey = `${mutation.target.nodeName}-${attr.name}`;
+
+                if (mutationKey) {
+                  mutationHash[mutationKey] = (mutationHash[mutationKey] || 0) + 1;
+
+                  if (mutationHash[mutationKey] >= 10) {
+                    observer.disconnect();
+                    resolve(`Repeated mutation detected for ${mutationKey}, exiting.`);
+                  }
+                }
+              });
+            }
+          });
+
+          // If no mutations occur for 1 second, resolve
+          timeout = setTimeout(() => {
+            observer.disconnect();
+            resolve('DOM stabilized after mutations.');
+          }, 1000);
+        });
+
+        // Final timeout to avoid infinite waiting
+        timeout = setTimeout(() => {
+          observer.disconnect();
+          resolve('Observer timeout reached, exiting.');
+        }, OBSERVER_TIMEOUT);
+
+        observer.observe(document.documentElement, { childList: true, subtree: true, attributes: true });
+      });
+    }, OBSERVER_TIMEOUT), // Pass OBSERVER_TIMEOUT dynamically to the browser context
   ]);
 };
 
+
 function isValidHttpUrl(urlString) {
   const pattern = /^(http|https):\/\/[^ "]+$/;
   return pattern.test(urlString);

package/src/constants/constants.ts

@@ -186,7 +186,7 @@ export class UrlsCrawled {
   error: { url: string }[] = [];
   exceededRequests: string[] = [];
   forbidden: string[] = [];
-  userExcluded: string[] = [];
+  userExcluded: { url: string; actualUrl: string; pageTitle: string }[] = [];
   everything: string[] = [];
 
   constructor(urlsCrawled?: Partial<UrlsCrawled>) {

package/src/crawlers/crawlDomain.ts

@@ -125,14 +125,6 @@ const crawlDomain = async ({
 
   const httpsAgent = new https.Agent({ rejectUnauthorized: false });
 
-  if (isBlacklistedUrl) {
-    guiInfoLog(guiInfoStatusTypes.SKIPPED, {
-      numScanned: urlsCrawled.scanned.length,
-      urlScanned: url,
-    });
-    return;
-  }
-
   // Boolean to omit axe scan for basic auth URL
   let isBasicAuth = false;
   let authHeader = '';
@@ -608,13 +600,13 @@ const crawlDomain = async ({
         }
 
         await waitForPageLoaded(page, 10000);
-        let actualUrl = request.url;
+        let actualUrl = page.url() || request.loadedUrl || request.url;
 
         if (page.url() !== 'about:blank') {
           actualUrl = page.url();
         }
 
-        if (isBlacklisted(actualUrl, blacklistedPatterns) || (isUrlPdf(actualUrl) && !isScanPdfs)) {
+        if (!isFollowStrategy(url, actualUrl, strategy) && (isBlacklisted(actualUrl, blacklistedPatterns) || (isUrlPdf(actualUrl) && !isScanPdfs))) {
           guiInfoLog(guiInfoStatusTypes.SKIPPED, {
             numScanned: urlsCrawled.scanned.length,
             urlScanned: actualUrl,
@@ -683,8 +675,13 @@ const crawlDomain = async ({
           return;
         }
 
-        if (blacklistedPatterns && isSkippedUrl(actualUrl, blacklistedPatterns)) {
-          urlsCrawled.userExcluded.push(request.url);
+        if (!isFollowStrategy(url, actualUrl, strategy) && blacklistedPatterns && isSkippedUrl(actualUrl, blacklistedPatterns)) {
+          urlsCrawled.userExcluded.push({
+            url: request.url,
+            pageTitle: request.url,
+            actualUrl: actualUrl,
+          });
+
           await enqueueProcess(page, enqueueLinks, browserContext);
           return;
         }
@@ -709,18 +706,18 @@ const crawlDomain = async ({
 
         if (isScanHtml) {
           // For deduplication, if the URL is redirected, we want to store the original URL and the redirected URL (actualUrl)
-          const isRedirected = !areLinksEqual(request.loadedUrl, request.url);
+          const isRedirected = !areLinksEqual(actualUrl, request.url);
 
           // check if redirected link is following strategy (same-domain/same-hostname)
           const isLoadedUrlFollowStrategy = isFollowStrategy(
-            request.loadedUrl,
+            actualUrl,
             request.url,
             strategy,
           );
           if (isRedirected && !isLoadedUrlFollowStrategy) {
             urlsCrawled.notScannedRedirects.push({
               fromUrl: request.url,
-              toUrl: request.loadedUrl, // i.e. actualUrl
+              toUrl: actualUrl, // i.e. actualUrl
             });
             return;
           }
@@ -729,13 +726,13 @@ const crawlDomain = async ({
 
           if (isRedirected) {
             const isLoadedUrlInCrawledUrls = urlsCrawled.scanned.some(
-              item => (item.actualUrl || item.url) === request.loadedUrl,
+              item => (item.actualUrl || item.url) === actualUrl,
             );
 
             if (isLoadedUrlInCrawledUrls) {
               urlsCrawled.notScannedRedirects.push({
                 fromUrl: request.url,
-                toUrl: request.loadedUrl, // i.e. actualUrl
+                toUrl: actualUrl, // i.e. actualUrl
               });
               return;
             }
@@ -750,16 +747,16 @@ const crawlDomain = async ({
               urlsCrawled.scanned.push({
                 url: urlWithoutAuth(request.url),
                 pageTitle: results.pageTitle,
-                actualUrl: request.loadedUrl, // i.e. actualUrl
+                actualUrl: actualUrl, // i.e. actualUrl
               });
 
               urlsCrawled.scannedRedirects.push({
                 fromUrl: urlWithoutAuth(request.url),
-                toUrl: request.loadedUrl, // i.e. actualUrl
+                toUrl: actualUrl, // i.e. actualUrl
               });
 
               results.url = request.url;
-              results.actualUrl = request.loadedUrl;
+              results.actualUrl = actualUrl;
               await dataset.pushData(results);
             }
           } else {

package/src/crawlers/crawlLocalFile.ts

@@ -153,6 +153,8 @@ const crawlLocalFile = async (
     await page.goto(request.url);
     const results = await runAxeScript({ includeScreenshots, page, randomToken });
 
+    const actualUrl = page.url() || request.loadedUrl || request.url;
+
     guiInfoLog(guiInfoStatusTypes.SCANNED, {
       numScanned: urlsCrawled.scanned.length,
       urlScanned: request.url,
@@ -161,16 +163,16 @@ const crawlLocalFile = async (
     urlsCrawled.scanned.push({
       url: request.url,
       pageTitle: results.pageTitle,
-      actualUrl: request.loadedUrl, // i.e. actualUrl
+      actualUrl: actualUrl, // i.e. actualUrl
     });
 
     urlsCrawled.scannedRedirects.push({
       fromUrl: request.url,
-      toUrl: request.loadedUrl, // i.e. actualUrl
+      toUrl: actualUrl, // i.e. actualUrl
     });
 
     results.url = request.url;
-    // results.actualUrl = request.loadedUrl;
+    results.actualUrl = actualUrl;
 
     await dataset.pushData(results);
   } else {

package/src/crawlers/crawlSitemap.ts

@@ -18,7 +18,7 @@ import {
   waitForPageLoaded,
   isFilePath,
 } from '../constants/common.js';
-import { areLinksEqual, isWhitelistedContentType } from '../utils.js';
+import { areLinksEqual, isWhitelistedContentType, isFollowStrategy } from '../utils.js';
 import { handlePdfDownload, runPdfScan, mapPdfScanResults } from './pdfScanFunc.js';
 import { guiInfoLog } from '../logs.js';
 
@@ -161,21 +161,67 @@ const crawlSitemap = async (
       ],
     },
     requestList,
+    postNavigationHooks: [
+      async ({ page, request }) => {
+        try {
+          // Wait for a quiet period in the DOM, but with safeguards
+          await page.evaluate(() => {
+            return new Promise((resolve) => {
+              let timeout;
+              let mutationCount = 0;
+              const MAX_MUTATIONS = 250; // Prevent infinite mutations
+              const OBSERVER_TIMEOUT = 5000; // Hard timeout to exit
+
+              const observer = new MutationObserver(() => {
+                clearTimeout(timeout);
+
+                mutationCount++;
+                if (mutationCount > MAX_MUTATIONS) {
+                  observer.disconnect();
+                  resolve('Too many mutations detected, exiting.');
+                  return;
+                }
+
+                timeout = setTimeout(() => {
+                  observer.disconnect();
+                  resolve('DOM stabilized after mutations.');
+                }, 1000);
+              });
+
+              timeout = setTimeout(() => {
+                observer.disconnect();
+                resolve('Observer timeout reached, exiting.');
+              }, OBSERVER_TIMEOUT); // Ensure the observer stops after X seconds
+
+              observer.observe(document.documentElement, { childList: true, subtree: true });
+
+            });
+          });
+        } catch (err) {
+          // Handle page navigation errors gracefully
+          if (err.message.includes('was destroyed')) {
+            return; // Page navigated or closed, no need to handle
+          }
+          throw err; // Rethrow unknown errors
+        }
+      },
+    ],
+
     preNavigationHooks: isBasicAuth
       ? [
-          async ({ page }) => {
-            await page.setExtraHTTPHeaders({
-              Authorization: authHeader,
-              ...extraHTTPHeaders,
-            });
-          },
-        ]
+        async ({ page }) => {
+          await page.setExtraHTTPHeaders({
+            Authorization: authHeader,
+            ...extraHTTPHeaders,
+          });
+        },
+      ]
       : [
-          async () => {
-            preNavigationHooks(extraHTTPHeaders);
-            // insert other code here
-          },
-        ],
+        async () => {
+          preNavigationHooks(extraHTTPHeaders);
+          // insert other code here
+        },
+      ],
     requestHandlerTimeoutSecs: 90,
     requestHandler: async ({ page, request, response, sendRequest }) => {
       await waitForPageLoaded(page, 10000);
@@ -191,7 +237,7 @@ const crawlSitemap = async (
         request.url = currentUrl.href;
       }
 
-      const actualUrl = request.loadedUrl || request.url;
+      const actualUrl = page.url() || request.loadedUrl || request.url;
 
       if (urlsCrawled.scanned.length >= maxRequestsPerCrawl) {
         crawler.autoscaledPool.abort();
@@ -223,8 +269,17 @@ const crawlSitemap = async (
       const contentType = response.headers()['content-type'];
       const status = response.status();
 
-      if (blacklistedPatterns && isSkippedUrl(actualUrl, blacklistedPatterns)) {
-        urlsCrawled.userExcluded.push(request.url);
+      if (blacklistedPatterns && !isFollowStrategy(actualUrl, request.url, "same-hostname") && isSkippedUrl(actualUrl, blacklistedPatterns)) {
+        urlsCrawled.userExcluded.push({
+          url: request.url,
+          pageTitle: request.url,
+          actualUrl: actualUrl,
+        });
+
+        guiInfoLog(guiInfoStatusTypes.SKIPPED, {
+          numScanned: urlsCrawled.scanned.length,
+          urlScanned: request.url,
+        });
         return;
       }
 
@@ -255,16 +310,16 @@ const crawlSitemap = async (
           urlScanned: request.url,
         });
 
-        const isRedirected = !areLinksEqual(request.loadedUrl, request.url);
+        const isRedirected = !areLinksEqual(page.url(), request.url);
         if (isRedirected) {
           const isLoadedUrlInCrawledUrls = urlsCrawled.scanned.some(
-            item => (item.actualUrl || item.url.href) === request.loadedUrl,
+            item => (item.actualUrl || item.url.href) === page,
           );
 
           if (isLoadedUrlInCrawledUrls) {
             urlsCrawled.notScannedRedirects.push({
               fromUrl: request.url,
-              toUrl: request.loadedUrl, // i.e. actualUrl
+              toUrl: actualUrl, // i.e. actualUrl
             });
             return;
           }
@@ -272,16 +327,16 @@ const crawlSitemap = async (
           urlsCrawled.scanned.push({
             url: urlWithoutAuth(request.url),
             pageTitle: results.pageTitle,
-            actualUrl: request.loadedUrl, // i.e. actualUrl
+            actualUrl: actualUrl, // i.e. actualUrl
           });
 
           urlsCrawled.scannedRedirects.push({
             fromUrl: urlWithoutAuth(request.url),
-            toUrl: request.loadedUrl, // i.e. actualUrl
+            toUrl: actualUrl,
           });
 
           results.url = request.url;
-          results.actualUrl = request.loadedUrl;
+          results.actualUrl = actualUrl;
         } else {
           urlsCrawled.scanned.push({
             url: urlWithoutAuth(request.url),

package/src/crawlers/custom/utils.ts

@@ -152,7 +152,12 @@ export const processPage = async (page, processPageParams) => {
       window.confirm('Page has been excluded, would you still like to proceed with the scan?'),
     );
     if (!continueScan) {
-      urlsCrawled.userExcluded.push(pageUrl);
+      urlsCrawled.userExcluded.push({
+        url: pageUrl,
+        pageTitle: pageUrl,
+        actualUrl: pageUrl,
+      });
+
       return;
     }
   }
@@ -396,7 +401,7 @@ export const initNewPage = async (page, pageClosePromises, processPageParams, pa
   // eslint-disable-next-line no-underscore-dangle
   const pageId = page._guid;
 
-  page.on('dialog', () => {});
+  page.on('dialog', () => { });
 
   const pageClosePromise = new Promise(resolve => {
     page.on('close', () => {