@govtechsg/oobee 0.10.21 → 0.10.29

package/src/crawlers/commonCrawlerFunc.ts

@@ -1,14 +1,14 @@
 import crawlee, { CrawlingContext, PlaywrightGotoOptions } from 'crawlee';
 import axe, { AxeResults, ImpactValue, NodeResult, Result, resultGroups, TagValue } from 'axe-core';
-import xPathToCss from 'xpath-to-css';
-import { Page } from 'playwright';
+import { BrowserContext, Page } from 'playwright';
+import { xPathToCss } from '../xPathToCss.js';
 import {
   axeScript,
   guiInfoStatusTypes,
   RuleFlags,
   saflyIconSelector,
 } from '../constants/constants.js';
-import { consoleLogger, guiInfoLog, silentLogger } from '../logs.js';
+import { guiInfoLog, silentLogger } from '../logs.js';
 import { takeScreenshotForHTMLElements } from '../screenshotFunc/htmlScreenshotFunc.js';
 import { isFilePath } from '../constants/common.js';
 import { customAxeConfig } from './customAxeFunctions.js';
@@ -208,63 +208,70 @@ export const runAxeScript = async ({
   selectors?: string[];
   ruleset?: RuleFlags[];
 }) => {
-  // Checking for DOM mutations before proceeding to scan
-  await page.evaluate(() => {
-    return new Promise(resolve => {
-      let timeout: NodeJS.Timeout;
-      let mutationCount = 0;
-      const MAX_MUTATIONS = 100;
-      const MAX_SAME_MUTATION_LIMIT = 10;
-      const mutationHash = {};
-
-      const observer = new MutationObserver(mutationsList => {
-        clearTimeout(timeout);
+  const browserContext: BrowserContext = page.context();
+  const requestUrl = page.url();
+
+  try {
+    // Checking for DOM mutations before proceeding to scan
+    await page.evaluate(() => {
+      return new Promise(resolve => {
+        let timeout: NodeJS.Timeout;
+        let mutationCount = 0;
+        const MAX_MUTATIONS = 250;
+        const MAX_SAME_MUTATION_LIMIT = 10;
+        const mutationHash = {};
+
+        const observer = new MutationObserver(mutationsList => {
+          clearTimeout(timeout);
+
+          mutationCount += 1;
+
+          if (mutationCount > MAX_MUTATIONS) {
+            observer.disconnect();
+            resolve('Too many mutations detected');
+          }
 
-        mutationCount += 1;
+          // To handle scenario where DOM elements are constantly changing and unable to exit
+          mutationsList.forEach(mutation => {
+            let mutationKey: string;
 
-        if (mutationCount > MAX_MUTATIONS) {
-          observer.disconnect();
-          resolve('Too many mutations detected');
-        }
+            if (mutation.target instanceof Element) {
+              Array.from(mutation.target.attributes).forEach(attr => {
+                mutationKey = `${mutation.target.nodeName}-${attr.name}`;
 
-        // To handle scenario where DOM elements are constantly changing and unable to exit
-        mutationsList.forEach(mutation => {
-          let mutationKey: string;
-
-          if (mutation.target instanceof Element) {
-            Array.from(mutation.target.attributes).forEach(attr => {
-              mutationKey = `${mutation.target.nodeName}-${attr.name}`;
+                if (mutationKey) {
+                  if (!mutationHash[mutationKey]) {
+                    mutationHash[mutationKey] = 1;
+                  } else {
+                    mutationHash[mutationKey] += 1;
+                  }
 
-              if (mutationKey) {
-                if (!mutationHash[mutationKey]) {
-                  mutationHash[mutationKey] = 1;
-                } else {
-                  mutationHash[mutationKey] += 1;
+                  if (mutationHash[mutationKey] >= MAX_SAME_MUTATION_LIMIT) {
+                    observer.disconnect();
+                    resolve(`Repeated mutation detected for ${mutationKey}`);
+                  }
                 }
+              });
+            }
+          });
 
-                if (mutationHash[mutationKey] >= MAX_SAME_MUTATION_LIMIT) {
-                  observer.disconnect();
-                  resolve(`Repeated mutation detected for ${mutationKey}`);
-                }
-              }
-            });
-          }
+          timeout = setTimeout(() => {
+            observer.disconnect();
+            resolve('DOM stabilized after mutations.');
+          }, 1000);
         });
 
         timeout = setTimeout(() => {
           observer.disconnect();
-          resolve('DOM stabilized after mutations.');
+          resolve('No mutations detected, exit from idle state');
         }, 1000);
-      });
-
-      timeout = setTimeout(() => {
-        observer.disconnect();
-        resolve('No mutations detected, exit from idle state');
-      }, 1000);
 
-      observer.observe(document, { childList: true, subtree: true, attributes: true });
+        observer.observe(document, { childList: true, subtree: true, attributes: true });
+      });
     });
-  });
+  } catch (e) {
+    silentLogger.warn(`Error while checking for DOM mutations: ${e}`);
+  }
 
   page.on('console', msg => {
     const type = msg.type();
@@ -350,24 +357,28 @@ export const runAxeScript = async ({
                 return !node.dataset.flagged; // fail any element with a data-flagged attribute set to true
               },
             },
-            {
-              ...customAxeConfig.checks[2],
-              evaluate: (_node: HTMLElement) => {
-                if (gradingReadabilityFlag === '') {
-                  return true; // Pass if no readability issues
-                }
-                // Dynamically update the grading messages
-                const gradingCheck = customAxeConfig.checks.find(
-                  check => check.id === 'oobee-grading-text-contents',
-                );
-                if (gradingCheck) {
-                  gradingCheck.metadata.messages.incomplete = `The text content is potentially difficult to read, with a Flesch-Kincaid Reading Ease score of ${gradingReadabilityFlag
-                    }.\nThe target passing score is above 50, indicating content readable by university students and lower grade levels.\nA higher score reflects better readability.`;
-                }
-
-                // Fail if readability issues are detected
-              },
-            },
+            ...(enableWcagAaa
+              ? [
+                {
+                  ...customAxeConfig.checks[2],
+                  evaluate: (_node: HTMLElement) => {
+                    if (gradingReadabilityFlag === '') {
+                      return true; // Pass if no readability issues
+                    }
+                    // Dynamically update the grading messages
+                    const gradingCheck = customAxeConfig.checks.find(
+                      check => check.id === 'oobee-grading-text-contents',
+                    );
+                    if (gradingCheck) {
+                      gradingCheck.metadata.messages.incomplete = `The text content is potentially difficult to read, with a Flesch-Kincaid Reading Ease score of ${gradingReadabilityFlag
+                        }.\nThe target passing score is above 50, indicating content readable by university students and lower grade levels.\nA higher score reflects better readability.`;
+                    }
+
+                    // Fail if readability issues are detected
+                  },
+                },
+              ]
+              : []),
           ],
           rules: customAxeConfig.rules
             .filter(rule => (disableOobee ? !rule.id.startsWith('oobee') : true))
@@ -409,123 +420,66 @@ export const runAxeScript = async ({
             const escapedCssSelectors =
               oobeeAccessibleLabelFlaggedCssSelectors.map(escapeCSSSelector);
 
-            function frameCheck(cssSelector: string): { doc: Document; remainingSelector: string } {
+            function framesCheck(cssSelector: string): {
+              doc: Document;
+              remainingSelector: string;
+            } {
               let doc = document; // Start with the main document
-              let frameSelector = ""; // To store the frame part of the selector
+              let remainingSelector = ''; // To store the last part of the selector
+              let targetIframe = null;
 
-              // Extract the 'frame' part of the selector
-              let frameMatch = cssSelector.match(/(frame[^>]*>)/i);
-              if (frameMatch) {
-                frameSelector = frameMatch[1].replace(">", "").trim(); // Clean up the frame part
-                cssSelector = cssSelector.split(frameMatch[1])[1].trim(); // Remove the frame portion
-              }
+              // Split the selector into parts at "> html"
+              const diffParts = cssSelector.split(/\s*>\s*html\s*/);
 
-              let targetFrame = null; // Target frame element
-
-              // Locate the frame based on the extracted frameSelector
-              if (frameSelector.includes("first-of-type")) {
-                // Select the first frame
-                targetFrame = document.querySelector("frame:first-of-type");
-              } else if (frameSelector.includes("nth-of-type")) {
-                // Select the nth frame
-                let nthIndex = frameSelector.match(/nth-of-type\((\d+)\)/);
-                if (nthIndex) {
-                  let index = parseInt(nthIndex[1]) - 1; // Zero-based index
-                  targetFrame = document.querySelectorAll("frame")[index];
-                }
-              } else if (frameSelector.includes("#")) {
-                // Frame with a specific ID
-                let idMatch = frameSelector.match(/#([\w-]+)/);
-                if (idMatch) {
-                  targetFrame = document.getElementById(idMatch[1]);
-                }
-              } else if (frameSelector.includes('[name="')) {
-                // Frame with a specific name attribute
-                let nameMatch = frameSelector.match(/name="([\w-]+)"/);
-                if (nameMatch) {
-                  targetFrame = document.querySelector(`frame[name="${nameMatch[1]}"]`);
-                }
-              } else {
-                // Default to the first frame
-                targetFrame = document.querySelector("frame");
-              }
+              for (let i = 0; i < diffParts.length - 1; i++) {
+                let iframeSelector = `${diffParts[i].trim()}`;
 
-              // Update the document if the frame was found
-              if (targetFrame && targetFrame.contentDocument) {
-                doc = targetFrame.contentDocument;
-              } else {
-                console.warn("Frame not found or contentDocument inaccessible.");
-              }
-
-              return { doc, remainingSelector: cssSelector };
-            }
-
-            function iframeCheck(cssSelector: string): { doc: Document; remainingSelector: string } {
-              let doc = document; // Start with the main document
-              let iframeSelector = ""; // To store the iframe part of the selector
-
-              // Extract the 'iframe' part of the selector
-              let iframeMatch = cssSelector.match(/(iframe[^>]*>)/i);
-              if (iframeMatch) {
-                iframeSelector = iframeMatch[1].replace(">", "").trim(); // Clean up the iframe part
-                cssSelector = cssSelector.split(iframeMatch[1])[1].trim(); // Remove the iframe portion
-              }
+                // Add back '> html' to the current part
+                if (i > 0) {
+                  iframeSelector = `html > ${iframeSelector}`;
+                }
 
-              let targetIframe = null; // Target iframe element
-
-              // Locate the iframe based on the extracted iframeSelector
-              if (iframeSelector.includes("first-of-type")) {
-                // Select the first iframe
-                targetIframe = document.querySelector("iframe:first-of-type");
-              } else if (iframeSelector.includes("nth-of-type")) {
-                // Select the nth iframe
-                let nthIndex = iframeSelector.match(/nth-of-type\((\d+)\)/);
-                if (nthIndex) {
-                  let index = parseInt(nthIndex[1]) - 1; // Zero-based index
-                  targetIframe = document.querySelectorAll("iframe")[index];
+                let frameset = null;
+                // Find the iframe using the current document context
+                if (doc.querySelector('frameset')) {
+                  frameset = doc.querySelector('frameset');
                 }
-              } else if (iframeSelector.includes("#")) {
-                // Iframe with a specific ID
-                let idMatch = iframeSelector.match(/#([\w-]+)/);
-                if (idMatch) {
-                  targetIframe = document.getElementById(idMatch[1]);
+
+                if (frameset) {
+                  doc = frameset;
+                  iframeSelector = iframeSelector.split('body >')[1].trim();
                 }
-              } else if (iframeSelector.includes('[name="')) {
-                // Iframe with a specific name attribute
-                let nameMatch = iframeSelector.match(/name="([\w-]+)"/);
-                if (nameMatch) {
-                  targetIframe = document.querySelector(`iframe[name="${nameMatch[1]}"]`);
+                targetIframe = doc.querySelector(iframeSelector);
+
+                if (targetIframe && targetIframe.contentDocument) {
+                  // Update the document to the iframe's contentDocument
+                  doc = targetIframe.contentDocument;
+                } else {
+                  console.warn(
+                    `Iframe not found or contentDocument inaccessible for selector: ${iframeSelector}`,
+                  );
+                  return { doc, remainingSelector: cssSelector }; // Return original selector if iframe not found
                 }
-              } else {
-                // Default to the first iframe
-                targetIframe = document.querySelector("iframe");
               }
 
-              // Update the document if the iframe was found
-              if (targetIframe && targetIframe.contentDocument) {
-                doc = targetIframe.contentDocument;
-              } else {
-                console.warn("Iframe not found or contentDocument inaccessible.");
-              }
+              // The last part is the remaining CSS selector
+              remainingSelector = diffParts[diffParts.length - 1].trim();
 
-              return { doc, remainingSelector: cssSelector };
+              // Remove any leading '>' combinators from remainingSelector
+              remainingSelector = `html${remainingSelector}`;
+
+              return { doc, remainingSelector };
             }
 
             function findElementByCssSelector(cssSelector: string): string | null {
               let doc = document;
 
-              // Check if the selector includes 'frame' and update doc and selector
-              if (cssSelector.includes("frame")) {
-                const result = frameCheck(cssSelector);
-                doc = result.doc;
-                cssSelector = result.remainingSelector;
-              }
+              // Check if the selector includes 'frame' or 'iframe' and update doc and selector
 
-              // Check for iframe
-              if (cssSelector.includes("iframe")) {
-                const result = iframeCheck(cssSelector);
-                doc = result.doc;
-                cssSelector = result.remainingSelector;
+              if (/\s*>\s*html\s*/.test(cssSelector)) {
+                const inFrames = framesCheck(cssSelector);
+                doc = inFrames.doc;
+                cssSelector = inFrames.remainingSelector;
               }
 
               // Query the element in the document (including inside frames)
@@ -553,35 +507,42 @@ export const runAxeScript = async ({
                 }
               }
 
-              return element ? element.outerHTML : null;
+              if (element) {
+                return element.outerHTML;
+              }
+
+              console.warn(`Unable to find element for css selector: ${cssSelector}`);
+              return null;
             }
 
             // Add oobee violations to Axe's report
             const oobeeAccessibleLabelViolations = {
               id: 'oobee-accessible-label',
               impact: 'serious' as ImpactValue,
-              tags: ['wcag2a', 'wcag211', 'wcag243', 'wcag412'],
+              tags: ['wcag2a', 'wcag211', 'wcag412'],
               description: 'Ensures clickable elements have an accessible label.',
               help: 'Clickable elements (i.e. elements with mouse-click interaction) must have accessible labels.',
               helpUrl: 'https://www.deque.com/blog/accessible-aria-buttons',
-              nodes: escapedCssSelectors.map(cssSelector => ({
-                html: findElementByCssSelector(cssSelector),
-                target: [cssSelector],
-                impact: 'serious' as ImpactValue,
-                failureSummary:
-                  'Fix any of the following:\n  The clickable element does not have an accessible label.',
-                any: [
-                  {
-                    id: 'oobee-accessible-label',
-                    data: null,
-                    relatedNodes: [],
-                    impact: 'serious',
-                    message: 'The clickable element does not have an accessible label.',
-                  },
-                ],
-                all: [],
-                none: [],
-              })),
+              nodes: escapedCssSelectors
+                .map(cssSelector => ({
+                  html: findElementByCssSelector(cssSelector),
+                  target: [cssSelector],
+                  impact: 'serious' as ImpactValue,
+                  failureSummary:
+                    'Fix any of the following:\n  The clickable element does not have an accessible label.',
+                  any: [
+                    {
+                      id: 'oobee-accessible-label',
+                      data: null,
+                      relatedNodes: [],
+                      impact: 'serious',
+                      message: 'The clickable element does not have an accessible label.',
+                    },
+                  ],
+                  all: [],
+                  none: [],
+                }))
+                .filter(item => item.html),
             };
 
             results.violations = [...results.violations, oobeeAccessibleLabelViolations];
@@ -612,7 +573,18 @@ export const runAxeScript = async ({
     results.incomplete = await takeScreenshotForHTMLElements(results.incomplete, page, randomToken);
   }
 
-  const pageTitle = await page.evaluate(() => document.title);
+  let pageTitle = null;
+  try {
+    pageTitle = await page.evaluate(() => document.title);
+  } catch (e) {
+    silentLogger.warn(`Error while getting page title: ${e}`);
+    if (page.isClosed()) {
+      silentLogger.info(`Page was closed for ${requestUrl}, creating new page`);
+      page = await browserContext.newPage();
+      await page.goto(requestUrl, { waitUntil: 'domcontentloaded' });
+      pageTitle = await page.evaluate(() => document.title);
+    }
+  }
 
   return filterAxeResults(results, pageTitle, customFlowDetails);
 };
@@ -653,4 +625,4 @@ export const isUrlPdf = (url: string) => {
   }
   const parsedUrl = new URL(url);
   return /\.pdf($|\?|#)/i.test(parsedUrl.pathname) || /\.pdf($|\?|#)/i.test(parsedUrl.href);
-};
+};

package/src/crawlers/crawlDomain.ts

@@ -40,8 +40,7 @@ import {
 import { silentLogger, guiInfoLog } from '../logs.js';
 import { ViewportSettingsClass } from '../combine.js';
 
-const isBlacklisted = (url: string) => {
-  const blacklistedPatterns = getBlackListedPatterns(null);
+const isBlacklisted = (url: string, blacklistedPatterns: string[]) => {
   if (!blacklistedPatterns) {
     return false;
   }
@@ -122,7 +121,7 @@ const crawlDomain = async ({
   const isScanPdfs = ['all', 'pdf-only'].includes(fileTypes);
   const { maxConcurrency } = constants;
   const { playwrightDeviceDetailsObject } = viewportSettings;
-  const isBlacklistedUrl = isBlacklisted(url);
+  const isBlacklistedUrl = isBlacklisted(url, blacklistedPatterns);
 
   const httpsAgent = new https.Agent({ rejectUnauthorized: false });
 
@@ -315,7 +314,7 @@ const crawlDomain = async ({
 
     const isExcluded = (newPageUrl: string): boolean => {
       const isAlreadyScanned: boolean = urlsCrawled.scanned.some(item => item.url === newPageUrl);
-      const isBlacklistedUrl: boolean = isBlacklisted(newPageUrl);
+      const isBlacklistedUrl: boolean = isBlacklisted(newPageUrl, blacklistedPatterns);
       const isNotFollowStrategy: boolean = !isFollowStrategy(newPageUrl, initialPageUrl, strategy);
       return isAlreadyScanned || isBlacklistedUrl || isNotFollowStrategy;
     };
@@ -469,7 +468,7 @@ const crawlDomain = async ({
       launcher: constants.launcher,
       launchOptions: getPlaywrightLaunchOptions(browser),
       // Bug in Chrome which causes browser pool crash when userDataDirectory is set in non-headless mode
-      userDataDir,
+      ...(process.env.CRAWLEE_HEADLESS === '0' && { userDataDir }),
     },
     retryOnBlocked: true,
     browserPoolOptions: {
@@ -496,7 +495,7 @@ const crawlDomain = async ({
           return new Promise(resolve => {
             let timeout;
             let mutationCount = 0;
-            const MAX_MUTATIONS = 100;
+            const MAX_MUTATIONS = 250;
             const MAX_SAME_MUTATION_LIMIT = 10;
             const mutationHash = {};
 
@@ -568,31 +567,31 @@ const crawlDomain = async ({
     ],
     preNavigationHooks: isBasicAuth
       ? [
-          async ({ page, request }) => {
-            await page.setExtraHTTPHeaders({
-              Authorization: authHeader,
-              ...extraHTTPHeaders,
-            });
-            const processible = await isProcessibleUrl(request.url);
-            if (!processible) {
-              request.skipNavigation = true;
-              return null;
-            }
-          },
-        ]
+        async ({ page, request }) => {
+          await page.setExtraHTTPHeaders({
+            Authorization: authHeader,
+            ...extraHTTPHeaders,
+          });
+          const processible = await isProcessibleUrl(request.url);
+          if (!processible) {
+            request.skipNavigation = true;
+            return null;
+          }
+        },
+      ]
       : [
-          async ({ page, request }) => {
-            await page.setExtraHTTPHeaders({
-              ...extraHTTPHeaders,
-            });
+        async ({ page, request }) => {
+          await page.setExtraHTTPHeaders({
+            ...extraHTTPHeaders,
+          });
 
-            const processible = await isProcessibleUrl(request.url);
-            if (!processible) {
-              request.skipNavigation = true;
-              return null;
-            }
-          },
-        ],
+          const processible = await isProcessibleUrl(request.url);
+          if (!processible) {
+            request.skipNavigation = true;
+            return null;
+          }
+        },
+      ],
     requestHandlerTimeoutSecs: 90, // Allow each page to be processed by up from default 60 seconds
     requestHandler: async ({ page, request, response, crawler, sendRequest, enqueueLinks }) => {
       const browserContext: BrowserContext = page.context();
@@ -615,7 +614,7 @@ const crawlDomain = async ({
           actualUrl = page.url();
         }
 
-        if (isBlacklisted(actualUrl) || (isUrlPdf(actualUrl) && !isScanPdfs)) {
+        if (isBlacklisted(actualUrl, blacklistedPatterns) || (isUrlPdf(actualUrl) && !isScanPdfs)) {
           guiInfoLog(guiInfoStatusTypes.SKIPPED, {
             numScanned: urlsCrawled.scanned.length,
             urlScanned: actualUrl,

package/src/crawlers/crawlIntelligentSitemap.ts

@@ -50,7 +50,13 @@ const crawlIntelligentSitemap = async (
     const homeUrl = getHomeUrl(link);
     let sitemapLinkFound = false;
     let sitemapLink = '';
-    const chromiumBrowser = await chromium.launch({ headless: true, channel: 'chrome' });
+    const chromiumBrowser = await chromium.launch(
+      {
+        headless: false,
+        channel: 'chrome',
+        args: ['--headless=new', '--no-sandbox']
+      });
+
     const page = await chromiumBrowser.newPage();
     for (const path of sitemapPaths) {
       sitemapLink = homeUrl + path;

package/src/crawlers/crawlLocalFile.ts

@@ -143,7 +143,7 @@ const crawlLocalFile = async (
 
   if (!isUrlPdf(request.url)) {
     const browserContext = await constants.launcher.launchPersistentContext('', {
-      headless: process.env.CRAWLEE_HEADLESS === '1',
+      headless: false,
       ...getPlaywrightLaunchOptions(browser),
       ...playwrightDeviceDetailsObject,
     });

package/src/crawlers/crawlSitemap.ts

@@ -144,7 +144,7 @@ const crawlSitemap = async (
       launcher: constants.launcher,
       launchOptions: getPlaywrightLaunchOptions(browser),
       // Bug in Chrome which causes browser pool crash when userDataDirectory is set in non-headless mode
-      userDataDir,
+      ...(process.env.CRAWLEE_HEADLESS === '0' && { userDataDir }),
     },
     retryOnBlocked: true,
     browserPoolOptions: {