@govtechsg/oobee 0.10.20 → 0.10.28

package/src/crawlers/commonCrawlerFunc.ts

@@ -1,14 +1,14 @@
 import crawlee, { CrawlingContext, PlaywrightGotoOptions } from 'crawlee';
 import axe, { AxeResults, ImpactValue, NodeResult, Result, resultGroups, TagValue } from 'axe-core';
-import xPathToCss from 'xpath-to-css';
-import { Page } from 'playwright';
+import { xPathToCss } from '../xPathToCss.js';
+import { BrowserContext, Page } from 'playwright';
 import {
   axeScript,
   guiInfoStatusTypes,
   RuleFlags,
   saflyIconSelector,
 } from '../constants/constants.js';
-import { consoleLogger, guiInfoLog, silentLogger } from '../logs.js';
+import { guiInfoLog, silentLogger } from '../logs.js';
 import { takeScreenshotForHTMLElements } from '../screenshotFunc/htmlScreenshotFunc.js';
 import { isFilePath } from '../constants/common.js';
 import { customAxeConfig } from './customAxeFunctions.js';
@@ -208,63 +208,70 @@ export const runAxeScript = async ({
   selectors?: string[];
   ruleset?: RuleFlags[];
 }) => {
-  // Checking for DOM mutations before proceeding to scan
-  await page.evaluate(() => {
-    return new Promise(resolve => {
-      let timeout: NodeJS.Timeout;
-      let mutationCount = 0;
-      const MAX_MUTATIONS = 100;
-      const MAX_SAME_MUTATION_LIMIT = 10;
-      const mutationHash = {};
-
-      const observer = new MutationObserver(mutationsList => {
-        clearTimeout(timeout);
-
-        mutationCount += 1;
+  const browserContext: BrowserContext = page.context();
+  const requestUrl = page.url();
+
+  try {
+    // Checking for DOM mutations before proceeding to scan
+    await page.evaluate(() => {
+      return new Promise(resolve => {
+        let timeout: NodeJS.Timeout;
+        let mutationCount = 0;
+        const MAX_MUTATIONS = 250;
+        const MAX_SAME_MUTATION_LIMIT = 10;
+        const mutationHash = {};
+
+        const observer = new MutationObserver(mutationsList => {
+          clearTimeout(timeout);
+
+          mutationCount += 1;
+
+          if (mutationCount > MAX_MUTATIONS) {
+            observer.disconnect();
+            resolve('Too many mutations detected');
+          }
 
-        if (mutationCount > MAX_MUTATIONS) {
-          observer.disconnect();
-          resolve('Too many mutations detected');
-        }
+          // To handle scenario where DOM elements are constantly changing and unable to exit
+          mutationsList.forEach(mutation => {
+            let mutationKey: string;
 
-        // To handle scenario where DOM elements are constantly changing and unable to exit
-        mutationsList.forEach(mutation => {
-          let mutationKey: string;
+            if (mutation.target instanceof Element) {
+              Array.from(mutation.target.attributes).forEach(attr => {
+                mutationKey = `${mutation.target.nodeName}-${attr.name}`;
 
-          if (mutation.target instanceof Element) {
-            Array.from(mutation.target.attributes).forEach(attr => {
-              mutationKey = `${mutation.target.nodeName}-${attr.name}`;
+                if (mutationKey) {
+                  if (!mutationHash[mutationKey]) {
+                    mutationHash[mutationKey] = 1;
+                  } else {
+                    mutationHash[mutationKey] += 1;
+                  }
 
-              if (mutationKey) {
-                if (!mutationHash[mutationKey]) {
-                  mutationHash[mutationKey] = 1;
-                } else {
-                  mutationHash[mutationKey] += 1;
+                  if (mutationHash[mutationKey] >= MAX_SAME_MUTATION_LIMIT) {
+                    observer.disconnect();
+                    resolve(`Repeated mutation detected for ${mutationKey}`);
+                  }
                 }
+              });
+            }
+          });
 
-                if (mutationHash[mutationKey] >= MAX_SAME_MUTATION_LIMIT) {
-                  observer.disconnect();
-                  resolve(`Repeated mutation detected for ${mutationKey}`);
-                }
-              }
-            });
-          }
+          timeout = setTimeout(() => {
+            observer.disconnect();
+            resolve('DOM stabilized after mutations.');
+          }, 1000);
         });
 
         timeout = setTimeout(() => {
           observer.disconnect();
-          resolve('DOM stabilized after mutations.');
+          resolve('No mutations detected, exit from idle state');
         }, 1000);
-      });
 
-      timeout = setTimeout(() => {
-        observer.disconnect();
-        resolve('No mutations detected, exit from idle state');
-      }, 1000);
-
-      observer.observe(document, { childList: true, subtree: true, attributes: true });
+        observer.observe(document, { childList: true, subtree: true, attributes: true });
+      });
     });
-  });
+  } catch (e) {
+    silentLogger.warn(`Error while checking for DOM mutations: ${e}`);
+  }
 
   page.on('console', msg => {
     const type = msg.type();
@@ -409,123 +416,62 @@ export const runAxeScript = async ({
             const escapedCssSelectors =
               oobeeAccessibleLabelFlaggedCssSelectors.map(escapeCSSSelector);
 
-            function frameCheck(cssSelector: string): { doc: Document; remainingSelector: string } {
+            function framesCheck(cssSelector: string): { doc: Document; remainingSelector: string } {
               let doc = document; // Start with the main document
-              let frameSelector = ""; // To store the frame part of the selector
+              let remainingSelector = ""; // To store the last part of the selector
+              let targetIframe = null;
 
-              // Extract the 'frame' part of the selector
-              let frameMatch = cssSelector.match(/(frame[^>]*>)/i);
-              if (frameMatch) {
-                frameSelector = frameMatch[1].replace(">", "").trim(); // Clean up the frame part
-                cssSelector = cssSelector.split(frameMatch[1])[1].trim(); // Remove the frame portion
-              }
+              // Split the selector into parts at "> html"
+              const diffParts = cssSelector.split(/\s*>\s*html\s*/);
 
-              let targetFrame = null; // Target frame element
-
-              // Locate the frame based on the extracted frameSelector
-              if (frameSelector.includes("first-of-type")) {
-                // Select the first frame
-                targetFrame = document.querySelector("frame:first-of-type");
-              } else if (frameSelector.includes("nth-of-type")) {
-                // Select the nth frame
-                let nthIndex = frameSelector.match(/nth-of-type\((\d+)\)/);
-                if (nthIndex) {
-                  let index = parseInt(nthIndex[1]) - 1; // Zero-based index
-                  targetFrame = document.querySelectorAll("frame")[index];
-                }
-              } else if (frameSelector.includes("#")) {
-                // Frame with a specific ID
-                let idMatch = frameSelector.match(/#([\w-]+)/);
-                if (idMatch) {
-                  targetFrame = document.getElementById(idMatch[1]);
-                }
-              } else if (frameSelector.includes('[name="')) {
-                // Frame with a specific name attribute
-                let nameMatch = frameSelector.match(/name="([\w-]+)"/);
-                if (nameMatch) {
-                  targetFrame = document.querySelector(`frame[name="${nameMatch[1]}"]`);
-                }
-              } else {
-                // Default to the first frame
-                targetFrame = document.querySelector("frame");
-              }
-
-              // Update the document if the frame was found
-              if (targetFrame && targetFrame.contentDocument) {
-                doc = targetFrame.contentDocument;
-              } else {
-                console.warn("Frame not found or contentDocument inaccessible.");
-              }
+              for (let i = 0; i < diffParts.length - 1; i++) {
+                let iframeSelector = `${diffParts[i].trim()}`;
 
-              return { doc, remainingSelector: cssSelector };
-            }
-
-            function iframeCheck(cssSelector: string): { doc: Document; remainingSelector: string } {
-              let doc = document; // Start with the main document
-              let iframeSelector = ""; // To store the iframe part of the selector
-
-              // Extract the 'iframe' part of the selector
-              let iframeMatch = cssSelector.match(/(iframe[^>]*>)/i);
-              if (iframeMatch) {
-                iframeSelector = iframeMatch[1].replace(">", "").trim(); // Clean up the iframe part
-                cssSelector = cssSelector.split(iframeMatch[1])[1].trim(); // Remove the iframe portion
-              }
+                // Add back '> html' to the current part
+                if (i > 0) {
+                  iframeSelector = "html > " + iframeSelector;
+                }
 
-              let targetIframe = null; // Target iframe element
-
-              // Locate the iframe based on the extracted iframeSelector
-              if (iframeSelector.includes("first-of-type")) {
-                // Select the first iframe
-                targetIframe = document.querySelector("iframe:first-of-type");
-              } else if (iframeSelector.includes("nth-of-type")) {
-                // Select the nth iframe
-                let nthIndex = iframeSelector.match(/nth-of-type\((\d+)\)/);
-                if (nthIndex) {
-                  let index = parseInt(nthIndex[1]) - 1; // Zero-based index
-                  targetIframe = document.querySelectorAll("iframe")[index];
+                let frameset = null;
+                // Find the iframe using the current document context
+                if (doc.querySelector("frameset")) {
+                  frameset = doc.querySelector("frameset");
                 }
-              } else if (iframeSelector.includes("#")) {
-                // Iframe with a specific ID
-                let idMatch = iframeSelector.match(/#([\w-]+)/);
-                if (idMatch) {
-                  targetIframe = document.getElementById(idMatch[1]);
+
+                if (frameset) {
+                  doc = frameset;
+                  iframeSelector = iframeSelector.split("body >")[1].trim();
                 }
-              } else if (iframeSelector.includes('[name="')) {
-                // Iframe with a specific name attribute
-                let nameMatch = iframeSelector.match(/name="([\w-]+)"/);
-                if (nameMatch) {
-                  targetIframe = document.querySelector(`iframe[name="${nameMatch[1]}"]`);
+                targetIframe = doc.querySelector(iframeSelector);
+
+                if (targetIframe && targetIframe.contentDocument) {
+                  // Update the document to the iframe's contentDocument
+                  doc = targetIframe.contentDocument;
+                } else {
+                  console.warn(`Iframe not found or contentDocument inaccessible for selector: ${iframeSelector}`);
+                  return { doc, remainingSelector: cssSelector }; // Return original selector if iframe not found
                 }
-              } else {
-                // Default to the first iframe
-                targetIframe = document.querySelector("iframe");
               }
 
-              // Update the document if the iframe was found
-              if (targetIframe && targetIframe.contentDocument) {
-                doc = targetIframe.contentDocument;
-              } else {
-                console.warn("Iframe not found or contentDocument inaccessible.");
-              }
+              // The last part is the remaining CSS selector
+              remainingSelector = diffParts[diffParts.length - 1].trim();
+
+              // Remove any leading '>' combinators from remainingSelector
+              remainingSelector = "html" + remainingSelector;
 
-              return { doc, remainingSelector: cssSelector };
+              return { doc, remainingSelector };
             }
 
+
             function findElementByCssSelector(cssSelector: string): string | null {
               let doc = document;
 
-              // Check if the selector includes 'frame' and update doc and selector
-              if (cssSelector.includes("frame")) {
-                const result = frameCheck(cssSelector);
-                doc = result.doc;
-                cssSelector = result.remainingSelector;
-              }
+              // Check if the selector includes 'frame' or 'iframe' and update doc and selector
 
-              // Check for iframe
-              if (cssSelector.includes("iframe")) {
-                const result = iframeCheck(cssSelector);
-                doc = result.doc;
-                cssSelector = result.remainingSelector;
+              if (/\s*>\s*html\s*/.test(cssSelector)) {
+                let inFrames = framesCheck(cssSelector)
+                doc = inFrames.doc;
+                cssSelector = inFrames.remainingSelector;
               }
 
               // Query the element in the document (including inside frames)
@@ -553,14 +499,19 @@ export const runAxeScript = async ({
                 }
               }
 
-              return element ? element.outerHTML : null;
+              if (element) {
+                return element.outerHTML;
+              }
+
+              console.warn(`Unable to find element for css selector: ${cssSelector}`);
+              return null;
             }
 
             // Add oobee violations to Axe's report
             const oobeeAccessibleLabelViolations = {
               id: 'oobee-accessible-label',
               impact: 'serious' as ImpactValue,
-              tags: ['wcag2a', 'wcag211', 'wcag243', 'wcag412'],
+              tags: ['wcag2a', 'wcag211', 'wcag412'],
               description: 'Ensures clickable elements have an accessible label.',
               help: 'Clickable elements (i.e. elements with mouse-click interaction) must have accessible labels.',
               helpUrl: 'https://www.deque.com/blog/accessible-aria-buttons',
@@ -581,7 +532,7 @@ export const runAxeScript = async ({
                 ],
                 all: [],
                 none: [],
-              })),
+              })).filter(item => item.html)
             };
 
             results.violations = [...results.violations, oobeeAccessibleLabelViolations];
@@ -612,7 +563,18 @@ export const runAxeScript = async ({
     results.incomplete = await takeScreenshotForHTMLElements(results.incomplete, page, randomToken);
   }
 
-  const pageTitle = await page.evaluate(() => document.title);
+  let pageTitle = null;
+  try {
+    pageTitle = await page.evaluate(() => document.title);
+  } catch (e) {
+    silentLogger.warn(`Error while getting page title: ${e}`);
+    if (page.isClosed()) {
+      silentLogger.info(`Page was closed for ${requestUrl}, creating new page`);
+      page = await browserContext.newPage();
+      await page.goto(requestUrl, { waitUntil: 'domcontentloaded' });
+      pageTitle = await page.evaluate(() => document.title);
+    }
+  }
 
   return filterAxeResults(results, pageTitle, customFlowDetails);
 };
@@ -653,4 +615,4 @@ export const isUrlPdf = (url: string) => {
   }
   const parsedUrl = new URL(url);
   return /\.pdf($|\?|#)/i.test(parsedUrl.pathname) || /\.pdf($|\?|#)/i.test(parsedUrl.href);
-};
+};

package/src/crawlers/crawlDomain.ts

@@ -469,7 +469,7 @@ const crawlDomain = async ({
       launcher: constants.launcher,
       launchOptions: getPlaywrightLaunchOptions(browser),
       // Bug in Chrome which causes browser pool crash when userDataDirectory is set in non-headless mode
-      userDataDir,
+      ...(process.env.CRAWLEE_HEADLESS === '0' && { userDataDir }),
     },
     retryOnBlocked: true,
     browserPoolOptions: {
@@ -496,7 +496,7 @@ const crawlDomain = async ({
           return new Promise(resolve => {
             let timeout;
             let mutationCount = 0;
-            const MAX_MUTATIONS = 100;
+            const MAX_MUTATIONS = 250;
             const MAX_SAME_MUTATION_LIMIT = 10;
             const mutationHash = {};
 
@@ -568,38 +568,31 @@ const crawlDomain = async ({
     ],
     preNavigationHooks: isBasicAuth
       ? [
-          async ({ page, request }) => {
-            await page.setExtraHTTPHeaders({
-              Authorization: authHeader,
-              ...extraHTTPHeaders,
-            });
-            const processible = await isProcessibleUrl(request.url);
-            if (!processible) {
-              request.skipNavigation = true;
-              return null;
-            }
-          },
-        ]
+        async ({ page, request }) => {
+          await page.setExtraHTTPHeaders({
+            Authorization: authHeader,
+            ...extraHTTPHeaders,
+          });
+          const processible = await isProcessibleUrl(request.url);
+          if (!processible) {
+            request.skipNavigation = true;
+            return null;
+          }
+        },
+      ]
       : [
-          async (crawlingContext, gotoOptions) => {
-            const { page, request } = crawlingContext;
-
-            await page.setExtraHTTPHeaders({
-              ...extraHTTPHeaders,
-            });
-
-            Object.assign(gotoOptions, {
-              waitUntil: 'networkidle',
-              timeout: 30000,
-            });
+        async ({ page, request }) => {
+          await page.setExtraHTTPHeaders({
+            ...extraHTTPHeaders,
+          });
 
-            const processible = await isProcessibleUrl(request.url);
-            if (!processible) {
-              request.skipNavigation = true;
-              return null;
-            }
-          },
-        ],
+          const processible = await isProcessibleUrl(request.url);
+          if (!processible) {
+            request.skipNavigation = true;
+            return null;
+          }
+        },
+      ],
     requestHandlerTimeoutSecs: 90, // Allow each page to be processed by up from default 60 seconds
     requestHandler: async ({ page, request, response, crawler, sendRequest, enqueueLinks }) => {
       const browserContext: BrowserContext = page.context();

package/src/crawlers/crawlIntelligentSitemap.ts

@@ -50,7 +50,13 @@ const crawlIntelligentSitemap = async (
     const homeUrl = getHomeUrl(link);
     let sitemapLinkFound = false;
     let sitemapLink = '';
-    const chromiumBrowser = await chromium.launch({ headless: true, channel: 'chrome' });
+    const chromiumBrowser = await chromium.launch(
+      {
+        headless: false,
+        channel: 'chrome',
+        args: ['--headless=new', '--no-sandbox']
+      });
+
     const page = await chromiumBrowser.newPage();
     for (const path of sitemapPaths) {
       sitemapLink = homeUrl + path;

package/src/crawlers/crawlLocalFile.ts

@@ -143,7 +143,7 @@ const crawlLocalFile = async (
 
   if (!isUrlPdf(request.url)) {
     const browserContext = await constants.launcher.launchPersistentContext('', {
-      headless: process.env.CRAWLEE_HEADLESS === '1',
+      headless: false,
       ...getPlaywrightLaunchOptions(browser),
       ...playwrightDeviceDetailsObject,
     });

package/src/crawlers/crawlSitemap.ts

@@ -144,7 +144,7 @@ const crawlSitemap = async (
       launcher: constants.launcher,
       launchOptions: getPlaywrightLaunchOptions(browser),
       // Bug in Chrome which causes browser pool crash when userDataDirectory is set in non-headless mode
-      userDataDir,
+      ...(process.env.CRAWLEE_HEADLESS === '0' && { userDataDir }),
     },
     retryOnBlocked: true,
     browserPoolOptions: {