@govtechsg/oobee 0.10.34 → 0.10.39

package/src/constants/constants.ts

@@ -7,6 +7,7 @@ import os from 'os';
 import { spawnSync, execSync } from 'child_process';
 import { chromium } from 'playwright';
 import { silentLogger } from '../logs.js';
+import { PageInfo } from '../mergeAxeResults.js';
 
 const filename = fileURLToPath(import.meta.url);
 const dirname = path.dirname(filename);
@@ -177,16 +178,16 @@ export const basicAuthRegex = /^.*\/\/.*:.*@.*$/i;
 export const axeScript = path.join(dirname, '../../node_modules/axe-core/axe.min.js');
 export class UrlsCrawled {
   toScan: string[] = [];
-  scanned: { url: string; actualUrl: string; pageTitle: string }[] = [];
-  invalid: { url: string; actualUrl: string; pageTitle: string }[] = [];
+  scanned: PageInfo[] = [];
+  invalid: PageInfo[] = [];
   scannedRedirects: { fromUrl: string; toUrl: string }[] = [];
   notScannedRedirects: { fromUrl: string; toUrl: string }[] = [];
-  outOfDomain: string[] = [];
-  blacklisted: { url: string; actualUrl: string; pageTitle: string }[] = [];
-  error: { url: string }[] = [];
-  exceededRequests: string[] = [];
-  forbidden: { url: string; actualUrl: string; pageTitle: string }[] = [];
-  userExcluded: { url: string; actualUrl: string; pageTitle: string }[] = [];
+  outOfDomain: PageInfo[] = [];
+  blacklisted: PageInfo[] = [];
+  error: PageInfo[] = [];
+  exceededRequests: PageInfo[] = [];
+  forbidden: PageInfo[] = [];
+  userExcluded: PageInfo[] = [];
   everything: string[] = [];
 
   constructor(urlsCrawled?: Partial<UrlsCrawled>) {

package/src/constants/itemTypeDescription.ts

@@ -1,10 +1,10 @@
 const itemTypeDescription = {
   mustFix:
-    'Issues that need to be addressed promptly, as they create significant barriers for persons with disabilities and can prevent them from accessing essential content or features.',
+    'Must Fix issues includes WCAG A & AA success criteria (excluding those requiring review).',
   goodToFix:
-    'Issues that could pose certain challenges for persons with disabilities (PWDs), but are unlikely to completely hinder their access to essential content or features.',
+    'Good to Fix issues includes WCAG Level AAA success criteria issues and all best practice rules that do not necessarily conform to WCAG success criterion but are industry accepted practices that improve the user experience.',
   needsReview:
-    'Occurrences could potentially be false positives, requiring human validation for accuracy.',
+    'Manual Review Required occurrences could potentially be false positive, requiring human validation for accuracy.',
   passed: 'Occurrences that passed the automated checks.',
 };
 

package/src/crawlers/commonCrawlerFunc.ts

@@ -1,7 +1,6 @@
 import crawlee, { CrawlingContext, PlaywrightGotoOptions } from 'crawlee';
 import axe, { AxeResults, ImpactValue, NodeResult, Result, resultGroups, TagValue } from 'axe-core';
 import { BrowserContext, Page } from 'playwright';
-import { xPathToCss } from '../xPathToCss.js';
 import {
   axeScript,
   guiInfoStatusTypes,
@@ -11,10 +10,15 @@ import {
 import { guiInfoLog, silentLogger } from '../logs.js';
 import { takeScreenshotForHTMLElements } from '../screenshotFunc/htmlScreenshotFunc.js';
 import { isFilePath } from '../constants/common.js';
-import { customAxeConfig } from './customAxeFunctions.js';
-import { flagUnlabelledClickableElements } from './custom/flagUnlabelledClickableElements.js';
 import { extractAndGradeText } from './custom/extractAndGradeText.js';
 import { ItemsInfo } from '../mergeAxeResults.js';
+import { evaluateAltText } from './custom/evaluateAltText.js';
+import { escapeCssSelector } from './custom/escapeCssSelector.js';
+import { framesCheck } from './custom/framesCheck.js';
+import { findElementByCssSelector } from './custom/findElementByCssSelector.js';
+import { getAxeConfiguration } from './custom/getAxeConfiguration.js';
+import { flagUnlabelledClickableElements } from './custom/flagUnlabelledClickableElements.js';
+import { xPathToCss } from './custom/xPathToCss.js';
 
 // types
 interface AxeResultsWithScreenshot extends AxeResults {
@@ -86,17 +90,17 @@ export const filterAxeResults = (
     const conformance = tags.filter(tag => tag.startsWith('wcag') || tag === 'best-practice');
 
     // handle rare cases where conformance level is not the first element
-    const levels = ['wcag2a', 'wcag2aa', 'wcag2aaa'];
-    if (conformance[0] !== 'best-practice' && !levels.includes(conformance[0])) {
+    const wcagRegex = /^wcag\d+a+$/;
+
+    if (conformance[0] !== 'best-practice' && !wcagRegex.test(conformance[0])) {
       conformance.sort((a, b) => {
-        if (levels.includes(a)) {
-          return -1;
-        }
-        if (levels.includes(b)) {
-          return 1;
-        }
-
-        return 0;
+      if (wcagRegex.test(a) && !wcagRegex.test(b)) {
+        return -1;
+      }
+      if (!wcagRegex.test(a) && wcagRegex.test(b)) {
+        return 1;
+      }
+      return 0;
       });
     }
 
@@ -138,9 +142,13 @@ export const filterAxeResults = (
 
     nodes.forEach(node => {
       const { impact } = node;
+      const hasWcagA = conformance.some(tag => /^wcag\d*a$/.test(tag));
+      const hasWcagAA = conformance.some(tag => /^wcag\d*aa$/.test(tag));
+      // const hasWcagAAA = conformance.some(tag => /^wcag\d*aaa$/.test(tag));
+
       if (displayNeedsReview) {
         addTo(needsReview, node);
-      } else if (impact === 'critical' || impact === 'serious') {
+      } else if (hasWcagA || hasWcagAA) {
         addTo(mustFix, node);
       } else {
         addTo(goodToFix, node);
@@ -283,21 +291,6 @@ export const runAxeScript = async ({
   });
 
   const disableOobee = ruleset.includes(RuleFlags.DISABLE_OOBEE);
-  const oobeeAccessibleLabelFlaggedXpaths = disableOobee
-    ? []
-    : (await flagUnlabelledClickableElements(page)).map(item => item.xpath);
-  const oobeeAccessibleLabelFlaggedCssSelectors = oobeeAccessibleLabelFlaggedXpaths
-    .map(xpath => {
-      try {
-        const cssSelector = xPathToCss(xpath);
-        return cssSelector;
-      } catch (e) {
-        console.error('Error converting XPath to CSS: ', xpath, e);
-        return '';
-      }
-    })
-    .filter(item => item !== '');
-
   const enableWcagAaa = ruleset.includes(RuleFlags.ENABLE_WCAG_AAA);
 
   const gradingReadabilityFlag = await extractAndGradeText(page); // Ensure flag is obtained before proceeding
@@ -308,103 +301,52 @@ export const runAxeScript = async ({
     async ({
       selectors,
       saflyIconSelector,
-      customAxeConfig,
       disableOobee,
       enableWcagAaa,
-      oobeeAccessibleLabelFlaggedCssSelectors,
       gradingReadabilityFlag,
+      evaluateAltTextFunctionString,
+      escapeCssSelectorFunctionString,
+      framesCheckFunctionString,
+      findElementByCssSelectorFunctionString,
+      getAxeConfigurationFunctionString,
+      flagUnlabelledClickableElementsFunctionString,
+      xPathToCssFunctionString,
     }) => {
       try {
-        const evaluateAltText = (node: Element) => {
-          const altText = node.getAttribute('alt');
-          const confusingTexts = ['img', 'image', 'picture', 'photo', 'graphic'];
-
-          if (altText) {
-            const trimmedAltText = altText.trim().toLowerCase();
-            if (confusingTexts.includes(trimmedAltText)) {
-              return false;
-            }
-          }
-          return true;
-        };
-
-        // for css id selectors starting with a digit, escape it with the unicode character e.g. #123 -> #\31 23
-        const escapeCSSSelector = (selector: string) => {
-          try {
-            return selector.replace(
-              /([#\.])(\d)/g,
-              (_match, prefix, digit) => `${prefix}\\3${digit} `,
-            );
-          } catch (e) {
-            console.error(`error escaping css selector: ${selector}`, e);
-            return selector;
-          }
-        };
-
+        // Load functions into the browser context
+        eval(evaluateAltTextFunctionString);
+        eval(escapeCssSelectorFunctionString);
+        eval(framesCheckFunctionString);
+        eval(findElementByCssSelectorFunctionString);
+        eval(flagUnlabelledClickableElementsFunctionString);
+        eval(xPathToCssFunctionString);
+        eval(getAxeConfigurationFunctionString);
         // remove so that axe does not scan
         document.querySelector(saflyIconSelector)?.remove();
 
-        axe.configure({
-          branding: customAxeConfig.branding,
-          checks: [
-            {
-              ...customAxeConfig.checks[0],
-              evaluate: evaluateAltText,
-            },
-            {
-              ...customAxeConfig.checks[1],
-              evaluate: (node: HTMLElement) => {
-                return !node.dataset.flagged; // fail any element with a data-flagged attribute set to true
-              },
-            },
-            ...(enableWcagAaa
-              ? [
-                {
-                  ...customAxeConfig.checks[2],
-                  evaluate: (_node: HTMLElement) => {
-                    if (gradingReadabilityFlag === '') {
-                      return true; // Pass if no readability issues
-                    }
-                    // Dynamically update the grading messages
-                    const gradingCheck = customAxeConfig.checks.find(
-                      check => check.id === 'oobee-grading-text-contents',
-                    );
-                    if (gradingCheck) {
-                      gradingCheck.metadata.messages.incomplete = `The text content is potentially difficult to read, with a Flesch-Kincaid Reading Ease score of ${gradingReadabilityFlag
-                        }.\nThe target passing score is above 50, indicating content readable by university students and lower grade levels.\nA higher score reflects better readability.`;
-                    }
-
-                    // Fail if readability issues are detected
-                  },
-                },
-              ]
-              : []),
-          ],
-          rules: customAxeConfig.rules
-            .filter(rule => (disableOobee ? !rule.id.startsWith('oobee') : true))
-            .concat(
-              enableWcagAaa
-                ? [
-                  {
-                    id: 'color-contrast-enhanced',
-                    enabled: true,
-                    tags: ['wcag2aaa', 'wcag146'],
-                  },
-                  {
-                    id: 'identical-links-same-purpose',
-                    enabled: true,
-                    tags: ['wcag2aaa', 'wcag249'],
-                  },
-                  {
-                    id: 'meta-refresh-no-exceptions',
-                    enabled: true,
-                    tags: ['wcag2aaa', 'wcag224', 'wcag325'],
-                  },
-                ]
-                : [],
-            ),
+        const oobeeAccessibleLabelFlaggedXpaths = disableOobee
+          ? []
+          : (await flagUnlabelledClickableElements()).map(item => item.xpath);
+        const oobeeAccessibleLabelFlaggedCssSelectors = oobeeAccessibleLabelFlaggedXpaths
+          .map(xpath => {
+            try {
+              const cssSelector = xPathToCss(xpath);
+              return cssSelector;
+            } catch (e) {
+              console.error('Error converting XPath to CSS: ', xpath, e);
+              return '';
+            }
+          })
+          .filter(item => item !== '');
+
+        const axeConfig = getAxeConfiguration({
+          enableWcagAaa,
+          gradingReadabilityFlag,
+          disableOobee,
         });
 
+        axe.configure(axeConfig);
+
         // removed needsReview condition
         const defaultResultTypes: resultGroups[] = ['violations', 'passes', 'incomplete'];
 
@@ -418,102 +360,7 @@ export const runAxeScript = async ({
             }
             // handle css id selectors that start with a digit
             const escapedCssSelectors =
-              oobeeAccessibleLabelFlaggedCssSelectors.map(escapeCSSSelector);
-
-            function framesCheck(cssSelector: string): {
-              doc: Document;
-              remainingSelector: string;
-            } {
-              let doc = document; // Start with the main document
-              let remainingSelector = ''; // To store the last part of the selector
-              let targetIframe = null;
-
-              // Split the selector into parts at "> html"
-              const diffParts = cssSelector.split(/\s*>\s*html\s*/);
-
-              for (let i = 0; i < diffParts.length - 1; i++) {
-                let iframeSelector = `${diffParts[i].trim()}`;
-
-                // Add back '> html' to the current part
-                if (i > 0) {
-                  iframeSelector = `html > ${iframeSelector}`;
-                }
-
-                let frameset = null;
-                // Find the iframe using the current document context
-                if (doc.querySelector('frameset')) {
-                  frameset = doc.querySelector('frameset');
-                }
-
-                if (frameset) {
-                  doc = frameset;
-                  iframeSelector = iframeSelector.split('body >')[1].trim();
-                }
-                targetIframe = doc.querySelector(iframeSelector);
-
-                if (targetIframe && targetIframe.contentDocument) {
-                  // Update the document to the iframe's contentDocument
-                  doc = targetIframe.contentDocument;
-                } else {
-                  console.warn(
-                    `Iframe not found or contentDocument inaccessible for selector: ${iframeSelector}`,
-                  );
-                  return { doc, remainingSelector: cssSelector }; // Return original selector if iframe not found
-                }
-              }
-
-              // The last part is the remaining CSS selector
-              remainingSelector = diffParts[diffParts.length - 1].trim();
-
-              // Remove any leading '>' combinators from remainingSelector
-              remainingSelector = `html${remainingSelector}`;
-
-              return { doc, remainingSelector };
-            }
-
-            function findElementByCssSelector(cssSelector: string): string | null {
-              let doc = document;
-
-              // Check if the selector includes 'frame' or 'iframe' and update doc and selector
-
-              if (/\s*>\s*html\s*/.test(cssSelector)) {
-                const inFrames = framesCheck(cssSelector);
-                doc = inFrames.doc;
-                cssSelector = inFrames.remainingSelector;
-              }
-
-              // Query the element in the document (including inside frames)
-              let element = doc.querySelector(cssSelector);
-
-              // Handle Shadow DOM if the element is not found
-              if (!element) {
-                const shadowRoots = [];
-                const allElements = document.querySelectorAll('*');
-
-                // Look for elements with shadow roots
-                allElements.forEach(el => {
-                  if (el.shadowRoot) {
-                    shadowRoots.push(el.shadowRoot);
-                  }
-                });
-
-                // Search inside each shadow root for the element
-                for (const shadowRoot of shadowRoots) {
-                  const shadowElement = shadowRoot.querySelector(cssSelector);
-                  if (shadowElement) {
-                    element = shadowElement; // Found the element inside shadow DOM
-                    break;
-                  }
-                }
-              }
-
-              if (element) {
-                return element.outerHTML;
-              }
-
-              console.warn(`Unable to find element for css selector: ${cssSelector}`);
-              return null;
-            }
+              oobeeAccessibleLabelFlaggedCssSelectors.map(escapeCssSelector);
 
             // Add oobee violations to Axe's report
             const oobeeAccessibleLabelViolations = {
@@ -560,11 +407,17 @@ export const runAxeScript = async ({
     {
       selectors,
       saflyIconSelector,
-      customAxeConfig,
       disableOobee,
       enableWcagAaa,
-      oobeeAccessibleLabelFlaggedCssSelectors,
       gradingReadabilityFlag,
+      evaluateAltTextFunctionString: evaluateAltText.toString(),
+      escapeCssSelectorFunctionString: escapeCssSelector.toString(),
+      framesCheckFunctionString: framesCheck.toString(),
+      findElementByCssSelectorFunctionString: findElementByCssSelector.toString(),
+      getAxeConfigurationFunctionString: getAxeConfiguration.toString(),
+      flagUnlabelledClickableElementsFunctionString:
+        flagUnlabelledClickableElements.toString(),
+      xPathToCssFunctionString: xPathToCss.toString(),
     },
   );
 

package/src/crawlers/crawlDomain.ts

@@ -29,6 +29,7 @@ import {
   getBlackListedPatterns,
   urlWithoutAuth,
   waitForPageLoaded,
+  initModifiedUserAgent,
 } from '../constants/common.js';
 import { areLinksEqual, isFollowStrategy } from '../utils.js';
 import {
@@ -455,6 +456,8 @@ const crawlDomain = async ({
     userDataDir = process.env.CRAWLEE_HEADLESS !== '0' ? userDataDirectory : '';
   }
 
+  await initModifiedUserAgent(browser, playwrightDeviceDetailsObject);
+  
   const crawler = new crawlee.PlaywrightCrawler({
     launchContext: {
       launcher: constants.launcher,
@@ -847,7 +850,7 @@ const crawlDomain = async ({
         // when max pages have been scanned, scan will abort and all relevant pages still opened will close instantly.
         // a browser close error will then be flagged. Since this is an intended behaviour, this error will be excluded.
         if (!isAbortingScanNow) {
-          urlsCrawled.error.push({ url: request.url });
+            urlsCrawled.error.push({ url: request.url, pageTitle: request.url, actualUrl: request.url });
         }
       }
     },
@@ -856,7 +859,8 @@ const crawlDomain = async ({
         numScanned: urlsCrawled.scanned.length,
         urlScanned: request.url,
       });
-      urlsCrawled.error.push({ url: request.url });
+      urlsCrawled.error.push({ url: request.url, pageTitle: request.url, actualUrl: request.url });
+    
       crawlee.log.error(`Failed Request - ${request.url}: ${request.errorMessages}`);
     },
     maxRequestsPerCrawl: Infinity,

package/src/crawlers/crawlLocalFile.ts

@@ -10,6 +10,7 @@ import {
   isFilePath,
   convertLocalFileToPath,
   convertPathToLocalFile,
+  initModifiedUserAgent,
 } from '../constants/common.js';
 import { runPdfScan, mapPdfScanResults, doPdfScreenshots } from './pdfScanFunc.js';
 import { guiInfoLog } from '../logs.js';
@@ -142,6 +143,7 @@ const crawlLocalFile = async (
   uuidToPdfMapping[pdfFileName] = trimmedUrl;
 
   if (!isUrlPdf(request.url)) {
+    await initModifiedUserAgent(browser);
     const browserContext = await constants.launcher.launchPersistentContext('', {
       headless: false,
       ...getPlaywrightLaunchOptions(browser),

package/src/crawlers/crawlSitemap.ts

@@ -17,6 +17,7 @@ import {
   urlWithoutAuth,
   waitForPageLoaded,
   isFilePath,
+  initModifiedUserAgent,
 } from '../constants/common.js';
 import { areLinksEqual, isWhitelistedContentType, isFollowStrategy } from '../utils.js';
 import { handlePdfDownload, runPdfScan, mapPdfScanResults } from './pdfScanFunc.js';
@@ -139,6 +140,7 @@ const crawlSitemap = async (
     userDataDir = process.env.CRAWLEE_HEADLESS !== '0' ? userDataDirectory : '';
   }
 
+  await initModifiedUserAgent(browser, playwrightDeviceDetailsObject);
   const crawler = new crawlee.PlaywrightCrawler({
     launchContext: {
       launcher: constants.launcher,
@@ -271,8 +273,8 @@ const crawlSitemap = async (
         return;
       }
 
-      const contentType = response.headers()['content-type'];
-      const status = response.status();
+      const contentType = response?.headers?.()['content-type'] || '';
+      const status = response ? response.status() : 0;
 
       if (blacklistedPatterns && !isFollowStrategy(actualUrl, request.url, "same-hostname") && isSkippedUrl(actualUrl, blacklistedPatterns)) {
         urlsCrawled.userExcluded.push({
@@ -379,7 +381,7 @@ const crawlSitemap = async (
         numScanned: urlsCrawled.scanned.length,
         urlScanned: request.url,
       });
-      urlsCrawled.error.push({ url: request.url });
+      urlsCrawled.error.push(request.url);
       crawlee.log.error(`Failed Request - ${request.url}: ${request.errorMessages}`);
     },
     maxRequestsPerCrawl: Infinity,

package/src/crawlers/custom/escapeCssSelector.ts

@@ -0,0 +1,10 @@
+// for css id selectors starting with a digit, escape it with the unicode character e.g. #123 -> #\31 23
+export function escapeCssSelector(selector: string) {
+  try {
+    return selector.replace(/([#\.])(\d)/g, (_match, prefix, digit) => `${prefix}\\3${digit} `);
+  } catch (e) {
+    console.error(`error escaping css selector: ${selector}`, e);
+    return selector;
+  }
+}
+

package/src/crawlers/custom/evaluateAltText.ts

@@ -0,0 +1,13 @@
+export function evaluateAltText(node: Element) {
+  const altText = node.getAttribute('alt');
+  const confusingTexts = ['img', 'image', 'picture', 'photo', 'graphic'];
+
+  if (altText) {
+    const trimmedAltText = altText.trim().toLowerCase();
+    if (confusingTexts.includes(trimmedAltText)) {
+      return false;
+    }
+  }
+  return true;
+}
+

package/src/crawlers/custom/extractAndGradeText.ts

@@ -47,8 +47,6 @@ export async function extractAndGradeText(page: Page): Promise<string> {
     const result =
       readabilityScore === 0 || readabilityScore > 50 ? '' : readabilityScore.toString(); // Convert readabilityScore to string
 
-    const pageUrl = await page.url(); // Get the page URL
-
     return result;
   } catch (error) {
     console.error('Error extracting and grading text:', error);

package/src/crawlers/custom/extractText.ts

@@ -0,0 +1,28 @@
+export function extractText(): string[] {
+  try {
+    // Extract text content from all specified elements (e.g., paragraphs)
+    const elements = document.querySelectorAll('p'); // Adjust selector as needed
+    const extractedSentences: string[] = [];
+
+    elements.forEach(element => {
+      const text = element.innerText.trim();
+      // Split the text into individual sentences
+      const sentencePattern = /[^.!?]*[.!?]+/g; // Match sentences ending with ., !, or ?
+      const matches = text.match(sentencePattern);
+      if (matches) {
+        // Add only sentences that end with punctuation
+        matches.forEach(sentence => {
+          const trimmedSentence = sentence.trim(); // Trim whitespace from each sentence
+          if (trimmedSentence.length > 0) {
+            extractedSentences.push(trimmedSentence);
+          }
+        });
+      }
+    });
+
+    return extractedSentences;
+  } catch (error) {
+    console.error('Error extracting text:', error);
+    return []; // Return an empty string in case of an error
+  }
+}

package/src/crawlers/custom/findElementByCssSelector.ts

@@ -0,0 +1,46 @@
+import { framesCheck } from "./framesCheck.js";
+
+export function findElementByCssSelector(cssSelector: string): string | null {
+  let doc = document;
+
+  // Check if the selector includes 'frame' or 'iframe' and update doc and selector
+
+  if (/\s*>\s*html\s*/.test(cssSelector)) {
+    const inFrames = framesCheck(cssSelector);
+    doc = inFrames.doc;
+    cssSelector = inFrames.remainingSelector;
+  }
+
+  // Query the element in the document (including inside frames)
+  let element = doc.querySelector(cssSelector);
+
+  // Handle Shadow DOM if the element is not found
+  if (!element) {
+    const shadowRoots = [];
+    const allElements = document.querySelectorAll('*');
+
+    // Look for elements with shadow roots
+    allElements.forEach(el => {
+      if (el.shadowRoot) {
+        shadowRoots.push(el.shadowRoot);
+      }
+    });
+
+    // Search inside each shadow root for the element
+    for (const shadowRoot of shadowRoots) {
+      const shadowElement = shadowRoot.querySelector(cssSelector);
+      if (shadowElement) {
+        element = shadowElement; // Found the element inside shadow DOM
+        break;
+      }
+    }
+  }
+
+  if (element) {
+    return element.outerHTML;
+  }
+
+  console.warn(`Unable to find element for css selector: ${cssSelector}`);
+  return null;
+}
+