@govtechsg/oobee 0.10.20 → 0.10.28

package/package.json

@@ -1,13 +1,15 @@
 {
   "name": "@govtechsg/oobee",
   "main": "dist/npmIndex.js",
-  "version": "0.10.20",
+  "version": "0.10.28",
   "type": "module",
+  "author": "Government Technology Agency <info@tech.gov.sg>",
   "dependencies": {
     "@json2csv/node": "^7.0.3",
     "@napi-rs/canvas": "^0.1.53",
     "axe-core": "^4.10.2",
     "axios": "^1.7.4",
+    "base64-stream": "^1.0.0",
     "cheerio": "^1.0.0-rc.12",
     "crawlee": "^3.11.1",
     "ejs": "^3.1.9",
@@ -20,8 +22,8 @@
     "lodash": "^4.17.21",
     "mime-types": "^2.1.35",
     "minimatch": "^9.0.3",
-    "pdfjs-dist": "github:veraPDF/pdfjs-dist#v2.14.305-taggedPdf-0.1.11",
-    "playwright": "1.46.1",
+    "pdfjs-dist": "github:veraPDF/pdfjs-dist#v4.4.168-taggedPdf-0.1.20",
+    "playwright": "1.49.1",
     "prettier": "^3.1.0",
     "print-message": "^3.0.1",
     "safe-regex": "^2.1.1",
@@ -39,6 +41,7 @@
   "devDependencies": {
     "@eslint/eslintrc": "^3.0.2",
     "@eslint/js": "^9.6.0",
+    "@types/base64-stream": "^1.0.5",
     "@types/eslint__js": "^8.42.3",
     "@types/fs-extra": "^11.0.4",
     "@types/inquirer": "^9.0.7",
@@ -47,6 +50,7 @@
     "@types/validator": "^13.11.10",
     "@types/which": "^3.0.4",
     "@types/xml2js": "^0.4.14",
+    "browserify-zlib": "^0.2.0",
     "eslint": "^8.57.0",
     "eslint-config-airbnb-base": "^15.0.0",
     "eslint-config-prettier": "^8.6.0",
@@ -54,6 +58,7 @@
     "eslint-plugin-prettier": "^5.0.0",
     "globals": "^15.2.0",
     "jest": "^29.7.0",
+    "readable-stream": "^4.7.0",
     "typescript-eslint": "^8.3.0"
   },
   "overrides": {
@@ -82,7 +87,6 @@
     "lint": "eslint . --report-unused-disable-directives --max-warnings 0",
     "lint:fix": "eslint . --fix --report-unused-disable-directives --max-warnings 0"
   },
-  "author": "",
   "license": "MIT",
   "description": "Oobee is a customisable, automated accessibility testing tool that allows software development teams to assess whether their products are user-friendly to persons with disabilities (PWDs).",
   "repository": {
@@ -93,4 +97,4 @@
     "url": "https://github.com/GovTechSG/oobee/issues"
   },
   "homepage": "https://github.com/GovTechSG/oobee#readme"
-}
+}

package/scripts/decodeUnzipParse.js

@@ -0,0 +1,29 @@
+/* Run the below command to get /src/static/ejs/partials/scripts/decodeUnzipParse.ejs */
+/* ( echo '<script>'; npx browserify decodeUnzipParse.js --standalone decodeUnzipParse | npx uglify-js; echo '</script>' ) > decodeUnzipParse.ejs */
+const { Readable } = require('readable-stream');
+const { Base64Decode } = require('base64-stream');
+const { createGunzip } = require('browserify-zlib');
+const { parser } = require('stream-json');
+const Asm = require('stream-json/Assembler');
+
+module.exports = function parseGzipBase64Json(base64String) {
+  return new Promise((resolve, reject) => {
+    const pipeline = Readable.from([base64String])
+      .pipe(new Base64Decode())
+      .pipe(createGunzip())
+      .pipe(parser());
+
+    // Assembler to assemble the streamed JSON tokens
+    const assembler = Asm.connectTo(pipeline);
+
+    // On 'done' event, the entire JSON object is ready
+    assembler.on('done', asm => {
+      resolve(asm.current);
+    });
+
+    // If anything goes wrong at any stage, reject the promise
+    pipeline.on('error', err => {
+      reject(err);
+    });
+  });
+};

package/scripts/install_oobee_dependencies.command

@@ -1,6 +1,6 @@
 #!/bin/bash
 
-NODE_VERSION="20.10.0"
+NODE_VERSION="22.13.1"
 
 # Get current shell command
 SHELL_COMMAND=$(ps -o comm= -p $$)
@@ -98,4 +98,4 @@ if [ "$(uname -m)" = "arm64" ] && /usr/bin/pgrep oahd >/dev/null 2>&1; then
 fi
 
 echo "Build TypeScript"
-npm run build || true
+npm run build || true

package/scripts/install_oobee_dependencies.ps1

@@ -9,11 +9,11 @@ $ErrorActionPreference = 'Stop'
 # Install NodeJS binaries
 if (-Not (Test-Path nodejs-win\node.exe)) {
     Write-Output "Downloading Node"
-    Invoke-WebRequest -o ./nodejs-win.zip "https://nodejs.org/dist/v20.10.0/node-v20.10.0-win-x64.zip"     
+    Invoke-WebRequest -o ./nodejs-win.zip "https://nodejs.org/dist/v22.13.1/node-v22.13.1-win-x64.zip"     
     
     Write-Output "Unzip Node"
     Expand-Archive .\nodejs-win.zip -DestinationPath .
-    Rename-Item node-v20.10.0-win-x64 -NewName nodejs-win
+    Rename-Item node-v22.13.1-win-x64 -NewName nodejs-win
     Remove-Item -Force .\nodejs-win.zip
 }
 
@@ -107,4 +107,4 @@ if (Test-Path oobee) {
     } else {
         Write-Output "Could not find oobee"
     }
-}
+}

package/src/cli.ts

@@ -184,11 +184,10 @@ Usage: npm run cli -- -c <crawler> -d <device> -w <viewport> -u <url> OPTIONS`,
     return option;
   })
   .check(argvs => {
-    if (
-      (argvs.scanner === ScannerTypes.CUSTOM || argvs.scanner === ScannerTypes.LOCALFILE) &&
-      argvs.maxpages
-    ) {
-      throw new Error('-p or --maxpages is only available in website and sitemap scans.');
+    if (argvs.scanner === ScannerTypes.CUSTOM && argvs.maxpages) {
+      throw new Error(
+        '-p or --maxpages is only available in website, sitemap and local file scans.',
+      );
     }
     return true;
   })
@@ -209,6 +208,9 @@ const scanInit = async (argvs: Answers): Promise<string> => {
 
   const updatedArgvs = { ...argvs };
 
+  // Cannot use data.browser and data.isHeadless as the connectivity check comes first before prepareData
+  setHeadlessMode(updatedArgvs.browserToRun, updatedArgvs.headless);
+
   // let chromeDataDir = null;
   // let edgeDataDir = null;
   // Empty string for profile directory will use incognito mode in playwright
@@ -338,8 +340,6 @@ const scanInit = async (argvs: Answers): Promise<string> => {
     }
   }
 
-  setHeadlessMode(data.browser, data.isHeadless);
-
   const screenToScan = getScreenToScan(
     updatedArgvs.deviceChosen,
     updatedArgvs.customDevice,
@@ -394,7 +394,9 @@ const optionsAnswer: Answers = {
   blacklistedPatternsFilename: options.blacklistedPatternsFilename,
   playwrightDeviceDetailsObject: options.playwrightDeviceDetailsObject,
   ruleset: options.ruleset,
+  generateJsonFiles: options.generateJsonFiles,
 };
+
 await scanInit(optionsAnswer);
 process.exit(0);
 

package/src/combine.ts

@@ -48,18 +48,19 @@ const combineRun = async (details: Data, deviceToScan: string) => {
     maxRequestsPerCrawl,
     browser,
     userDataDirectory,
-    strategy,
-    specifiedMaxConcurrency,
+    strategy, // Allow subdomains: if checked, = 'same-domain'
+    specifiedMaxConcurrency, // Slow scan mode: if checked, = '1'
     fileTypes,
     blacklistedPatternsFilename,
-    includeScreenshots,
-    followRobots,
+    includeScreenshots, // Include screenshots: if checked, = 'true'
+    followRobots, // Adhere to robots.txt: if checked, = 'true'
     metadata,
     customFlowLabel = 'Custom Flow',
     extraHTTPHeaders,
     safeMode,
     zip,
-    ruleset,
+    ruleset, // Enable custom checks, Enable WCAG AAA: if checked, = 'enable-wcag-aaa')
+    generateJsonFiles,
   } = envDetails;
 
   process.env.CRAWLEE_LOG_LEVEL = 'ERROR';
@@ -90,6 +91,12 @@ const combineRun = async (details: Data, deviceToScan: string) => {
     crawlType: type,
     requestUrl: finalUrl,
     urlsCrawled: new UrlsCrawled(),
+    isIncludeScreenshots: envDetails.includeScreenshots,
+    isAllowSubdomains: envDetails.strategy,
+    isEnableCustomChecks: envDetails.ruleset,
+    isEnableWcagAaa: envDetails.ruleset,
+    isSlowScanMode: envDetails.specifiedMaxConcurrency,
+    isAdhereRobots: envDetails.followRobots,
   };
 
   const viewportSettings: ViewportSettingsClass = new ViewportSettingsClass(
@@ -214,6 +221,7 @@ const combineRun = async (details: Data, deviceToScan: string) => {
         undefined,
         scanDetails,
         zip,
+        generateJsonFiles,
       );
       const [name, email] = nameEmail.split(':');
 

package/src/constants/cliFunctions.ts

@@ -270,7 +270,9 @@ export const cliOptions: { [key: string]: Options } = {
     coerce: option => {
       const validChoices = Object.values(RuleFlags);
       const userChoices: string[] = option.split(',');
-      const invalidUserChoices = userChoices.filter(choice => !validChoices.includes(choice as RuleFlags));
+      const invalidUserChoices = userChoices.filter(
+        choice => !validChoices.includes(choice as RuleFlags),
+      );
       if (invalidUserChoices.length > 0) {
         printMessage(
           [
@@ -294,6 +296,41 @@ export const cliOptions: { [key: string]: Options } = {
       return userChoices;
     },
   },
+  g: {
+    alias: 'generateJsonFiles',
+    describe: `Generate two gzipped and base64-encoded JSON files containing the results of the accessibility scan:\n
+1. scanData.json.gz.b64: Provides an overview of the scan, including:
+   - WCAG compliance score
+   - Violated WCAG clauses
+   - Metadata (e.g., scan start and end times)
+   - Pages scanned and skipped
+2. scanItems.json.gz.b64: Contains detailed information about detected accessibility issues, including:
+   - Severity levels
+   - Issue descriptions
+   - Related WCAG guidelines
+   - URL of the pages violated the WCAG clauses
+Useful for in-depth analysis or integration with external reporting tools.\n
+To obtain the JSON files, you need to base64-decode the file followed by gunzip. For example:\n
+(macOS) base64 -D -i scanData.json.gz.b64 | gunzip > scanData.json\n
+(linux) base64 -d scanData.json.gz.b64 | gunzip > scanData.json\n
+`,
+    type: 'string',
+    requiresArg: true,
+    default: 'no',
+    demandOption: false,
+    coerce: value => {
+      const validYes = ['yes', 'y'];
+      const validNo = ['no', 'n'];
+
+      if (validYes.includes(value.toLowerCase())) {
+        return true;
+      }
+      if (validNo.includes(value.toLowerCase())) {
+        return false;
+      }
+      throw new Error(`Invalid value "${value}" for --generate. Use "yes", "y", "no", or "n".`);
+    },
+  },
 };
 
 export const configureReportSetting = (isEnabled: boolean): void => {

package/src/constants/common.ts

@@ -402,8 +402,16 @@ const checkUrlConnectivityWithBrowser = async (
     let browserContext;
 
     try {
+      // Temporary browserContextLaunchOptions to force headless mode during connectivity check
+      // If a user selects cli options h=no, the connectivity check should still proceed in headless
+      const launchOptions = getPlaywrightLaunchOptions(browserToRun);
+      const browserContextLaunchOptions = {
+        ...launchOptions,
+        args: [...launchOptions.args, '--headless=new'],
+      };
+
       browserContext = await constants.launcher.launchPersistentContext(clonedDataDir, {
-        ...getPlaywrightLaunchOptions(browserToRun),
+        ...browserContextLaunchOptions,
         ...(viewport && { viewport }),
         ...(userAgent && { userAgent }),
         ...(extraHTTPHeaders && { extraHTTPHeaders }),
@@ -437,6 +445,7 @@ const checkUrlConnectivityWithBrowser = async (
         silentLogger.info('Unable to detect networkidle');
       }
 
+      // This response state doesn't seem to work with the new headless=new flag
       if (response.status() === 401) {
         res.status = constants.urlCheckStatuses.unauthorised.code;
       } else {
@@ -459,8 +468,14 @@ const checkUrlConnectivityWithBrowser = async (
         res.content = responseFromUrl.content;
       }
     } catch (error) {
-      silentLogger.error(error);
-      res.status = constants.urlCheckStatuses.systemError.code;
+
+      // But this does work with the headless=new flag
+      if (error.message.includes('net::ERR_INVALID_AUTH_CREDENTIALS')) {
+        res.status = constants.urlCheckStatuses.unauthorised.code;
+      } else {
+        // enters here if input is not a URL or not using http/https protocols
+        res.status = constants.urlCheckStatuses.systemError.code;
+      }
     } finally {
       await browserContext.close();
     }
@@ -579,6 +594,7 @@ export const prepareData = async (argv: Answers): Promise<Data> => {
     safeMode,
     zip,
     ruleset,
+    generateJsonFiles,
   } = argv;
 
   // construct filename for scan results
@@ -625,6 +641,7 @@ export const prepareData = async (argv: Answers): Promise<Data> => {
     safeMode,
     zip,
     ruleset,
+    generateJsonFiles,
   };
 };
 
@@ -1768,15 +1785,24 @@ export const getPlaywrightLaunchOptions = (browser?: string): LaunchOptions => {
   if (browser) {
     channel = browser;
   }
+
+  // Set new headless mode as Chrome 132 does not support headless=old
+  if (process.env.CRAWLEE_HEADLESS === '1') constants.launchOptionsArgs.push('--headless=new');
+
   const options: LaunchOptions = {
     // Drop the --use-mock-keychain flag to allow MacOS devices
     // to use the cloned cookies.
-    ignoreDefaultArgs: ['--use-mock-keychain'],
+    ignoreDefaultArgs: ['--use-mock-keychain', '--headless'],
+    // necessary from Chrome 132 to use our own headless=new flag
     args: constants.launchOptionsArgs,
+    headless: false,
     ...(channel && { channel }), // Having no channel is equivalent to "chromium"
   };
+
+  // Necessary as Chrome 132 does not support headless=old
+  options.headless = false;
+
   if (proxy) {
-    options.headless = false;
     options.slowMo = 1000; // To ensure server-side rendered proxy page is loaded
   } else if (browser === BrowserTypes.EDGE && os.platform() === 'win32') {
     // edge should be in non-headless mode

package/src/constants/constants.ts

@@ -304,33 +304,35 @@ export const sitemapPaths = [
   '/sitemap_index.xml.xz',
 ];
 
+// Remember to update getWcagPassPercentage() in src/utils/utils.ts if you change this
 const wcagLinks = {
-  'WCAG 1.1.1': 'https://www.w3.org/TR/WCAG21/#non-text-content',
-  'WCAG 1.2.2': 'https://www.w3.org/TR/WCAG21/#captions-prerecorded',
-  'WCAG 1.3.1': 'https://www.w3.org/TR/WCAG21/#info-and-relationships',
-  // 'WCAG 1.3.4': 'https://www.w3.org/TR/WCAG21/#orientation', - TODO: review for veraPDF
-  'WCAG 1.3.5': 'https://www.w3.org/TR/WCAG21/#use-of-color',
-  'WCAG 1.4.1': 'https://www.w3.org/TR/WCAG21/#use-of-color',
-  'WCAG 1.4.2': 'https://www.w3.org/TR/WCAG21/#audio-control',
-  'WCAG 1.4.3': 'https://www.w3.org/TR/WCAG21/#contrast-minimum',
-  'WCAG 1.4.4': 'https://www.w3.org/TR/WCAG21/#resize-text',
-  'WCAG 1.4.6': 'https://www.w3.org/TR/WCAG21/#contrast-enhanced',
-  // 'WCAG 1.4.10': 'https://www.w3.org/TR/WCAG21/#reflow', - TODO: review for veraPDF
-  'WCAG 1.4.12': 'https://www.w3.org/TR/WCAG21/#text-spacing',
-  'WCAG 2.1.1': 'https://www.w3.org/TR/WCAG21/#pause-stop-hide',
-  'WCAG 2.2.1': 'https://www.w3.org/TR/WCAG21/#timing-adjustable',
-  'WCAG 2.2.2': 'https://www.w3.org/TR/WCAG21/#pause-stop-hide',
-  'WCAG 2.2.4': 'https://www.w3.org/TR/WCAG21/#interruptions',
-  'WCAG 2.4.1': 'https://www.w3.org/TR/WCAG21/#bypass-blocks',
-  'WCAG 2.4.2': 'https://www.w3.org/TR/WCAG21/#page-titled',
-  'WCAG 2.4.3': 'https://www.w3.org/TR/WCAG21/#focus-order',
-  'WCAG 2.4.4': 'https://www.w3.org/TR/WCAG21/#link-purpose-in-context',
-  'WCAG 2.4.9': 'https://www.w3.org/TR/WCAG21/#link-purpose-link-only',
+  'WCAG 1.1.1': 'https://www.w3.org/TR/WCAG22/#non-text-content',
+  'WCAG 1.2.2': 'https://www.w3.org/TR/WCAG22/#captions-prerecorded',
+  'WCAG 1.3.1': 'https://www.w3.org/TR/WCAG22/#info-and-relationships',
+  // 'WCAG 1.3.4': 'https://www.w3.org/TR/WCAG22/#orientation', - TODO: review for veraPDF
+  'WCAG 1.3.5': 'https://www.w3.org/TR/WCAG22/#use-of-color',
+  'WCAG 1.4.1': 'https://www.w3.org/TR/WCAG22/#use-of-color',
+  'WCAG 1.4.2': 'https://www.w3.org/TR/WCAG22/#audio-control',
+  'WCAG 1.4.3': 'https://www.w3.org/TR/WCAG22/#contrast-minimum',
+  'WCAG 1.4.4': 'https://www.w3.org/TR/WCAG22/#resize-text',
+  'WCAG 1.4.6': 'https://www.w3.org/TR/WCAG22/#contrast-enhanced', // AAA
+  // 'WCAG 1.4.10': 'https://www.w3.org/TR/WCAG22/#reflow', - TODO: review for veraPDF
+  'WCAG 1.4.12': 'https://www.w3.org/TR/WCAG22/#text-spacing',
+  'WCAG 2.1.1': 'https://www.w3.org/TR/WCAG22/#pause-stop-hide',
+  'WCAG 2.2.1': 'https://www.w3.org/TR/WCAG22/#timing-adjustable',
+  'WCAG 2.2.2': 'https://www.w3.org/TR/WCAG22/#pause-stop-hide',
+  'WCAG 2.2.4': 'https://www.w3.org/TR/WCAG22/#interruptions', // AAA
+  'WCAG 2.4.1': 'https://www.w3.org/TR/WCAG22/#bypass-blocks',
+  'WCAG 2.4.2': 'https://www.w3.org/TR/WCAG22/#page-titled',
+  'WCAG 2.4.4': 'https://www.w3.org/TR/WCAG22/#link-purpose-in-context',
+  'WCAG 2.4.9': 'https://www.w3.org/TR/WCAG22/#link-purpose-link-only', // AAA
   'WCAG 2.5.8': 'https://www.w3.org/TR/WCAG22/#target-size-minimum',
-  'WCAG 3.1.1': 'https://www.w3.org/TR/WCAG21/#language-of-page',
-  'WCAG 3.1.2': 'https://www.w3.org/TR/WCAG21/#labels-or-instructions',
-  'WCAG 3.2.5': 'https://www.w3.org/TR/WCAG21/#change-on-request',
-  'WCAG 4.1.2': 'https://www.w3.org/TR/WCAG21/#name-role-value',
+  'WCAG 3.1.1': 'https://www.w3.org/TR/WCAG22/#language-of-page',
+  'WCAG 3.1.2': 'https://www.w3.org/TR/WCAG22/#labels-or-instructions',
+  'WCAG 3.1.5': 'https://www.w3.org/TR/WCAG22/#reading-level', // AAA
+  'WCAG 3.2.5': 'https://www.w3.org/TR/WCAG22/#change-on-request', // AAA
+  'WCAG 3.3.2': 'https://www.w3.org/TR/WCAG22/#labels-or-instructions',
+  'WCAG 4.1.2': 'https://www.w3.org/TR/WCAG22/#name-role-value',
 };
 
 const urlCheckStatuses = {
@@ -423,7 +425,7 @@ export default {
 };
 
 export const rootPath = dirname;
-export const wcagWebPage = 'https://www.w3.org/TR/WCAG21/';
+export const wcagWebPage = 'https://www.w3.org/TR/WCAG22/';
 const latestAxeVersion = '4.9';
 export const axeVersion = latestAxeVersion;
 export const axeWebPage = `https://dequeuniversity.com/rules/axe/${latestAxeVersion}/`;

package/src/constants/questions.ts

@@ -1,6 +1,6 @@
 import { Question } from 'inquirer';
 import { Answers } from '../index.js';
-import { getUserDataTxt } from '../utils.js';
+import { getUserDataTxt, setHeadlessMode } from '../utils.js';
 import {
   checkUrl,
   deleteClonedProfiles,
@@ -79,6 +79,9 @@ const startScanQuestions = [
 
       const statuses = constants.urlCheckStatuses;
       const { browserToRun, clonedBrowserDataDir } = getBrowserToRun(BrowserTypes.CHROME);
+
+      setHeadlessMode(browserToRun, answers.headless);
+
       const playwrightDeviceDetailsObject = getPlaywrightDeviceDetailsObject(
         answers.deviceChosen,
         answers.customDevice,