@govtechsg/oobee 0.10.20 → 0.10.21

package/README.md

@@ -346,6 +346,24 @@ Options:
                                      ks
   [string] [choices: "default", "disable-oobee", "enable-wcag-aaa", "disable-oob
                                        ee,enable-wcag-aaa"] [default: "default"]
+  -g, --generateJsonFiles            Generate two JSON files containing the
+                                     results of the accessibility scan:
+                                     1. `scanData.json`: Provides an overview of
+                                        the scan, including:
+                                        - WCAG compliance score
+                                        - Violated WCAG clauses
+                                        - Metadata (e.g., scan start and end times)
+                                        - Pages scanned and skipped
+                                     2. `scanItems.json`: Contains detailed
+                                        information about detected accessibility
+                                        issues, including:
+                                        - Severity levels
+                                        - Issue descriptions
+                                        - Related WCAG guidelines
+                                        - URL of the pages violated the WCAG clauses
+                                     Useful for in-depth analysis or integration
+                                     with external reporting tools.
+                                     [string] [choices: "yes", "no"] [default: "no"]
 
 Examples:
   To scan sitemap of website:', 'npm run cli -- -c [ 1 | sitemap ] -u <url_lin

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@govtechsg/oobee",
   "main": "dist/npmIndex.js",
-  "version": "0.10.20",
+  "version": "0.10.21",
   "type": "module",
   "dependencies": {
     "@json2csv/node": "^7.0.3",
@@ -93,4 +93,4 @@
     "url": "https://github.com/GovTechSG/oobee/issues"
   },
   "homepage": "https://github.com/GovTechSG/oobee#readme"
-}
+}

package/src/cli.ts

@@ -184,11 +184,10 @@ Usage: npm run cli -- -c <crawler> -d <device> -w <viewport> -u <url> OPTIONS`,
     return option;
   })
   .check(argvs => {
-    if (
-      (argvs.scanner === ScannerTypes.CUSTOM || argvs.scanner === ScannerTypes.LOCALFILE) &&
-      argvs.maxpages
-    ) {
-      throw new Error('-p or --maxpages is only available in website and sitemap scans.');
+    if (argvs.scanner === ScannerTypes.CUSTOM && argvs.maxpages) {
+      throw new Error(
+        '-p or --maxpages is only available in website, sitemap and local file scans.',
+      );
     }
     return true;
   })
@@ -394,7 +393,9 @@ const optionsAnswer: Answers = {
   blacklistedPatternsFilename: options.blacklistedPatternsFilename,
   playwrightDeviceDetailsObject: options.playwrightDeviceDetailsObject,
   ruleset: options.ruleset,
+  generateJsonFiles: options.generateJsonFiles,
 };
+
 await scanInit(optionsAnswer);
 process.exit(0);
 

package/src/combine.ts

@@ -48,18 +48,19 @@ const combineRun = async (details: Data, deviceToScan: string) => {
     maxRequestsPerCrawl,
     browser,
     userDataDirectory,
-    strategy,
-    specifiedMaxConcurrency,
+    strategy, // Allow subdomains: if checked, = 'same-domain'
+    specifiedMaxConcurrency, // Slow scan mode: if checked, = '1'
     fileTypes,
     blacklistedPatternsFilename,
-    includeScreenshots,
-    followRobots,
+    includeScreenshots, // Include screenshots: if checked, = 'true'
+    followRobots, // Adhere to robots.txt: if checked, = 'true'
     metadata,
     customFlowLabel = 'Custom Flow',
     extraHTTPHeaders,
     safeMode,
     zip,
-    ruleset,
+    ruleset, // Enable custom checks, Enable WCAG AAA: if checked, = 'enable-wcag-aaa')
+    generateJsonFiles,
   } = envDetails;
 
   process.env.CRAWLEE_LOG_LEVEL = 'ERROR';
@@ -90,6 +91,12 @@ const combineRun = async (details: Data, deviceToScan: string) => {
     crawlType: type,
     requestUrl: finalUrl,
     urlsCrawled: new UrlsCrawled(),
+    isIncludeScreenshots: envDetails.includeScreenshots,
+    isAllowSubdomains: envDetails.strategy,
+    isEnableCustomChecks: envDetails.ruleset,
+    isEnableWcagAaa: envDetails.ruleset,
+    isSlowScanMode: envDetails.specifiedMaxConcurrency,
+    isAdhereRobots: envDetails.followRobots,
   };
 
   const viewportSettings: ViewportSettingsClass = new ViewportSettingsClass(
@@ -214,6 +221,7 @@ const combineRun = async (details: Data, deviceToScan: string) => {
         undefined,
         scanDetails,
         zip,
+        generateJsonFiles,
       );
       const [name, email] = nameEmail.split(':');
 

package/src/constants/cliFunctions.ts

@@ -270,7 +270,9 @@ export const cliOptions: { [key: string]: Options } = {
     coerce: option => {
       const validChoices = Object.values(RuleFlags);
       const userChoices: string[] = option.split(',');
-      const invalidUserChoices = userChoices.filter(choice => !validChoices.includes(choice as RuleFlags));
+      const invalidUserChoices = userChoices.filter(
+        choice => !validChoices.includes(choice as RuleFlags),
+      );
       if (invalidUserChoices.length > 0) {
         printMessage(
           [
@@ -294,6 +296,27 @@ export const cliOptions: { [key: string]: Options } = {
       return userChoices;
     },
   },
+  g: {
+    alias: 'generateJsonFiles',
+    describe:
+      'Generate JSON files in the results folder. Accepts "yes", "no", "y", or "n". Default is "no".',
+    type: 'string',
+    requiresArg: true,
+    default: 'no',
+    demandOption: false,
+    coerce: value => {
+      const validYes = ['yes', 'y'];
+      const validNo = ['no', 'n'];
+
+      if (validYes.includes(value.toLowerCase())) {
+        return true;
+      }
+      if (validNo.includes(value.toLowerCase())) {
+        return false;
+      }
+      throw new Error(`Invalid value "${value}" for --generate. Use "yes", "y", "no", or "n".`);
+    },
+  },
 };
 
 export const configureReportSetting = (isEnabled: boolean): void => {

package/src/constants/common.ts

@@ -579,6 +579,7 @@ export const prepareData = async (argv: Answers): Promise<Data> => {
     safeMode,
     zip,
     ruleset,
+    generateJsonFiles,
   } = argv;
 
   // construct filename for scan results
@@ -625,6 +626,7 @@ export const prepareData = async (argv: Answers): Promise<Data> => {
     safeMode,
     zip,
     ruleset,
+    generateJsonFiles,
   };
 };
 

package/src/crawlers/crawlDomain.ts

@@ -581,18 +581,11 @@ const crawlDomain = async ({
           },
         ]
       : [
-          async (crawlingContext, gotoOptions) => {
-            const { page, request } = crawlingContext;
-
+          async ({ page, request }) => {
             await page.setExtraHTTPHeaders({
               ...extraHTTPHeaders,
             });
 
-            Object.assign(gotoOptions, {
-              waitUntil: 'networkidle',
-              timeout: 30000,
-            });
-
             const processible = await isProcessibleUrl(request.url);
             if (!processible) {
               request.skipNavigation = true;

package/src/index.ts

@@ -51,6 +51,7 @@ export type Answers = {
   exportDirectory: string;
   zip: string;
   ruleset: RuleFlags[];
+  generateJsonFiles: boolean;
 };
 
 export type Data = {
@@ -80,6 +81,7 @@ export type Data = {
   userDataDirectory?: string;
   zip?: string;
   ruleset: RuleFlags[];
+  generateJsonFiles: boolean;
 };
 
 const userData = getUserDataTxt();

package/src/mergeAxeResults.ts

@@ -24,8 +24,6 @@ import { consoleLogger, silentLogger } from './logs.js';
 import itemTypeDescription from './constants/itemTypeDescription.js';
 import { oobeeAiHtmlETL, oobeeAiRules } from './constants/oobeeAi.js';
 
-const cwd = process.cwd();
-
 export type ItemsInfo = {
   html: string;
   message: string;
@@ -84,10 +82,12 @@ type AllIssues = {
   cypressScanAboutMetadata: string;
   wcagLinks: { [key: string]: string };
   [key: string]: any;
+  advancedScanOptionsSummaryItems: { [key: string]: boolean };
 };
 
 const filename = fileURLToPath(import.meta.url);
 const dirname = path.dirname(filename);
+const BUFFER_LIMIT = 100 * 1024 * 1024; // 100MB size
 
 const extractFileNames = async (directory: string): Promise<string[]> => {
   ensureDirSync(directory);
@@ -214,8 +214,8 @@ const compileHtmlWithEJS = async (allIssues, storagePath, htmlFilename = 'report
   const injectScript = `
   <script>
     try {
-      const base64DecodeChunkedWithDecoder = (data, chunkSize = 1024 * 1024) => {
-      const encodedChunks = data.split('.');
+      const base64DecodeChunkedWithDecoder = (data, chunkSize = ${BUFFER_LIMIT}) => {
+      const encodedChunks = data.split('|');
       const decoder = new TextDecoder();
       const jsonParts = [];
 
@@ -259,7 +259,7 @@ const splitHtmlAndCreateFiles = async (htmlFilePath, storagePath) => {
       throw new Error('Marker comment not found in the HTML file.');
     }
 
-    const topContent = htmlContent.slice(0, splitIndex + splitMarker.length) + '\n\n';
+    const topContent = `${htmlContent.slice(0, splitIndex + splitMarker.length)}\n\n`;
     const bottomContent = htmlContent.slice(splitIndex + splitMarker.length);
 
     const topFilePath = path.join(storagePath, 'report-partial-top.htm.txt');
@@ -293,8 +293,6 @@ const writeHTML = async (allIssues, storagePath, htmlFilename = 'report') => {
 
   outputStream.write(prefixData);
 
-  // Create a readable stream for the input file with a highWaterMark set to 10MB
-  const BUFFER_LIMIT = 10 * 1024 * 1024; // 10 MB
   const inputStream = fs.createReadStream(inputFilePath, {
     encoding: 'utf-8',
     highWaterMark: BUFFER_LIMIT,
@@ -457,15 +455,14 @@ function writeLargeJsonToFile(obj, filePath) {
   });
 }
 
-const base64Encode = async (data, num) => {
+const base64Encode = async (data, num, storagePath, generateJsonFiles) => {
   try {
-    const tempFilename =
+    const tempFilePath =
       num === 1
-        ? `scanItems_${uuidv4()}.json`
+        ? path.join(storagePath, 'scanItems.json')
         : num === 2
-          ? `scanData_${uuidv4()}.json`
-          : `${uuidv4()}.json`;
-    const tempFilePath = path.join(process.cwd(), tempFilename);
+          ? path.join(storagePath, 'scanData.json')
+          : path.join(storagePath, `${uuidv4()}.json`);
 
     await writeLargeJsonToFile(data, tempFilePath);
 
@@ -473,16 +470,28 @@ const base64Encode = async (data, num) => {
     const outputFilePath = path.join(process.cwd(), outputFilename);
 
     try {
-      const CHUNK_SIZE = 1024 * 1024; // 1MB chunks
       const readStream = fs.createReadStream(tempFilePath, {
         encoding: 'utf8',
-        highWaterMark: CHUNK_SIZE,
+        highWaterMark: BUFFER_LIMIT,
       });
       const writeStream = fs.createWriteStream(outputFilePath, { encoding: 'utf8' });
 
+      let previousChunk = null;
+
       for await (const chunk of readStream) {
         const encodedChunk = Buffer.from(chunk).toString('base64');
-        writeStream.write(`${encodedChunk}.`);
+
+        if (previousChunk !== null) {
+          // Note: Notice the pipe symbol `|`, it is intended to be here as a delimiter
+          // for the scenario where there are chunking happens
+          writeStream.write(`${previousChunk}|`);
+        }
+
+        previousChunk = encodedChunk;
+      }
+
+      if (previousChunk !== null) {
+        writeStream.write(previousChunk);
       }
 
       await new Promise((resolve, reject) => {
@@ -492,9 +501,11 @@ const base64Encode = async (data, num) => {
 
       return outputFilePath;
     } finally {
-      await fs.promises
-        .unlink(tempFilePath)
-        .catch(err => console.error('Temp file delete error:', err));
+      if (!generateJsonFiles) {
+        await fs.promises
+          .unlink(tempFilePath)
+          .catch(err => console.error('Temp file delete error:', err));
+      }
     }
   } catch (error) {
     console.error('Error encoding data to Base64:', error);
@@ -520,10 +531,10 @@ const streamEncodedDataToFile = async (inputFilePath, writeStream, appendComma)
   }
 };
 
-const writeBase64 = async (allIssues, storagePath) => {
+const writeBase64 = async (allIssues, storagePath, generateJsonFiles) => {
   const { items, ...rest } = allIssues;
-  const encodedScanItemsPath = await base64Encode(items, 1);
-  const encodedScanDataPath = await base64Encode(rest, 2);
+  const encodedScanItemsPath = await base64Encode(items, 1, storagePath, generateJsonFiles);
+  const encodedScanDataPath = await base64Encode(rest, 2, storagePath, generateJsonFiles);
 
   const filePath = path.join(storagePath, 'scanDetails.csv');
   const directoryPath = path.dirname(filePath);
@@ -766,6 +777,7 @@ const generateArtifacts = async (
   cypressScanAboutMetadata,
   scanDetails,
   zip = undefined, // optional
+  generateJsonFiles = false,
 ) => {
   const intermediateDatasetsPath = `${randomToken}/datasets/${randomToken}`;
   const phAppVersion = getVersion();
@@ -835,6 +847,17 @@ const generateArtifacts = async (
     },
     cypressScanAboutMetadata,
     wcagLinks: constants.wcagLinks,
+    // Populate boolean values for id="advancedScanOptionsSummary"
+    advancedScanOptionsSummaryItems: {
+      showIncludeScreenshots: [true].includes(scanDetails.isIncludeScreenshots),
+      showAllowSubdomains: [true].includes(scanDetails.isAllowSubdomains),
+      showEnableCustomChecks: ['default', 'enable-wcag-aaa'].includes(
+        scanDetails.isEnableCustomChecks?.[0],
+      ),
+      showEnableWcagAaa: (scanDetails.isEnableWcagAaa || []).includes('enable-wcag-aaa'),
+      showSlowScanMode: [1].includes(scanDetails.isSlowScanMode),
+      showAdhereRobots: [true].includes(scanDetails.isAdhereRobots),
+    },
   };
 
   const allFiles = await extractFileNames(intermediateDatasetsPath);
@@ -870,6 +893,9 @@ const generateArtifacts = async (
   }
 
   allIssues.wcagPassPercentage = getWcagPassPercentage(allIssues.wcagViolations);
+  consoleLogger.info(
+    `advancedScanOptionsSummaryItems is ${allIssues.advancedScanOptionsSummaryItems}`,
+  );
 
   const getAxeImpactCount = (allIssues: AllIssues) => {
     const impactCount = {
@@ -908,7 +934,7 @@ const generateArtifacts = async (
   }
 
   await writeCsv(allIssues, storagePath);
-  await writeBase64(allIssues, storagePath);
+  await writeBase64(allIssues, storagePath, generateJsonFiles);
   await writeSummaryHTML(allIssues, storagePath);
   await writeHTML(allIssues, storagePath);
   await retryFunction(() => writeSummaryPdf(storagePath, pagesScanned.length), 1);

package/src/static/ejs/partials/components/scanAbout.ejs

@@ -225,6 +225,71 @@
         </svg>
       </span>
     </li>
+
+    <li>
+      <svg aria-label="Advanced options scan summary" xmlns="http://www.w3.org/2000/svg" width="18" height="18" viewBox="0 0 18 18" fill="none">
+        <rect x="0.87" y="0.87" width="16.26" height="16.26" rx="3.13" stroke="#93928D" stroke-width="1.74"/>
+        <path d="M5 9L7.5 11.5L13 6" stroke="#93928D" stroke-width="1.74" stroke-linecap="square"/>
+      </svg>
+      <div>
+        <div class="d-flex flex-row justify-content-center align-items-center gap-3">
+          <button id="advancedScanOptionsSummaryTitle" onclick="toggleAdvanceScanSummary()">Advanced scan summary
+          </button>
+        </div>
+
+      </div>
+    </li>
+    <ul id="advancedScanOptionsSummary" class="d-none mb-3">
+      <li id="showIncludeScreenshots"class="d-flex flex-row">
+          <svg aria-label="Include screenshots was checked" xmlns="http://www.w3.org/2000/svg" width="18" height="18" viewBox="0 0 18 18" fill="none">
+            <rect x="0.87" y="0.87" width="16.26" height="16.26" rx="3.13" fill="#93928D" stroke="#93928D" stroke-width="1.74"/>
+            <path d="M5 9L7.5 11.5L13 6" fill="none" stroke="#ffffff" stroke-width="1.74" stroke-linecap="square"/>
+          </svg>
+        <div>Include screenshots
+        </div>
+      </li>
+      <li id="showAllowSubdomains"class="d-flex flex-row">
+          <svg aria-label="Allow subdomains was checked" xmlns="http://www.w3.org/2000/svg" width="18" height="18" viewBox="0 0 18 18" fill="none">
+            <rect x="0.87" y="0.87" width="16.26" height="16.26" rx="3.13" fill="#93928D" stroke="#93928D" stroke-width="1.74"/>
+            <path d="M5 9L7.5 11.5L13 6" fill="none" stroke="#ffffff" stroke-width="1.74" stroke-linecap="square"/>
+          </svg>
+        <div>Allow subdomains for scans
+        </div>
+      </li>
+      <li id="showEnableCustomChecks"class="d-flex flex-row">
+        <svg aria-label="Enable custom checks was checked" xmlns="http://www.w3.org/2000/svg" width="18" height="18" viewBox="0 0 18 18" fill="none">
+          <rect x="0.87" y="0.87" width="16.26" height="16.26" rx="3.13" fill="#93928D" stroke="#93928D" stroke-width="1.74"/>
+          <path d="M5 9L7.5 11.5L13 6" fill="none" stroke="#ffffff" stroke-width="1.74" stroke-linecap="square"/>
+        </svg>
+      <div>Enable custom checks
+      </div>
+    </li>
+    <li id="showEnableWcagAaa"class="d-flex flex-row">
+        <svg aria-label="Enable WCAG AAA checks was checked" xmlns="http://www.w3.org/2000/svg" width="18" height="18" viewBox="0 0 18 18" fill="none">
+          <rect x="0.87" y="0.87" width="16.26" height="16.26" rx="3.13" fill="#93928D" stroke="#93928D" stroke-width="1.74"/>
+          <path d="M5 9L7.5 11.5L13 6" fill="none" stroke="#ffffff" stroke-width="1.74" stroke-linecap="square"/>
+        </svg>
+      <div>Enable WCAG AAA checks
+      </div>
+    </li>
+      <li id="showSlowScanMode"class="d-flex flex-row">
+          <svg aria-label="Slow scan mode was checked" xmlns="http://www.w3.org/2000/svg" width="18" height="18" viewBox="0 0 18 18" fill="none">
+            <rect x="0.87" y="0.87" width="16.26" height="16.26" rx="3.13" fill="#93928D" stroke="#93928D" stroke-width="1.74"/>
+            <path d="M5 9L7.5 11.5L13 6" fill="none" stroke="#ffffff" stroke-width="1.74" stroke-linecap="square"/>
+          </svg>
+        <div>Slow scan mode
+        </div>
+      </li>
+
+      <li id="showAdhereRobots"class="d-flex flex-row">
+          <svg aria-label="Adhere to robots.txt was checked" xmlns="http://www.w3.org/2000/svg" width="18" height="18" viewBox="0 0 18 18" fill="none">
+            <rect x="0.87" y="0.87" width="16.26" height="16.26" rx="3.13" fill="#93928D" stroke="#93928D" stroke-width="1.74"/>
+            <path d="M5 9L7.5 11.5L13 6" fill="none" stroke="#ffffff" stroke-width="1.74" stroke-linecap="square"/>
+          </svg>
+        <div>Adhere to robots.txt
+        </div>
+      </li>
+    </ul>
     <li>
       <svg
         aria-label="Scan engine"

package/src/static/ejs/partials/scripts/scanAboutScript.ejs

@@ -0,0 +1,38 @@
+<%# functions to handle interaction and ui for advancedScanOptionsSummary in scanAbout.ejs.
+component %>
+
+<script>
+  let optionsToCheck = scanData.advancedScanOptionsSummaryItems;
+
+  document.querySelectorAll('#advancedScanOptionsSummary li').forEach(liElement => {
+    liElement.classList.add('d-none');
+  });
+
+  function toggleAdvanceScanSummary() {
+    const chevron = document.getElementById('advancedScanOptionsSummaryTitle');
+    const advancedScanOptionsSummary = document.getElementById('advancedScanOptionsSummary');
+
+    const isHidden = advancedScanOptionsSummary.classList.toggle('d-none');
+
+    chevron.classList.toggle('chevron-rotated', !isHidden);
+
+    if (!isHidden) {
+      showScanOptions(optionsToCheck);
+    }
+  }
+
+  function showScanOptions(options) {
+    document.querySelectorAll('#advancedScanOptionsSummary li').forEach(liElement => {
+      liElement.classList.add('d-none');
+    });
+
+    for (const key in options) {
+      if (options[key] === true) {
+        const liElement = document.getElementById(key);
+        if (liElement) {
+          liElement.classList.remove('d-none');
+        }
+      }
+    }
+  }
+</script>

package/src/static/ejs/partials/styles/styles.ejs

@@ -755,7 +755,8 @@
     margin: 1.5rem 0 1rem 0;
   }
 
-  button#wcagModalToggle {
+  button#wcagModalToggle,
+  button#advancedScanOptionsSummaryTitle {
     background: none;
     border: 0;
     padding: 0;
@@ -962,6 +963,26 @@
     width: 1.125rem;
   }
 
+  #advancedScanOptionsSummary li > svg {
+    margin-left: 2rem;
+  }
+
+  #advancedScanOptionsSummaryTitle::after {
+    content: '';
+    display: inline-block;
+    width: 12px;
+    height: 12px;
+    background: url('data:image/svg+xml;utf8,<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 9 16" fill="none"><path d="M1.03847 16C0.833084 16 0.632306 15.9388 0.461529 15.8241C0.290753 15.7095 0.157649 15.5465 0.0790493 15.3558C0.000449621 15.1651 -0.0201154 14.9553 0.0199549 14.7529C0.0600251 14.5505 0.158931 14.3645 0.304165 14.2186L6.49293 7.99975L0.304165 1.78088C0.109639 1.58514 0.000422347 1.31979 0.000518839 1.04315C0.000615331 0.766523 0.110018 0.501248 0.30468 0.30564C0.499341 0.110032 0.763331 9.70251e-05 1.03862 6.41929e-08C1.31392 -9.68968e-05 1.57798 0.109652 1.77278 0.305123L8.69586 7.26187C8.8906 7.45757 9 7.72299 9 7.99975C9 8.2765 8.8906 8.54192 8.69586 8.73763L1.77278 15.6944C1.67646 15.7914 1.562 15.8684 1.43598 15.9208C1.30996 15.9733 1.17487 16.0002 1.03847 16Z" fill="%23006B8C" transform="rotate(90, 4.5, 8)"/></svg>');
+    background-size: contain;
+    background-repeat: no-repeat;
+    transform: scaleY(1);
+    margin-left: 0.5rem;
+  }
+
+  #advancedScanOptionsSummaryTitle.chevron-rotated::after {
+    transform: scaleY(-1);
+  }
+
   #footer {
     padding: 0.75rem 1rem;
     box-shadow: 0 -0.25rem 10px #736ccb1a;

package/src/static/ejs/report.ejs

@@ -25,6 +25,7 @@
     <%- include('partials/header') %> <%- include('partials/main') %> <%-
     include('partials/scripts/popper') %> <%- include('partials/scripts/bootstrap') %> <%-
     include('partials/scripts/highlightjs') %> <%- include('partials/scripts/utils') %> <%-
+    include('partials/scripts/scanAboutScript') %> <%-
     include('partials/scripts/categorySelectorDropdownScript') %> <%-
     include('partials/scripts/categorySummary') %> <%- include('partials/scripts/ruleOffcanvas') %>
     <%- include('partials/scripts/screenshotLightbox')%>