@govtechsg/oobee 0.10.39 → 0.10.43

package/src/utils.ts

@@ -2,29 +2,31 @@ import { execSync, spawnSync } from 'child_process';
 import path from 'path';
 import os from 'os';
 import fs from 'fs-extra';
+import axe, { Rule } from 'axe-core';
 import constants, {
   BrowserTypes,
   destinationPath,
   getIntermediateScreenshotsPath,
 } from './constants/constants.js';
-import { silentLogger } from './logs.js';
+import { consoleLogger, silentLogger } from './logs.js';
+import { getAxeConfiguration } from './crawlers/custom/getAxeConfiguration.js';
 
 export const getVersion = () => {
-  const loadJSON = filePath =>
+  const loadJSON = (filePath: string): { version: string } =>
     JSON.parse(fs.readFileSync(new URL(filePath, import.meta.url)).toString());
   const versionNum = loadJSON('../package.json').version;
 
   return versionNum;
 };
 
-export const getHost = url => new URL(url).host;
+export const getHost = (url: string): string => new URL(url).host;
 
 export const getCurrentDate = () => {
   const date = new Date();
   return `${date.getFullYear()}-${date.getMonth() + 1}-${date.getDate()}`;
 };
 
-export const isWhitelistedContentType = contentType => {
+export const isWhitelistedContentType = (contentType: string): boolean => {
   const whitelist = ['text/html'];
   return whitelist.filter(type => contentType.trim().startsWith(type)).length === 1;
 };
@@ -42,7 +44,7 @@ export const getStoragePath = (randomToken: string): string => {
   return `${constants.exportDirectory}/${randomToken}`;
 };
 
-export const createDetailsAndLogs = async randomToken => {
+export const createDetailsAndLogs = async (randomToken: string): Promise<void> => {
   const storagePath = getStoragePath(randomToken);
   const logPath = `logs/${randomToken}`;
   try {
@@ -56,20 +58,24 @@ export const createDetailsAndLogs = async randomToken => {
           await fs.copy('errors.txt', `${logPath}/${randomToken}.txt`);
         } catch (error) {
           if (error.code === 'EBUSY') {
-            console.log(
+            consoleLogger.error(
               `Unable to copy the file from 'errors.txt' to '${logPath}/${randomToken}.txt' because it is currently in use.`,
             );
-            console.log(
+            consoleLogger.error(
               'Please close any applications that might be using this file and try again.',
             );
           } else {
-            console.log(`An unexpected error occurred while copying the file: ${error.message}`);
+            consoleLogger.error(
+              `An unexpected error occurred while copying the file: ${error.message}`,
+            );
           }
         }
       }
     });
   } catch (error) {
-    console.log(`An error occurred while setting up storage or log directories: ${error.message}`);
+    consoleLogger.error(
+      `An error occurred while setting up storage or log directories: ${error.message}`,
+    );
   }
 };
 
@@ -96,7 +102,7 @@ export const getUserDataTxt = () => {
   return null;
 };
 
-export const writeToUserDataTxt = async (key, value) => {
+export const writeToUserDataTxt = async (key: string, value: string): Promise<void> => {
   const textFilePath = getUserDataFilePath();
 
   // Create file if it doesn't exist
@@ -113,25 +119,27 @@ export const writeToUserDataTxt = async (key, value) => {
   }
 };
 
-export const createAndUpdateResultsFolders = async randomToken => {
+export const createAndUpdateResultsFolders = async (randomToken: string): Promise<void> => {
   const storagePath = getStoragePath(randomToken);
   await fs.ensureDir(`${storagePath}`);
 
   const intermediatePdfResultsPath = `${randomToken}/${constants.pdfScanResultFileName}`;
 
-  const transferResults = async (intermPath, resultFile) => {
+  const transferResults = async (intermPath: string, resultFile: string): Promise<void> => {
     try {
       if (fs.existsSync(intermPath)) {
         await fs.copy(intermPath, `${storagePath}/${resultFile}`);
       }
     } catch (error) {
       if (error.code === 'EBUSY') {
-        console.log(
+        consoleLogger.error(
           `Unable to copy the file from ${intermPath} to ${storagePath}/${resultFile} because it is currently in use.`,
         );
-        console.log('Please close any applications that might be using this file and try again.');
+        consoleLogger.error(
+          'Please close any applications that might be using this file and try again.',
+        );
       } else {
-        console.log(
+        consoleLogger.error(
           `An unexpected error occurred while copying the file from ${intermPath} to ${storagePath}/${resultFile}: ${error.message}`,
         );
       }
@@ -141,20 +149,20 @@ export const createAndUpdateResultsFolders = async randomToken => {
   await Promise.all([transferResults(intermediatePdfResultsPath, constants.pdfScanResultFileName)]);
 };
 
-export const createScreenshotsFolder = randomToken => {
+export const createScreenshotsFolder = (randomToken: string): void => {
   const storagePath = getStoragePath(randomToken);
   const intermediateScreenshotsPath = getIntermediateScreenshotsPath(randomToken);
   if (fs.existsSync(intermediateScreenshotsPath)) {
     fs.readdir(intermediateScreenshotsPath, (err, files) => {
       if (err) {
-        console.log(`Screenshots were not moved successfully: ${err.message}`);
+        consoleLogger.error(`Screenshots were not moved successfully: ${err.message}`);
       }
 
       if (!fs.existsSync(destinationPath(storagePath))) {
         try {
           fs.mkdirSync(destinationPath(storagePath), { recursive: true });
         } catch (error) {
-          console.error('Screenshots folder was not created successfully:', error);
+          consoleLogger.error('Screenshots folder was not created successfully:', error);
         }
       }
 
@@ -167,32 +175,20 @@ export const createScreenshotsFolder = randomToken => {
 
       fs.rmdir(intermediateScreenshotsPath, rmdirErr => {
         if (rmdirErr) {
-          console.log(rmdirErr);
+          consoleLogger.error(rmdirErr);
         }
       });
     });
   }
 };
 
-export const cleanUp = async pathToDelete => {
+export const cleanUp = (pathToDelete: string): void => {
   fs.removeSync(pathToDelete);
 };
 
-/* istanbul ignore next */
-// export const getFormattedTime = () =>
-//   new Date().toLocaleTimeString('en-GB', {
-//     year: 'numeric',
-//     month: 'short',
-//     day: 'numeric',
-//     hour12: true,
-//     hour: 'numeric',
-//     minute: '2-digit',
-//     timeZoneName: "longGeneric",
-//   });
-
 export const getWcagPassPercentage = (
   wcagViolations: string[],
-  showEnableWcagAaa: boolean
+  showEnableWcagAaa: boolean,
 ): {
   passPercentageAA: string;
   totalWcagChecksAA: number;
@@ -201,20 +197,28 @@ export const getWcagPassPercentage = (
   totalWcagChecksAAandAAA: number;
   totalWcagViolationsAAandAAA: number;
 } => {
-
   // These AAA rules should not be counted as WCAG Pass Percentage only contains A and AA
   const wcagAAALinks = ['WCAG 1.4.6', 'WCAG 2.2.4', 'WCAG 2.4.9', 'WCAG 3.1.5', 'WCAG 3.2.5'];
   const wcagAAA = ['wcag146', 'wcag224', 'wcag249', 'wcag315', 'wcag325'];
-  
+
   const wcagLinksAAandAAA = constants.wcagLinks;
-  
+
   const wcagViolationsAAandAAA = showEnableWcagAaa ? wcagViolations.length : null;
   const totalChecksAAandAAA = showEnableWcagAaa ? Object.keys(wcagLinksAAandAAA).length : null;
-  const passedChecksAAandAAA = showEnableWcagAaa ? totalChecksAAandAAA - wcagViolationsAAandAAA : null;
-  const passPercentageAAandAAA = showEnableWcagAaa ? (totalChecksAAandAAA === 0 ? 0 : (passedChecksAAandAAA / totalChecksAAandAAA) * 100) : null;
+  const passedChecksAAandAAA = showEnableWcagAaa
+    ? totalChecksAAandAAA - wcagViolationsAAandAAA
+    : null;
+  // eslint-disable-next-line no-nested-ternary
+  const passPercentageAAandAAA = showEnableWcagAaa
+    ? totalChecksAAandAAA === 0
+      ? 0
+      : (passedChecksAAandAAA / totalChecksAAandAAA) * 100
+    : null;
 
   const wcagViolationsAA = wcagViolations.filter(violation => !wcagAAA.includes(violation)).length;
-  const totalChecksAA = Object.keys(wcagLinksAAandAAA).filter(key => !wcagAAALinks.includes(key)).length;
+  const totalChecksAA = Object.keys(wcagLinksAAandAAA).filter(
+    key => !wcagAAALinks.includes(key),
+  ).length;
   const passedChecksAA = totalChecksAA - wcagViolationsAA;
   const passPercentageAA = totalChecksAA === 0 ? 0 : (passedChecksAA / totalChecksAA) * 100;
 
@@ -228,7 +232,294 @@ export const getWcagPassPercentage = (
   };
 };
 
-export const getFormattedTime = inputDate => {
+export type IssueCategory = 'mustFix' | 'goodToFix' | 'needsReview' | 'passed';
+
+export interface IssueDetail {
+  ruleId: string;
+  wcagConformance: string[];
+  occurrencesMustFix?: number;
+  occurrencesGoodToFix?: number;
+  occurrencesNeedsReview?: number;
+  occurrencesPassed: number;
+}
+
+export interface PageDetail {
+  pageTitle: string;
+  url: string;
+  totalOccurrencesFailedIncludingNeedsReview: number;
+  totalOccurrencesFailedExcludingNeedsReview: number;
+  totalOccurrencesMustFix?: number;
+  totalOccurrencesGoodToFix?: number;
+  totalOccurrencesNeedsReview: number;
+  totalOccurrencesPassed: number;
+  occurrencesExclusiveToNeedsReview: boolean;
+  typesOfIssuesCount: number;
+  typesOfIssuesExcludingNeedsReviewCount: number;
+  categoriesPresent: IssueCategory[];
+  conformance?: string[]; // WCAG levels as flexible strings
+  typesOfIssues: IssueDetail[];
+}
+
+export interface ScanPagesDetail {
+  oobeeAppVersion?: string;
+  pagesAffected: PageDetail[];
+  pagesNotAffected: PageDetail[];
+  scannedPagesCount: number;
+  pagesNotScanned: PageDetail[];
+  pagesNotScannedCount: number;
+}
+
+export const getProgressPercentage = (
+  scanPagesDetail: ScanPagesDetail,
+  showEnableWcagAaa: boolean,
+): {
+  averageProgressPercentageAA: string;
+  averageProgressPercentageAAandAAA: string;
+} => {
+  const pages = scanPagesDetail.pagesAffected || [];
+
+  const progressPercentagesAA = pages.map((page: PageDetail) => {
+    const violations: string[] = page.conformance;
+    return getWcagPassPercentage(violations, showEnableWcagAaa).passPercentageAA;
+  });
+
+  const progressPercentagesAAandAAA = pages.map((page: PageDetail) => {
+    const violations: string[] = page.conformance;
+    return getWcagPassPercentage(violations, showEnableWcagAaa).passPercentageAAandAAA;
+  });
+
+  const totalAA = progressPercentagesAA.reduce((sum, p) => sum + parseFloat(p), 0);
+  const avgAA = progressPercentagesAA.length ? totalAA / progressPercentagesAA.length : 0;
+
+  const totalAAandAAA = progressPercentagesAAandAAA.reduce((sum, p) => sum + parseFloat(p), 0);
+  const avgAAandAAA = progressPercentagesAAandAAA.length
+    ? totalAAandAAA / progressPercentagesAAandAAA.length
+    : 0;
+
+  return {
+    averageProgressPercentageAA: avgAA.toFixed(2),
+    averageProgressPercentageAAandAAA: avgAAandAAA.toFixed(2),
+  };
+};
+
+export const getTotalRulesCount = async (
+  enableWcagAaa: boolean,
+  disableOobee: boolean,
+): Promise<{
+  totalRulesMustFix: number;
+  totalRulesGoodToFix: number;
+  totalRulesMustFixAndGoodToFix: number;
+}> => {
+  const axeConfig = getAxeConfiguration({
+    enableWcagAaa,
+    gradingReadabilityFlag: '',
+    disableOobee,
+  });
+
+  // Get default rules from axe-core
+  const defaultRules = axe.getRules();
+
+  // Merge custom rules with default rules, converting RuleMetadata to Rule
+  const mergedRules: Rule[] = defaultRules.map(defaultRule => {
+    const customRule = axeConfig.rules.find(r => r.id === defaultRule.ruleId);
+    if (customRule) {
+      // Merge properties from customRule into defaultRule (RuleMetadata) to create a Rule
+      return {
+        id: defaultRule.ruleId,
+        enabled: customRule.enabled,
+        selector: customRule.selector,
+        any: customRule.any,
+        tags: defaultRule.tags,
+        metadata: customRule.metadata, // Use custom metadata if it exists
+      };
+    }
+    // Convert defaultRule (RuleMetadata) to Rule
+    return {
+      id: defaultRule.ruleId,
+      enabled: true, // Default to true if not overridden
+      tags: defaultRule.tags,
+      // No metadata here, since defaultRule.metadata might not exist
+    };
+  });
+
+  // Add any custom rules that don't override the default rules
+  axeConfig.rules.forEach(customRule => {
+    if (!mergedRules.some(mergedRule => mergedRule.id === customRule.id)) {
+      // Ensure customRule is of type Rule
+      const rule: Rule = {
+        id: customRule.id,
+        enabled: customRule.enabled,
+        selector: customRule.selector,
+        any: customRule.any,
+        tags: customRule.tags,
+        metadata: customRule.metadata,
+        // Add other properties if needed
+      };
+      mergedRules.push(rule);
+    }
+  });
+
+  // Apply the merged configuration to axe-core
+  axe.configure({ ...axeConfig, rules: mergedRules });
+
+  // ... (rest of your logic)
+  let totalRulesMustFix = 0;
+  let totalRulesGoodToFix = 0;
+
+  const wcagRegex = /^wcag\d+a+$/;
+
+  // Use mergedRules instead of rules to check enabled property
+  mergedRules.forEach(rule => {
+    if (!rule.enabled) {
+      return;
+    }
+
+    if (rule.id === 'frame-tested') return; // Ignore 'frame-tested' rule
+
+    const tags = rule.tags || [];
+
+    // Skip experimental and deprecated rules
+    if (tags.includes('experimental') || tags.includes('deprecated')) {
+      return;
+    }
+
+    const conformance = tags.filter(tag => tag.startsWith('wcag') || tag === 'best-practice');
+
+    // Ensure conformance level is sorted correctly
+    if (
+      conformance.length > 0 &&
+      conformance[0] !== 'best-practice' &&
+      !wcagRegex.test(conformance[0])
+    ) {
+      conformance.sort((a, b) => {
+        if (wcagRegex.test(a) && !wcagRegex.test(b)) {
+          return -1;
+        }
+        if (!wcagRegex.test(a) && wcagRegex.test(b)) {
+          return 1;
+        }
+        return 0;
+      });
+    }
+
+    if (conformance.includes('best-practice')) {
+      // console.log(`${totalRulesMustFix} Good To Fix: ${rule.id}`);
+
+      totalRulesGoodToFix += 1; // Categorized as "Good to Fix"
+    } else {
+      // console.log(`${totalRulesMustFix} Must Fix: ${rule.id}`);
+
+      totalRulesMustFix += 1; // Otherwise, it's "Must Fix"
+    }
+  });
+
+  return {
+    totalRulesMustFix,
+    totalRulesGoodToFix,
+    totalRulesMustFixAndGoodToFix: totalRulesMustFix + totalRulesGoodToFix,
+  };
+};
+
+export const getIssuesPercentage = async (
+  scanPagesDetail: ScanPagesDetail,
+  enableWcagAaa: boolean,
+  disableOobee: boolean,
+): Promise<{
+  avgTypesOfIssuesPercentageOfTotalRulesAtMustFix: string;
+  avgTypesOfIssuesPercentageOfTotalRulesAtGoodToFix: string;
+  avgTypesOfIssuesPercentageOfTotalRulesAtMustFixAndGoodToFix: string;
+  totalRulesMustFix: number;
+  totalRulesGoodToFix: number;
+  totalRulesMustFixAndGoodToFix: number;
+  avgTypesOfIssuesCountAtMustFix: string;
+  avgTypesOfIssuesCountAtGoodToFix: string;
+  avgTypesOfIssuesCountAtMustFixAndGoodToFix: string;
+  pagesAffectedPerRule: Record<string, number>;
+  pagesPercentageAffectedPerRule: Record<string, string>;
+}> => {
+  const pages = scanPagesDetail.pagesAffected || [];
+  const totalPages = pages.length;
+
+  const pagesAffectedPerRule: Record<string, number> = {};
+
+  pages.forEach(page => {
+    page.typesOfIssues.forEach(issue => {
+      if ((issue.occurrencesMustFix || issue.occurrencesGoodToFix) > 0) {
+        pagesAffectedPerRule[issue.ruleId] = (pagesAffectedPerRule[issue.ruleId] || 0) + 1;
+      }
+    });
+  });
+
+  const pagesPercentageAffectedPerRule: Record<string, string> = {};
+  Object.entries(pagesAffectedPerRule).forEach(([ruleId, count]) => {
+    pagesPercentageAffectedPerRule[ruleId] =
+      totalPages > 0 ? ((count / totalPages) * 100).toFixed(2) : '0.00';
+  });
+
+  const typesOfIssuesCountAtMustFix = pages.map(
+    page => page.typesOfIssues.filter(issue => (issue.occurrencesMustFix || 0) > 0).length,
+  );
+
+  const typesOfIssuesCountAtGoodToFix = pages.map(
+    page => page.typesOfIssues.filter(issue => (issue.occurrencesGoodToFix || 0) > 0).length,
+  );
+
+  const typesOfIssuesCountSumMustFixAndGoodToFix = pages.map(
+    (_, index) =>
+      (typesOfIssuesCountAtMustFix[index] || 0) + (typesOfIssuesCountAtGoodToFix[index] || 0),
+  );
+
+  const { totalRulesMustFix, totalRulesGoodToFix, totalRulesMustFixAndGoodToFix } =
+    await getTotalRulesCount(enableWcagAaa, disableOobee);
+
+  const avgMustFixPerPage =
+    totalPages > 0
+      ? typesOfIssuesCountAtMustFix.reduce((sum, count) => sum + count, 0) / totalPages
+      : 0;
+
+  const avgGoodToFixPerPage =
+    totalPages > 0
+      ? typesOfIssuesCountAtGoodToFix.reduce((sum, count) => sum + count, 0) / totalPages
+      : 0;
+
+  const avgMustFixAndGoodToFixPerPage =
+    totalPages > 0
+      ? typesOfIssuesCountSumMustFixAndGoodToFix.reduce((sum, count) => sum + count, 0) / totalPages
+      : 0;
+
+  const avgTypesOfIssuesPercentageOfTotalRulesAtMustFix =
+    totalRulesMustFix > 0 ? ((avgMustFixPerPage / totalRulesMustFix) * 100).toFixed(2) : '0.00';
+
+  const avgTypesOfIssuesPercentageOfTotalRulesAtGoodToFix =
+    totalRulesGoodToFix > 0
+      ? ((avgGoodToFixPerPage / totalRulesGoodToFix) * 100).toFixed(2)
+      : '0.00';
+
+  const avgTypesOfIssuesPercentageOfTotalRulesAtMustFixAndGoodToFix =
+    totalRulesMustFixAndGoodToFix > 0
+      ? ((avgMustFixAndGoodToFixPerPage / totalRulesMustFixAndGoodToFix) * 100).toFixed(2)
+      : '0.00';
+
+  const avgTypesOfIssuesCountAtMustFix = avgMustFixPerPage.toFixed(2);
+  const avgTypesOfIssuesCountAtGoodToFix = avgGoodToFixPerPage.toFixed(2);
+  const avgTypesOfIssuesCountAtMustFixAndGoodToFix = avgMustFixAndGoodToFixPerPage.toFixed(2);
+
+  return {
+    avgTypesOfIssuesCountAtMustFix,
+    avgTypesOfIssuesCountAtGoodToFix,
+    avgTypesOfIssuesCountAtMustFixAndGoodToFix,
+    avgTypesOfIssuesPercentageOfTotalRulesAtMustFix,
+    avgTypesOfIssuesPercentageOfTotalRulesAtGoodToFix,
+    avgTypesOfIssuesPercentageOfTotalRulesAtMustFixAndGoodToFix,
+    totalRulesMustFix,
+    totalRulesGoodToFix,
+    totalRulesMustFixAndGoodToFix,
+    pagesAffectedPerRule,
+    pagesPercentageAffectedPerRule,
+  };
+};
+
+export const getFormattedTime = (inputDate: Date): string => {
   if (inputDate) {
     return inputDate.toLocaleTimeString('en-GB', {
       year: 'numeric',
@@ -250,7 +541,7 @@ export const getFormattedTime = inputDate => {
   });
 };
 
-export const formatDateTimeForMassScanner = date => {
+export const formatDateTimeForMassScanner = (date: Date): string => {
   // Format date and time parts separately
   const year = date.getFullYear().toString().slice(-2); // Get the last two digits of the year
   const month = `0${date.getMonth() + 1}`.slice(-2); // Month is zero-indexed
@@ -271,14 +562,13 @@ export const setHeadlessMode = (browser: string, isHeadless: boolean): void => {
   } else {
     process.env.CRAWLEE_HEADLESS = '0';
   }
-
 };
 
-export const setThresholdLimits = setWarnLevel => {
+export const setThresholdLimits = (setWarnLevel: string): void => {
   process.env.WARN_LEVEL = setWarnLevel;
 };
 
-export const zipResults = (zipName, resultsPath) => {
+export const zipResults = (zipName: string, resultsPath: string): void => {
   // Check prior zip file exist and remove
   if (fs.existsSync(zipName)) {
     fs.unlinkSync(zipName);
@@ -309,9 +599,9 @@ export const zipResults = (zipName, resultsPath) => {
 
 // areLinksEqual compares 2 string URLs and ignores comparison of 'www.' and url protocol
 // i.e. 'http://google.com' and 'https://www.google.com' returns true
-export const areLinksEqual = (link1, link2) => {
+export const areLinksEqual = (link1: string, link2: string): boolean => {
   try {
-    const format = link => {
+    const format = (link: string): URL => {
       return new URL(link.replace(/www\./, ''));
     };
     const l1 = format(link1);
@@ -336,7 +626,7 @@ export const randomThreeDigitNumberString = () => {
   return String(threeDigitNumber);
 };
 
-export const isFollowStrategy = (link1, link2, rule) => {
+export const isFollowStrategy = (link1: string, link2: string, rule: string): boolean => {
   const parsedLink1 = new URL(link1);
   const parsedLink2 = new URL(link2);
   if (rule === 'same-domain') {
@@ -347,17 +637,17 @@ export const isFollowStrategy = (link1, link2, rule) => {
   return parsedLink1.hostname === parsedLink2.hostname;
 };
 
-/* eslint-disable no-await-in-loop */
-export const retryFunction = async (func, maxAttempt) => {
+export const retryFunction = async <T>(func: () => Promise<T>, maxAttempt: number): Promise<T> => {
   let attemptCount = 0;
   while (attemptCount < maxAttempt) {
     attemptCount += 1;
     try {
+      // eslint-disable-next-line no-await-in-loop
       const result = await func();
       return result;
     } catch (error) {
       silentLogger.error(`(Attempt count: ${attemptCount} of ${maxAttempt}) ${error}`);
     }
   }
+  throw new Error('Maximum number of attempts reached');
 };
-/* eslint-enable no-await-in-loop */