@govtechsg/oobee 0.10.58 → 0.10.62

package/DETAILS.md

@@ -41,7 +41,7 @@ Note: Level AAA are disabled by default.  Please specify `enable-wcag-aaa` in ru
 | WCAG 1.4.2  | A     | Yes      |             |              |
 | WCAG 1.4.3  | AA    | Yes      |             |              |
 | WCAG 1.4.4  | AA    | Yes      |             |              |
-| WCAG 1.4.6  | AAA   | Yes      |             |              |
+| WCAG 1.4.6  | AAA   |          | Yes         |              |
 | WCAG 1.4.12 | AA    | Yes      |             |              |
 | WCAG 2.1.1  | A     | Yes      |             |              |
 | WCAG 2.1.3  | AAA   | Yes *    |             |              |

package/README.md

@@ -86,6 +86,7 @@ verapdf --version
 | OOBEE_VERBOSE | When set to `true`, log output goes to console | `false` |
 | OOBEE_FAST_CRAWLER| When set to `true`, increases scan concurrency at a rapid rate.  Experimental, may cause system stability issues on low-powered devices. | `false`|
 | OOBEE_VALIDATE_URL| When set to `true`, validates if URLs are valid and exits. | `false` |
+| OOBEE_LOGS_PATH | When set, logs are written to this path. |  |
 | WARN_LEVEL | Only used in tests. |  |
 
 #### Environment variables used internally (Do not set)

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@govtechsg/oobee",
   "main": "dist/npmIndex.js",
-  "version": "0.10.58",
+  "version": "0.10.62",
   "type": "module",
   "author": "Government Technology Agency <info@tech.gov.sg>",
   "dependencies": {
@@ -69,7 +69,8 @@
     "ansi-regex": "^5.0.1",
     "tough-cookie": "^5.0.0-rc.2",
     "micromatch": "github:micromatch/micromatch.git#4.0.8",
-    "brace-expansion": "^1.1.12"
+    "brace-expansion": "^1.1.12",
+    "tmp": "0.2.4"
   },
   "optionalDependencies": {
     "@napi-rs/canvas-darwin-arm64": "^0.1.53",

package/src/cli.ts

@@ -5,18 +5,14 @@ import printMessage from 'print-message';
 import { devices } from 'playwright';
 import { fileURLToPath } from 'url';
 import path from 'path';
-import { cleanUp, setHeadlessMode, getVersion, getStoragePath } from './utils.js';
+import { cleanUp, setHeadlessMode, getVersion, getStoragePath, listenForCleanUp, cleanUpAndExit } from './utils.js';
 import {
   checkUrl,
   prepareData,
   getFileSitemap,
   validEmail,
   validName,
-  getBrowserToRun,
-  getPlaywrightDeviceDetailsObject,
-  deleteClonedProfiles,
   getScreenToScan,
-  getClonedProfilesWithRandomToken,
   validateDirPath,
   validateFilePath,
   validateCustomFlowLabel,
@@ -57,20 +53,20 @@ Usage: npm run cli -- -c <crawler> -d <device> -w <viewport> -u <url> OPTIONS`,
         [`Invalid device. Please provide an existing device to start the scan.`],
         messageOptions,
       );
-      process.exit(1);
+      cleanUpAndExit(1);
     }
     return option;
   })
   .coerce('w', option => {
     if (!option || Number.isNaN(option)) {
       printMessage([`Invalid viewport width. Please provide a number. `], messageOptions);
-      process.exit(1);
+      cleanUpAndExit(1);
     } else if (option < 320 || option > 1080) {
       printMessage(
         ['Invalid viewport width! Please provide a viewport width between 320-1080 pixels.'],
         messageOptions,
       );
-      process.exit(1);
+      cleanUpAndExit(1);
     }
     return option;
   })
@@ -80,7 +76,7 @@ Usage: npm run cli -- -c <crawler> -d <device> -w <viewport> -u <url> OPTIONS`,
         [`Invalid maximum number of pages. Please provide a positive integer.`],
         messageOptions,
       );
-      process.exit(1);
+      cleanUpAndExit(1);
     }
     return option;
   })
@@ -90,7 +86,7 @@ Usage: npm run cli -- -c <crawler> -d <device> -w <viewport> -u <url> OPTIONS`,
         [`Invalid number for max concurrency. Please provide a positive integer.`],
         messageOptions,
       );
-      process.exit(1);
+      cleanUpAndExit(1);
     }
     return option;
   })
@@ -100,23 +96,23 @@ Usage: npm run cli -- -c <crawler> -d <device> -w <viewport> -u <url> OPTIONS`,
         [`Invalid format. Please provide your name and email address separated by ":"`],
         messageOptions,
       );
-      process.exit(1);
+      cleanUpAndExit(1);
     }
     const [name, email] = nameEmail.split(':');
     if (name === '' || name === undefined || name === null) {
       printMessage([`Please provide your name.`], messageOptions);
-      process.exit(1);
+      cleanUpAndExit(1);
     }
     if (!validName(name)) {
       printMessage([`Invalid name. Please provide a valid name.`], messageOptions);
-      process.exit(1);
+      cleanUpAndExit(1);
     }
     if (!validEmail(email)) {
       printMessage(
         [`Invalid email address. Please provide a valid email address.`],
         messageOptions,
       );
-      process.exit(1);
+      cleanUpAndExit(1);
     }
     return nameEmail;
   })
@@ -124,7 +120,7 @@ Usage: npm run cli -- -c <crawler> -d <device> -w <viewport> -u <url> OPTIONS`,
     const validationErrors = validateDirPath(option);
     if (validationErrors) {
       printMessage([`Invalid exportDirectory directory path. ${validationErrors}`], messageOptions);
-      process.exit(1);
+      cleanUpAndExit(1);
     }
     return option;
   })
@@ -136,7 +132,7 @@ Usage: npm run cli -- -c <crawler> -d <device> -w <viewport> -u <url> OPTIONS`,
       return validateFilePath(option, dirname);
     } catch (err) {
       printMessage([`Invalid blacklistedPatternsFilename file path. ${err}`], messageOptions);
-      process.exit(1);
+      cleanUpAndExit(1);
     }
   })
   .coerce('i', option => {
@@ -146,7 +142,7 @@ Usage: npm run cli -- -c <crawler> -d <device> -w <viewport> -u <url> OPTIONS`,
         [`Invalid value for fileTypes. Please provide valid keywords: ${choices.join(', ')}.`],
         messageOptions,
       );
-      process.exit(1);
+      cleanUpAndExit(1);
     }
     return option;
   })
@@ -154,7 +150,7 @@ Usage: npm run cli -- -c <crawler> -d <device> -w <viewport> -u <url> OPTIONS`,
     const { isValid, errorMessage } = validateCustomFlowLabel(option);
     if (!isValid) {
       printMessage([errorMessage], messageOptions);
-      process.exit(1);
+      cleanUpAndExit(1);
     }
     return option;
   })
@@ -165,7 +161,7 @@ Usage: npm run cli -- -c <crawler> -d <device> -w <viewport> -u <url> OPTIONS`,
         [`Invalid value for additional. Please provide valid keywords: ${choices.join(', ')}.`],
         messageOptions,
       );
-      process.exit(1);
+      cleanUpAndExit(1);
     }
     return option;
   })
@@ -202,12 +198,12 @@ Usage: npm run cli -- -c <crawler> -d <device> -w <viewport> -u <url> OPTIONS`,
         ['Invalid scan duration. Please provide a positive number of seconds.'],
         messageOptions,
       );
-      process.exit(1);
+      cleanUpAndExit(1);
     }
     return duration;
   })
   .check(argvs => {
-    if (argvs.scanner === ScannerTypes.CUSTOM && argvs.scanDuration > 0) {
+    if (argvs.scanner === ScannerTypes.CUSTOM && typeof argvs.scanDuration === 'number' && argvs.scanDuration > 0) {
       throw new Error('-l or --scanDuration is not allowed for custom flow scans.');
     }
     return true;
@@ -225,48 +221,30 @@ const scanInit = async (argvs: Answers): Promise<string> => {
 
   // Cannot use data.browser and data.isHeadless as the connectivity check comes first before prepareData
   setHeadlessMode(updatedArgvs.browserToRun, updatedArgvs.headless);
-
-  // let chromeDataDir = null;
-  // let edgeDataDir = null;
-  // Empty string for profile directory will use incognito mode in playwright
-  let clonedDataDir = '';
   const statuses = constants.urlCheckStatuses;
 
-  const { browserToRun, clonedBrowserDataDir } = getBrowserToRun(updatedArgvs.browserToRun, true);
-  updatedArgvs.browserToRun = browserToRun;
-  clonedDataDir = clonedBrowserDataDir;
-
-  if (updatedArgvs.customDevice === 'Desktop' || updatedArgvs.customDevice === 'Mobile') {
-    updatedArgvs.deviceChosen = argvs.customDevice;
-    delete updatedArgvs.customDevice;
-  }
+  const data = await prepareData(updatedArgvs);
 
-  // Creating the playwrightDeviceDetailObject
-  // for use in crawlDomain & crawlSitemap's preLaunchHook
-  updatedArgvs.playwrightDeviceDetailsObject = getPlaywrightDeviceDetailsObject(
-    updatedArgvs.deviceChosen,
-    updatedArgvs.customDevice,
-    updatedArgvs.viewportWidth,
-  );
+  // Executes cleanUp script if error encountered
+  listenForCleanUp(data.randomToken);
 
   const res = await checkUrl(
-    updatedArgvs.scanner,
-    updatedArgvs.url,
-    updatedArgvs.browserToRun,
-    clonedDataDir,
-    updatedArgvs.playwrightDeviceDetailsObject,
-    isCustomFlow,
-    parseHeaders(updatedArgvs.header),
+    data.type,
+    data.entryUrl,
+    data.browser,
+    data.userDataDirectory,
+    data.playwrightDeviceDetailsObject,
+    data.extraHTTPHeaders
   );
 
   if (res.httpStatus) consoleLogger.info(`Connectivity Check HTTP Response Code: ${res.httpStatus}`);
 
   switch (res.status) {
     case statuses.success.code: {
-      updatedArgvs.finalUrl = res.url;
+      data.url = res.url;
       if (process.env.OOBEE_VALIDATE_URL) {
         console.log('Url is valid');
-        process.exit(0);
+        cleanUpAndExit(0, data.randomToken);
       }
 
       break;
@@ -274,17 +252,17 @@ const scanInit = async (argvs: Answers): Promise<string> => {
     case statuses.unauthorised.code: {
       printMessage([statuses.unauthorised.message], messageOptions);
       consoleLogger.info(statuses.unauthorised.message);
-      process.exit(res.status);
+      cleanUpAndExit(res.status);
     }
     case statuses.cannotBeResolved.code: {
       printMessage([statuses.cannotBeResolved.message], messageOptions);
       consoleLogger.info(statuses.cannotBeResolved.message);
-      process.exit(res.status);
+      cleanUpAndExit(res.status);
     }
     case statuses.systemError.code: {
       printMessage([statuses.systemError.message], messageOptions);
       consoleLogger.info(statuses.systemError.message);
-      process.exit(res.status);
+      cleanUpAndExit(res.status);
     }
     case statuses.invalidUrl.code: {
       if (
@@ -293,67 +271,49 @@ const scanInit = async (argvs: Answers): Promise<string> => {
       ) {
         printMessage([statuses.invalidUrl.message], messageOptions);
         consoleLogger.info(statuses.invalidUrl.message);
-        process.exit(res.status);
+        cleanUpAndExit(res.status);
       }
 
       const finalFilePath = getFileSitemap(updatedArgvs.url);
       if (finalFilePath) {
-        updatedArgvs.isLocalFileScan = true;
-        updatedArgvs.finalUrl = finalFilePath;
+        data.isLocalFileScan = true;
+        data.url = finalFilePath;
 
         if (process.env.OOBEE_VALIDATE_URL) {
           console.log('Url is valid');
-          process.exit(0);
+          cleanUpAndExit(0);
         }
       } else if (updatedArgvs.scanner === ScannerTypes.LOCALFILE) {
         printMessage([statuses.notALocalFile.message], messageOptions);
         consoleLogger.info(statuses.notALocalFile.message);
-        process.exit(statuses.notALocalFile.code);
+        cleanUpAndExit(statuses.notALocalFile.code);
       } else if (updatedArgvs.scanner !== ScannerTypes.SITEMAP) {
         printMessage([statuses.notASitemap.message], messageOptions);
         consoleLogger.info(statuses.notASitemap.message);
-        process.exit(statuses.notASitemap.code);
+        cleanUpAndExit(statuses.notASitemap.code);
       }
       break;
     }
     case statuses.notASitemap.code: {
       printMessage([statuses.notASitemap.message], messageOptions);
       consoleLogger.info(statuses.notASitemap.message);
-      process.exit(res.status);
+      cleanUpAndExit(res.status);
     }
     case statuses.notALocalFile.code: {
       printMessage([statuses.notALocalFile.message], messageOptions);
       consoleLogger.info(statuses.notALocalFile.message);
-      process.exit(res.status);
+      cleanUpAndExit(res.status);
     }
     case statuses.browserError.code: {
       printMessage([statuses.browserError.message], messageOptions);
       consoleLogger.info(statuses.browserError.message);
-      process.exit(res.status);
+      cleanUpAndExit(res.status);
     }
     default:
       break;
   }
 
-  if (updatedArgvs.scanner === ScannerTypes.WEBSITE && !updatedArgvs.strategy) {
-    updatedArgvs.strategy = 'same-domain';
-  }
-
-  const data = await prepareData(updatedArgvs);
-
-  // File clean up after url check
-  // files will clone a second time below if url check passes
   if (process.env.OOBEE_VERBOSE) {
-    deleteClonedProfiles(data.browser, data.randomToken);
-  } else {
-    deleteClonedProfiles(data.browser); // first deletion
-  }
-
-  if (updatedArgvs.exportDirectory) {
-    constants.exportDirectory = updatedArgvs.exportDirectory;
-  }
-
-  if (process.env.RUNNING_FROM_PH_GUI || process.env.OOBEE_VERBOSE) {
     const randomTokenMessage = {
       type: 'randomToken',
       payload: `${data.randomToken}`,
@@ -364,29 +324,16 @@ const scanInit = async (argvs: Answers): Promise<string> => {
   }
 
   const screenToScan = getScreenToScan(
-    updatedArgvs.deviceChosen,
-    updatedArgvs.customDevice,
-    updatedArgvs.viewportWidth,
+    data.deviceChosen,
+    data.customDevice,
+    data.viewportWidth,
   );
 
-  // Clone profiles a second time
-  clonedDataDir = getClonedProfilesWithRandomToken(data.browser, data.randomToken);
-  data.userDataDirectory = clonedDataDir;
-
   printMessage([`Oobee version: ${appVersion}`, 'Starting scan...'], messageOptions);
-
+  consoleLogger.info(`Oobee version: ${appVersion}`); 
+  
   await combineRun(data, screenToScan);
 
-  // Delete cloned directory
-  if (process.env.OOBEE_VERBOSE) {
-    deleteClonedProfiles(data.browser, data.randomToken);
-  } else {
-    deleteClonedProfiles(data.browser); // second deletion
-  }
-
-  // Delete dataset and request queues
-  cleanUp(data.randomToken);
-
   return getStoragePath(data.randomToken);
 };
 
@@ -422,6 +369,6 @@ const optionsAnswer: Answers = {
 };
 
 await scanInit(optionsAnswer);
-process.exit(0);
+cleanUpAndExit(0);
 
 export default options;

package/src/combine.ts

@@ -5,9 +5,9 @@ import crawlDomain from './crawlers/crawlDomain.js';
 import crawlLocalFile from './crawlers/crawlLocalFile.js';
 import crawlIntelligentSitemap from './crawlers/crawlIntelligentSitemap.js';
 import generateArtifacts from './mergeAxeResults.js';
-import { getHost, createAndUpdateResultsFolders, createDetailsAndLogs } from './utils.js';
+import { getHost, createAndUpdateResultsFolders, createDetailsAndLogs, cleanUp, cleanUpAndExit } from './utils.js';
 import { ScannerTypes, UrlsCrawled } from './constants/constants.js';
-import { getBlackListedPatterns, submitForm, urlWithoutAuth } from './constants/common.js';
+import { getBlackListedPatterns, submitForm } from './constants/common.js';
 import { consoleLogger, silentLogger } from './logs.js';
 import runCustom from './crawlers/runCustom.js';
 import { alertMessageOptions } from './constants/cliFunctions.js';
@@ -55,7 +55,7 @@ const combineRun = async (details: Data, deviceToScan: string) => {
     includeScreenshots, // Include screenshots: if checked, = 'true'
     followRobots, // Adhere to robots.txt: if checked, = 'true'
     metadata,
-    customFlowLabel = 'Custom Flow',
+    customFlowLabel = 'None',
     extraHTTPHeaders,
     safeMode,
     zip,
@@ -67,6 +67,12 @@ const combineRun = async (details: Data, deviceToScan: string) => {
   process.env.CRAWLEE_LOG_LEVEL = 'ERROR';
   process.env.CRAWLEE_STORAGE_DIR = randomToken;
 
+  if (process.env.CRAWLEE_SYSTEM_INFO_V2 === undefined) {
+  // Set the environment variable to enable system info v2
+  // Resolves issue with when wmic is not installed on Windows
+    process.env.CRAWLEE_SYSTEM_INFO_V2 = '1';
+  }
+  
   const host = type === ScannerTypes.SITEMAP || type === ScannerTypes.LOCALFILE ? '' : getHost(url);
 
   let blacklistedPatterns: string[] | null = null;
@@ -74,12 +80,12 @@ const combineRun = async (details: Data, deviceToScan: string) => {
     blacklistedPatterns = getBlackListedPatterns(blacklistedPatternsFilename);
   } catch (error) {
     consoleLogger.error(error);
-    process.exit(1);
+    cleanUpAndExit(1);
   }
 
   // remove basic-auth credentials from URL
   const finalUrl = !(type === ScannerTypes.SITEMAP || type === ScannerTypes.LOCALFILE)
-    ? urlWithoutAuth(url)
+    ? new URL(url)
     : new URL(pathToFileURL(url));
 
   // Use the string version of finalUrl to reduce logic at submitForm
@@ -207,7 +213,7 @@ const combineRun = async (details: Data, deviceToScan: string) => {
 
     default:
       consoleLogger.error(`type: ${type} not defined`);
-      process.exit(1);
+      cleanUpAndExit(1);
   }
 
   scanDetails.endTime = new Date();
@@ -252,10 +258,16 @@ const combineRun = async (details: Data, deviceToScan: string) => {
         metadata,
       );
     } else {
+
+      // No page were scanned because the URL loaded does not meet the crawler requirements
       printMessage([`No pages were scanned.`], alertMessageOptions);
+      cleanUpAndExit(1, randomToken, true);
     }
   } else {
+
+    // No page were scanned because the URL loaded does not meet the crawler requirements
     printMessage([`No pages were scanned.`], alertMessageOptions);
+    cleanUpAndExit(1, randomToken, true);
   }
 };
 

package/src/constants/cliFunctions.ts

@@ -1,6 +1,7 @@
 import { Options } from 'yargs';
 import printMessage from 'print-message';
 import { BrowserTypes, RuleFlags, ScannerTypes } from './constants.js';
+import { cleanUpAndExit } from '../utils.js';
 
 export const messageOptions = {
   border: false,
@@ -53,7 +54,7 @@ export const cliOptions: { [key: string]: Options } = {
             ],
             messageOptions,
           );
-          process.exit(1);
+          cleanUpAndExit(1);
           return null;
       }
     },
@@ -158,7 +159,7 @@ export const cliOptions: { [key: string]: Options } = {
             ],
             messageOptions,
           );
-          process.exit(1);
+          cleanUpAndExit(1);
           return null;
       }
     },
@@ -282,7 +283,7 @@ export const cliOptions: { [key: string]: Options } = {
           ],
           messageOptions,
         );
-        process.exit(1);
+        cleanUpAndExit(1);
       }
       if (userChoices.length > 1 && userChoices.includes('default')) {
         printMessage(
@@ -291,7 +292,7 @@ export const cliOptions: { [key: string]: Options } = {
           ],
           messageOptions,
         );
-        process.exit(1);
+        cleanUpAndExit(1);
       }
       return userChoices;
     },