@govtechsg/oobee 0.10.58 → 0.10.61

package/DETAILS.md

@@ -41,7 +41,7 @@ Note: Level AAA are disabled by default.  Please specify `enable-wcag-aaa` in ru
 | WCAG 1.4.2  | A     | Yes      |             |              |
 | WCAG 1.4.3  | AA    | Yes      |             |              |
 | WCAG 1.4.4  | AA    | Yes      |             |              |
-| WCAG 1.4.6  | AAA   | Yes      |             |              |
+| WCAG 1.4.6  | AAA   |          | Yes         |              |
 | WCAG 1.4.12 | AA    | Yes      |             |              |
 | WCAG 2.1.1  | A     | Yes      |             |              |
 | WCAG 2.1.3  | AAA   | Yes *    |             |              |

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@govtechsg/oobee",
   "main": "dist/npmIndex.js",
-  "version": "0.10.58",
+  "version": "0.10.61",
   "type": "module",
   "author": "Government Technology Agency <info@tech.gov.sg>",
   "dependencies": {

package/src/cli.ts

@@ -12,11 +12,8 @@ import {
   getFileSitemap,
   validEmail,
   validName,
-  getBrowserToRun,
-  getPlaywrightDeviceDetailsObject,
   deleteClonedProfiles,
   getScreenToScan,
-  getClonedProfilesWithRandomToken,
   validateDirPath,
   validateFilePath,
   validateCustomFlowLabel,
@@ -207,7 +204,7 @@ Usage: npm run cli -- -c <crawler> -d <device> -w <viewport> -u <url> OPTIONS`,
     return duration;
   })
   .check(argvs => {
-    if (argvs.scanner === ScannerTypes.CUSTOM && argvs.scanDuration > 0) {
+    if (argvs.scanner === ScannerTypes.CUSTOM && typeof argvs.scanDuration === 'number' && argvs.scanDuration > 0) {
       throw new Error('-l or --scanDuration is not allowed for custom flow scans.');
     }
     return true;
@@ -225,45 +222,26 @@ const scanInit = async (argvs: Answers): Promise<string> => {
 
   // Cannot use data.browser and data.isHeadless as the connectivity check comes first before prepareData
   setHeadlessMode(updatedArgvs.browserToRun, updatedArgvs.headless);
-
-  // let chromeDataDir = null;
-  // let edgeDataDir = null;
-  // Empty string for profile directory will use incognito mode in playwright
-  let clonedDataDir = '';
   const statuses = constants.urlCheckStatuses;
 
-  const { browserToRun, clonedBrowserDataDir } = getBrowserToRun(updatedArgvs.browserToRun, true);
-  updatedArgvs.browserToRun = browserToRun;
-  clonedDataDir = clonedBrowserDataDir;
-
-  if (updatedArgvs.customDevice === 'Desktop' || updatedArgvs.customDevice === 'Mobile') {
-    updatedArgvs.deviceChosen = argvs.customDevice;
-    delete updatedArgvs.customDevice;
-  }
+  const data = await prepareData(updatedArgvs);
 
-  // Creating the playwrightDeviceDetailObject
-  // for use in crawlDomain & crawlSitemap's preLaunchHook
-  updatedArgvs.playwrightDeviceDetailsObject = getPlaywrightDeviceDetailsObject(
-    updatedArgvs.deviceChosen,
-    updatedArgvs.customDevice,
-    updatedArgvs.viewportWidth,
-  );
+  constants.userDataDirectory = data.userDataDirectory;
 
   const res = await checkUrl(
-    updatedArgvs.scanner,
-    updatedArgvs.url,
-    updatedArgvs.browserToRun,
-    clonedDataDir,
-    updatedArgvs.playwrightDeviceDetailsObject,
-    isCustomFlow,
-    parseHeaders(updatedArgvs.header),
+    data.type,
+    data.entryUrl,
+    data.browser,
+    data.userDataDirectory,
+    data.playwrightDeviceDetailsObject,
+    data.extraHTTPHeaders
   );
 
   if (res.httpStatus) consoleLogger.info(`Connectivity Check HTTP Response Code: ${res.httpStatus}`);
 
   switch (res.status) {
     case statuses.success.code: {
-      updatedArgvs.finalUrl = res.url;
+      data.url = res.url;
       if (process.env.OOBEE_VALIDATE_URL) {
         console.log('Url is valid');
         process.exit(0);
@@ -298,8 +276,8 @@ const scanInit = async (argvs: Answers): Promise<string> => {
 
       const finalFilePath = getFileSitemap(updatedArgvs.url);
       if (finalFilePath) {
-        updatedArgvs.isLocalFileScan = true;
-        updatedArgvs.finalUrl = finalFilePath;
+        data.isLocalFileScan = true;
+        data.url = finalFilePath;
 
         if (process.env.OOBEE_VALIDATE_URL) {
           console.log('Url is valid');
@@ -335,25 +313,9 @@ const scanInit = async (argvs: Answers): Promise<string> => {
       break;
   }
 
-  if (updatedArgvs.scanner === ScannerTypes.WEBSITE && !updatedArgvs.strategy) {
-    updatedArgvs.strategy = 'same-domain';
-  }
-
-  const data = await prepareData(updatedArgvs);
+  deleteClonedProfiles(data.browser, data.randomToken);
 
-  // File clean up after url check
-  // files will clone a second time below if url check passes
   if (process.env.OOBEE_VERBOSE) {
-    deleteClonedProfiles(data.browser, data.randomToken);
-  } else {
-    deleteClonedProfiles(data.browser); // first deletion
-  }
-
-  if (updatedArgvs.exportDirectory) {
-    constants.exportDirectory = updatedArgvs.exportDirectory;
-  }
-
-  if (process.env.RUNNING_FROM_PH_GUI || process.env.OOBEE_VERBOSE) {
     const randomTokenMessage = {
       type: 'randomToken',
       payload: `${data.randomToken}`,
@@ -364,25 +326,16 @@ const scanInit = async (argvs: Answers): Promise<string> => {
   }
 
   const screenToScan = getScreenToScan(
-    updatedArgvs.deviceChosen,
-    updatedArgvs.customDevice,
-    updatedArgvs.viewportWidth,
+    data.deviceChosen,
+    data.customDevice,
+    data.viewportWidth,
   );
 
-  // Clone profiles a second time
-  clonedDataDir = getClonedProfilesWithRandomToken(data.browser, data.randomToken);
-  data.userDataDirectory = clonedDataDir;
-
   printMessage([`Oobee version: ${appVersion}`, 'Starting scan...'], messageOptions);
 
   await combineRun(data, screenToScan);
 
-  // Delete cloned directory
-  if (process.env.OOBEE_VERBOSE) {
-    deleteClonedProfiles(data.browser, data.randomToken);
-  } else {
-    deleteClonedProfiles(data.browser); // second deletion
-  }
+  deleteClonedProfiles(data.browser, data.randomToken);
 
   // Delete dataset and request queues
   cleanUp(data.randomToken);

package/src/combine.ts

@@ -5,9 +5,9 @@ import crawlDomain from './crawlers/crawlDomain.js';
 import crawlLocalFile from './crawlers/crawlLocalFile.js';
 import crawlIntelligentSitemap from './crawlers/crawlIntelligentSitemap.js';
 import generateArtifacts from './mergeAxeResults.js';
-import { getHost, createAndUpdateResultsFolders, createDetailsAndLogs } from './utils.js';
+import { getHost, createAndUpdateResultsFolders, createDetailsAndLogs, cleanUp } from './utils.js';
 import { ScannerTypes, UrlsCrawled } from './constants/constants.js';
-import { getBlackListedPatterns, submitForm, urlWithoutAuth } from './constants/common.js';
+import { getBlackListedPatterns, submitForm } from './constants/common.js';
 import { consoleLogger, silentLogger } from './logs.js';
 import runCustom from './crawlers/runCustom.js';
 import { alertMessageOptions } from './constants/cliFunctions.js';
@@ -55,7 +55,7 @@ const combineRun = async (details: Data, deviceToScan: string) => {
     includeScreenshots, // Include screenshots: if checked, = 'true'
     followRobots, // Adhere to robots.txt: if checked, = 'true'
     metadata,
-    customFlowLabel = 'Custom Flow',
+    customFlowLabel = 'None',
     extraHTTPHeaders,
     safeMode,
     zip,
@@ -67,6 +67,12 @@ const combineRun = async (details: Data, deviceToScan: string) => {
   process.env.CRAWLEE_LOG_LEVEL = 'ERROR';
   process.env.CRAWLEE_STORAGE_DIR = randomToken;
 
+  if (process.env.CRAWLEE_SYSTEM_INFO_V2 === undefined) {
+  // Set the environment variable to enable system info v2
+  // Resolves issue with when wmic is not installed on Windows
+    process.env.CRAWLEE_SYSTEM_INFO_V2 = '1';
+  }
+  
   const host = type === ScannerTypes.SITEMAP || type === ScannerTypes.LOCALFILE ? '' : getHost(url);
 
   let blacklistedPatterns: string[] | null = null;
@@ -79,7 +85,7 @@ const combineRun = async (details: Data, deviceToScan: string) => {
 
   // remove basic-auth credentials from URL
   const finalUrl = !(type === ScannerTypes.SITEMAP || type === ScannerTypes.LOCALFILE)
-    ? urlWithoutAuth(url)
+    ? new URL(url)
     : new URL(pathToFileURL(url));
 
   // Use the string version of finalUrl to reduce logic at submitForm
@@ -252,10 +258,18 @@ const combineRun = async (details: Data, deviceToScan: string) => {
         metadata,
       );
     } else {
+
+      // No page were scanned because the URL loaded does not meet the crawler requirements
       printMessage([`No pages were scanned.`], alertMessageOptions);
+      cleanUp(randomToken);
+      process.exit(1);
     }
   } else {
+
+    // No page were scanned because the URL loaded does not meet the crawler requirements
     printMessage([`No pages were scanned.`], alertMessageOptions);
+    cleanUp(randomToken);
+    process.exit(1);
   }
 };