@govtechsg/oobee 0.10.58 → 0.10.61

package/src/index.ts

@@ -97,10 +97,7 @@ const runScan = async (answers: Answers) => {
     answers.customDevice,
     answers.viewportWidth,
   );
-  const { browserToRun } = getBrowserToRun(BrowserTypes.CHROME);
-  deleteClonedProfiles(browserToRun);
-  answers.browserToRun = browserToRun;
-
+    
   if (!answers.nameEmail) {
     answers.nameEmail = `${userData.name}:${userData.email}`;
   }
@@ -116,7 +113,7 @@ const runScan = async (answers: Answers) => {
   await combineRun(data, screenToScan);
 
   // Delete cloned directory
-  deleteClonedProfiles(data.browser);
+  deleteClonedProfiles(data.browser, data.randomToken);
 
   // Delete dataset and request queues
   cleanUp(data.randomToken);

package/src/logs.ts

@@ -2,7 +2,6 @@
 /* eslint-disable no-shadow */
 import { createLogger, format, transports } from 'winston';
 import { guiInfoStatusTypes } from './constants/constants.js';
-import { urlWithoutAuth } from './constants/common.js';
 
 const { combine, timestamp, printf } = format;
 
@@ -54,7 +53,7 @@ export const guiInfoLog = (status: string, data: { numScanned?: number; urlScann
       case guiInfoStatusTypes.DUPLICATE:
         console.log(
           `crawling::${data.numScanned || 0}::${status}::${
-            urlWithoutAuth(data.urlScanned) || 'no url provided'
+            data.urlScanned || 'no url provided'
           }`,
         );
         break;

package/src/mergeAxeResults.ts

@@ -15,7 +15,7 @@ import { pipeline } from 'stream/promises';
 // @ts-ignore
 import * as Sentry from '@sentry/node';
 import constants, { ScannerTypes, sentryConfig, setSentryUser } from './constants/constants.js';
-import { urlWithoutAuth } from './constants/common.js';
+import { getBrowserToRun, getPlaywrightLaunchOptions } from './constants/common.js';
 
 import {
   createScreenshotsFolder,
@@ -961,29 +961,19 @@ const writeScanDetailsCsv = async (
   });
 };
 
-let browserChannel = 'chrome';
+let browserChannel = getBrowserToRun().browserToRun;
 
-if (os.platform() === 'win32') {
-  browserChannel = 'msedge';
-}
-
-if (os.platform() === 'linux') {
-  browserChannel = 'chromium';
-}
-
-const writeSummaryPdf = async (storagePath: string, pagesScanned: number, filename = 'summary') => {
+const writeSummaryPdf = async (storagePath: string, pagesScanned: number, filename = 'summary', browser: string, userDataDirectory: string) => {
   const htmlFilePath = `${storagePath}/${filename}.html`;
   const fileDestinationPath = `${storagePath}/${filename}.pdf`;
-  const browser = await chromium.launch({
-    headless: false,
-    channel: browserChannel,
-    args: ['--headless=new', '--no-sandbox'],
-  });
 
-  const context = await browser.newContext({
-    ignoreHTTPSErrors: true,
-    serviceWorkers: 'block',
-  });
+  const effectiveUserDataDirectory = process.env.CRAWLEE_HEADLESS === '1'
+    ? userDataDirectory
+    : '';
+  const context = await constants.launcher.launchPersistentContext(effectiveUserDataDirectory, {
+        headless: process.env.CRAWLEE_HEADLESS === '1',
+        ...getPlaywrightLaunchOptions(browser),
+      });
 
   const page = await context.newPage();
 
@@ -1008,8 +998,7 @@ const writeSummaryPdf = async (storagePath: string, pagesScanned: number, filena
 
   await page.close();
 
-  await context.close();
-  await browser.close();
+  await context.close().catch(() => {});
 
   if (pagesScanned < 2000) {
     fs.unlinkSync(htmlFilePath);
@@ -1721,15 +1710,10 @@ const generateArtifacts = async (
   zip: string = undefined, // optional
   generateJsonFiles = false,
 ) => {
-  const intermediateDatasetsPath = `${randomToken}/datasets/${randomToken}`;
+  const intermediateDatasetsPath = `${getStoragePath(randomToken)}/crawlee`;
   const oobeeAppVersion = getVersion();
   const storagePath = getStoragePath(randomToken);
 
-  urlScanned =
-    scanType === ScannerTypes.SITEMAP || scanType === ScannerTypes.LOCALFILE
-      ? urlScanned
-      : urlWithoutAuth(urlScanned);
-
   const formatAboutStartTime = (dateString: string) => {
     const utcStartTimeDate = new Date(dateString);
     const formattedStartTime = utcStartTimeDate.toLocaleTimeString('en-GB', {
@@ -1851,10 +1835,18 @@ const generateArtifacts = async (
 
   printMessage([
     'Scan Summary',
+    `Oobee App Version: ${allIssues.oobeeAppVersion}`,
     '',
     `Site Name: ${allIssues.siteName}`,
     `URL: ${allIssues.urlScanned}`,
     `Pages Scanned: ${allIssues.totalPagesScanned}`,
+    `Start Time: ${allIssues.startTime}`,
+    `End Time: ${allIssues.endTime}`,
+    `Elapsed Time: ${(new Date(allIssues.endTime).getTime() - new Date(allIssues.startTime).getTime()) / 1000}s`,
+    `Device: ${allIssues.deviceChosen}`,
+    `Viewport: ${allIssues.viewport}`,
+    `Scan Type: ${allIssues.scanType}`,
+    `Label: ${allIssues.customFlowLabel || 'N/A'}`,
     '',
     `Must Fix: ${allIssues.items.mustFix.rules.length} ${Object.keys(allIssues.items.mustFix.rules).length === 1 ? 'issue' : 'issues'} / ${allIssues.items.mustFix.totalItems} ${allIssues.items.mustFix.totalItems === 1 ? 'occurrence' : 'occurrences'}`,
     `Good to Fix: ${allIssues.items.goodToFix.rules.length} ${Object.keys(allIssues.items.goodToFix.rules).length === 1 ? 'issue' : 'issues'} / ${allIssues.items.goodToFix.totalItems} ${allIssues.items.goodToFix.totalItems === 1 ? 'occurrence' : 'occurrences'}`,
@@ -1892,7 +1884,11 @@ const generateArtifacts = async (
   consoleLogger.info(`End Time: ${allIssues.endTime}`);
   const elapsedSeconds = (new Date(allIssues.endTime).getTime() - new Date(allIssues.startTime).getTime()) / 1000;
   consoleLogger.info(`Elapsed Time: ${elapsedSeconds}s`);
-
+  consoleLogger.info(`Device: ${allIssues.deviceChosen}`);
+  consoleLogger.info(`Viewport: ${allIssues.viewport}`);
+  consoleLogger.info(`Scan Type: ${allIssues.scanType}`);
+  consoleLogger.info(`Label: ${allIssues.customFlowLabel || 'N/A'}`);
+  
   const getAxeImpactCount = (allIssues: AllIssues) => {
     const impactCount = {
       critical: 0,
@@ -1986,7 +1982,16 @@ const generateArtifacts = async (
     ]);
   }
 
-  await retryFunction(() => writeSummaryPdf(storagePath, pagesScanned.length), 1);
+  // Should consider refactor constants.userDataDirectory to be a parameter in future
+  await retryFunction(() => writeSummaryPdf(storagePath, pagesScanned.length, 'summary', browserChannel, constants.userDataDirectory), 1);
+
+  const foldersToRemove = ['crawlee', 'logs'];
+  for (const folder of foldersToRemove) {
+    const folderPath = path.join(storagePath, folder);
+    if (await fs.pathExists(folderPath)) {
+      await fs.remove(folderPath);
+    }
+  }
 
   // Take option if set
   if (typeof zip === 'string') {

package/src/npmIndex.ts

@@ -10,7 +10,6 @@ import {
   getBrowserToRun,
   getPlaywrightLaunchOptions,
   submitForm,
-  urlWithoutAuth,
 } from './constants/common.js';
 import { createCrawleeSubFolders, filterAxeResults } from './crawlers/commonCrawlerFunc.js';
 import { createAndUpdateResultsFolders, createDetailsAndLogs } from './utils.js';
@@ -201,11 +200,12 @@ export const init = async ({
     res: { pageUrl: string; pageTitle: string; axeScanResults: AxeResults },
     metadata: string,
     elementsToClick: string[],
+    randomToken: string,
   ) => {
     throwErrorIfTerminated();
     if (includeScreenshots) {
       // use chrome by default
-      const { browserToRun, clonedBrowserDataDir } = getBrowserToRun(BrowserTypes.CHROME);
+      const { browserToRun, clonedBrowserDataDir } = getBrowserToRun(BrowserTypes.CHROME, false, randomToken);
       const browserContext = await constants.launcher.launchPersistentContext(
         clonedBrowserDataDir,
         { viewport: viewportSettings, ...getPlaywrightLaunchOptions(browserToRun) },
@@ -237,7 +237,7 @@ export const init = async ({
       );
 
       await browserContext.close();
-      deleteClonedProfiles(browserToRun);
+      deleteClonedProfiles(browserToRun, randomToken);
     }
     const pageIndex = urlsCrawled.scanned.length + 1;
     const filteredResults = filterAxeResults(res.axeScanResults, res.pageTitle, {
@@ -245,7 +245,7 @@ export const init = async ({
       metadata,
     });
     urlsCrawled.scanned.push({
-      url: urlWithoutAuth(res.pageUrl).toString(),
+      url: res.pageUrl.toString(),
       actualUrl: 'tbd',
       pageTitle: `${pageIndex}: ${res.pageTitle}`,
     });

package/src/utils.ts

@@ -11,6 +11,7 @@ import constants, {
 } from './constants/constants.js';
 import { consoleLogger, silentLogger } from './logs.js';
 import { getAxeConfiguration } from './crawlers/custom/getAxeConfiguration.js';
+import { constant } from 'lodash';
 
 export const getVersion = () => {
   const loadJSON = (filePath: string): { version: string } =>
@@ -33,21 +34,57 @@ export const isWhitelistedContentType = (contentType: string): boolean => {
 };
 
 export const getStoragePath = (randomToken: string): string => {
-  if (process.env.OOBEE_VERBOSE_STORAGE_PATH) {
-    return `${process.env.OOBEE_VERBOSE_STORAGE_PATH}/${randomToken}`;
+  // If exportDirectory is set, use it
+  if (constants.exportDirectory) {
+    return constants.exportDirectory;
   }
-  if (constants.exportDirectory === process.cwd()) {
-    return `results/${randomToken}`;
+
+  // Otherwise, use the current working directory
+  let storagePath = path.join(process.cwd(), 'results', randomToken);
+
+  // Ensure storagePath is writable; if directory doesn't exist, try to create it in Documents or home directory
+  const isWritable = (() => {
+    try {
+      if (!fs.existsSync(storagePath)) {
+        fs.mkdirSync(storagePath, { recursive: true });
+      }
+      fs.accessSync(storagePath, fs.constants.W_OK);
+      return true;
+    } catch {
+      return false;
+    }
+  })();
+
+  if (!isWritable) {
+    if (os.platform() === 'win32') {
+      // Use Documents folder on Windows
+      const documentsPath = path.join(process.env.USERPROFILE || process.env.HOMEPATH || '', 'Documents');
+      storagePath = path.join(documentsPath, 'Oobee', randomToken);
+    } else if (os.platform() === 'darwin') {
+      // Use Documents folder on Mac
+      const documentsPath = path.join(process.env.HOME || '', 'Documents');
+      storagePath = path.join(documentsPath, 'Oobee', randomToken);
+    } else {
+      // Use home directory for Linux/other
+      const homePath = process.env.HOME || '';
+      storagePath = path.join(homePath, 'Oobee', randomToken);
+    }
+    consoleLogger.warn(`Warning: Cannot write to cwd, writing to ${storagePath}`);
+
   }
-  if (!path.isAbsolute(constants.exportDirectory)) {
-    constants.exportDirectory = path.resolve(process.cwd(), constants.exportDirectory);
+
+  if (!fs.existsSync(storagePath)) {
+    fs.mkdirSync(storagePath, { recursive: true });
   }
-  return `${constants.exportDirectory}/${randomToken}`;
+
+  constants.exportDirectory = storagePath;
+  return storagePath;
+
 };
 
 export const createDetailsAndLogs = async (randomToken: string): Promise<void> => {
   const storagePath = getStoragePath(randomToken);
-  const logPath = `logs/${randomToken}`;
+  const logPath = `${getStoragePath(randomToken)}/logs`;
   try {
     await fs.ensureDir(storagePath);
 
@@ -193,8 +230,11 @@ export const createScreenshotsFolder = (randomToken: string): void => {
   }
 };
 
-export const cleanUp = (pathToDelete: string): void => {
-  fs.removeSync(pathToDelete);
+export const cleanUp = (randomToken: string): void => {
+  fs.removeSync(randomToken);
+  fs.removeSync(path.join(process.env.APPDATA || '/tmp', randomToken));
+  fs.removeSync(path.join(getStoragePath(randomToken),'crawlee'));
+  fs.removeSync(path.join(getStoragePath(randomToken),'logs'));
 };
 
 export const getWcagPassPercentage = (
@@ -711,16 +751,18 @@ export const zipResults = (zipName: string, resultsPath: string): void => {
     fs.unlinkSync(zipName);
   }
 
+  // Check if user specified absolute or relative path
+  const zipFilePath = path.isAbsolute(zipName) ? zipName : path.join(process.cwd(), zipName);
+
+
   if (os.platform() === 'win32') {
     execSync(
-      `Get-ChildItem -Path "${resultsPath}\\*.*" -Recurse | Compress-Archive -DestinationPath "${zipName}"`,
-      { shell: 'powershell.exe' },
+      `Get-ChildItem -Path "*.*" -Recurse | Compress-Archive -DestinationPath "${zipFilePath}"`,
+      { shell: 'powershell.exe', cwd: resultsPath },
     );
   } else {
     // Get zip command in Mac and Linux
     const command = '/usr/bin/zip';
-    // Check if user specified absolute or relative path
-    const zipFilePath = path.isAbsolute(zipName) ? zipName : path.join(process.cwd(), zipName);
 
     // To zip up files recursively (-r) in the results folder path and write it to user's specified path
     const args = ['-r', zipFilePath, '.'];