@govtechsg/oobee 0.10.58 → 0.10.62

package/src/crawlers/crawlSitemap.ts

@@ -17,15 +17,15 @@ import {
   getLinksFromSitemap,
   getPlaywrightLaunchOptions,
   isSkippedUrl,
-  urlWithoutAuth,
   waitForPageLoaded,
   isFilePath,
-  initModifiedUserAgent,
 } from '../constants/common.js';
-import { areLinksEqual, isWhitelistedContentType, isFollowStrategy } from '../utils.js';
+import { areLinksEqual, isWhitelistedContentType, register } from '../utils.js';
 import { handlePdfDownload, runPdfScan, mapPdfScanResults } from './pdfScanFunc.js';
 import { guiInfoLog } from '../logs.js';
 import { ViewportSettingsClass } from '../combine.js';
+import * as path from 'path';
+import fsp from 'fs/promises';
 
 const crawlSitemap = async ({
   sitemapUrl,
@@ -70,50 +70,19 @@ const crawlSitemap = async ({
   let dataset: crawlee.Dataset;
   let urlsCrawled: UrlsCrawled;
 
-  // Boolean to omit axe scan for basic auth URL
-  let isBasicAuth: boolean;
-  let basicAuthPage = 0;
-  let finalLinks = [];
-  let authHeader = '';
-
   if (fromCrawlIntelligentSitemap) {
     dataset = datasetFromIntelligent;
     urlsCrawled = urlsCrawledFromIntelligent;
   } else {
     ({ dataset } = await createCrawleeSubFolders(randomToken));
     urlsCrawled = { ...constants.urlsCrawledObj };
-
-    if (!fs.existsSync(randomToken)) {
-      fs.mkdirSync(randomToken);
-    }
   }
 
-  let parsedUrl;
-  let username = '';
-  let password = '';
-
   if (!crawledFromLocalFile && isFilePath(sitemapUrl)) {
     console.log('Local file crawling not supported for sitemap. Please provide a valid URL.');
     return;
   }
 
-  if (isFilePath(sitemapUrl)) {
-    parsedUrl = sitemapUrl;
-  } else {
-    parsedUrl = new URL(sitemapUrl);
-    if (parsedUrl.username !== '' && parsedUrl.password !== '') {
-      isBasicAuth = true;
-      username = decodeURIComponent(parsedUrl.username);
-      password = decodeURIComponent(parsedUrl.password);
-
-      // Create auth header
-      authHeader = `Basic ${Buffer.from(`${username}:${password}`).toString('base64')}`;
-
-      parsedUrl.username = '';
-      parsedUrl.password = '';
-    }
-  }
-
   const linksFromSitemap = await getLinksFromSitemap(
     sitemapUrl,
     maxRequestsPerCrawl,
@@ -121,29 +90,11 @@ const crawlSitemap = async ({
     userDataDirectory,
     userUrlInputFromIntelligent,
     fromCrawlIntelligentSitemap,
-    username,
-    password,
+    extraHTTPHeaders,
   );
-  /**
-   * Regex to match http://username:password@hostname.com
-   * utilised in scan strategy to ensure subsequent URLs within the same domain are scanned.
-   * First time scan with original `url` containing credentials is strictly to authenticate for browser session
-   * subsequent URLs are without credentials.
-   * basicAuthPage is set to -1 for basic auth URL to ensure it is not counted towards maxRequestsPerCrawl
-   */
 
   sitemapUrl = encodeURI(sitemapUrl);
 
-  if (isBasicAuth) {
-    // request to basic auth URL to authenticate for browser session
-    finalLinks.push(new Request({ url: sitemapUrl, uniqueKey: `auth:${sitemapUrl}` }));
-    const finalUrl = `${sitemapUrl.split('://')[0]}://${sitemapUrl.split('@')[1]}`;
-
-    // obtain base URL without credentials so that subsequent URLs within the same domain can be scanned
-    finalLinks.push(new Request({ url: finalUrl }));
-    basicAuthPage = -2;
-  }
-
   const pdfDownloads: Promise<void>[] = [];
   const uuidToPdfMapping: Record<string, string> = {};
   const isScanHtml = ['all', 'html-only'].includes(fileTypes);
@@ -151,36 +102,43 @@ const crawlSitemap = async ({
   const { playwrightDeviceDetailsObject } = viewportSettings;
   const { maxConcurrency } = constants;
 
-  finalLinks = [...finalLinks, ...linksFromSitemap];
-
   const requestList = await RequestList.open({
-    sources: finalLinks,
+    sources: linksFromSitemap,
   });
 
-  let userDataDir = '';
-  if (userDataDirectory) {
-    userDataDir = process.env.CRAWLEE_HEADLESS !== '0' ? userDataDirectory : '';
-  }
-
-  await initModifiedUserAgent(browser, playwrightDeviceDetailsObject);
-  const crawler = new crawlee.PlaywrightCrawler({
+  const crawler = register(new crawlee.PlaywrightCrawler({
     launchContext: {
       launcher: constants.launcher,
       launchOptions: getPlaywrightLaunchOptions(browser),
       // Bug in Chrome which causes browser pool crash when userDataDirectory is set in non-headless mode
-      ...(process.env.CRAWLEE_HEADLESS === '0' && { userDataDir }),
+      ...(process.env.CRAWLEE_HEADLESS === '1' && { userDataDir: userDataDirectory }),
     },
     retryOnBlocked: true,
     browserPoolOptions: {
       useFingerprints: false,
       preLaunchHooks: [
-        async (_pageId: string, launchContext: LaunchContext) => {
+        async (_pageId, launchContext) => {
+          const baseDir = userDataDirectory; // e.g., /Users/young/.../Chrome/oobee-...
+
+          // Ensure base exists
+          await fsp.mkdir(baseDir, { recursive: true });
+
+          // Create a unique subdir per browser
+          const subProfileDir = path.join(baseDir, `profile-${Date.now()}-${Math.random().toString(36).slice(2, 8)}`);
+          await fsp.mkdir(subProfileDir, { recursive: true });
+
+          // Assign to Crawlee's launcher
+          launchContext.userDataDir = subProfileDir;
+
+          // Safely extend launchOptions
           launchContext.launchOptions = {
             ...launchContext.launchOptions,
-            bypassCSP: true,
             ignoreHTTPSErrors: true,
             ...playwrightDeviceDetailsObject,
           };
+
+          // Optionally log for debugging
+          // console.log(`[HOOK] Using userDataDir: ${subProfileDir}`);
         },
       ],
     },
@@ -193,7 +151,7 @@ const crawlSitemap = async ({
             return new Promise(resolve => {
               let timeout;
               let mutationCount = 0;
-              const MAX_MUTATIONS = 250; // stop if things never quiet down
+              const MAX_MUTATIONS = 500; // stop if things never quiet down
               const OBSERVER_TIMEOUT = 5000; // hard cap on total wait
 
               const observer = new MutationObserver(() => {
@@ -252,15 +210,7 @@ const crawlSitemap = async ({
           return;
         }
 
-        // Set headers if basic auth
-        if (isBasicAuth) {
-          await page.setExtraHTTPHeaders({
-            Authorization: authHeader,
-            ...extraHTTPHeaders,
-          });
-        } else {
-          preNavigationHooks(extraHTTPHeaders);
-        }
+        preNavigationHooks(extraHTTPHeaders);
       },
     ],
     requestHandlerTimeoutSecs: 90,
@@ -282,17 +232,6 @@ const crawlSitemap = async ({
         return;
       }
 
-      // Set basic auth header if needed
-      if (isBasicAuth) {
-        await page.setExtraHTTPHeaders({
-          Authorization: authHeader,
-        });
-        const currentUrl = new URL(request.url);
-        currentUrl.username = username;
-        currentUrl.password = password;
-        request.url = currentUrl.href;
-      }
-
       await waitForPageLoaded(page, 10000);
 
       const actualUrl = page.url() || request.loadedUrl || request.url;
@@ -341,9 +280,7 @@ const crawlSitemap = async ({
       const contentType = response?.headers?.()['content-type'] || '';
       const status = response ? response.status() : 0;
 
-      if (basicAuthPage < 0) {
-        basicAuthPage += 1;
-      } else if (isScanHtml && status < 300 && isWhitelistedContentType(contentType)) {
+      if (isScanHtml && status < 300 && isWhitelistedContentType(contentType)) {
         const isRedirected = !areLinksEqual(page.url(), request.url);
         const isLoadedUrlInCrawledUrls = urlsCrawled.scanned.some(
           item => (item.actualUrl || item.url) === page.url(),
@@ -382,13 +319,13 @@ const crawlSitemap = async ({
         });
 
         urlsCrawled.scanned.push({
-          url: urlWithoutAuth(request.url),
+          url: request.url,
           pageTitle: results.pageTitle,
           actualUrl, // i.e. actualUrl
         });
 
         urlsCrawled.scannedRedirects.push({
-          fromUrl: urlWithoutAuth(request.url),
+          fromUrl: request.url,
           toUrl: actualUrl,
         });
 
@@ -421,9 +358,6 @@ const crawlSitemap = async ({
       }
     },
     failedRequestHandler: async ({ request, response, error }) => {
-      if (isBasicAuth && request.url) {
-        request.url = `${request.url.split('://')[0]}://${request.url.split('@')[1]}`;
-      }
 
       // check if scanned pages have reached limit due to multi-instances of handler running
       if (urlsCrawled.scanned.length >= maxRequestsPerCrawl) {
@@ -461,7 +395,7 @@ const crawlSitemap = async ({
         scaleDownStepRatio: 0.1,       // Scale down slower
       },
     }),
-  });
+  }));
 
   await crawler.run();
 

package/src/crawlers/custom/utils.ts

@@ -6,7 +6,7 @@ import path from 'path';
 import { runAxeScript } from '../commonCrawlerFunc.js';
 import { consoleLogger, guiInfoLog, silentLogger } from '../../logs.js';
 import { guiInfoStatusTypes } from '../../constants/constants.js';
-import { isSkippedUrl, urlWithoutAuth } from '../../constants/common.js';
+import { isSkippedUrl } from '../../constants/common.js';
 
 //! For Cypress Test
 // env to check if Cypress test is running
@@ -77,8 +77,8 @@ export const screenshotFullPage = async (page, screenshotsDir: string, screensho
       window.scrollTo(0, 0);
     });
 
-    consoleLogger.info(`Screenshot page at: ${urlWithoutAuth(page.url())}`);
-    silentLogger.info(`Screenshot page at: ${urlWithoutAuth(page.url())}`);
+    consoleLogger.info(`Screenshot page at: ${page.url()}`);
+    consoleLogger.info(`Screenshot page at: ${page.url()}`);
 
     await page.screenshot({
       timeout: 5000,
@@ -116,7 +116,7 @@ export const runAxeScan = async (
   await dataset.pushData(result);
 
   urlsCrawled.scanned.push({
-    url: urlWithoutAuth(page.url()),
+    url: page.url(),
     pageTitle: result.pageTitle,
     pageImagePath: customFlowDetails.pageImagePath,
   });
@@ -469,7 +469,7 @@ export const initNewPage = async (page, pageClosePromises, processPageParams, pa
       consoleLogger.info(`Overlay state: ${existingOverlay}`);
     } catch {
       consoleLogger.info('Error in adding overlay menu to page');
-      silentLogger.info('Error in adding overlay menu to page');
+      consoleLogger.info('Error in adding overlay menu to page');
     }
   });
 

package/src/crawlers/pdfScanFunc.ts

@@ -15,6 +15,7 @@ import constants, {
   STATUS_CODE_METADATA,
   UrlsCrawled,
 } from '../constants/constants.js';
+import { cleanUpAndExit } from '../utils.js';
 
 const require = createRequire(import.meta.url);
 
@@ -233,7 +234,7 @@ const getVeraExecutable = () => {
     const veraPdfExeNotFoundError =
       'Could not find veraPDF executable.  Please ensure veraPDF is installed at current directory.';
     consoleLogger.error(veraPdfExeNotFoundError);
-    silentLogger.error(veraPdfExeNotFoundError);
+    consoleLogger.error(veraPdfExeNotFoundError);
   }
   return veraPdfExe;
 };
@@ -355,7 +356,7 @@ export const runPdfScan = async (randomToken: string) => {
     'profiles/veraPDF-validation-profiles-rel-1.26/PDF_UA/WCAG-2-2.xml',
   )}"`;
   if (!veraPdfExe || !veraPdfProfile) {
-    process.exit(1);
+    cleanUpAndExit(1);
   }
 
   const intermediateFolder = randomToken; // NOTE: assumes this folder is already created for crawlee

package/src/crawlers/runCustom.ts

@@ -1,7 +1,7 @@
 /* eslint-env browser */
 import { chromium } from 'playwright';
 import { createCrawleeSubFolders } from './commonCrawlerFunc.js';
-import { cleanUp } from '../utils.js';
+import { cleanUpAndExit, register} from '../utils.js';
 import constants, {
   getIntermediateScreenshotsPath,
   guiInfoStatusTypes,
@@ -48,7 +48,6 @@ const runCustom = async (
   includeScreenshots: boolean,
 ) => {
   // checks and delete datasets path if it already exists
-  cleanUp(randomToken);
   process.env.CRAWLEE_STORAGE_DIR = randomToken;
 
   const urlsCrawled: UrlsCrawled = { ...constants.urlsCrawledObj };
@@ -83,6 +82,8 @@ const runCustom = async (
       ...viewportSettings.playwrightDeviceDetailsObject,
     });
 
+    register(context);
+
     // Detection of new page
     context.on('page', async newPage => {
       await initNewPage(newPage, pageClosePromises, processPageParams, pagesDict);
@@ -107,7 +108,7 @@ const runCustom = async (
     await allPagesClosedPromise(pageClosePromises);
   } catch (error) {
     log(`PLAYWRIGHT EXECUTION ERROR ${error}`);
-    process.exit(1);
+    cleanUpAndExit(1, randomToken, true);
   }
 
   guiInfoLog(guiInfoStatusTypes.COMPLETED, {});

package/src/index.ts

@@ -7,6 +7,8 @@ import {
   cleanUp,
   getUserDataTxt,
   writeToUserDataTxt,
+  listenForCleanUp,
+  cleanUpAndExit,
 } from './utils.js';
 import {
   prepareData,
@@ -97,10 +99,7 @@ const runScan = async (answers: Answers) => {
     answers.customDevice,
     answers.viewportWidth,
   );
-  const { browserToRun } = getBrowserToRun(BrowserTypes.CHROME);
-  deleteClonedProfiles(browserToRun);
-  answers.browserToRun = browserToRun;
-
+    
   if (!answers.nameEmail) {
     answers.nameEmail = `${userData.name}:${userData.email}`;
   }
@@ -109,19 +108,19 @@ const runScan = async (answers: Answers) => {
   answers.metadata = '{}';
 
   const data: Data = await prepareData(answers);
+  
+  // Executes cleanUp script if error encountered
+  listenForCleanUp(data.randomToken);
+  
   data.userDataDirectory = getClonedProfilesWithRandomToken(data.browser, data.randomToken);
 
   printMessage(['Scanning website...'], messageOptions);
 
   await combineRun(data, screenToScan);
 
-  // Delete cloned directory
-  deleteClonedProfiles(data.browser);
-
   // Delete dataset and request queues
-  cleanUp(data.randomToken);
+  cleanUpAndExit(0, data.randomToken);
 
-  process.exit(0);
 };
 
 if (userData) {

package/src/logs.ts

@@ -2,7 +2,8 @@
 /* eslint-disable no-shadow */
 import { createLogger, format, transports } from 'winston';
 import { guiInfoStatusTypes } from './constants/constants.js';
-import { urlWithoutAuth } from './constants/common.js';
+import path from 'path';
+import { randomUUID } from 'crypto';
 
 const { combine, timestamp, printf } = format;
 
@@ -21,12 +22,32 @@ const logFormat = printf(({ timestamp, level, message }) => {
 // transport: storage device for logs
 // Enabled for console and storing into files; Files are overwritten each time
 // All logs in combined.txt, error in errors.txt
+const uuid = randomUUID();
+let basePath: string;
+
+if (process.env.OOBEE_LOGS_PATH) {
+  basePath = process.env.OOBEE_LOGS_PATH;
+} else if (process.platform === 'win32') {
+  basePath = path.join(process.env.APPDATA, 'Oobee');
+} else if (process.platform === 'darwin') {
+  basePath = path.join(process.env.HOME, 'Library', 'Application Support', 'Oobee');
+} else {
+  basePath = path.join(process.cwd());
+}
+
+export const errorsTxtPath = path.join(basePath, `${uuid}.txt`);
 
 const consoleLogger = createLogger({
   silent: !(process.env.RUNNING_FROM_PH_GUI || process.env.OOBEE_VERBOSE),
   format: combine(timestamp({ format: 'YYYY-MM-DD HH:mm:ss' }), logFormat),
-  transports:
-    process.env.RUNNING_FROM_PH_GUI || process.env.OOBEE_VERBOSE ? [new transports.Console()] : [],
+  transports: [
+    new transports.Console({ level: 'info' }),
+    new transports.File({
+      filename: errorsTxtPath,
+      level: 'info',
+      handleExceptions: true,
+    }),
+  ],
 });
 
 // No display in consoles, this will mostly be used within the interactive script to avoid disrupting the flow
@@ -35,9 +56,10 @@ const consoleLogger = createLogger({
 const silentLogger = createLogger({
   format: combine(timestamp({ format: 'YYYY-MM-DD HH:mm:ss' }), logFormat),
   transports: [
-    process.env.OOBEE_VERBOSE || process.env.RUNNING_FROM_PH_GUI
-      ? new transports.Console({ handleExceptions: true })
-      : new transports.File({ filename: 'errors.txt', level: 'warn', handleExceptions: true }),
+    new transports.File({ 
+      filename: errorsTxtPath,
+      level: 'warn', 
+      handleExceptions: true }),
   ].filter(Boolean),
 });
 
@@ -47,16 +69,17 @@ export const guiInfoLog = (status: string, data: { numScanned?: number; urlScann
     switch (status) {
       case guiInfoStatusTypes.COMPLETED:
         console.log('Scan completed');
+        silentLogger.info('Scan completed');
         break;
       case guiInfoStatusTypes.SCANNED:
       case guiInfoStatusTypes.SKIPPED:
       case guiInfoStatusTypes.ERROR:
       case guiInfoStatusTypes.DUPLICATE:
-        console.log(
-          `crawling::${data.numScanned || 0}::${status}::${
-            urlWithoutAuth(data.urlScanned) || 'no url provided'
-          }`,
-        );
+        const msg = `crawling::${data.numScanned || 0}::${status}::${
+            data.urlScanned || 'no url provided'
+          }`;
+        console.log(msg);
+        silentLogger.info(msg);
         break;
       default:
         console.log(`Status provided to gui info log not recognized: ${status}`);
@@ -65,4 +88,6 @@ export const guiInfoLog = (status: string, data: { numScanned?: number; urlScann
   }
 };
 
+consoleLogger.info(`Logger writing to: ${errorsTxtPath}`);
+
 export { logFormat, consoleLogger, silentLogger };

package/src/mergeAxeResults.ts

@@ -15,7 +15,7 @@ import { pipeline } from 'stream/promises';
 // @ts-ignore
 import * as Sentry from '@sentry/node';
 import constants, { ScannerTypes, sentryConfig, setSentryUser } from './constants/constants.js';
-import { urlWithoutAuth } from './constants/common.js';
+import { getBrowserToRun, getPlaywrightLaunchOptions } from './constants/common.js';
 
 import {
   createScreenshotsFolder,
@@ -29,6 +29,7 @@ import {
   getWcagCriteriaMap,
   categorizeWcagCriteria,
   getUserDataTxt,
+  register
 } from './utils.js';
 import { consoleLogger, silentLogger } from './logs.js';
 import itemTypeDescription from './constants/itemTypeDescription.js';
@@ -961,29 +962,21 @@ const writeScanDetailsCsv = async (
   });
 };
 
-let browserChannel = 'chrome';
+let browserChannel = getBrowserToRun().browserToRun;
 
-if (os.platform() === 'win32') {
-  browserChannel = 'msedge';
-}
-
-if (os.platform() === 'linux') {
-  browserChannel = 'chromium';
-}
-
-const writeSummaryPdf = async (storagePath: string, pagesScanned: number, filename = 'summary') => {
+const writeSummaryPdf = async (storagePath: string, pagesScanned: number, filename = 'summary', browser: string, userDataDirectory: string) => {
   const htmlFilePath = `${storagePath}/${filename}.html`;
   const fileDestinationPath = `${storagePath}/${filename}.pdf`;
-  const browser = await chromium.launch({
-    headless: false,
-    channel: browserChannel,
-    args: ['--headless=new', '--no-sandbox'],
-  });
 
-  const context = await browser.newContext({
-    ignoreHTTPSErrors: true,
-    serviceWorkers: 'block',
-  });
+  const effectiveUserDataDirectory = process.env.CRAWLEE_HEADLESS === '1'
+    ? userDataDirectory
+    : '';
+  const context = await constants.launcher.launchPersistentContext(effectiveUserDataDirectory, {
+        headless: process.env.CRAWLEE_HEADLESS === '1',
+        ...getPlaywrightLaunchOptions(browser),
+      });
+
+  register(context);
 
   const page = await context.newPage();
 
@@ -1008,8 +1001,7 @@ const writeSummaryPdf = async (storagePath: string, pagesScanned: number, filena
 
   await page.close();
 
-  await context.close();
-  await browser.close();
+  await context.close().catch(() => {});
 
   if (pagesScanned < 2000) {
     fs.unlinkSync(htmlFilePath);
@@ -1721,14 +1713,9 @@ const generateArtifacts = async (
   zip: string = undefined, // optional
   generateJsonFiles = false,
 ) => {
-  const intermediateDatasetsPath = `${randomToken}/datasets/${randomToken}`;
-  const oobeeAppVersion = getVersion();
   const storagePath = getStoragePath(randomToken);
-
-  urlScanned =
-    scanType === ScannerTypes.SITEMAP || scanType === ScannerTypes.LOCALFILE
-      ? urlScanned
-      : urlWithoutAuth(urlScanned);
+  const intermediateDatasetsPath = `${storagePath}/crawlee`;
+  const oobeeAppVersion = getVersion();
 
   const formatAboutStartTime = (dateString: string) => {
     const utcStartTimeDate = new Date(dateString);
@@ -1851,10 +1838,18 @@ const generateArtifacts = async (
 
   printMessage([
     'Scan Summary',
+    `Oobee App Version: ${allIssues.oobeeAppVersion}`,
     '',
     `Site Name: ${allIssues.siteName}`,
     `URL: ${allIssues.urlScanned}`,
     `Pages Scanned: ${allIssues.totalPagesScanned}`,
+    `Start Time: ${allIssues.startTime}`,
+    `End Time: ${allIssues.endTime}`,
+    `Elapsed Time: ${(new Date(allIssues.endTime).getTime() - new Date(allIssues.startTime).getTime()) / 1000}s`,
+    `Device: ${allIssues.deviceChosen}`,
+    `Viewport: ${allIssues.viewport}`,
+    `Scan Type: ${allIssues.scanType}`,
+    `Label: ${allIssues.customFlowLabel || 'N/A'}`,
     '',
     `Must Fix: ${allIssues.items.mustFix.rules.length} ${Object.keys(allIssues.items.mustFix.rules).length === 1 ? 'issue' : 'issues'} / ${allIssues.items.mustFix.totalItems} ${allIssues.items.mustFix.totalItems === 1 ? 'occurrence' : 'occurrences'}`,
     `Good to Fix: ${allIssues.items.goodToFix.rules.length} ${Object.keys(allIssues.items.goodToFix.rules).length === 1 ? 'issue' : 'issues'} / ${allIssues.items.goodToFix.totalItems} ${allIssues.items.goodToFix.totalItems === 1 ? 'occurrence' : 'occurrences'}`,
@@ -1892,7 +1887,11 @@ const generateArtifacts = async (
   consoleLogger.info(`End Time: ${allIssues.endTime}`);
   const elapsedSeconds = (new Date(allIssues.endTime).getTime() - new Date(allIssues.startTime).getTime()) / 1000;
   consoleLogger.info(`Elapsed Time: ${elapsedSeconds}s`);
-
+  consoleLogger.info(`Device: ${allIssues.deviceChosen}`);
+  consoleLogger.info(`Viewport: ${allIssues.viewport}`);
+  consoleLogger.info(`Scan Type: ${allIssues.scanType}`);
+  consoleLogger.info(`Label: ${allIssues.customFlowLabel || 'N/A'}`);
+  
   const getAxeImpactCount = (allIssues: AllIssues) => {
     const impactCount = {
       critical: 0,
@@ -1986,7 +1985,14 @@ const generateArtifacts = async (
     ]);
   }
 
-  await retryFunction(() => writeSummaryPdf(storagePath, pagesScanned.length), 1);
+  // Should consider refactor constants.userDataDirectory to be a parameter in future
+  await retryFunction(() => writeSummaryPdf(storagePath, pagesScanned.length, 'summary', browserChannel, constants.userDataDirectory), 1);
+
+  try {
+    fs.rmSync(path.join(storagePath, 'crawlee'), { recursive: true, force: true });
+  } catch (error) {
+    consoleLogger.warn(`Unable to force remove crawlee folder: ${error.message}`);
+  }
 
   // Take option if set
   if (typeof zip === 'string') {

package/src/npmIndex.ts

@@ -10,7 +10,6 @@ import {
   getBrowserToRun,
   getPlaywrightLaunchOptions,
   submitForm,
-  urlWithoutAuth,
 } from './constants/common.js';
 import { createCrawleeSubFolders, filterAxeResults } from './crawlers/commonCrawlerFunc.js';
 import { createAndUpdateResultsFolders, createDetailsAndLogs } from './utils.js';
@@ -201,11 +200,12 @@ export const init = async ({
     res: { pageUrl: string; pageTitle: string; axeScanResults: AxeResults },
     metadata: string,
     elementsToClick: string[],
+    randomToken: string,
   ) => {
     throwErrorIfTerminated();
     if (includeScreenshots) {
       // use chrome by default
-      const { browserToRun, clonedBrowserDataDir } = getBrowserToRun(BrowserTypes.CHROME);
+      const { browserToRun, clonedBrowserDataDir } = getBrowserToRun(BrowserTypes.CHROME, false, randomToken);
       const browserContext = await constants.launcher.launchPersistentContext(
         clonedBrowserDataDir,
         { viewport: viewportSettings, ...getPlaywrightLaunchOptions(browserToRun) },
@@ -237,7 +237,7 @@ export const init = async ({
       );
 
       await browserContext.close();
-      deleteClonedProfiles(browserToRun);
+      deleteClonedProfiles(browserToRun, randomToken);
     }
     const pageIndex = urlsCrawled.scanned.length + 1;
     const filteredResults = filterAxeResults(res.axeScanResults, res.pageTitle, {
@@ -245,7 +245,7 @@ export const init = async ({
       metadata,
     });
     urlsCrawled.scanned.push({
-      url: urlWithoutAuth(res.pageUrl).toString(),
+      url: res.pageUrl.toString(),
       actualUrl: 'tbd',
       pageTitle: `${pageIndex}: ${res.pageTitle}`,
     });

package/src/screenshotFunc/htmlScreenshotFunc.ts

@@ -22,7 +22,7 @@ export const takeScreenshotForHTMLElements = async (
   for (const violation of violations) {
     if (screenshotCount >= maxScreenshots) {
       /*
-      silentLogger.warn(
+      consoleLogger.warn(
         `Skipping screenshots for ${violation.id} as maxScreenshots (${maxScreenshots}) exceeded. You can increase it by specifying a higher value when calling takeScreenshotForHTMLElements.`,
       );
       */
@@ -34,7 +34,7 @@ export const takeScreenshotForHTMLElements = async (
 
     // Check if rule ID is 'oobee-grading-text-contents' and skip screenshot logic
     if (rule === 'oobee-grading-text-contents') {
-      // silentLogger.info('Skipping screenshot for rule oobee-grading-text-contents');
+      // consoleLogger.info('Skipping screenshot for rule oobee-grading-text-contents');
       newViolations.push(violation); // Make sure it gets added
       continue;
     }
@@ -59,13 +59,13 @@ export const takeScreenshotForHTMLElements = async (
               nodeWithScreenshotPath.screenshotPath = screenshotPath;
               screenshotCount++;
             } else {
-              // silentLogger.info(`Element at ${currLocator} is not visible`);
+              // consoleLogger.info(`Element at ${currLocator} is not visible`);
             }
 
             break; // Stop looping after finding the first visible locator
           }
         } catch (e) {
-          // silentLogger.info(`Unable to take element screenshot at ${selector}`);
+          // consoleLogger.info(`Unable to take element screenshot at ${selector}`);
         }
       }
       newViolationNodes.push(nodeWithScreenshotPath);