@govtechsg/oobee 0.10.58 → 0.10.61

package/src/constants/constants.ts

@@ -452,7 +452,7 @@ const reserveFileNameKeywords = [
 
 export default {
   cliZipFileName: 'oobee-scan-results.zip',
-  exportDirectory: `${process.cwd()}`,
+  exportDirectory: undefined,
   maxRequestsPerCrawl,
   maxConcurrency: 25,
   urlsCrawledObj,
@@ -466,6 +466,7 @@ export default {
   reserveFileNameKeywords,
   wcagLinks,
   robotsTxtUrls: null,
+  userDataDirectory: null, // This will be set later in the code
 };
 
 export const rootPath = dirname;

package/src/constants/questions.ts

@@ -1,6 +1,6 @@
 import { Question } from 'inquirer';
 import { Answers } from '../index.js';
-import { getUserDataTxt, setHeadlessMode } from '../utils.js';
+import { getUserDataTxt, randomThreeDigitNumberString, setHeadlessMode } from '../utils.js';
 import {
   checkUrl,
   deleteClonedProfiles,
@@ -15,6 +15,7 @@ import {
   parseHeaders,
 } from './common.js';
 import constants, { BrowserTypes, ScannerTypes } from './constants.js';
+import { random } from 'lodash';
 
 const userData = getUserDataTxt();
 
@@ -78,8 +79,15 @@ const startScanQuestions = [
         process.exit(1);
       }
 
+      // construct filename for scan results
+      const [date, time] = new Date().toLocaleString('sv').replaceAll(/-|:/g, '').split(' ');
+      const domain = new URL(url).hostname;
+      let resultFilename: string;
+      const randomThreeDigitNumber = randomThreeDigitNumberString();
+      resultFilename = `${date}_${time}_${domain}_${randomThreeDigitNumber}`;
+  
       const statuses = constants.urlCheckStatuses;
-      const { browserToRun, clonedBrowserDataDir } = getBrowserToRun(BrowserTypes.CHROME);
+      const { browserToRun, clonedBrowserDataDir } = getBrowserToRun(BrowserTypes.CHROME, false, resultFilename);
 
       setHeadlessMode(browserToRun, answers.headless);
 
@@ -95,11 +103,11 @@ const startScanQuestions = [
         browserToRun,
         clonedBrowserDataDir,
         playwrightDeviceDetailsObject,
-        answers.scanner === ScannerTypes.CUSTOM,
         parseHeaders(answers.header),
       );
 
-      deleteClonedProfiles(browserToRun);
+      deleteClonedProfiles(browserToRun, resultFilename);
+      
       switch (res.status) {
         case statuses.success.code:
           answers.finalUrl = res.url;

package/src/crawlers/commonCrawlerFunc.ts

@@ -21,6 +21,9 @@ import { getAxeConfiguration } from './custom/getAxeConfiguration.js';
 import { flagUnlabelledClickableElements } from './custom/flagUnlabelledClickableElements.js';
 import xPathToCss from './custom/xPathToCss.js';
 import type { Response as PlaywrightResponse } from 'playwright';
+import fs from 'fs';
+import { getStoragePath } from '../utils.js';
+import path from 'path';
 
 // types
 interface AxeResultsWithScreenshot extends AxeResults {
@@ -254,7 +257,7 @@ export const runAxeScript = async ({
       return new Promise(resolve => {
         let timeout: NodeJS.Timeout;
         let mutationCount = 0;
-        const MAX_MUTATIONS = 250;
+        const MAX_MUTATIONS = 500;
         const MAX_SAME_MUTATION_LIMIT = 10;
         const mutationHash: Record<string, number> = {};
 
@@ -476,8 +479,11 @@ export const runAxeScript = async ({
 export const createCrawleeSubFolders = async (
   randomToken: string,
 ): Promise<{ dataset: crawlee.Dataset; requestQueue: crawlee.RequestQueue }> => {
-  const dataset = await crawlee.Dataset.open(randomToken);
-  const requestQueue = await crawlee.RequestQueue.open(randomToken);
+
+  const crawleeDir = path.join(getStoragePath(randomToken),"crawlee");
+
+  const dataset = await crawlee.Dataset.open(crawleeDir);
+  const requestQueue = await crawlee.RequestQueue.open(crawleeDir);
   return { dataset, requestQueue };
 };
 

package/src/crawlers/crawlDomain.ts

@@ -27,9 +27,7 @@ import {
   isSkippedUrl,
   isDisallowedInRobotsTxt,
   getUrlsFromRobotsTxt,
-  urlWithoutAuth,
   waitForPageLoaded,
-  initModifiedUserAgent,
 } from '../constants/common.js';
 import { areLinksEqual, isFollowStrategy } from '../utils.js';
 import {
@@ -40,6 +38,8 @@ import {
 } from './pdfScanFunc.js';
 import { consoleLogger, guiInfoLog, silentLogger } from '../logs.js';
 import { ViewportSettingsClass } from '../combine.js';
+import * as path from 'path';
+import fsp from 'fs/promises';
 
 const isBlacklisted = (url: string, blacklistedPatterns: string[]) => {
   if (!blacklistedPatterns) {
@@ -115,10 +115,6 @@ const crawlDomain = async ({
 
   ({ requestQueue } = await createCrawleeSubFolders(randomToken));
 
-  if (!fs.existsSync(randomToken)) {
-    fs.mkdirSync(randomToken);
-  }
-
   const pdfDownloads: Promise<void>[] = [];
   const uuidToPdfMapping: Record<string, string> = {};
   const isScanHtml = ['all', 'html-only'].includes(fileTypes);
@@ -126,45 +122,11 @@ const crawlDomain = async ({
   const { maxConcurrency } = constants;
   const { playwrightDeviceDetailsObject } = viewportSettings;
 
-  const httpsAgent = new https.Agent({ rejectUnauthorized: false });
-
-  // Boolean to omit axe scan for basic auth URL
-  let isBasicAuth = false;
-  let authHeader = '';
-
-  // Test basic auth and add auth header if auth exist
-  const parsedUrl = new URL(url);
-  let username: string;
-  let password: string;
-  if (parsedUrl.username !== '' && parsedUrl.password !== '') {
-    isBasicAuth = true;
-    username = decodeURIComponent(parsedUrl.username);
-    password = decodeURIComponent(parsedUrl.password);
-
-    // Create auth header
-    authHeader = `Basic ${Buffer.from(`${username}:${password}`).toString('base64')}`;
-
-    // Remove username from parsedUrl
-    parsedUrl.username = '';
-    parsedUrl.password = '';
-    // Send the finalUrl without credentials by setting auth header instead
-    const finalUrl = parsedUrl.toString();
-
-    await requestQueue.addRequest({
-      url: finalUrl,
-      skipNavigation: isUrlPdf(finalUrl),
-      headers: {
-        Authorization: authHeader,
-      },
-      label: finalUrl,
-    });
-  } else {
-    await requestQueue.addRequest({
-      url,
-      skipNavigation: isUrlPdf(url),
-      label: url,
-    });
-  }
+  await requestQueue.addRequest({
+    url,
+    skipNavigation: isUrlPdf(url),
+    label: url,
+  });
 
   const enqueueProcess = async (
     page: Page,
@@ -377,31 +339,40 @@ const crawlDomain = async ({
 
   let isAbortingScanNow = false;
 
-  let userDataDir = '';
-  if (userDataDirectory) {
-    userDataDir = process.env.CRAWLEE_HEADLESS !== '0' ? userDataDirectory : '';
-  }
-
-  await initModifiedUserAgent(browser, playwrightDeviceDetailsObject);
-
   const crawler = new crawlee.PlaywrightCrawler({
     launchContext: {
       launcher: constants.launcher,
       launchOptions: getPlaywrightLaunchOptions(browser),
       // Bug in Chrome which causes browser pool crash when userDataDirectory is set in non-headless mode
-      ...(process.env.CRAWLEE_HEADLESS === '0' && { userDataDir }),
+      ...(process.env.CRAWLEE_HEADLESS === '1' && { userDataDir: userDataDirectory }),
     },
     retryOnBlocked: true,
     browserPoolOptions: {
       useFingerprints: false,
       preLaunchHooks: [
         async (_pageId, launchContext) => {
+          const baseDir = userDataDirectory; // e.g., /Users/young/.../Chrome/oobee-...
+
+          // Ensure base exists
+          await fsp.mkdir(baseDir, { recursive: true });
+
+          // Create a unique subdir per browser
+          const subProfileDir = path.join(baseDir, `profile-${Date.now()}-${Math.random().toString(36).slice(2, 8)}`);
+          await fsp.mkdir(subProfileDir, { recursive: true });
+
+          // Assign to Crawlee's launcher
+          launchContext.userDataDir = subProfileDir;
+
+          // Safely extend launchOptions
           launchContext.launchOptions = {
             ...launchContext.launchOptions,
-            bypassCSP: true,
             ignoreHTTPSErrors: true,
             ...playwrightDeviceDetailsObject,
+            ...(extraHTTPHeaders && { extraHTTPHeaders }),
           };
+
+          // Optionally log for debugging
+          // console.log(`[HOOK] Using userDataDir: ${subProfileDir}`);
         },
       ],
     },
@@ -414,7 +385,7 @@ const crawlDomain = async ({
           return new Promise(resolve => {
             let timeout;
             let mutationCount = 0;
-            const MAX_MUTATIONS = 250; // stop if things never quiet down
+            const MAX_MUTATIONS = 500; // stop if things never quiet down
             const OBSERVER_TIMEOUT = 5000; // hard cap on total wait
 
             const observer = new MutationObserver(() => {
@@ -464,33 +435,10 @@ const crawlDomain = async ({
         }
       },
     ],
-    preNavigationHooks: [ async({ page, request}) => {
-      if (isBasicAuth) {
-        await page.setExtraHTTPHeaders({
-          Authorization: authHeader,
-          ...extraHTTPHeaders,
-        });
-      } else {
-        await page.setExtraHTTPHeaders({
-          ...extraHTTPHeaders,
-        });
-      }
-    }],
     requestHandlerTimeoutSecs: 90, // Allow each page to be processed by up from default 60 seconds
     requestHandler: async ({ page, request, response, crawler, sendRequest, enqueueLinks }) => {
       const browserContext: BrowserContext = page.context();
       try {
-        // Set basic auth header if needed
-        if (isBasicAuth) {
-          await page.setExtraHTTPHeaders({
-            Authorization: authHeader,
-          });
-          const currentUrl = new URL(request.url);
-          currentUrl.username = username;
-          currentUrl.password = password;
-          request.url = currentUrl.href;
-        }
-
         await waitForPageLoaded(page, 10000);
         let actualUrl = page.url() || request.loadedUrl || request.url;
 
@@ -652,13 +600,13 @@ const crawlDomain = async ({
               });
 
               urlsCrawled.scanned.push({
-                url: urlWithoutAuth(request.url),
+                url: request.url,
                 pageTitle: results.pageTitle,
                 actualUrl, // i.e. actualUrl
               });
 
               urlsCrawled.scannedRedirects.push({
-                fromUrl: urlWithoutAuth(request.url),
+                fromUrl: request.url,
                 toUrl: actualUrl, // i.e. actualUrl
               });
 
@@ -671,10 +619,10 @@ const crawlDomain = async ({
             if (urlsCrawled.scanned.length < maxRequestsPerCrawl) {
               guiInfoLog(guiInfoStatusTypes.SCANNED, {
                 numScanned: urlsCrawled.scanned.length,
-                urlScanned: urlWithoutAuth(request.url),
+                urlScanned: request.url,
               });
               urlsCrawled.scanned.push({
-                url: urlWithoutAuth(request.url),
+                url: request.url,
                 actualUrl: request.url,
                 pageTitle: results.pageTitle,
               });
@@ -695,7 +643,7 @@ const crawlDomain = async ({
           });
         }
 
-        if (followRobots) await getUrlsFromRobotsTxt(request.url, browser);
+        if (followRobots) await getUrlsFromRobotsTxt(request.url, browser, userDataDirectory, extraHTTPHeaders);
         await enqueueProcess(page, enqueueLinks, browserContext);
       } catch (e) {
         try {

package/src/crawlers/crawlIntelligentSitemap.ts

@@ -7,6 +7,7 @@ import crawlDomain from './crawlDomain.js';
 import crawlSitemap from './crawlSitemap.js';
 import { EnqueueStrategy } from 'crawlee';
 import { ViewportSettingsClass } from '../combine.js';
+import { getPlaywrightLaunchOptions } from '../constants/common.js';
 
 const crawlIntelligentSitemap = async (
   url: string,
@@ -36,9 +37,6 @@ const crawlIntelligentSitemap = async (
   let sitemapUrl;
 
   ({ dataset } = await createCrawleeSubFolders(randomToken));
-  if (!fs.existsSync(randomToken)) {
-    fs.mkdirSync(randomToken);
-  }
 
   function getHomeUrl(parsedUrl: string) {
     const urlObject = new URL(parsedUrl);
@@ -48,15 +46,21 @@ const crawlIntelligentSitemap = async (
     return `${urlObject.protocol}//${urlObject.hostname}${urlObject.port ? `:${urlObject.port}` : ''}`;
   }
 
-  async function findSitemap(link: string) {
+  async function findSitemap(link: string, userDataDirectory: string, extraHTTPHeaders: Record<string, string>) {
     const homeUrl = getHomeUrl(link);
     let sitemapLink = '';
-    const chromiumBrowser = await chromium.launch({
-      headless: false,
-      channel: 'chrome',
-      args: ['--headless=new', '--no-sandbox'],
+
+    const effectiveUserDataDirectory = process.env.CRAWLEE_HEADLESS === '1'
+        ? userDataDirectory
+        : '';
+    const context = await constants.launcher.launchPersistentContext(effectiveUserDataDirectory, {
+        headless: process.env.CRAWLEE_HEADLESS === '1',
+        ...getPlaywrightLaunchOptions(browser),
+        ...(extraHTTPHeaders && { extraHTTPHeaders }),
     });
-    const page = await chromiumBrowser.newPage();
+
+    const page = await context.newPage();
+
     for (const path of sitemapPaths) {
       sitemapLink = homeUrl + path;
       if (await checkUrlExists(page, sitemapLink)) {
@@ -64,7 +68,8 @@ const crawlIntelligentSitemap = async (
         break;
       }
     }
-    await chromiumBrowser.close();
+    await page.close();
+    await context.close().catch(() => { });
     return sitemapExist ? sitemapLink : '';
   }
 
@@ -79,7 +84,7 @@ const crawlIntelligentSitemap = async (
   };
 
   try {
-    sitemapUrl = await findSitemap(url);
+    sitemapUrl = await findSitemap(url, userDataDirectory, extraHTTPHeaders);
   } catch (error) {
     consoleLogger.error(error);
   }

package/src/crawlers/crawlLocalFile.ts

@@ -13,7 +13,6 @@ import {
   isFilePath,
   convertLocalFileToPath,
   convertPathToLocalFile,
-  initModifiedUserAgent,
 } from '../constants/common.js';
 import { runPdfScan, mapPdfScanResults, doPdfScreenshots } from './pdfScanFunc.js';
 import { guiInfoLog } from '../logs.js';
@@ -74,9 +73,6 @@ export const crawlLocalFile = async ({
     ({ dataset } = await createCrawleeSubFolders(randomToken));
     urlsCrawled = { ...constants.urlsCrawledObj };
 
-    if (!fs.existsSync(randomToken)) {
-      fs.mkdirSync(randomToken);
-    }
   }
 
   // Check if the sitemapUrl is a local file and if it exists
@@ -136,16 +132,6 @@ export const crawlLocalFile = async ({
     console.log(e);
   }
 
-  if (basicAuthRegex.test(sitemapUrl)) {
-    isBasicAuth = true;
-    // request to basic auth URL to authenticate for browser session
-    finalLinks.push(new Request({ url: sitemapUrl, uniqueKey: `auth:${sitemapUrl}` }));
-    const finalUrl = `${sitemapUrl.split('://')[0]}://${sitemapUrl.split('@')[1]}`;
-    // obtain base URL without credentials so that subsequent URLs within the same domain can be scanned
-    finalLinks.push(new Request({ url: finalUrl }));
-    basicAuthPage = -2;
-  }
-
   const uuidToPdfMapping: Record<string, string> = {}; // key and value of string type
 
   finalLinks = [...finalLinks, ...linksFromSitemap];
@@ -165,9 +151,12 @@ export const crawlLocalFile = async ({
   let shouldAbort = false;
 
   if (!isUrlPdf(request.url)) {
-    await initModifiedUserAgent(browser);
-    const browserContext = await constants.launcher.launchPersistentContext('', {
-      headless: false,
+    const effectiveUserDataDirectory = process.env.CRAWLEE_HEADLESS === '1'
+      ? userDataDirectory
+      : '';
+
+    const browserContext = await constants.launcher.launchPersistentContext(effectiveUserDataDirectory, {
+      headless: process.env.CRAWLEE_HEADLESS === '1',
       ...getPlaywrightLaunchOptions(browser),
       ...playwrightDeviceDetailsObject,
     });

package/src/crawlers/crawlSitemap.ts

@@ -17,15 +17,15 @@ import {
   getLinksFromSitemap,
   getPlaywrightLaunchOptions,
   isSkippedUrl,
-  urlWithoutAuth,
   waitForPageLoaded,
   isFilePath,
-  initModifiedUserAgent,
 } from '../constants/common.js';
 import { areLinksEqual, isWhitelistedContentType, isFollowStrategy } from '../utils.js';
 import { handlePdfDownload, runPdfScan, mapPdfScanResults } from './pdfScanFunc.js';
 import { guiInfoLog } from '../logs.js';
 import { ViewportSettingsClass } from '../combine.js';
+import * as path from 'path';
+import fsp from 'fs/promises';
 
 const crawlSitemap = async ({
   sitemapUrl,
@@ -70,50 +70,19 @@ const crawlSitemap = async ({
   let dataset: crawlee.Dataset;
   let urlsCrawled: UrlsCrawled;
 
-  // Boolean to omit axe scan for basic auth URL
-  let isBasicAuth: boolean;
-  let basicAuthPage = 0;
-  let finalLinks = [];
-  let authHeader = '';
-
   if (fromCrawlIntelligentSitemap) {
     dataset = datasetFromIntelligent;
     urlsCrawled = urlsCrawledFromIntelligent;
   } else {
     ({ dataset } = await createCrawleeSubFolders(randomToken));
     urlsCrawled = { ...constants.urlsCrawledObj };
-
-    if (!fs.existsSync(randomToken)) {
-      fs.mkdirSync(randomToken);
-    }
   }
 
-  let parsedUrl;
-  let username = '';
-  let password = '';
-
   if (!crawledFromLocalFile && isFilePath(sitemapUrl)) {
     console.log('Local file crawling not supported for sitemap. Please provide a valid URL.');
     return;
   }
 
-  if (isFilePath(sitemapUrl)) {
-    parsedUrl = sitemapUrl;
-  } else {
-    parsedUrl = new URL(sitemapUrl);
-    if (parsedUrl.username !== '' && parsedUrl.password !== '') {
-      isBasicAuth = true;
-      username = decodeURIComponent(parsedUrl.username);
-      password = decodeURIComponent(parsedUrl.password);
-
-      // Create auth header
-      authHeader = `Basic ${Buffer.from(`${username}:${password}`).toString('base64')}`;
-
-      parsedUrl.username = '';
-      parsedUrl.password = '';
-    }
-  }
-
   const linksFromSitemap = await getLinksFromSitemap(
     sitemapUrl,
     maxRequestsPerCrawl,
@@ -121,29 +90,11 @@ const crawlSitemap = async ({
     userDataDirectory,
     userUrlInputFromIntelligent,
     fromCrawlIntelligentSitemap,
-    username,
-    password,
+    extraHTTPHeaders,
   );
-  /**
-   * Regex to match http://username:password@hostname.com
-   * utilised in scan strategy to ensure subsequent URLs within the same domain are scanned.
-   * First time scan with original `url` containing credentials is strictly to authenticate for browser session
-   * subsequent URLs are without credentials.
-   * basicAuthPage is set to -1 for basic auth URL to ensure it is not counted towards maxRequestsPerCrawl
-   */
 
   sitemapUrl = encodeURI(sitemapUrl);
 
-  if (isBasicAuth) {
-    // request to basic auth URL to authenticate for browser session
-    finalLinks.push(new Request({ url: sitemapUrl, uniqueKey: `auth:${sitemapUrl}` }));
-    const finalUrl = `${sitemapUrl.split('://')[0]}://${sitemapUrl.split('@')[1]}`;
-
-    // obtain base URL without credentials so that subsequent URLs within the same domain can be scanned
-    finalLinks.push(new Request({ url: finalUrl }));
-    basicAuthPage = -2;
-  }
-
   const pdfDownloads: Promise<void>[] = [];
   const uuidToPdfMapping: Record<string, string> = {};
   const isScanHtml = ['all', 'html-only'].includes(fileTypes);
@@ -151,36 +102,43 @@ const crawlSitemap = async ({
   const { playwrightDeviceDetailsObject } = viewportSettings;
   const { maxConcurrency } = constants;
 
-  finalLinks = [...finalLinks, ...linksFromSitemap];
-
   const requestList = await RequestList.open({
-    sources: finalLinks,
+    sources: linksFromSitemap,
   });
 
-  let userDataDir = '';
-  if (userDataDirectory) {
-    userDataDir = process.env.CRAWLEE_HEADLESS !== '0' ? userDataDirectory : '';
-  }
-
-  await initModifiedUserAgent(browser, playwrightDeviceDetailsObject);
   const crawler = new crawlee.PlaywrightCrawler({
     launchContext: {
       launcher: constants.launcher,
       launchOptions: getPlaywrightLaunchOptions(browser),
       // Bug in Chrome which causes browser pool crash when userDataDirectory is set in non-headless mode
-      ...(process.env.CRAWLEE_HEADLESS === '0' && { userDataDir }),
+      ...(process.env.CRAWLEE_HEADLESS === '1' && { userDataDir: userDataDirectory }),
     },
     retryOnBlocked: true,
     browserPoolOptions: {
       useFingerprints: false,
       preLaunchHooks: [
-        async (_pageId: string, launchContext: LaunchContext) => {
+        async (_pageId, launchContext) => {
+          const baseDir = userDataDirectory; // e.g., /Users/young/.../Chrome/oobee-...
+
+          // Ensure base exists
+          await fsp.mkdir(baseDir, { recursive: true });
+
+          // Create a unique subdir per browser
+          const subProfileDir = path.join(baseDir, `profile-${Date.now()}-${Math.random().toString(36).slice(2, 8)}`);
+          await fsp.mkdir(subProfileDir, { recursive: true });
+
+          // Assign to Crawlee's launcher
+          launchContext.userDataDir = subProfileDir;
+
+          // Safely extend launchOptions
           launchContext.launchOptions = {
             ...launchContext.launchOptions,
-            bypassCSP: true,
             ignoreHTTPSErrors: true,
             ...playwrightDeviceDetailsObject,
           };
+
+          // Optionally log for debugging
+          // console.log(`[HOOK] Using userDataDir: ${subProfileDir}`);
         },
       ],
     },
@@ -193,7 +151,7 @@ const crawlSitemap = async ({
             return new Promise(resolve => {
               let timeout;
               let mutationCount = 0;
-              const MAX_MUTATIONS = 250; // stop if things never quiet down
+              const MAX_MUTATIONS = 500; // stop if things never quiet down
               const OBSERVER_TIMEOUT = 5000; // hard cap on total wait
 
               const observer = new MutationObserver(() => {
@@ -252,15 +210,7 @@ const crawlSitemap = async ({
           return;
         }
 
-        // Set headers if basic auth
-        if (isBasicAuth) {
-          await page.setExtraHTTPHeaders({
-            Authorization: authHeader,
-            ...extraHTTPHeaders,
-          });
-        } else {
-          preNavigationHooks(extraHTTPHeaders);
-        }
+        preNavigationHooks(extraHTTPHeaders);
       },
     ],
     requestHandlerTimeoutSecs: 90,
@@ -282,17 +232,6 @@ const crawlSitemap = async ({
         return;
       }
 
-      // Set basic auth header if needed
-      if (isBasicAuth) {
-        await page.setExtraHTTPHeaders({
-          Authorization: authHeader,
-        });
-        const currentUrl = new URL(request.url);
-        currentUrl.username = username;
-        currentUrl.password = password;
-        request.url = currentUrl.href;
-      }
-
       await waitForPageLoaded(page, 10000);
 
       const actualUrl = page.url() || request.loadedUrl || request.url;
@@ -341,9 +280,7 @@ const crawlSitemap = async ({
       const contentType = response?.headers?.()['content-type'] || '';
       const status = response ? response.status() : 0;
 
-      if (basicAuthPage < 0) {
-        basicAuthPage += 1;
-      } else if (isScanHtml && status < 300 && isWhitelistedContentType(contentType)) {
+      if (isScanHtml && status < 300 && isWhitelistedContentType(contentType)) {
         const isRedirected = !areLinksEqual(page.url(), request.url);
         const isLoadedUrlInCrawledUrls = urlsCrawled.scanned.some(
           item => (item.actualUrl || item.url) === page.url(),
@@ -382,13 +319,13 @@ const crawlSitemap = async ({
         });
 
         urlsCrawled.scanned.push({
-          url: urlWithoutAuth(request.url),
+          url: request.url,
           pageTitle: results.pageTitle,
           actualUrl, // i.e. actualUrl
         });
 
         urlsCrawled.scannedRedirects.push({
-          fromUrl: urlWithoutAuth(request.url),
+          fromUrl: request.url,
           toUrl: actualUrl,
         });
 
@@ -421,9 +358,6 @@ const crawlSitemap = async ({
       }
     },
     failedRequestHandler: async ({ request, response, error }) => {
-      if (isBasicAuth && request.url) {
-        request.url = `${request.url.split('://')[0]}://${request.url.split('@')[1]}`;
-      }
 
       // check if scanned pages have reached limit due to multi-instances of handler running
       if (urlsCrawled.scanned.length >= maxRequestsPerCrawl) {

package/src/crawlers/custom/utils.ts

@@ -6,7 +6,7 @@ import path from 'path';
 import { runAxeScript } from '../commonCrawlerFunc.js';
 import { consoleLogger, guiInfoLog, silentLogger } from '../../logs.js';
 import { guiInfoStatusTypes } from '../../constants/constants.js';
-import { isSkippedUrl, urlWithoutAuth } from '../../constants/common.js';
+import { isSkippedUrl } from '../../constants/common.js';
 
 //! For Cypress Test
 // env to check if Cypress test is running
@@ -77,8 +77,8 @@ export const screenshotFullPage = async (page, screenshotsDir: string, screensho
       window.scrollTo(0, 0);
     });
 
-    consoleLogger.info(`Screenshot page at: ${urlWithoutAuth(page.url())}`);
-    silentLogger.info(`Screenshot page at: ${urlWithoutAuth(page.url())}`);
+    consoleLogger.info(`Screenshot page at: ${page.url()}`);
+    silentLogger.info(`Screenshot page at: ${page.url()}`);
 
     await page.screenshot({
       timeout: 5000,
@@ -116,7 +116,7 @@ export const runAxeScan = async (
   await dataset.pushData(result);
 
   urlsCrawled.scanned.push({
-    url: urlWithoutAuth(page.url()),
+    url: page.url(),
     pageTitle: result.pageTitle,
     pageImagePath: customFlowDetails.pageImagePath,
   });