@govtechsg/oobee 0.10.58 → 0.10.61

package/src/constants/common.ts

@@ -119,7 +119,7 @@ export const validateFilePath = (filePath: string, cliDir: string) => {
 
     return absolutePath;
   } catch {
-    throw new Error(`Please ensure path provided exists: ${absolutePath}`);
+    throw new Error(`Please ensure path provided exists and writable: ${absolutePath}`);
   }
 };
 
@@ -277,110 +277,11 @@ export const sanitizeUrlInput = (url: string): { isValid: boolean; url: string }
   return { isValid: false, url: sanitizeUrl };
 };
 
-const requestToUrl = async (
-  url: string,
-  isCustomFlow: boolean,
-  extraHTTPHeaders: Record<string, string>,
-) => {
-  // User-Agent is modified to emulate a browser to handle cases where some sites ban non browser agents, resulting in a 403 error
-  const res = new RES();
-  const parsedUrl = new URL(url);
-  await axios
-    .get(parsedUrl.href, {
-      headers: {
-        ...extraHTTPHeaders,
-        'User-Agent': devices['Desktop Chrome HiDPI'].userAgent,
-        Host: parsedUrl.host,
-      },
-      auth: {
-        username: decodeURIComponent(parsedUrl.username),
-        password: decodeURIComponent(parsedUrl.password),
-      },
-      httpsAgent,
-      timeout: 5000,
-    })
-    .then(async response => {
-      let redirectUrl = response.request.res.responseUrl;
-      redirectUrl = new URL(redirectUrl).href;
-      res.status = constants.urlCheckStatuses.success.code;
-      let data;
-      if (typeof response.data === 'string' || response.data instanceof String) {
-        data = response.data;
-      } else if (typeof response.data === 'object' && response.data !== null) {
-        try {
-          data = JSON.stringify(response.data);
-        } catch (error) {
-          console.log('Error converting object to JSON:', error);
-        }
-      } else {
-        console.log('Unsupported data type:', typeof response.data);
-      }
-      const modifiedHTML = data.replace(/<noscript>[\s\S]*?<\/noscript>/gi, '');
-
-      const metaRefreshMatch =
-        /<meta\s+http-equiv="refresh"\s+content="(?:\d+;)?\s*url=(?:'([^']*)'|"([^"]*)"|([^>]*))"/i.exec(
-          modifiedHTML,
-        );
-
-      const hasMetaRefresh = metaRefreshMatch && metaRefreshMatch.length > 1;
-
-      if (redirectUrl != null && (hasMetaRefresh || !isCustomFlow)) {
-        res.url = redirectUrl;
-      } else {
-        res.url = url;
-      }
-
-      if (hasMetaRefresh) {
-        let urlOrRelativePath;
-
-        for (let i = 1; i < metaRefreshMatch.length; i++) {
-          if (metaRefreshMatch[i] !== undefined && metaRefreshMatch[i] !== null) {
-            urlOrRelativePath = metaRefreshMatch[i];
-            break; // Stop the loop once the first non-null value is found
-          }
-        }
-
-        if (urlOrRelativePath.includes('URL=')) {
-          res.url = urlOrRelativePath.split('URL=').pop();
-        } else {
-          const pathname = res.url.substring(0, res.url.lastIndexOf('/'));
-          res.url = new URL(urlOrRelativePath, pathname).toString();
-        }
-      }
-
-      res.content = response.data;
-    })
-    .catch(async error => {
-      if (error.code === 'ECONNABORTED' || error.code === 'ERR_FR_TOO_MANY_REDIRECTS') {
-        res.status = constants.urlCheckStatuses.axiosTimeout.code;
-      } else if (error.response) {
-        if (error.response.status === 401) {
-          // enters here if URL is protected by basic auth
-          res.status = constants.urlCheckStatuses.unauthorised.code;
-        } else {
-          // enters here if server responds with a status other than 2xx
-          // the scan should still proceed even if error codes are received, so that accessibility scans for error pages can be done too
-          res.status = constants.urlCheckStatuses.success.code;
-        }
-        res.url = url;
-        res.content = error.response.data;
-        return res;
-      } else if (error.request) {
-        // enters here if URL cannot be accessed
-        res.status = constants.urlCheckStatuses.cannotBeResolved.code;
-      } else {
-        res.status = constants.urlCheckStatuses.systemError.code;
-      }
-    });
-  return res;
-};
-
 const checkUrlConnectivityWithBrowser = async (
   url: string,
   browserToRun: string,
   clonedDataDir: string,
   playwrightDeviceDetailsObject: DeviceDescriptor,
-  isCustomFlow: boolean,
   extraHTTPHeaders: Record<string, string>,
 ) => {
   const res = new RES();
@@ -391,27 +292,18 @@ const checkUrlConnectivityWithBrowser = async (
     return res;
   }
 
-  let viewport = null;
-  let userAgent = null;
-  if ('viewport' in playwrightDeviceDetailsObject) viewport = playwrightDeviceDetailsObject.viewport;
-  if ('userAgent' in playwrightDeviceDetailsObject) userAgent = playwrightDeviceDetailsObject.userAgent;
-
   // Ensure Accept header for non-html content fallback
   extraHTTPHeaders['Accept'] ||= 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8';
 
-  const launchOptions = getPlaywrightLaunchOptions(browserToRun);
-  const browserContextLaunchOptions = {
-    ...launchOptions,
-    args: [...launchOptions.args, '--headless=new'],
-  };
+  await initModifiedUserAgent(browserToRun, playwrightDeviceDetailsObject, clonedDataDir);
 
   let browserContext;
   try {
     browserContext = await constants.launcher.launchPersistentContext(clonedDataDir, {
-      ...browserContextLaunchOptions,
-      ...(viewport && { viewport }),
-      ...(userAgent && { userAgent }),
       ...(extraHTTPHeaders && { extraHTTPHeaders }),
+      ignoreHTTPSErrors: true,
+      ...getPlaywrightLaunchOptions(browserToRun),
+      ...playwrightDeviceDetailsObject,
     });
   } catch (err) {
     printMessage([`Unable to launch browser\n${err}`], messageOptions);
@@ -422,48 +314,77 @@ const checkUrlConnectivityWithBrowser = async (
   try {
     const page = await browserContext.newPage();
 
-    // Skip Playwright for PDF (use raw request instead)
-    if (isUrlPdf(url)) {
-      return await requestToUrl(url, false, extraHTTPHeaders);
+    // STEP 1: HEAD request before actual navigation
+    let statusCode = 0;
+    let contentType = '';
+    let disposition = '';
+
+    try {
+      const headResp = await page.request.fetch(url, {
+        method: 'HEAD',
+        headers: extraHTTPHeaders,
+      });
+
+      statusCode = headResp.status();
+      contentType = headResp.headers()['content-type'] || '';
+      disposition = headResp.headers()['content-disposition'] || '';
+
+      // If it looks like a downloadable file, skip goto entirely
+      if (
+        contentType.includes('pdf') ||
+        contentType.includes('octet-stream') ||
+        disposition.includes('attachment')
+      ) {
+        res.status = statusCode === 401
+          ? constants.urlCheckStatuses.unauthorised.code
+          : constants.urlCheckStatuses.success.code;
+
+        res.httpStatus = statusCode;
+        res.url = url;
+        res.content = ''; // Don't try to render binary
+
+        await browserContext.close();
+        return res;
+      }
+    } catch (e) {
+      consoleLogger.info(`HEAD request failed: ${e.message}`);
+      res.status = constants.urlCheckStatuses.systemError.code;
+      await browserContext.close();
+      return res;
     }
 
+    // STEP 2: Safe to proceed with navigation
     const response = await page.goto(url, {
       timeout: 30000,
-      ...(proxy && { waitUntil: 'commit' }),
+      waitUntil: 'commit', // Don't wait for full load
     });
 
-    try {
-      await page.waitForLoadState('networkidle', { timeout: 10000 });
-    } catch {
-      consoleLogger.info('Unable to detect networkidle');
-    }
-
-    const status = response.status();
-    res.status = status === 401
+    const finalStatus = statusCode || (response?.status?.() ?? 0);
+    res.status = finalStatus === 401
       ? constants.urlCheckStatuses.unauthorised.code
       : constants.urlCheckStatuses.success.code;
 
-    // Store the status code
-    res.httpStatus = response?.status?.() ?? 0;
+    res.httpStatus = finalStatus;
+    res.url = page.url();
 
-    // Store final navigated URL
-    res.url = isCustomFlow ? url : page.url();
-
-    // Check content type to determine how to extract content
-    const contentType = response.headers()['content-type'] || '';
-
-    if (contentType.includes('xml') || res.url.endsWith('.xml')) {
-      // Fetch raw content to avoid Playwright's HTML-wrapped <pre> behavior
-      const rawResponse = await requestToUrl(res.url, true, extraHTTPHeaders);
-      res.content = rawResponse.content;
+    contentType = response?.headers()?.['content-type'] || '';
+    if (contentType.includes('pdf') || contentType.includes('octet-stream')) {
+      res.content = ''; // Avoid triggering render/download
     } else {
-      res.content = await page.content(); // rendered DOM
+      try {
+        await page.waitForLoadState('networkidle', { timeout: 10000 });
+      } catch {
+        consoleLogger.info('Unable to detect networkidle');
+      }
+
+      res.content = await page.content();
     }
 
   } catch (error) {
     if (error.message.includes('net::ERR_INVALID_AUTH_CREDENTIALS')) {
       res.status = constants.urlCheckStatuses.unauthorised.code;
     } else {
+      console.log(error);
       res.status = constants.urlCheckStatuses.systemError.code;
     }
   } finally {
@@ -501,7 +422,6 @@ export const checkUrl = async (
   browser: string,
   clonedDataDir: string,
   playwrightDeviceDetailsObject: DeviceDescriptor,
-  isCustomFlow: boolean,
   extraHTTPHeaders: Record<string, string>,
 ) => {
   const res = await checkUrlConnectivityWithBrowser(
@@ -509,7 +429,6 @@ export const checkUrl = async (
     browser,
     clonedDataDir,
     playwrightDeviceDetailsObject,
-    isCustomFlow,
     extraHTTPHeaders,
   );
 
@@ -555,18 +474,16 @@ export const prepareData = async (argv: Answers): Promise<Data> => {
   if (isEmptyObject(argv)) {
     throw Error('No inputs should be provided');
   }
-  const {
+  let {
     scanner,
     headless,
     url,
     deviceChosen,
     customDevice,
     viewportWidth,
-    playwrightDeviceDetailsObject,
     maxpages,
     strategy,
     isLocalFileScan,
-    finalUrl,
     browserToRun,
     nameEmail,
     customFlowLabel,
@@ -578,32 +495,72 @@ export const prepareData = async (argv: Answers): Promise<Data> => {
     followRobots,
     header,
     safeMode,
+    exportDirectory,
     zip,
     ruleset,
     generateJsonFiles,
     scanDuration
   } = argv;
 
+  const extraHTTPHeaders = parseHeaders(header);
+
+  // Set default username and password for basic auth
+  let username = '';
+  let password = '';
+
+  // Remove credentials from URL if not a local file scan
+  url = argv.isLocalFileScan 
+    ? url 
+    : (() => {
+        const temp = new URL(url);
+        username = temp.username;
+        password = temp.password;
+
+        if (username !== '' || password !== '') {
+          extraHTTPHeaders['Authorization'] = `Basic ${Buffer.from(`${username}:${password}`).toString('base64')}`;
+        } 
+
+        temp.username = '';
+        temp.password = '';
+        return temp.toString();
+      })();
+
   // construct filename for scan results
   const [date, time] = new Date().toLocaleString('sv').replaceAll(/-|:/g, '').split(' ');
   const domain = argv.isLocalFileScan ? path.basename(argv.url) : new URL(argv.url).hostname;
+
   const sanitisedLabel = customFlowLabel ? `_${customFlowLabel.replaceAll(' ', '_')}` : '';
   let resultFilename: string;
   const randomThreeDigitNumber = randomThreeDigitNumberString();
-  if (process.env.OOBEE_VERBOSE) {
-    resultFilename = `${date}_${time}${sanitisedLabel}_${domain}_${randomThreeDigitNumber}`;
-  } else {
-    resultFilename = `${date}_${time}${sanitisedLabel}_${domain}`;
+  resultFilename = `${date}_${time}${sanitisedLabel}_${domain}_${randomThreeDigitNumber}`;
+
+  // Set exported directory
+  if (exportDirectory) {
+    constants.exportDirectory = path.join(exportDirectory, resultFilename);
   }
+  
+  // Creating the playwrightDeviceDetailObject
+  deviceChosen = customDevice === 'Desktop' || customDevice === 'Mobile' ? customDevice : deviceChosen;
+  
+  const playwrightDeviceDetailsObject = getPlaywrightDeviceDetailsObject(
+    deviceChosen,
+    customDevice,
+    viewportWidth,
+  );
+
+  const { browserToRun: resolvedBrowser, clonedBrowserDataDir } = getBrowserToRun(browserToRun, true, resultFilename);
+  browserToRun = resolvedBrowser;
+
+  const resolvedUserDataDirectory = getClonedProfilesWithRandomToken(browserToRun, resultFilename);
 
   if (followRobots) {
     constants.robotsTxtUrls = {};
-    await getUrlsFromRobotsTxt(url, browserToRun);
+    await getUrlsFromRobotsTxt(url, browserToRun, resolvedUserDataDirectory, extraHTTPHeaders);
   }
 
   return {
     type: scanner,
-    url: finalUrl,
+    url: url,
     entryUrl: url,
     isHeadless: headless,
     deviceChosen,
@@ -624,8 +581,9 @@ export const prepareData = async (argv: Answers): Promise<Data> => {
     includeScreenshots: !(additional === 'none'),
     metadata,
     followRobots,
-    extraHTTPHeaders: parseHeaders(header),
+    extraHTTPHeaders: extraHTTPHeaders,
     safeMode,
+    userDataDirectory: resolvedUserDataDirectory,
     zip,
     ruleset,
     generateJsonFiles,
@@ -633,7 +591,7 @@ export const prepareData = async (argv: Answers): Promise<Data> => {
   };
 };
 
-export const getUrlsFromRobotsTxt = async (url: string, browserToRun: string): Promise<void> => {
+export const getUrlsFromRobotsTxt = async (url: string, browserToRun: string, userDataDirectory: string, extraHTTPHeaders: Record<string, string>): Promise<void> => {
   if (!constants.robotsTxtUrls) return;
 
   const domain = new URL(url).origin;
@@ -642,22 +600,18 @@ export const getUrlsFromRobotsTxt = async (url: string, browserToRun: string): P
 
   let robotsTxt: string;
   try {
-    if (proxy) {
-      robotsTxt = await getRobotsTxtViaPlaywright(robotsUrl, browserToRun);
-    } else {
-      robotsTxt = await getRobotsTxtViaAxios(robotsUrl);
-    }
+    robotsTxt = await getRobotsTxtViaPlaywright(robotsUrl, browserToRun, userDataDirectory, extraHTTPHeaders);
+    consoleLogger.info(`Fetched robots.txt from ${robotsUrl}`);
   } catch (e) {
     // if robots.txt is not found, do nothing
+    consoleLogger.info(`Unable to fetch robots.txt from ${robotsUrl}`);
   }
-  console.log('robotsTxt', robotsTxt);
+
   if (!robotsTxt) {
     constants.robotsTxtUrls[domain] = {};
     return;
   }
-
-  console.log('Found robots.txt: ', robotsUrl);
-
+  
   const lines = robotsTxt.split(/\r?\n/);
   let shouldCapture = false;
   const disallowedUrls = [];
@@ -705,30 +659,30 @@ export const getUrlsFromRobotsTxt = async (url: string, browserToRun: string): P
   constants.robotsTxtUrls[domain] = { disallowedUrls, allowedUrls };
 };
 
-const getRobotsTxtViaPlaywright = async (robotsUrl: string, browser: string): Promise<string> => {
-  const browserContext = await constants.launcher.launchPersistentContext('', {
+const getRobotsTxtViaPlaywright = async (robotsUrl: string, browser: string, userDataDirectory: string, extraHTTPHeaders: Record<string, string>): Promise<string> => {
+
+  let robotsDataDir = '';
+  // Bug in Chrome which causes browser pool crash when userDataDirectory is set in non-headless mode
+  if (process.env.CRAWLEE_HEADLESS === '1') {
+    // Create robots own user data directory else SingletonLock: File exists (17) with crawlDomain or crawlSitemap's own browser
+    const robotsDataDir = path.join(userDataDirectory, 'robots');
+    if (!fs.existsSync(robotsDataDir)) {
+      fs.mkdirSync(robotsDataDir, { recursive: true });
+    }
+  }
+
+  const browserContext = await constants.launcher.launchPersistentContext(robotsDataDir, {
     ...getPlaywrightLaunchOptions(browser),
+    ...(extraHTTPHeaders && { extraHTTPHeaders }),
   });
 
   const page = await browserContext.newPage();
-  await page.goto(robotsUrl, { waitUntil: 'networkidle', timeout: 30000 });
 
+  await page.goto(robotsUrl, { waitUntil: 'networkidle', timeout: 30000 });
   const robotsTxt: string | null = await page.evaluate(() => document.body.textContent);
   return robotsTxt;
 };
 
-const getRobotsTxtViaAxios = async (robotsUrl: string): Promise<string> => {
-  const instance = axios.create({
-    httpsAgent: new https.Agent({
-      rejectUnauthorized: false,
-      keepAlive: true,
-    }),
-  });
-
-  const robotsTxt = (await (await instance.get(robotsUrl, { timeout: 2000 })).data) as string;
-  return robotsTxt;
-};
-
 export const isDisallowedInRobotsTxt = (url: string): boolean => {
   if (!constants.robotsTxtUrls) return;
 
@@ -760,8 +714,7 @@ export const getLinksFromSitemap = async (
   userDataDirectory: string,
   userUrlInput: string,
   isIntelligent: boolean,
-  username: string,
-  password: string,
+  extraHTTPHeaders: Record<string, string>,
 ) => {
   const scannedSitemaps = new Set<string>();
   const urls: Record<string, Request> = {}; // dictionary of requests to urls to be scanned
@@ -772,11 +725,6 @@ export const getLinksFromSitemap = async (
     if (!url) return;
     if (isDisallowedInRobotsTxt(url)) return;
 
-    // add basic auth credentials to the URL
-    username !== '' && password !== ''
-      ? (url = addBasicAuthCredentials(url, username, password))
-      : url;
-
     url = convertPathToLocalFile(url);
 
     let request;
@@ -791,13 +739,6 @@ export const getLinksFromSitemap = async (
     urls[url] = request;
   };
 
-  const addBasicAuthCredentials = (url: string, username: string, password: string) => {
-    const urlObject = new URL(url);
-    urlObject.username = username;
-    urlObject.password = password;
-    return urlObject.toString();
-  };
-
   const calculateCloseness = (sitemapUrl: string) => {
     // Remove 'http://', 'https://', and 'www.' prefixes from the URLs
     const normalizedSitemapUrl = sitemapUrl.replace(/^(https?:\/\/)?(www\.)?/, '');
@@ -868,16 +809,10 @@ export const getLinksFromSitemap = async (
     finalUserDataDirectory = '';
   }
 
-  const fetchUrls = async (url: string) => {
+  const fetchUrls = async (url: string, extraHTTPHeaders: Record<string, string>) => {
     let data;
     let sitemapType;
-    let isBasicAuth = false;
-
-    let username = '';
-    let password = '';
-
-    let parsedUrl;
-
+   
     if (scannedSitemaps.has(url)) {
       // Skip processing if the sitemap has already been scanned
       return;
@@ -893,17 +828,9 @@ export const getLinksFromSitemap = async (
       if (!fs.existsSync(url)) {
         return;
       }
-      parsedUrl = url;
+
     } else if (isValidHttpUrl(url)) {
-      parsedUrl = new URL(url);
-
-      if (parsedUrl.username !== '' && parsedUrl.password !== '') {
-        isBasicAuth = true;
-        username = decodeURIComponent(parsedUrl.username);
-        password = decodeURIComponent(parsedUrl.password);
-        parsedUrl.username = '';
-        parsedUrl.password = '';
-      }
+      // Do nothing, url is valid
     } else {
       printMessage([`Invalid Url/Filepath: ${url}`], messageOptions);
       return;
@@ -915,12 +842,17 @@ export const getLinksFromSitemap = async (
         {
           ...getPlaywrightLaunchOptions(browser),
           // Not necessary to parse http_credentials as I am parsing it directly in URL
+          // Bug in Chrome which causes browser pool crash when userDataDirectory is set in non-headless mode
+          ...(process.env.CRAWLEE_HEADLESS === '1' && { userDataDir: userDataDirectory }),
+          ...(extraHTTPHeaders && { extraHTTPHeaders }),
         },
       );
 
       const page = await browserContext.newPage();
+
       await page.goto(url, { waitUntil: 'networkidle', timeout: 60000 });
-      if (constants.launcher === webkit) {
+
+      if (await page.locator('body').count() > 0) {
         data = await page.locator('body').innerText();
       } else {
         const urlSet = page.locator('urlset');
@@ -948,35 +880,14 @@ export const getLinksFromSitemap = async (
         addToUrlList(url);
         return;
       }
-      if (proxy) {
-        await getDataUsingPlaywright();
-      } else {
-        try {
-          const instance = axios.create({
-            httpsAgent: new https.Agent({
-              rejectUnauthorized: false,
-              keepAlive: true,
-            }),
-            auth: {
-              username,
-              password,
-            },
-          });
-          try {
-            data = await (await instance.get(url, { timeout: 80000 })).data;
-          } catch {
-            return; // to skip the error
-          }
-        } catch (error) {
-          if (error.code === 'ECONNABORTED') {
-            await getDataUsingPlaywright();
-          }
-        }
-      }
+
+      await getDataUsingPlaywright();
+
     } else {
       url = convertLocalFileToPath(url);
       data = fs.readFileSync(url, 'utf8');
     }
+
     const $ = cheerio.load(data, { xml: true });
 
     // This case is when the document is not an XML format document
@@ -1012,7 +923,7 @@ export const getLinksFromSitemap = async (
             break;
           }
           if (childSitemapUrlText.endsWith('.xml') || childSitemapUrlText.endsWith('.txt')) {
-            await fetchUrls(childSitemapUrlText); // Recursive call for nested sitemaps
+            await fetchUrls(childSitemapUrlText, extraHTTPHeaders); // Recursive call for nested sitemaps
           } else {
             addToUrlList(childSitemapUrlText); // Add regular URLs to the list
           }
@@ -1037,7 +948,7 @@ export const getLinksFromSitemap = async (
   };
 
   try {
-    await fetchUrls(sitemapUrl);
+    await fetchUrls(sitemapUrl, extraHTTPHeaders);
   } catch (e) {
     consoleLogger.error(e);
   }
@@ -1086,20 +997,26 @@ export const validName = (name: string) => {
  * @returns object consisting of browser to run and cloned data directory
  */
 export const getBrowserToRun = (
-  preferredBrowser: BrowserTypes,
+  preferredBrowser?: BrowserTypes,
   isCli = false,
+  randomToken?: string
 ): { browserToRun: BrowserTypes; clonedBrowserDataDir: string } => {
+
+  if (!randomToken) {
+    randomToken = '';
+  }
+  
   const platform = os.platform();
 
   // Prioritise Chrome on Windows and Mac platforms if user does not specify a browser
   if (!preferredBrowser && (os.platform() === 'win32' || os.platform() === 'darwin')) {
     preferredBrowser = BrowserTypes.CHROME;
+  } else {
+    printMessage([`Preferred browser ${preferredBrowser}`], messageOptions);
   }
 
-  printMessage([`Preferred browser ${preferredBrowser}`], messageOptions);
-
   if (preferredBrowser === BrowserTypes.CHROME) {
-    const chromeData = getChromeData();
+    const chromeData = getChromeData(randomToken);
     if (chromeData) return chromeData;
 
     if (platform === 'darwin') {
@@ -1113,7 +1030,7 @@ export const getBrowserToRun = (
       if (isCli)
         printMessage(['Unable to use Chrome, falling back to Edge browser...'], messageOptions);
 
-      const edgeData = getEdgeData();
+      const edgeData = getEdgeData(randomToken);
       if (edgeData) return edgeData;
 
       if (isCli)
@@ -1125,12 +1042,12 @@ export const getBrowserToRun = (
       printMessage(['Unable to use Chrome, falling back to Chromium browser...'], messageOptions);
     }
   } else if (preferredBrowser === BrowserTypes.EDGE) {
-    const edgeData = getEdgeData();
+    const edgeData = getEdgeData(randomToken);
     if (edgeData) return edgeData;
 
     if (isCli)
       printMessage(['Unable to use Edge, falling back to Chrome browser...'], messageOptions);
-    const chromeData = getChromeData();
+    const chromeData = getChromeData(randomToken);
     if (chromeData) return chromeData;
 
     if (platform === 'darwin') {
@@ -1161,7 +1078,7 @@ export const getBrowserToRun = (
   // defaults to chromium
   return {
     browserToRun: BrowserTypes.CHROMIUM,
-    clonedBrowserDataDir: cloneChromiumProfiles(),
+    clonedBrowserDataDir: cloneChromiumProfiles(randomToken),
   };
 };
 
@@ -1181,9 +1098,9 @@ export const getClonedProfilesWithRandomToken = (browser: string, randomToken: s
   return cloneChromiumProfiles(randomToken);
 };
 
-export const getChromeData = () => {
+export const getChromeData = (randomToken: string) => {
   const browserDataDir = getDefaultChromeDataDir();
-  const clonedBrowserDataDir = cloneChromeProfiles();
+  const clonedBrowserDataDir = cloneChromeProfiles(randomToken);
   if (browserDataDir && clonedBrowserDataDir) {
     const browserToRun = BrowserTypes.CHROME;
     return { browserToRun, clonedBrowserDataDir };
@@ -1191,9 +1108,9 @@ export const getChromeData = () => {
   return null;
 };
 
-export const getEdgeData = () => {
+export const getEdgeData = (randomToken: string) => {
   const browserDataDir = getDefaultEdgeDataDir();
-  const clonedBrowserDataDir = cloneEdgeProfiles();
+  const clonedBrowserDataDir = cloneEdgeProfiles(randomToken);
   if (browserDataDir && clonedBrowserDataDir) {
     const browserToRun = BrowserTypes.EDGE;
     return { browserToRun, clonedBrowserDataDir };
@@ -1397,7 +1314,7 @@ const cloneLocalStateFile = (options: GlobOptionsWithFileTypesFalse, destDir: st
  * @param {string} randomToken - random token to append to the cloned directory
  * @returns {string} cloned data directory, null if any of the sub files failed to copy
  */
-export const cloneChromeProfiles = (randomToken?: string): string => {
+export const cloneChromeProfiles = (randomToken: string): string => {
   const baseDir = getDefaultChromeDataDir();
 
   if (!baseDir) {
@@ -1406,18 +1323,10 @@ export const cloneChromeProfiles = (randomToken?: string): string => {
 
   let destDir;
 
-  if (randomToken) {
-    destDir = path.join(baseDir, `oobee-${randomToken}`);
-  } else {
-    destDir = path.join(baseDir, 'oobee');
-  }
+  destDir = path.join(baseDir, `oobee-${randomToken}`);
 
   if (fs.existsSync(destDir)) {
-    if (process.env.OOBEE_VERBOSE) {
       deleteClonedChromeProfiles(randomToken);
-    } else {
-      deleteClonedChromeProfiles();
-    }
   }
 
   if (!fs.existsSync(destDir)) {
@@ -1435,10 +1344,13 @@ export const cloneChromeProfiles = (randomToken?: string): string => {
     return destDir;
   }
 
-  return null;
+  consoleLogger.error('Failed to clone Chrome profiles. You may be logged out of your accounts.');
+
+  // For future reference, return a null instead to halt the scan
+  return destDir;
 };
 
-export const cloneChromiumProfiles = (randomToken?: string): string => {
+export const cloneChromiumProfiles = (randomToken: string): string => {
   const baseDir = getDefaultChromiumDataDir();
 
   if (!baseDir) {
@@ -1447,10 +1359,10 @@ export const cloneChromiumProfiles = (randomToken?: string): string => {
 
   let destDir: string;
 
-  if (randomToken) {
-    destDir = path.join(baseDir, `oobee-${randomToken}`);
-  } else {
-    destDir = path.join(baseDir, 'oobee');
+  destDir = path.join(baseDir, `oobee-${randomToken}`);
+
+  if (fs.existsSync(destDir)) {
+      deleteClonedChromiumProfiles(randomToken);
   }
 
   if (!fs.existsSync(destDir)) {
@@ -1468,7 +1380,7 @@ export const cloneChromiumProfiles = (randomToken?: string): string => {
  * @param {string} randomToken - random token to append to the cloned directory
  * @returns {string} cloned data directory, null if any of the sub files failed to copy
  */
-export const cloneEdgeProfiles = (randomToken?: string): string => {
+export const cloneEdgeProfiles = (randomToken: string): string => {
   const baseDir = getDefaultEdgeDataDir();
 
   if (!baseDir) {
@@ -1477,18 +1389,10 @@ export const cloneEdgeProfiles = (randomToken?: string): string => {
 
   let destDir;
 
-  if (randomToken) {
-    destDir = path.join(baseDir, `oobee-${randomToken}`);
-  } else {
-    destDir = path.join(baseDir, 'oobee');
-  }
+  destDir = path.join(baseDir, `oobee-${randomToken}`);
 
   if (fs.existsSync(destDir)) {
-    if (process.env.OOBEE_VERBOSE) {
       deleteClonedEdgeProfiles(randomToken);
-    } else {
-      deleteClonedEdgeProfiles();
-    }
   }
 
   if (!fs.existsSync(destDir)) {
@@ -1507,10 +1411,13 @@ export const cloneEdgeProfiles = (randomToken?: string): string => {
     return destDir;
   }
 
-  return null;
+  consoleLogger.error('Failed to clone Edge profiles. You may be logged out of your accounts.');
+
+  // For future reference, return a null instead to halt the scan
+  return destDir;
 };
 
-export const deleteClonedProfiles = (browser: string, randomToken?: string): void => {
+export const deleteClonedProfiles = (browser: string, randomToken: string): void => {
   if (browser === BrowserTypes.CHROME) {
     deleteClonedChromeProfiles(randomToken);
   } else if (browser === BrowserTypes.EDGE) {
@@ -1565,9 +1472,7 @@ export const deleteClonedChromeProfiles = (randomToken?: string): void => {
  * @returns null
  */
 export const deleteClonedEdgeProfiles = (randomToken?: string): void => {
-  if (process.env.OOBEE_VERBOSE) {
-    return;
-  }
+
   const baseDir = getDefaultEdgeDataDir();
 
   if (!baseDir) {
@@ -1716,13 +1621,9 @@ export const submitFormViaPlaywright = async (
   } finally {
     await browserContext.close();
     if (proxy && browserToRun === BrowserTypes.EDGE) {
-      if (!process.env.OOBEE_VERBOSE) {
-        deleteClonedEdgeProfiles();
-      }
+        deleteClonedEdgeProfiles(clonedDir);
     } else if (proxy && browserToRun === BrowserTypes.CHROME) {
-      if (!process.env.OOBEE_VERBOSE) {
-        deleteClonedChromeProfiles();
-      }
+        deleteClonedChromeProfiles(clonedDir);
     }
   }
 };
@@ -1781,7 +1682,9 @@ export const submitForm = async (
 export async function initModifiedUserAgent(
   browser?: string,
   playwrightDeviceDetailsObject?: object,
+  userDataDirectory?: string,
 ) {
+
   const isHeadless = process.env.CRAWLEE_HEADLESS === '1';
 
   // If headless mode is enabled, ensure the headless flag is set.
@@ -1798,7 +1701,11 @@ export async function initModifiedUserAgent(
   };
 
   // Launch a temporary persistent context with an empty userDataDir to mimic your production browser setup.
-  const browserContext = await constants.launcher.launchPersistentContext('', launchOptions);
+  const effectiveUserDataDirectory = process.env.CRAWLEE_HEADLESS === '1'
+  ? userDataDirectory
+  : '';
+
+  const browserContext = await constants.launcher.launchPersistentContext(effectiveUserDataDirectory, launchOptions);
   const page = await browserContext.newPage();
 
   // Retrieve the default user agent.
@@ -1856,13 +1763,6 @@ export const getPlaywrightLaunchOptions = (browser?: string): LaunchOptions => {
   return options;
 };
 
-export const urlWithoutAuth = (url: string): string => {
-  const parsedUrl = new URL(url);
-  parsedUrl.username = '';
-  parsedUrl.password = '';
-  return parsedUrl.toString();
-};
-
 export const waitForPageLoaded = async (page: Page, timeout = 10000) => {
   const OBSERVER_TIMEOUT = timeout; // Ensure observer timeout does not exceed the main timeout
 
@@ -1887,7 +1787,7 @@ export const waitForPageLoaded = async (page: Page, timeout = 10000) => {
 
         let timeout: NodeJS.Timeout;
         let mutationCount = 0;
-        const MAX_MUTATIONS = 250;
+        const MAX_MUTATIONS = 500;
         const mutationHash: Record<string, number> = {};
 
         const observer = new MutationObserver(mutationsList => {