@govtechsg/oobee 0.10.58 → 0.10.62

package/src/constants/constants.ts

@@ -5,10 +5,11 @@ import { globSync } from 'glob';
 import which from 'which';
 import os from 'os';
 import { spawnSync, execSync } from 'child_process';
-import { chromium } from 'playwright';
+import { Browser, BrowserContext, chromium } from 'playwright';
 import * as Sentry from '@sentry/node';
 import { consoleLogger, silentLogger } from '../logs.js';
 import { PageInfo } from '../mergeAxeResults.js';
+import { PlaywrightCrawler } from 'crawlee';
 
 const filename = fileURLToPath(import.meta.url);
 const dirname = path.dirname(filename);
@@ -136,7 +137,7 @@ export const getDefaultChromiumDataDir = () => {
     }
     return null;
   } catch (error) {
-    silentLogger.error(`Error in getDefaultChromiumDataDir(): ${error}`);
+    consoleLogger.error(`Error in getDefaultChromiumDataDir(): ${error}`);
   }
 };
 
@@ -227,45 +228,68 @@ if (fs.existsSync('/.dockerenv')) {
   launchOptionsArgs = ['--disable-gpu', '--no-sandbox', '--disable-dev-shm-usage'];
 }
 
-export const getProxy = (): { type: string; url: string } | null => {
-  if (os.platform() === 'win32') {
-    let internetSettings: string[];
-    try {
-      internetSettings = execSync(
-        'Get-ItemProperty -Path "Registry::HKCU\\Software\\Microsoft\\Windows\\CurrentVersion\\Internet Settings"',
-        { shell: 'powershell.exe' },
-      )
-        .toString()
-        .split('\n');
-    } catch (e) {
-      console.log(e.toString());
-      silentLogger.error(e.toString());
+type ProxyInfo = { type: 'autoConfig' | 'manualProxy'; url: string } | null;
+
+function queryRegKey(key: string): Record<string, string> {
+  try {
+    const out = execSync(`reg query "${key}"`, { encoding: 'utf8', stdio: ['ignore', 'pipe', 'pipe'] });
+    const values: Record<string, string> = {};
+    for (const line of out.split(/\r?\n/)) {
+      const parts = line.trim().split(/\s{2,}/);
+      if (parts.length >= 3) {
+        const [name, _type, ...rest] = parts;
+        values[name] = rest.join(' ');
+      }
     }
+    return values;
+  } catch {
+    return {};
+  }
+}
 
-    const getSettingValue = (settingName: string) =>
-      internetSettings
-        .find(s => s.startsWith(settingName))
-        // split only once at with ':' as the delimiter
-        ?.split(/:(.*)/s)[1]
-        ?.trim();
+function parseDwordFlag(v: unknown): number {
+  if (v == null) return 0;
+  const s = String(v).trim();
+  // Handles "1", "0", "0x1", "0x0"
+  if (/^0x[0-9a-f]+$/i.test(s)) return parseInt(s, 16);
+  if (/^\d+$/.test(s)) return parseInt(s, 10);
+  return 0;
+}
 
-    if (getSettingValue('AutoConfigURL')) {
-      return { type: 'autoConfig', url: getSettingValue('AutoConfigURL') };
-    }
-    if (getSettingValue('ProxyEnable') === '1') {
-      return { type: 'manualProxy', url: getSettingValue('ProxyServer') };
-    }
-    return null;
+function normalizePacUrl(u: string): string {
+  const s = u.trim();
+  // If it lacks a scheme, assume http:// (Chrome requires a full URL)
+  return /^(https?|file):/i.test(s) ? s : `http://${s}`;
+}
+
+export const getProxy = (): ProxyInfo => {
+  if (os.platform() !== 'win32') return null;
+
+  const values = queryRegKey('HKCU\\Software\\Microsoft\\Windows\\CurrentVersion\\Internet Settings');
+  const pacUrlRaw = (values['AutoConfigURL'] || '').trim();
+  const proxyEnableRaw = (values['ProxyEnable'] || '').trim();
+  const proxyServerRaw = (values['ProxyServer'] || '').trim();
+
+  // 1) PAC beats manual proxy if present
+  if (pacUrlRaw) {
+    return { type: 'autoConfig', url: normalizePacUrl(pacUrlRaw) };
+  }
+
+  // 2) Manual proxy only if enabled
+  const enabled = parseDwordFlag(proxyEnableRaw) === 1;
+  if (enabled && proxyServerRaw) {
+    return { type: 'manualProxy', url: proxyServerRaw };
   }
-  // develop for mac
+
   return null;
 };
 
+// Usage
 export const proxy = getProxy();
 
-if (proxy && proxy.type === 'autoConfig') {
+if (proxy?.type === 'autoConfig') {
   launchOptionsArgs.push(`--proxy-pac-url=${proxy.url}`);
-} else if (proxy && proxy.type === 'manualProxy') {
+} else if (proxy?.type === 'manualProxy') {
   launchOptionsArgs.push(`--proxy-server=${proxy.url}`);
 }
 
@@ -405,6 +429,7 @@ const urlCheckStatuses = {
   },
   axiosTimeout: { code: 18, message: 'Axios timeout exceeded. Falling back on browser checks.' },
   notALocalFile: { code: 19, message: 'Provided filepath is not a local html or sitemap file.' },
+  terminationRequested: { code: 15, message: 'Termination requested.' }
 };
 
 /* eslint-disable no-unused-vars */
@@ -452,7 +477,7 @@ const reserveFileNameKeywords = [
 
 export default {
   cliZipFileName: 'oobee-scan-results.zip',
-  exportDirectory: `${process.cwd()}`,
+  exportDirectory: undefined,
   maxRequestsPerCrawl,
   maxConcurrency: 25,
   urlsCrawledObj,
@@ -466,6 +491,14 @@ export default {
   reserveFileNameKeywords,
   wcagLinks,
   robotsTxtUrls: null,
+  userDataDirectory: null, // This will be set later in the code
+  randomToken: null, // This will be set later in the code
+  // Track all active Crawlee / Playwright resources for cleanup
+  resources: {
+      crawlers: new Set<PlaywrightCrawler>(),
+      browserContexts: new Set<BrowserContext>(),
+      browsers: new Set<Browser>(),
+    },
 };
 
 export const rootPath = dirname;

package/src/constants/questions.ts

@@ -1,6 +1,6 @@
 import { Question } from 'inquirer';
 import { Answers } from '../index.js';
-import { getUserDataTxt, setHeadlessMode } from '../utils.js';
+import { getUserDataTxt, randomThreeDigitNumberString, setHeadlessMode } from '../utils.js';
 import {
   checkUrl,
   deleteClonedProfiles,
@@ -15,6 +15,7 @@ import {
   parseHeaders,
 } from './common.js';
 import constants, { BrowserTypes, ScannerTypes } from './constants.js';
+import { random } from 'lodash';
 
 const userData = getUserDataTxt();
 
@@ -78,8 +79,15 @@ const startScanQuestions = [
         process.exit(1);
       }
 
+      // construct filename for scan results
+      const [date, time] = new Date().toLocaleString('sv').replaceAll(/-|:/g, '').split(' ');
+      const domain = new URL(url).hostname;
+      let resultFilename: string;
+      const randomThreeDigitNumber = randomThreeDigitNumberString();
+      resultFilename = `${date}_${time}_${domain}_${randomThreeDigitNumber}`;
+  
       const statuses = constants.urlCheckStatuses;
-      const { browserToRun, clonedBrowserDataDir } = getBrowserToRun(BrowserTypes.CHROME);
+      const { browserToRun, clonedBrowserDataDir } = getBrowserToRun(BrowserTypes.CHROME, false, resultFilename);
 
       setHeadlessMode(browserToRun, answers.headless);
 
@@ -95,11 +103,9 @@ const startScanQuestions = [
         browserToRun,
         clonedBrowserDataDir,
         playwrightDeviceDetailsObject,
-        answers.scanner === ScannerTypes.CUSTOM,
         parseHeaders(answers.header),
       );
-
-      deleteClonedProfiles(browserToRun);
+      
       switch (res.status) {
         case statuses.success.code:
           answers.finalUrl = res.url;

package/src/crawlers/commonCrawlerFunc.ts

@@ -21,6 +21,9 @@ import { getAxeConfiguration } from './custom/getAxeConfiguration.js';
 import { flagUnlabelledClickableElements } from './custom/flagUnlabelledClickableElements.js';
 import xPathToCss from './custom/xPathToCss.js';
 import type { Response as PlaywrightResponse } from 'playwright';
+import fs from 'fs';
+import { getStoragePath } from '../utils.js';
+import path from 'path';
 
 // types
 interface AxeResultsWithScreenshot extends AxeResults {
@@ -254,7 +257,7 @@ export const runAxeScript = async ({
       return new Promise(resolve => {
         let timeout: NodeJS.Timeout;
         let mutationCount = 0;
-        const MAX_MUTATIONS = 250;
+        const MAX_MUTATIONS = 500;
         const MAX_SAME_MUTATION_LIMIT = 10;
         const mutationHash: Record<string, number> = {};
 
@@ -315,9 +318,9 @@ export const runAxeScript = async ({
   page.on('console', msg => {
     const type = msg.type();
     if (type === 'error') {
-      silentLogger.log({ level: 'error', message: msg.text() });
+      consoleLogger.log({ level: 'error', message: msg.text() });
     } else {
-      silentLogger.log({ level: 'info', message: msg.text() });
+      consoleLogger.log({ level: 'info', message: msg.text() });
     }
   });
   */
@@ -476,8 +479,11 @@ export const runAxeScript = async ({
 export const createCrawleeSubFolders = async (
   randomToken: string,
 ): Promise<{ dataset: crawlee.Dataset; requestQueue: crawlee.RequestQueue }> => {
-  const dataset = await crawlee.Dataset.open(randomToken);
-  const requestQueue = await crawlee.RequestQueue.open(randomToken);
+
+  const crawleeDir = path.join(getStoragePath(randomToken),"crawlee");
+
+  const dataset = await crawlee.Dataset.open(crawleeDir);
+  const requestQueue = await crawlee.RequestQueue.open(crawleeDir);
   return { dataset, requestQueue };
 };
 

package/src/crawlers/crawlDomain.ts

@@ -27,11 +27,9 @@ import {
   isSkippedUrl,
   isDisallowedInRobotsTxt,
   getUrlsFromRobotsTxt,
-  urlWithoutAuth,
   waitForPageLoaded,
-  initModifiedUserAgent,
 } from '../constants/common.js';
-import { areLinksEqual, isFollowStrategy } from '../utils.js';
+import { areLinksEqual, isFollowStrategy, register } from '../utils.js';
 import {
   handlePdfDownload,
   runPdfScan,
@@ -40,6 +38,8 @@ import {
 } from './pdfScanFunc.js';
 import { consoleLogger, guiInfoLog, silentLogger } from '../logs.js';
 import { ViewportSettingsClass } from '../combine.js';
+import * as path from 'path';
+import fsp from 'fs/promises';
 
 const isBlacklisted = (url: string, blacklistedPatterns: string[]) => {
   if (!blacklistedPatterns) {
@@ -115,10 +115,6 @@ const crawlDomain = async ({
 
   ({ requestQueue } = await createCrawleeSubFolders(randomToken));
 
-  if (!fs.existsSync(randomToken)) {
-    fs.mkdirSync(randomToken);
-  }
-
   const pdfDownloads: Promise<void>[] = [];
   const uuidToPdfMapping: Record<string, string> = {};
   const isScanHtml = ['all', 'html-only'].includes(fileTypes);
@@ -126,45 +122,11 @@ const crawlDomain = async ({
   const { maxConcurrency } = constants;
   const { playwrightDeviceDetailsObject } = viewportSettings;
 
-  const httpsAgent = new https.Agent({ rejectUnauthorized: false });
-
-  // Boolean to omit axe scan for basic auth URL
-  let isBasicAuth = false;
-  let authHeader = '';
-
-  // Test basic auth and add auth header if auth exist
-  const parsedUrl = new URL(url);
-  let username: string;
-  let password: string;
-  if (parsedUrl.username !== '' && parsedUrl.password !== '') {
-    isBasicAuth = true;
-    username = decodeURIComponent(parsedUrl.username);
-    password = decodeURIComponent(parsedUrl.password);
-
-    // Create auth header
-    authHeader = `Basic ${Buffer.from(`${username}:${password}`).toString('base64')}`;
-
-    // Remove username from parsedUrl
-    parsedUrl.username = '';
-    parsedUrl.password = '';
-    // Send the finalUrl without credentials by setting auth header instead
-    const finalUrl = parsedUrl.toString();
-
-    await requestQueue.addRequest({
-      url: finalUrl,
-      skipNavigation: isUrlPdf(finalUrl),
-      headers: {
-        Authorization: authHeader,
-      },
-      label: finalUrl,
-    });
-  } else {
-    await requestQueue.addRequest({
-      url,
-      skipNavigation: isUrlPdf(url),
-      label: url,
-    });
-  }
+  await requestQueue.addRequest({
+    url,
+    skipNavigation: isUrlPdf(url),
+    label: url,
+  });
 
   const enqueueProcess = async (
     page: Page,
@@ -377,31 +339,40 @@ const crawlDomain = async ({
 
   let isAbortingScanNow = false;
 
-  let userDataDir = '';
-  if (userDataDirectory) {
-    userDataDir = process.env.CRAWLEE_HEADLESS !== '0' ? userDataDirectory : '';
-  }
-
-  await initModifiedUserAgent(browser, playwrightDeviceDetailsObject);
-
-  const crawler = new crawlee.PlaywrightCrawler({
+  const crawler = register(new crawlee.PlaywrightCrawler({
     launchContext: {
       launcher: constants.launcher,
       launchOptions: getPlaywrightLaunchOptions(browser),
       // Bug in Chrome which causes browser pool crash when userDataDirectory is set in non-headless mode
-      ...(process.env.CRAWLEE_HEADLESS === '0' && { userDataDir }),
+      ...(process.env.CRAWLEE_HEADLESS === '1' && { userDataDir: userDataDirectory }),
     },
     retryOnBlocked: true,
     browserPoolOptions: {
       useFingerprints: false,
       preLaunchHooks: [
         async (_pageId, launchContext) => {
+          const baseDir = userDataDirectory; // e.g., /Users/young/.../Chrome/oobee-...
+
+          // Ensure base exists
+          await fsp.mkdir(baseDir, { recursive: true });
+
+          // Create a unique subdir per browser
+          const subProfileDir = path.join(baseDir, `profile-${Date.now()}-${Math.random().toString(36).slice(2, 8)}`);
+          await fsp.mkdir(subProfileDir, { recursive: true });
+
+          // Assign to Crawlee's launcher
+          launchContext.userDataDir = subProfileDir;
+
+          // Safely extend launchOptions
           launchContext.launchOptions = {
             ...launchContext.launchOptions,
-            bypassCSP: true,
             ignoreHTTPSErrors: true,
             ...playwrightDeviceDetailsObject,
+            ...(extraHTTPHeaders && { extraHTTPHeaders }),
           };
+
+          // Optionally log for debugging
+          // console.log(`[HOOK] Using userDataDir: ${subProfileDir}`);
         },
       ],
     },
@@ -414,7 +385,7 @@ const crawlDomain = async ({
           return new Promise(resolve => {
             let timeout;
             let mutationCount = 0;
-            const MAX_MUTATIONS = 250; // stop if things never quiet down
+            const MAX_MUTATIONS = 500; // stop if things never quiet down
             const OBSERVER_TIMEOUT = 5000; // hard cap on total wait
 
             const observer = new MutationObserver(() => {
@@ -464,33 +435,10 @@ const crawlDomain = async ({
         }
       },
     ],
-    preNavigationHooks: [ async({ page, request}) => {
-      if (isBasicAuth) {
-        await page.setExtraHTTPHeaders({
-          Authorization: authHeader,
-          ...extraHTTPHeaders,
-        });
-      } else {
-        await page.setExtraHTTPHeaders({
-          ...extraHTTPHeaders,
-        });
-      }
-    }],
     requestHandlerTimeoutSecs: 90, // Allow each page to be processed by up from default 60 seconds
     requestHandler: async ({ page, request, response, crawler, sendRequest, enqueueLinks }) => {
       const browserContext: BrowserContext = page.context();
       try {
-        // Set basic auth header if needed
-        if (isBasicAuth) {
-          await page.setExtraHTTPHeaders({
-            Authorization: authHeader,
-          });
-          const currentUrl = new URL(request.url);
-          currentUrl.username = username;
-          currentUrl.password = password;
-          request.url = currentUrl.href;
-        }
-
         await waitForPageLoaded(page, 10000);
         let actualUrl = page.url() || request.loadedUrl || request.url;
 
@@ -652,13 +600,13 @@ const crawlDomain = async ({
               });
 
               urlsCrawled.scanned.push({
-                url: urlWithoutAuth(request.url),
+                url: request.url,
                 pageTitle: results.pageTitle,
                 actualUrl, // i.e. actualUrl
               });
 
               urlsCrawled.scannedRedirects.push({
-                fromUrl: urlWithoutAuth(request.url),
+                fromUrl: request.url,
                 toUrl: actualUrl, // i.e. actualUrl
               });
 
@@ -671,10 +619,10 @@ const crawlDomain = async ({
             if (urlsCrawled.scanned.length < maxRequestsPerCrawl) {
               guiInfoLog(guiInfoStatusTypes.SCANNED, {
                 numScanned: urlsCrawled.scanned.length,
-                urlScanned: urlWithoutAuth(request.url),
+                urlScanned: request.url,
               });
               urlsCrawled.scanned.push({
-                url: urlWithoutAuth(request.url),
+                url: request.url,
                 actualUrl: request.url,
                 pageTitle: results.pageTitle,
               });
@@ -695,7 +643,7 @@ const crawlDomain = async ({
           });
         }
 
-        if (followRobots) await getUrlsFromRobotsTxt(request.url, browser);
+        if (followRobots) await getUrlsFromRobotsTxt(request.url, browser, userDataDirectory, extraHTTPHeaders);
         await enqueueProcess(page, enqueueLinks, browserContext);
       } catch (e) {
         try {
@@ -775,7 +723,7 @@ const crawlDomain = async ({
         scaleDownStepRatio: 0.1,       // Scale down slower
       },
     }),
-  });
+  }));
 
   await crawler.run();
 

package/src/crawlers/crawlIntelligentSitemap.ts

@@ -7,6 +7,8 @@ import crawlDomain from './crawlDomain.js';
 import crawlSitemap from './crawlSitemap.js';
 import { EnqueueStrategy } from 'crawlee';
 import { ViewportSettingsClass } from '../combine.js';
+import { getPlaywrightLaunchOptions } from '../constants/common.js';
+import { register } from '../utils.js';
 
 const crawlIntelligentSitemap = async (
   url: string,
@@ -36,9 +38,6 @@ const crawlIntelligentSitemap = async (
   let sitemapUrl;
 
   ({ dataset } = await createCrawleeSubFolders(randomToken));
-  if (!fs.existsSync(randomToken)) {
-    fs.mkdirSync(randomToken);
-  }
 
   function getHomeUrl(parsedUrl: string) {
     const urlObject = new URL(parsedUrl);
@@ -48,15 +47,22 @@ const crawlIntelligentSitemap = async (
     return `${urlObject.protocol}//${urlObject.hostname}${urlObject.port ? `:${urlObject.port}` : ''}`;
   }
 
-  async function findSitemap(link: string) {
+  async function findSitemap(link: string, userDataDirectory: string, extraHTTPHeaders: Record<string, string>) {
     const homeUrl = getHomeUrl(link);
     let sitemapLink = '';
-    const chromiumBrowser = await chromium.launch({
-      headless: false,
-      channel: 'chrome',
-      args: ['--headless=new', '--no-sandbox'],
+
+    const effectiveUserDataDirectory = process.env.CRAWLEE_HEADLESS === '1'
+        ? userDataDirectory
+        : '';
+    const context = await constants.launcher.launchPersistentContext(effectiveUserDataDirectory, {
+        headless: process.env.CRAWLEE_HEADLESS === '1',
+        ...getPlaywrightLaunchOptions(browser),
+        ...(extraHTTPHeaders && { extraHTTPHeaders }),
     });
-    const page = await chromiumBrowser.newPage();
+    register(context);
+
+    const page = await context.newPage();
+
     for (const path of sitemapPaths) {
       sitemapLink = homeUrl + path;
       if (await checkUrlExists(page, sitemapLink)) {
@@ -64,7 +70,8 @@ const crawlIntelligentSitemap = async (
         break;
       }
     }
-    await chromiumBrowser.close();
+    await page.close();
+    await context.close().catch(() => { });
     return sitemapExist ? sitemapLink : '';
   }
 
@@ -79,7 +86,7 @@ const crawlIntelligentSitemap = async (
   };
 
   try {
-    sitemapUrl = await findSitemap(url);
+    sitemapUrl = await findSitemap(url, userDataDirectory, extraHTTPHeaders);
   } catch (error) {
     consoleLogger.error(error);
   }

package/src/crawlers/crawlLocalFile.ts

@@ -13,11 +13,11 @@ import {
   isFilePath,
   convertLocalFileToPath,
   convertPathToLocalFile,
-  initModifiedUserAgent,
 } from '../constants/common.js';
 import { runPdfScan, mapPdfScanResults, doPdfScreenshots } from './pdfScanFunc.js';
 import { guiInfoLog } from '../logs.js';
 import crawlSitemap from './crawlSitemap.js';
+import { register } from '../utils.js';
 
 export const crawlLocalFile = async ({
   url,
@@ -74,9 +74,6 @@ export const crawlLocalFile = async ({
     ({ dataset } = await createCrawleeSubFolders(randomToken));
     urlsCrawled = { ...constants.urlsCrawledObj };
 
-    if (!fs.existsSync(randomToken)) {
-      fs.mkdirSync(randomToken);
-    }
   }
 
   // Check if the sitemapUrl is a local file and if it exists
@@ -136,16 +133,6 @@ export const crawlLocalFile = async ({
     console.log(e);
   }
 
-  if (basicAuthRegex.test(sitemapUrl)) {
-    isBasicAuth = true;
-    // request to basic auth URL to authenticate for browser session
-    finalLinks.push(new Request({ url: sitemapUrl, uniqueKey: `auth:${sitemapUrl}` }));
-    const finalUrl = `${sitemapUrl.split('://')[0]}://${sitemapUrl.split('@')[1]}`;
-    // obtain base URL without credentials so that subsequent URLs within the same domain can be scanned
-    finalLinks.push(new Request({ url: finalUrl }));
-    basicAuthPage = -2;
-  }
-
   const uuidToPdfMapping: Record<string, string> = {}; // key and value of string type
 
   finalLinks = [...finalLinks, ...linksFromSitemap];
@@ -165,13 +152,18 @@ export const crawlLocalFile = async ({
   let shouldAbort = false;
 
   if (!isUrlPdf(request.url)) {
-    await initModifiedUserAgent(browser);
-    const browserContext = await constants.launcher.launchPersistentContext('', {
-      headless: false,
+    const effectiveUserDataDirectory = process.env.CRAWLEE_HEADLESS === '1'
+      ? userDataDirectory
+      : '';
+
+    const browserContext = await constants.launcher.launchPersistentContext(effectiveUserDataDirectory, {
+      headless: process.env.CRAWLEE_HEADLESS === '1',
       ...getPlaywrightLaunchOptions(browser),
       ...playwrightDeviceDetailsObject,
     });
 
+    register(browserContext);
+
     const timeoutId = scanDuration > 0
     ? setTimeout(() => {
         console.log(`Crawl duration of ${scanDuration}s exceeded. Aborting local file scan.`);