@govtechsg/oobee 0.10.63 → 0.10.65

package/Dockerfile

@@ -2,9 +2,14 @@
 # Node version is v22
 FROM mcr.microsoft.com/playwright:v1.50.1-noble
 
-# Installation of packages for oobee and runner
-RUN apt-get update && apt-get install -y zip git
-
+# Installation of packages for oobee and runner (locked versions from build log)
+RUN apt-get update && apt-get install -y \
+    git=1:2.43.0-1ubuntu7.3 \
+    git-man=1:2.43.0-1ubuntu7.3 \
+    unzip=6.0-28ubuntu4.1 \
+    zip=3.0-13ubuntu0.2 \
+ && rm -rf /var/lib/apt/lists/*
+ 
 WORKDIR /app/oobee
 
 # Clone oobee repository

package/README.md

@@ -88,6 +88,8 @@ verapdf --version
 | OOBEE_VALIDATE_URL| When set to `true`, validates if URLs are valid and exits. | `false` |
 | OOBEE_LOGS_PATH | When set, logs are written to this path. |  |
 | WARN_LEVEL | Only used in tests. |  |
+| OOBEE_DISABLE_BROWSER_DOWNLOAD | Experimental flag to disable file downloads on Chrome/Chromium/Edge.  Does not affect Local File scan | |
+| OOBEE_SLOWMO | Experimental flag to slow down web browser behaviour by specified duration (in miliseconds) | |
 
 #### Environment variables used internally (Do not set)
 Do not set these environment variables or behaviour might change unexpectedly.

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@govtechsg/oobee",
   "main": "dist/npmIndex.js",
-  "version": "0.10.63",
+  "version": "0.10.65",
   "type": "module",
   "author": "Government Technology Agency <info@tech.gov.sg>",
   "dependencies": {
@@ -20,6 +20,7 @@
     "https": "^1.0.0",
     "inquirer": "^9.2.12",
     "jsdom": "^21.1.2",
+    "jszip": "^3.10.1",
     "lodash": "^4.17.21",
     "mime-types": "^2.1.35",
     "minimatch": "^9.0.3",

package/src/cli.ts

@@ -5,7 +5,7 @@ import printMessage from 'print-message';
 import { devices } from 'playwright';
 import { fileURLToPath } from 'url';
 import path from 'path';
-import { cleanUp, setHeadlessMode, getVersion, getStoragePath, listenForCleanUp, cleanUpAndExit } from './utils.js';
+import { setHeadlessMode, getVersion, getStoragePath, listenForCleanUp, cleanUpAndExit } from './utils.js';
 import {
   checkUrl,
   prepareData,
@@ -16,7 +16,6 @@ import {
   validateDirPath,
   validateFilePath,
   validateCustomFlowLabel,
-  parseHeaders,
 } from './constants/common.js';
 import constants, { ScannerTypes } from './constants/constants.js';
 import { cliOptions, messageOptions } from './constants/cliFunctions.js';

package/src/combine.ts

@@ -5,7 +5,7 @@ import crawlDomain from './crawlers/crawlDomain.js';
 import crawlLocalFile from './crawlers/crawlLocalFile.js';
 import crawlIntelligentSitemap from './crawlers/crawlIntelligentSitemap.js';
 import generateArtifacts from './mergeAxeResults.js';
-import { getHost, createAndUpdateResultsFolders, createDetailsAndLogs, cleanUp, cleanUpAndExit } from './utils.js';
+import { getHost, createAndUpdateResultsFolders, cleanUpAndExit } from './utils.js';
 import { ScannerTypes, UrlsCrawled } from './constants/constants.js';
 import { getBlackListedPatterns, submitForm } from './constants/common.js';
 import { consoleLogger, silentLogger } from './logs.js';
@@ -218,7 +218,7 @@ const combineRun = async (details: Data, deviceToScan: string) => {
 
   scanDetails.endTime = new Date();
   scanDetails.urlsCrawled = urlsCrawledObj;
-  await createDetailsAndLogs(randomToken);
+
   if (scanDetails.urlsCrawled) {
     if (scanDetails.urlsCrawled.scanned.length > 0) {
       await createAndUpdateResultsFolders(randomToken);

package/src/constants/common.ts

@@ -22,18 +22,18 @@ import constants, {
   getDefaultChromeDataDir,
   getDefaultEdgeDataDir,
   getDefaultChromiumDataDir,
-  proxy,
   // Legacy code start - Google Sheets submission
   formDataFields,
   // Legacy code end - Google Sheets submission
   ScannerTypes,
   BrowserTypes,
 } from './constants.js';
-import { consoleLogger, silentLogger } from '../logs.js';
+import { consoleLogger } from '../logs.js';
 import { isUrlPdf } from '../crawlers/commonCrawlerFunc.js';
 import { cleanUpAndExit, randomThreeDigitNumberString, register } from '../utils.js';
 import { Answers, Data } from '../index.js';
 import { DeviceDescriptor } from '../types/types.js';
+import { getProxyInfo, proxyInfoToResolution, ProxySettings } from '../proxyService.js';
 
 // validateDirPath validates a provided directory path
 // returns null if no error
@@ -304,6 +304,7 @@ const checkUrlConnectivityWithBrowser = async (
       ignoreHTTPSErrors: true,
       ...getPlaywrightLaunchOptions(browserToRun),
       ...playwrightDeviceDetailsObject,
+      ...(process.env.OOBEE_DISABLE_BROWSER_DOWNLOAD && { acceptDownloads: false }),
     });
 
     register(browserContext);
@@ -485,7 +486,7 @@ export const prepareData = async (argv: Answers): Promise<Data> => {
     viewportWidth,
     maxpages,
     strategy,
-    isLocalFileScan,
+    isLocalFileScan = false,
     browserToRun,
     nameEmail,
     customFlowLabel,
@@ -510,6 +511,10 @@ export const prepareData = async (argv: Answers): Promise<Data> => {
   let username = '';
   let password = '';
 
+  if (isFilePath(url)) {
+    argv.isLocalFileScan = true;
+  }
+
   // Remove credentials from URL if not a local file scan
   url = argv.isLocalFileScan 
     ? url 
@@ -550,7 +555,7 @@ export const prepareData = async (argv: Answers): Promise<Data> => {
     viewportWidth,
   );
 
-  const { browserToRun: resolvedBrowser, clonedBrowserDataDir } = getBrowserToRun(browserToRun, true, resultFilename);
+  const { browserToRun: resolvedBrowser, clonedBrowserDataDir } = getBrowserToRun(resultFilename, browserToRun, true);
   browserToRun = resolvedBrowser;
 
   const resolvedUserDataDirectory = getClonedProfilesWithRandomToken(browserToRun, resultFilename);
@@ -1005,14 +1010,10 @@ export const validName = (name: string) => {
  * @returns object consisting of browser to run and cloned data directory
  */
 export const getBrowserToRun = (
+  randomToken: string,
   preferredBrowser?: BrowserTypes,
   isCli = false,
-  randomToken?: string
 ): { browserToRun: BrowserTypes; clonedBrowserDataDir: string } => {
-
-  if (!randomToken) {
-    randomToken = '';
-  }
   
   const platform = os.platform();
 
@@ -1597,15 +1598,8 @@ export const submitFormViaPlaywright = async (
   userDataDirectory: string,
   finalUrl: string,
 ) => {
-  const dirName = `clone-${Date.now()}`;
-  let clonedDir = null;
-  if (proxy && browserToRun === BrowserTypes.EDGE) {
-    clonedDir = cloneEdgeProfiles(dirName);
-  } else if (proxy && browserToRun === BrowserTypes.CHROME) {
-    clonedDir = cloneChromeProfiles(dirName);
-  }
   const browserContext = await constants.launcher.launchPersistentContext(
-    clonedDir || userDataDirectory,
+    userDataDirectory,
     {
       ...getPlaywrightLaunchOptions(browserToRun),
     },
@@ -1618,7 +1612,7 @@ export const submitFormViaPlaywright = async (
   try {
     await page.goto(finalUrl, {
       timeout: 30000,
-      ...(proxy && { waitUntil: 'commit' }),
+      waitUntil: 'commit',
     });
 
     try {
@@ -1630,11 +1624,6 @@ export const submitFormViaPlaywright = async (
     consoleLogger.error(error);
   } finally {
     await browserContext.close();
-    if (proxy && browserToRun === BrowserTypes.EDGE) {
-        deleteClonedEdgeProfiles(clonedDir);
-    } else if (proxy && browserToRun === BrowserTypes.CHROME) {
-        deleteClonedChromeProfiles(clonedDir);
-    }
   }
 };
 
@@ -1673,19 +1662,17 @@ export const submitForm = async (
     finalUrl += `&${formDataFields.redirectUrlField}=${scannedUrl}`;
   }
 
-  if (proxy) {
-    await submitFormViaPlaywright(browserToRun, userDataDirectory, finalUrl);
-  } else {
-    try {
-      await axios.get(finalUrl, { timeout: 2000 });
-    } catch (error) {
-      if (error.code === 'ECONNABORTED') {
-        if (browserToRun || constants.launcher === webkit) {
-          await submitFormViaPlaywright(browserToRun, userDataDirectory, finalUrl);
-        }
+
+  try {
+    await axios.get(finalUrl, { timeout: 2000 });
+  } catch (error) {
+    if (error.code === 'ECONNABORTED') {
+      if (browserToRun || constants.launcher === webkit) {
+        await submitFormViaPlaywright(browserToRun, userDataDirectory, finalUrl);
       }
     }
   }
+
 };
 // Legacy code end - Google Sheets submission
 
@@ -1736,42 +1723,61 @@ export async function initModifiedUserAgent(
   // console.log('Modified User Agent:', modifiedUA);
 }
 
+const cacheProxyInfo = getProxyInfo();
+
 /**
  * @param {string} browser browser name ("chrome" or "edge", null for chromium, the default Playwright browser)
  * @returns playwright launch options object. For more details: https://playwright.dev/docs/api/class-browsertype#browser-type-launch
  */
 export const getPlaywrightLaunchOptions = (browser?: string): LaunchOptions => {
-  let channel: string;
-  if (browser) {
-    channel = browser;
-  }
+  const channel = browser || undefined;
+
+  const resolution = proxyInfoToResolution(cacheProxyInfo);
+
+  // Start with your base args
+  const finalArgs = [...constants.launchOptionsArgs];
 
-  // Set new headless mode as Chrome 132 does not support headless=old
-  // Also mute audio
+  // Headless flags (unchanged)
   if (process.env.CRAWLEE_HEADLESS === '1') {
-    constants.launchOptionsArgs.push('--headless=new');
-    constants.launchOptionsArgs.push('--mute-audio');
+    if (!finalArgs.includes('--headless=new')) finalArgs.push('--headless=new');
+    if (!finalArgs.includes('--mute-audio')) finalArgs.push('--mute-audio');
+  }
+
+  // Map resolution to Playwright options
+  let proxyOpt: ProxySettings | undefined;
+  switch (resolution.kind) {
+    case 'manual':
+      proxyOpt = resolution.settings;
+      break;
+    case 'pac': {
+      finalArgs.push(`--proxy-pac-url=${resolution.pacUrl}`);
+      if (resolution.bypass) finalArgs.push(`--proxy-bypass-list=${resolution.bypass}`);
+      break;
+    }
+    case 'none':
+      // nothing
+      break;
   }
 
   const options: LaunchOptions = {
-    // Drop the --use-mock-keychain flag to allow MacOS devices
-    // to use the cloned cookies.
     ignoreDefaultArgs: ['--use-mock-keychain', '--headless'],
-    // necessary from Chrome 132 to use our own headless=new flag
-    args: constants.launchOptionsArgs,
+    args: finalArgs,
     headless: false,
-    ...(channel && { channel }), // Having no channel is equivalent to "chromium"
+    ...(channel && { channel }),
+    ...(proxyOpt ? { proxy: proxyOpt } : {}),
   };
 
-  // Necessary as Chrome 132 does not support headless=old
-  options.headless = false;
+  // SlowMo (unchanged)
+  if (!options.slowMo && process.env.OOBEE_SLOWMO && Number(process.env.OOBEE_SLOWMO) >= 1) {
+    options.slowMo = Number(process.env.OOBEE_SLOWMO);
+    consoleLogger.info(`Enabled browser slowMo with value: ${process.env.OOBEE_SLOWMO}ms`);
+  }
 
-  if (proxy) {
-    options.slowMo = 1000; // To ensure server-side rendered proxy page is loaded
-  } else if (browser === BrowserTypes.EDGE && os.platform() === 'win32') {
-    // edge should be in non-headless mode
+  // Edge on Windows should not be headless (unchanged)
+  if (browser === BrowserTypes.EDGE && os.platform() === 'win32') {
     options.headless = false;
   }
+
   return options;
 };
 

package/src/constants/constants.ts

@@ -141,14 +141,53 @@ export const getDefaultChromiumDataDir = () => {
   }
 };
 
-export const removeQuarantineFlag = function (searchPath: string) {
-  if (os.platform() === 'darwin') {
-    const execPaths = globSync(searchPath, { absolute: true, nodir: true });
-    if (execPaths.length > 0) {
-      execPaths.forEach(filePath => spawnSync('xattr', ['-d', 'com.apple.quarantine', filePath]));
+export function removeQuarantineFlag(searchPattern: string, allowedRoot = process.cwd()) {
+  if (os.platform() !== 'darwin') return;
+
+  const matches = globSync(searchPattern, {
+    absolute: true,
+    nodir: true,
+    dot: true,
+    follow: false, // don't follow symlinks
+  });
+
+  const root = path.resolve(allowedRoot);
+
+  for (const p of matches) {
+    const resolved = path.resolve(p);
+
+    // Ensure the file is under the allowed root (containment check)
+    if (!resolved.startsWith(root + path.sep)) continue;
+
+    // lstat: skip if not a regular file or if it's a symlink
+    let st: fs.Stats;
+    try {
+      st = fs.lstatSync(resolved);
+    } catch {
+      continue;
+    }
+    if (!st.isFile() || st.isSymbolicLink()) continue;
+
+    // basic filename sanity: no control chars
+    const base = path.basename(resolved);
+    if (/[\x00-\x1F]/.test(base)) continue;
+
+    // Use absolute binary path and terminate options with "--"
+    const proc = spawnSync('/usr/bin/xattr', ['-d', 'com.apple.quarantine', '--', resolved], {
+      stdio: ['ignore', 'ignore', 'pipe'],
+    });
+
+    // Optional: inspect errors (common benign case is "No such xattr")
+    if (proc.status !== 0) {
+      const err = proc.stderr?.toString() || '';
+      // swallow benign errors; otherwise log if you have a logger
+      if (!/No such xattr/i.test(err)) {
+        // console.warn(`xattr failed for ${resolved}: ${err.trim()}`);
+      }
     }
   }
-};
+}
+
 
 export const getExecutablePath = function (dir: string, file: string): string {
   let execPaths = globSync(`${dir}/${file}`, { absolute: true, nodir: true });
@@ -228,71 +267,6 @@ if (fs.existsSync('/.dockerenv')) {
   launchOptionsArgs = ['--disable-gpu', '--no-sandbox', '--disable-dev-shm-usage'];
 }
 
-type ProxyInfo = { type: 'autoConfig' | 'manualProxy'; url: string } | null;
-
-function queryRegKey(key: string): Record<string, string> {
-  try {
-    const out = execSync(`reg query "${key}"`, { encoding: 'utf8', stdio: ['ignore', 'pipe', 'pipe'] });
-    const values: Record<string, string> = {};
-    for (const line of out.split(/\r?\n/)) {
-      const parts = line.trim().split(/\s{2,}/);
-      if (parts.length >= 3) {
-        const [name, _type, ...rest] = parts;
-        values[name] = rest.join(' ');
-      }
-    }
-    return values;
-  } catch {
-    return {};
-  }
-}
-
-function parseDwordFlag(v: unknown): number {
-  if (v == null) return 0;
-  const s = String(v).trim();
-  // Handles "1", "0", "0x1", "0x0"
-  if (/^0x[0-9a-f]+$/i.test(s)) return parseInt(s, 16);
-  if (/^\d+$/.test(s)) return parseInt(s, 10);
-  return 0;
-}
-
-function normalizePacUrl(u: string): string {
-  const s = u.trim();
-  // If it lacks a scheme, assume http:// (Chrome requires a full URL)
-  return /^(https?|file):/i.test(s) ? s : `http://${s}`;
-}
-
-export const getProxy = (): ProxyInfo => {
-  if (os.platform() !== 'win32') return null;
-
-  const values = queryRegKey('HKCU\\Software\\Microsoft\\Windows\\CurrentVersion\\Internet Settings');
-  const pacUrlRaw = (values['AutoConfigURL'] || '').trim();
-  const proxyEnableRaw = (values['ProxyEnable'] || '').trim();
-  const proxyServerRaw = (values['ProxyServer'] || '').trim();
-
-  // 1) PAC beats manual proxy if present
-  if (pacUrlRaw) {
-    return { type: 'autoConfig', url: normalizePacUrl(pacUrlRaw) };
-  }
-
-  // 2) Manual proxy only if enabled
-  const enabled = parseDwordFlag(proxyEnableRaw) === 1;
-  if (enabled && proxyServerRaw) {
-    return { type: 'manualProxy', url: proxyServerRaw };
-  }
-
-  return null;
-};
-
-// Usage
-export const proxy = getProxy();
-
-if (proxy?.type === 'autoConfig') {
-  launchOptionsArgs.push(`--proxy-pac-url=${proxy.url}`);
-} else if (proxy?.type === 'manualProxy') {
-  launchOptionsArgs.push(`--proxy-server=${proxy.url}`);
-}
-
 export const impactOrder = {
   minor: 0,
   moderate: 1,

package/src/constants/questions.ts

@@ -81,13 +81,26 @@ const startScanQuestions = [
 
       // construct filename for scan results
       const [date, time] = new Date().toLocaleString('sv').replaceAll(/-|:/g, '').split(' ');
-      const domain = new URL(url).hostname;
+      let domain = '';
+      try {
+        domain = new URL(url).hostname;
+      } catch (error) {
+        // If the input is a local filepath, try to resolve it
+        const finalFilePath = getFileSitemap(url);
+        if (finalFilePath) {
+          answers.isLocalFileScan = true;
+          answers.finalUrl = finalFilePath;
+          return true;
+        }
+        return 'Invalid URL';
+      }
+      
       let resultFilename: string;
       const randomThreeDigitNumber = randomThreeDigitNumberString();
       resultFilename = `${date}_${time}_${domain}_${randomThreeDigitNumber}`;
   
       const statuses = constants.urlCheckStatuses;
-      const { browserToRun, clonedBrowserDataDir } = getBrowserToRun(BrowserTypes.CHROME, false, resultFilename);
+      const { browserToRun, clonedBrowserDataDir } = getBrowserToRun(resultFilename, BrowserTypes.CHROME, false);
 
       setHeadlessMode(browserToRun, answers.headless);
 

package/src/crawlers/crawlDomain.ts

@@ -368,6 +368,7 @@ const crawlDomain = async ({
             ...launchContext.launchOptions,
             ignoreHTTPSErrors: true,
             ...playwrightDeviceDetailsObject,
+            ...(process.env.OOBEE_DISABLE_BROWSER_DOWNLOAD && { acceptDownloads: false }),
             ...(extraHTTPHeaders && { extraHTTPHeaders }),
           };
 

package/src/crawlers/crawlIntelligentSitemap.ts

@@ -41,9 +41,6 @@ const crawlIntelligentSitemap = async (
 
   function getHomeUrl(parsedUrl: string) {
     const urlObject = new URL(parsedUrl);
-    if (urlObject.username && urlObject.password) {
-      return `${urlObject.protocol}//${urlObject.username}:${urlObject.password}@${urlObject.hostname}${urlObject.port ? `:${urlObject.port}` : ''}`;
-    }
     return `${urlObject.protocol}//${urlObject.hostname}${urlObject.port ? `:${urlObject.port}` : ''}`;
   }
 

package/src/crawlers/crawlLocalFile.ts

@@ -6,6 +6,7 @@ import constants, {
   guiInfoStatusTypes,
   basicAuthRegex,
   UrlsCrawled,
+  STATUS_CODE_METADATA,
 } from '../constants/constants.js';
 import { ViewportSettingsClass } from '../combine.js';
 import {
@@ -17,7 +18,7 @@ import {
 import { runPdfScan, mapPdfScanResults, doPdfScreenshots } from './pdfScanFunc.js';
 import { guiInfoLog } from '../logs.js';
 import crawlSitemap from './crawlSitemap.js';
-import { register } from '../utils.js';
+import { getPdfStoragePath, getStoragePath, register } from '../utils.js';
 
 export const crawlLocalFile = async ({
   url,
@@ -59,7 +60,7 @@ export const crawlLocalFile = async ({
   let dataset: any;
   let urlsCrawled: UrlsCrawled;
   let linksFromSitemap = [];
-  let sitemapUrl = url; 
+  let sitemapUrl: string; 
 
   // Boolean to omit axe scan for basic auth URL
   let isBasicAuth: boolean;
@@ -76,10 +77,13 @@ export const crawlLocalFile = async ({
 
   }
 
+  // Checks if its in the right file format, and change it before placing into linksFromSitemap
+  url = convertLocalFileToPath(url);
+
   // Check if the sitemapUrl is a local file and if it exists
-  if (!isFilePath(sitemapUrl) || !fs.existsSync(sitemapUrl)) {
+  if (!fs.existsSync(url) && !isFilePath(url)) {
     // Convert to an absolute path
-    let normalizedPath = path.resolve(sitemapUrl);
+    let normalizedPath = path.resolve(url);
 
     // Normalize the path to handle different path separators
     normalizedPath = path.normalize(normalizedPath);
@@ -90,17 +94,15 @@ export const crawlLocalFile = async ({
     }
 
     // At this point, normalizedPath is a valid and existing file path
-    sitemapUrl = normalizedPath;
+    url = normalizedPath;
   }
 
-  // Checks if its in the right file format, and change it before placing into linksFromSitemap
-  convertLocalFileToPath(sitemapUrl);
-
   // XML Files
-  if (!(sitemapUrl.match(/\.xml$/i) || sitemapUrl.match(/\.txt$/i))) {
-    linksFromSitemap = [new Request({ url: sitemapUrl })];
+  if (!(url.match(/\.xml$/i) || url.match(/\.txt$/i))) {
+    linksFromSitemap = [new Request({ url: url })];
     // Non XML file
   } else {
+    sitemapUrl = url;
     // Put it to crawlSitemap function to handle xml files
     const updatedUrlsCrawled = await crawlSitemap({
       sitemapUrl,
@@ -127,12 +129,6 @@ export const crawlLocalFile = async ({
     return urlsCrawled;
   }
 
-  try {
-    sitemapUrl = encodeURI(sitemapUrl);
-  } catch (e) {
-    console.log(e);
-  }
-
   const uuidToPdfMapping: Record<string, string> = {}; // key and value of string type
 
   finalLinks = [...finalLinks, ...linksFromSitemap];
@@ -142,16 +138,10 @@ export const crawlLocalFile = async ({
   });
 
   const request = linksFromSitemap[0];
-  const pdfFileName = path.basename(request.url);
-  const trimmedUrl: string = request.url;
-  const destinationFilePath: string = `${randomToken}/${pdfFileName}`;
-  const data: Buffer = fs.readFileSync(trimmedUrl);
-  fs.writeFileSync(destinationFilePath, data);
-  uuidToPdfMapping[pdfFileName] = trimmedUrl;
 
   let shouldAbort = false;
 
-  if (!isUrlPdf(request.url)) {
+  if (!isUrlPdf(url)) {
     const effectiveUserDataDirectory = process.env.CRAWLEE_HEADLESS === '1'
       ? userDataDirectory
       : '';
@@ -160,6 +150,7 @@ export const crawlLocalFile = async ({
       headless: process.env.CRAWLEE_HEADLESS === '1',
       ...getPlaywrightLaunchOptions(browser),
       ...playwrightDeviceDetailsObject,
+      ...(process.env.OOBEE_DISABLE_BROWSER_DOWNLOAD && { acceptDownloads: false }),
     });
 
     register(browserContext);
@@ -172,8 +163,8 @@ export const crawlLocalFile = async ({
     : null;
 
     const page = await browserContext.newPage();
-    request.url = convertPathToLocalFile(request.url);
-    await page.goto(request.url);
+    url = convertPathToLocalFile(url);
+    await page.goto(url);
 
     if (shouldAbort) {
       console.warn('Scan aborted due to timeout before page scan.');
@@ -184,33 +175,39 @@ export const crawlLocalFile = async ({
 
     const results = await runAxeScript({ includeScreenshots, page, randomToken });
 
-    const actualUrl = page.url() || request.loadedUrl || request.url;
+    const actualUrl = page.url() || request.loadedUrl || url;
 
     guiInfoLog(guiInfoStatusTypes.SCANNED, {
       numScanned: urlsCrawled.scanned.length,
-      urlScanned: request.url,
+      urlScanned: url,
     });
 
     urlsCrawled.scanned.push({
-      url: request.url,
+      url: url,
       pageTitle: results.pageTitle,
       actualUrl: actualUrl, // i.e. actualUrl
     });
 
     urlsCrawled.scannedRedirects.push({
-      fromUrl: request.url,
+      fromUrl: url,
       toUrl: actualUrl, // i.e. actualUrl
     });
 
-    results.url = request.url;
+    results.url = url;
     results.actualUrl = actualUrl;
 
     await dataset.pushData(results);
   } else {
+
+    const pdfFileName = path.basename(url);
+    const destinationFilePath: string = path.join(getPdfStoragePath(randomToken), pdfFileName);
+    fs.copyFileSync(url, destinationFilePath);
+    uuidToPdfMapping[pdfFileName] = url;
+
     urlsCrawled.scanned.push({
-      url: trimmedUrl,
+      url: url,
       pageTitle: pdfFileName,
-      actualUrl: trimmedUrl,
+      actualUrl: url,
     });
 
     await runPdfScan(randomToken);

package/src/crawlers/crawlSitemap.ts

@@ -21,7 +21,7 @@ import {
   isFilePath,
 } from '../constants/common.js';
 import { areLinksEqual, isWhitelistedContentType, register } from '../utils.js';
-import { handlePdfDownload, runPdfScan, mapPdfScanResults } from './pdfScanFunc.js';
+import { handlePdfDownload, runPdfScan, mapPdfScanResults, doPdfScreenshots } from './pdfScanFunc.js';
 import { guiInfoLog } from '../logs.js';
 import { ViewportSettingsClass } from '../combine.js';
 import * as path from 'path';
@@ -135,6 +135,7 @@ const crawlSitemap = async ({
             ...launchContext.launchOptions,
             ignoreHTTPSErrors: true,
             ...playwrightDeviceDetailsObject,
+            ...(process.env.OOBEE_DISABLE_BROWSER_DOWNLOAD && { acceptDownloads: false }),
           };
 
           // Optionally log for debugging
@@ -412,11 +413,11 @@ const crawlSitemap = async ({
     const pdfResults = await mapPdfScanResults(randomToken, uuidToPdfMapping);
 
     // get screenshots from pdf docs
-    // if (includeScreenshots) {
-    //   await Promise.all(pdfResults.map(
-    //     async result => await doPdfScreenshots(randomToken, result)
-    //   ));
-    // }
+    if (includeScreenshots) {
+      await Promise.all(pdfResults.map(
+        async result => await doPdfScreenshots(randomToken, result)
+      ));
+    }
 
     // push results for each pdf document to key value store
     await Promise.all(pdfResults.map(result => dataset.pushData(result)));