@govtechsg/oobee 0.10.39 → 0.10.43

package/src/crawlers/custom/xPathToCss.ts

@@ -1,12 +1,12 @@
-export function xPathToCss(expr: string) {
-  const isValidXPath = expr =>
+export default function xPathToCss(expr: string) {
+  const isValidXPath = (expr: string) =>
     typeof expr !== 'undefined' &&
     expr.replace(/[\s-_=]/g, '') !== '' &&
     expr.length ===
-    expr.replace(
-      /[-_\w:.]+\(\)\s*=|=\s*[-_\w:.]+\(\)|\sor\s|\sand\s|\[(?:[^\/\]]+[\/\[]\/?.+)+\]|starts-with\(|\[.*last\(\)\s*[-\+<>=].+\]|number\(\)|not\(|count\(|text\(|first\(|normalize-space|[^\/]following-sibling|concat\(|descendant::|parent::|self::|child::|/gi,
-      '',
-    ).length;
+      expr.replace(
+        /[-_\w:.]+\(\)\s*=|=\s*[-_\w:.]+\(\)|\sor\s|\sand\s|\[(?:[^\/\]]+[\/\[]\/?.+)+\]|starts-with\(|\[.*last\(\)\s*[-\+<>=].+\]|number\(\)|not\(|count\(|text\(|first\(|normalize-space|[^\/]following-sibling|concat\(|descendant::|parent::|self::|child::|/gi,
+        '',
+      ).length;
 
   const getValidationRegex = () => {
     let regex =
@@ -30,7 +30,7 @@ export function xPathToCss(expr: string) {
       value: '\\s*[\\w/:][-/\\w\\s,:;.]*',
     };
 
-    Object.keys(subRegexes).forEach(key => {
+    Object.keys(subRegexes).forEach((key: keyof typeof subRegexes) => {
       regex = regex.replace(new RegExp(`%\\(${key}\\)s`, 'gi'), subRegexes[key]);
     });
 
@@ -42,14 +42,14 @@ export function xPathToCss(expr: string) {
     return new RegExp(regex, 'gi');
   };
 
-  const preParseXpath = expr =>
+  const preParseXpath = (expr: string) =>
     expr.replace(
       /contains\s*\(\s*concat\(["']\s+["']\s*,\s*@class\s*,\s*["']\s+["']\)\s*,\s*["']\s+([a-zA-Z0-9-_]+)\s+["']\)/gi,
       '@class="$1"',
     );
 
-  function escapeCssIdSelectors(cssSelector) {
-    return cssSelector.replace(/#([^ >]+)/g, (match, id) => {
+  function escapeCssIdSelectors(cssSelector: string) {
+    return cssSelector.replace(/#([^ >]+)/g, (_match, id) => {
       // Escape special characters in the id part
       return `#${id.replace(/[!"#$%&'()*+,./:;<=>?@[\\\]^`{|}~]/g, '\\$&')}`;
     });

package/src/crawlers/pdfScanFunc.ts

@@ -256,30 +256,63 @@ export const handlePdfDownload = (
 
   pdfDownloads.push(
     new Promise<void>(async resolve => {
-      const bufs = [];
-      let pdfResponse: ReadStream;
+      let bufs: Buffer[] = [];
+      let buf: Buffer;
 
       if (isFilePath(url)) {
-        // Read the file from the file system
+        // Read from local file system
         const filePath = new URL(url).pathname;
-        pdfResponse = fs.createReadStream(filePath, { encoding: 'binary' });
+        const pdfResponse = fs.createReadStream(filePath, { encoding: 'binary' });
+
+        const downloadFile = fs.createWriteStream(`${randomToken}/${pdfFileName}.pdf`, {
+          flags: 'a',
+        });
+
+        pdfResponse.on('data', (chunk: Buffer) => {
+          downloadFile.write(chunk, 'binary');
+          bufs.push(Buffer.from(chunk));
+        });
+
+        pdfResponse.on('end', () => {
+          downloadFile.end();
+          buf = Buffer.concat(bufs);
+
+          if (isPDF(buf)) {
+            guiInfoLog(guiInfoStatusTypes.SCANNED, {
+              numScanned: urlsCrawled.scanned.length,
+              urlScanned: request.url,
+            });
+            urlsCrawled.scanned.push({
+              url: request.url,
+              pageTitle,
+              actualUrl: url,
+            });
+          } else {
+            guiInfoLog(guiInfoStatusTypes.SKIPPED, {
+              numScanned: urlsCrawled.scanned.length,
+              urlScanned: request.url,
+            });
+            urlsCrawled.invalid.push({
+              url: request.url,
+              pageTitle: url,
+              actualUrl: url,
+            });
+          }
+
+          resolve();
+        });
       } else {
-        // Send HTTP/HTTPS request
-        pdfResponse = await sendRequest({ responseType: 'buffer', isStream: true });
-        pdfResponse.setEncoding('binary');
-      }
-      const downloadFile = fs.createWriteStream(`${randomToken}/${pdfFileName}.pdf`, {
-        flags: 'a',
-      });
+        // Download from remote URL
+        const response = await sendRequest({ responseType: 'buffer' });
+        buf = Buffer.isBuffer(response) ? response : response.body;
 
-      pdfResponse.on('data', (chunk: Buffer) => {
-        downloadFile.write(chunk, 'binary');
-        bufs.push(Buffer.from(chunk));
-      });
+        const downloadFile = fs.createWriteStream(`${randomToken}/${pdfFileName}.pdf`, {
+          flags: 'a',
+        });
 
-      pdfResponse.on('end', () => {
+        downloadFile.write(buf, 'binary');
         downloadFile.end();
-        const buf = Buffer.concat(bufs);
+
         if (isPDF(buf)) {
           guiInfoLog(guiInfoStatusTypes.SCANNED, {
             numScanned: urlsCrawled.scanned.length,
@@ -298,11 +331,12 @@ export const handlePdfDownload = (
           urlsCrawled.invalid.push({
             url: request.url,
             pageTitle: url,
-            actualUrl: url, // i.e. actualUrl
+            actualUrl: url,
           });
         }
+
         resolve();
-      });
+      }
     }),
   );
 
@@ -374,14 +408,21 @@ export const mapPdfScanResults = async (
       const { itemDetails, validationResult } = jobs[jobIdx];
       const { name: fileName } = itemDetails;
 
-      const uuid = fileName
-        .split(os.platform() === 'win32' ? '\\' : '/')
-        .pop()
-        .split('.')[0];
-      const url = uuidToUrlMapping[uuid];
-      const pageTitle = decodeURI(url).split('/').pop();
-      const filePath = `${randomToken}/${uuid}.pdf`;
+      const rawFileName = fileName.split(os.platform() === 'win32' ? '\\' : '/').pop();
+      const fileNameWithoutExt = rawFileName.replace(/\.pdf$/i, '');
 
+      const url =
+        uuidToUrlMapping[rawFileName] || // exact match like 'Some-filename.pdf'
+        uuidToUrlMapping[fileNameWithoutExt] || // uuid-based key like 'a9f7ebbd-5a90...'
+        `file://${fileName}`; // fallback
+
+      const filePath = `${randomToken}/${rawFileName}`;
+
+
+      const pageTitle = decodeURI(url).split('/').pop();
+      translated.url = url;
+      translated.pageTitle = pageTitle;
+      
       translated.url = url;
       translated.pageTitle = pageTitle;
       translated.filePath = filePath;

package/src/crawlers/runCustom.ts

@@ -48,7 +48,7 @@ const runCustom = async (
   includeScreenshots: boolean,
 ) => {
   // checks and delete datasets path if it already exists
-  await cleanUp(randomToken);
+  cleanUp(randomToken);
   process.env.CRAWLEE_STORAGE_DIR = randomToken;
 
   const urlsCrawled: UrlsCrawled = { ...constants.urlsCrawledObj };

package/src/index.ts

@@ -1,6 +1,4 @@
 #!/usr/bin/env node
-/* eslint-disable func-names */
-/* eslint-disable no-param-reassign */
 import printMessage from 'print-message';
 import inquirer from 'inquirer';
 import { EnqueueStrategy } from 'crawlee';
@@ -22,6 +20,7 @@ import {
 import questions from './constants/questions.js';
 import combineRun from './combine.js';
 import { BrowserTypes, RuleFlags, ScannerTypes } from './constants/constants.js';
+import { DeviceDescriptor } from './types/types.js';
 
 export type Answers = {
   headless: boolean;
@@ -32,7 +31,7 @@ export type Answers = {
   scanner: ScannerTypes;
   url: string;
   clonedBrowserDataDir: string;
-  playwrightDeviceDetailsObject: object;
+  playwrightDeviceDetailsObject: DeviceDescriptor;
   nameEmail: string;
   fileTypes: string;
   metadata: string;
@@ -61,7 +60,7 @@ export type Data = {
   deviceChosen: string;
   customDevice: string;
   viewportWidth: number;
-  playwrightDeviceDetailsObject: object;
+  playwrightDeviceDetailsObject: DeviceDescriptor;
   maxRequestsPerCrawl: number;
   strategy: EnqueueStrategy;
   isLocalFileScan: boolean;

package/src/logs.ts

@@ -40,7 +40,7 @@ const silentLogger = createLogger({
 });
 
 // guiInfoLogger feeds the gui information via console log and is mainly used for scanning process
-export const guiInfoLog = (status, data) => {
+export const guiInfoLog = (status: string, data: { numScanned?: number; urlScanned?: string }) => {
   if (process.env.RUNNING_FROM_PH_GUI || process.env.OOBEE_VERBOSE) {
     switch (status) {
       case guiInfoStatusTypes.COMPLETED: