@govtechsg/oobee 0.10.63 → 0.10.65

package/src/utils.ts

@@ -11,8 +11,8 @@ import constants, {
 } from './constants/constants.js';
 import { consoleLogger, errorsTxtPath, silentLogger } from './logs.js';
 import { getAxeConfiguration } from './crawlers/custom/getAxeConfiguration.js';
-import { constant } from 'lodash';
-import { errors } from 'playwright';
+import JSZip from 'jszip';
+import { createReadStream, createWriteStream } from 'fs';
 
 export const getVersion = () => {
   const loadJSON = (filePath: string): { version: string } =>
@@ -34,6 +34,15 @@ export const isWhitelistedContentType = (contentType: string): boolean => {
   return whitelist.filter(type => contentType.trim().startsWith(type)).length === 1;
 };
 
+export const getPdfStoragePath = (randomToken: string): string => {
+  const storagePath = getStoragePath(randomToken);
+  const pdfStoragePath = path.join(storagePath, 'pdfs');
+  if (!fs.existsSync(pdfStoragePath)) {
+    fs.mkdirSync(pdfStoragePath, { recursive: true });
+  }
+  return pdfStoragePath;
+};
+
 export const getStoragePath = (randomToken: string): string => {
   // If exportDirectory is set, use it
   if (constants.exportDirectory) {
@@ -83,41 +92,6 @@ export const getStoragePath = (randomToken: string): string => {
 
 };
 
-export const createDetailsAndLogs = async (randomToken: string): Promise<void> => {
-  const storagePath = getStoragePath(randomToken);
-  const logPath = `${storagePath}}/logs`;
-  try {
-    await fs.ensureDir(storagePath);
-
-    // update logs
-    await fs.ensureDir(logPath);
-    await fs.pathExists('errors.txt').then(async exists => {
-      if (exists) {
-        try {
-          await fs.copy('errors.txt', `${logPath}/${randomToken}.txt`);
-        } catch (error) {
-          if (error.code === 'EBUSY') {
-            consoleLogger.error(
-              `Unable to copy the file from 'errors.txt' to '${logPath}/${randomToken}.txt' because it is currently in use.`,
-            );
-            consoleLogger.error(
-              'Please close any applications that might be using this file and try again.',
-            );
-          } else {
-            consoleLogger.error(
-              `An unexpected error occurred while copying the file: ${error.message}`,
-            );
-          }
-        }
-      }
-    });
-  } catch (error) {
-    consoleLogger.error(
-      `An error occurred while setting up storage or log directories: ${error.message}`,
-    );
-  }
-};
-
 export const getUserDataFilePath = () => {
   const platform = os.platform();
   if (platform === 'win32') {
@@ -401,6 +375,12 @@ export const cleanUp = async (randomToken?: string, isError: boolean = false): P
       consoleLogger.warn(`Unable to force remove crawlee folder: ${error.message}`);
     }
 
+    try {
+      fs.rmSync(path.join(storagePath, 'pdfs'), { recursive: true, force: true });
+    } catch (error) {
+      consoleLogger.warn(`Unable to force remove pdfs folder: ${error.message}`);
+    }
+    
     let deleteErrorLogFile = true;
 
     if (isError) {
@@ -992,35 +972,53 @@ export const setThresholdLimits = (setWarnLevel: string): void => {
   process.env.WARN_LEVEL = setWarnLevel;
 };
 
-export const zipResults = (zipName: string, resultsPath: string): void => {
-  // Check prior zip file exist and remove
-  if (fs.existsSync(zipName)) {
-    fs.unlinkSync(zipName);
-  }
+export const zipResults = async (zipName: string, resultsPath: string): Promise<void> => {
+  // Resolve and validate the output path
+  const zipFilePath = path.isAbsolute(zipName) ? zipName : path.join(resultsPath, zipName);
 
-  // Check if user specified absolute or relative path
-  const zipFilePath = path.isAbsolute(zipName) ? zipName : path.join(process.cwd(), zipName);
+  // Ensure parent dir exists
+  fs.mkdirSync(path.dirname(zipFilePath), { recursive: true });
 
+  // Remove any prior file atomically
+  try { fs.unlinkSync(zipFilePath); } catch { /* ignore if not exists */ }
 
-  if (os.platform() === 'win32') {
-    execSync(
-      `tar.exe -a -c -f "${zipFilePath}" *`,
-      { cwd: resultsPath },
-    );
-  } else {
-    // Get zip command in Mac and Linux
-    const command = '/usr/bin/zip';
+  // CWD must exist and be a directory
+  const stats = fs.statSync(resultsPath);
+  if (!stats.isDirectory()) {
+    throw new Error(`resultsPath is not a directory: ${resultsPath}`);
+  }
+  async function addFolderToZip(folderPath: string, zipFolder: JSZip): Promise<void> {
+    const items = await fs.readdir(folderPath);
+    for (const item of items) {
+      const fullPath = path.join(folderPath, item);
+      const stats = await fs.stat(fullPath);
+      if (stats.isDirectory()) {
+        const folder = zipFolder.folder(item);
+        await addFolderToZip(fullPath, folder);
+      } else {
+        // Add file as a stream so that it doesn't load the entire file into memory
+        zipFolder.file(item, createReadStream(fullPath));
+      }
+    }
+  }
 
-    // To zip up files recursively (-r) in the results folder path and write it to user's specified path
-    const args = ['-r', zipFilePath, '.'];
+  await addFolderToZip(resultsPath, new JSZip());
 
-    // Change working directory only for the zip command
-    const options = {
-      cwd: resultsPath,
-    };
+  const zip = new JSZip();
+  await addFolderToZip(resultsPath, zip);
 
-    spawnSync(command, args, options);
-  }
+  const zipStream = zip.generateNodeStream({
+    type: 'nodebuffer',
+    streamFiles: true,
+    compression: 'DEFLATE',
+  });
+
+  await new Promise((resolve, reject) => {
+    const outStream = createWriteStream(zipFilePath);
+    zipStream.pipe(outStream)
+      .on('finish', resolve)
+      .on('error', reject);
+  });
 };
 
 // areLinksEqual compares 2 string URLs and ignores comparison of 'www.' and url protocol