@govtechsg/oobee 0.10.58 → 0.10.62

package/src/static/ejs/partials/scripts/utils.ejs

@@ -12,17 +12,14 @@
   // extract tagname and attribute name from html tag
   // e.g. ["input", "type", "value", "role"] from <input type="text" value="..." role="..." />
   const getHtmlTagAndAttributes = (htmlString) => {
-    const regex = /<(\w+)(\s+(\w+)(\s*=\s*"[^"]*")?)*\s*\/?>/;
-    const match = htmlString.match(regex); // check if structure of html tag is valid
-
-    if (match) {
-      const tagName = match[1];
-      const attributes = match[0]
-        .match(/\w+\s*=\s*"[^"]*"/g) // extract attributes e.g. ['type="text"', 'value="..."']
-        .map((attr) => attr.match(/(\w+)\s*=/)[1]); // get the name e.g. "type" from each
-      return [tagName, ...attributes];
-    }
-    return [];
+    const tagMatch = htmlString.match(/^<\s*(\w+)/); // Get tag name
+    if (!tagMatch) return [];
+
+    const tagName = tagMatch[1];
+    const attrMatches = [...htmlString.matchAll(/(\w[\w-]*)\s*=\s*"[^"]*"/g)];
+    const attributes = attrMatches.map(match => match[1]);
+
+    return [tagName, ...attributes];
   };
 
   const rulesUsingRoles = [

package/src/utils.ts

@@ -9,8 +9,10 @@ import constants, {
   destinationPath,
   getIntermediateScreenshotsPath,
 } from './constants/constants.js';
-import { consoleLogger, silentLogger } from './logs.js';
+import { consoleLogger, errorsTxtPath, silentLogger } from './logs.js';
 import { getAxeConfiguration } from './crawlers/custom/getAxeConfiguration.js';
+import { constant } from 'lodash';
+import { errors } from 'playwright';
 
 export const getVersion = () => {
   const loadJSON = (filePath: string): { version: string } =>
@@ -33,21 +35,57 @@ export const isWhitelistedContentType = (contentType: string): boolean => {
 };
 
 export const getStoragePath = (randomToken: string): string => {
-  if (process.env.OOBEE_VERBOSE_STORAGE_PATH) {
-    return `${process.env.OOBEE_VERBOSE_STORAGE_PATH}/${randomToken}`;
+  // If exportDirectory is set, use it
+  if (constants.exportDirectory) {
+    return constants.exportDirectory;
   }
-  if (constants.exportDirectory === process.cwd()) {
-    return `results/${randomToken}`;
+
+  // Otherwise, use the current working directory
+  let storagePath = path.join(process.cwd(), 'results', randomToken);
+
+  // Ensure storagePath is writable; if directory doesn't exist, try to create it in Documents or home directory
+  const isWritable = (() => {
+    try {
+      if (!fs.existsSync(storagePath)) {
+        fs.mkdirSync(storagePath, { recursive: true });
+      }
+      fs.accessSync(storagePath, fs.constants.W_OK);
+      return true;
+    } catch {
+      return false;
+    }
+  })();
+
+  if (!isWritable) {
+    if (os.platform() === 'win32') {
+      // Use Documents folder on Windows
+      const documentsPath = path.join(process.env.USERPROFILE || process.env.HOMEPATH || '', 'Documents');
+      storagePath = path.join(documentsPath, 'Oobee', randomToken);
+    } else if (os.platform() === 'darwin') {
+      // Use Documents folder on Mac
+      const documentsPath = path.join(process.env.HOME || '', 'Documents');
+      storagePath = path.join(documentsPath, 'Oobee', randomToken);
+    } else {
+      // Use home directory for Linux/other
+      const homePath = process.env.HOME || '';
+      storagePath = path.join(homePath, 'Oobee', randomToken);
+    }
+    consoleLogger.warn(`Warning: Cannot write to cwd, writing to ${storagePath}`);
+
   }
-  if (!path.isAbsolute(constants.exportDirectory)) {
-    constants.exportDirectory = path.resolve(process.cwd(), constants.exportDirectory);
+
+  if (!fs.existsSync(storagePath)) {
+    fs.mkdirSync(storagePath, { recursive: true });
   }
-  return `${constants.exportDirectory}/${randomToken}`;
+
+  constants.exportDirectory = storagePath;
+  return storagePath;
+
 };
 
 export const createDetailsAndLogs = async (randomToken: string): Promise<void> => {
   const storagePath = getStoragePath(randomToken);
-  const logPath = `logs/${randomToken}`;
+  const logPath = `${storagePath}}/logs`;
   try {
     await fs.ensureDir(storagePath);
 
@@ -193,8 +231,257 @@ export const createScreenshotsFolder = (randomToken: string): void => {
   }
 };
 
-export const cleanUp = (pathToDelete: string): void => {
-  fs.removeSync(pathToDelete);
+
+let __shuttingDown = false;
+let __stopAllLock: Promise<void> | null = null;
+
+/**
+ * Register a resource so it can be stopped later.
+ * Supports Crawlee crawlers, Playwright BrowserContexts, and Browsers.
+ */
+export function register(resource: any) {
+  const name = resource?.constructor?.name;
+
+  if (name?.endsWith('Crawler')) {
+    constants.resources.crawlers.add(resource);
+  } else if (name === 'BrowserContext') {
+    constants.resources.browserContexts.add(resource);
+  } else if (name === 'Browser') {
+    constants.resources.browsers.add(resource);
+  }
+
+  return resource;
+}
+
+/**
+ * Stops or tears down all tracked resources.
+ * @param mode "graceful" (finish in-flight), "abort" (drop in-flight), or "teardown" (close immediately)
+ * @param timeoutMs Max time to wait before forcing shutdown
+ */
+export async function stopAll({ mode = 'graceful', timeoutMs = 10_000 } = {}) {
+  if (__stopAllLock) return __stopAllLock; // prevent overlap
+  __stopAllLock = (async () => {
+    const timeout = (ms: number) => new Promise(res => setTimeout(res, ms));
+    consoleLogger.info(`Stop browsers starting, mode=${mode}, timeoutMs=${timeoutMs}`);
+
+    // --- Crawlers ---
+    for (const c of [...constants.resources.crawlers]) {
+      try {
+        const pool = (c as any).autoscaledPool;
+        if (pool && typeof pool.isRunning !== 'undefined' && !pool.isRunning) {
+          consoleLogger.info('Skipping crawler (already stopped)');
+          continue;
+        }
+
+        consoleLogger.info(`Closing crawler (${mode})...`);
+        if (mode === 'graceful') {
+          if (typeof c.stop === 'function') {
+            await Promise.race([c.stop(), timeout(timeoutMs)]);
+          }
+        } else if (mode === 'abort') {
+          pool?.abort?.();
+        } else {
+          if (typeof c.teardown === 'function') {
+            await Promise.race([c.teardown(), timeout(timeoutMs)]);
+          }
+        }
+        consoleLogger.info(`Crawler closed (${mode})`);
+      } catch (err) {
+        consoleLogger.warn(`Error stopping crawler: ${(err as Error).message}`);
+      } finally {
+        constants.resources.crawlers.delete(c);
+      }
+    }
+
+    // --- BrowserContexts ---
+    for (const ctx of [...constants.resources.browserContexts]) {
+      // compute once so we can also use in finally
+      const pagesArr = typeof ctx.pages === 'function' ? ctx.pages() : [];
+      const hasOpenPages = Array.isArray(pagesArr) && pagesArr.length > 0;
+
+      try {
+        const browser = typeof ctx.browser === 'function' ? ctx.browser() : null;
+        if (browser && (browser as any).isClosed?.()) {
+          consoleLogger.info('Skipping BrowserContext (browser already closed)');
+          continue;
+        }
+
+        // ➜ Graceful: don't kill contexts that are still doing work
+        if (mode === 'graceful' && hasOpenPages) {
+          consoleLogger.info(`Skipping BrowserContext in graceful (has ${pagesArr.length} open page(s))`);
+          continue; // leave it for the teardown pass
+        }
+
+        // (Optional speed-up) close pages first if any
+        if (hasOpenPages) {
+          consoleLogger.info(`Closing ${pagesArr.length} page(s) before context close...`);
+          for (const p of pagesArr) {
+            try { await Promise.race([p.close(), timeout(1500)]); } catch {}
+          }
+        }
+
+        consoleLogger.info('Closing BrowserContext...');
+        if (typeof ctx.close === 'function') {
+          await Promise.race([ctx.close(), timeout(timeoutMs)]);
+        }
+        consoleLogger.info('BrowserContext closed');
+
+        // also close its browser (persistent contexts)
+        const b = browser;
+        if (b && !(b as any).isClosed?.()) {
+          consoleLogger.info('Closing Browser (from context.browser())...');
+          if (typeof b.close === 'function') {
+            await Promise.race([b.close(), timeout(timeoutMs)]);
+          }
+          consoleLogger.info('Browser closed (from context.browser())');
+        }
+      } catch (err) {
+        consoleLogger.warn(`Error closing BrowserContext: ${(err as Error).message}`);
+      } finally {
+        // only delete from the set if we actually closed it (or tried to)
+        if (!(mode === 'graceful' && hasOpenPages)) {
+          constants.resources.browserContexts.delete(ctx);
+        }
+      }
+    }
+
+    // --- Browsers ---
+    for (const b of [...constants.resources.browsers]) {
+      try {
+        if ((b as any).isClosed?.()) {
+          consoleLogger.info('Skipping Browser (already closed)');
+          continue;
+        }
+
+        consoleLogger.info('Closing Browser...');
+        if (typeof b.close === 'function') {
+          await Promise.race([b.close(), timeout(timeoutMs)]);
+        }
+        consoleLogger.info('Browser closed');
+      } catch (err) {
+        consoleLogger.warn(`Error closing Browser: ${(err as Error).message}`);
+      } finally {
+        constants.resources.browsers.delete(b);
+      }
+    }
+
+    consoleLogger.info(`Stop browsers finished for mode=${mode}`);
+  })();
+
+  try {
+    await __stopAllLock;
+  } finally {
+    __stopAllLock = null;
+  }
+}
+
+export const cleanUp = async (randomToken?: string, isError: boolean = false): Promise<void> => {
+
+  if (isError) {
+    await stopAll({ mode: 'graceful', timeoutMs: 8000 });
+    await stopAll({ mode: 'teardown', timeoutMs: 4000 });
+  }
+  
+  if (randomToken === undefined && constants.randomToken) {
+    randomToken = constants.randomToken;
+  }
+
+  if (constants.userDataDirectory) try {
+    fs.rmSync(constants.userDataDirectory, { recursive: true, force: true });
+  } catch (error) {
+    consoleLogger.warn(`Unable to force remove userDataDirectory: ${error.message}`);
+  }
+
+  if (randomToken !== undefined) {
+    const storagePath = getStoragePath(randomToken);
+
+    try {
+      fs.rmSync(path.join(storagePath, 'crawlee'), { recursive: true, force: true });
+    } catch (error) {
+      consoleLogger.warn(`Unable to force remove crawlee folder: ${error.message}`);
+    }
+
+    let deleteErrorLogFile = true;
+
+    if (isError) {
+      let logsPath = storagePath;
+
+      if (process.env.OOBEE_LOGS_PATH) {
+        logsPath = process.env.OOBEE_LOGS_PATH;
+      }
+
+      if (fs.existsSync(errorsTxtPath)) {
+        try {
+          const logFilePath = path.join(logsPath, `logs-${randomToken}.txt`);
+          fs.copyFileSync(errorsTxtPath, logFilePath);
+          console.log(`An error occured. Log file is located at: ${logFilePath}`);
+
+        } catch (copyError) {
+          consoleLogger.error(`Error copying errors file during cleanup: ${copyError.message}`);
+          console.log(`An error occured. Log file is located at: ${errorsTxtPath}`);
+          deleteErrorLogFile = false; // Do not delete the log file if copy failed
+        }
+    
+        if (deleteErrorLogFile && fs.existsSync(errorsTxtPath)) {
+          try {
+            fs.unlinkSync(errorsTxtPath);
+          } catch (error) {
+            consoleLogger.warn(`Unable to delete log file ${errorsTxtPath}: ${error.message}`);
+          }
+        }
+
+      }
+
+    } 
+    
+    if (fs.existsSync(storagePath) && fs.readdirSync(storagePath).length === 0) {
+      try {
+        fs.rmdirSync(storagePath);
+        consoleLogger.info(`Deleted empty storage path: ${storagePath}`);
+    
+      } catch (error) {
+        consoleLogger.warn(`Error deleting empty storage path ${storagePath}: ${error.message}`);
+      }
+    }
+
+    consoleLogger.info(`Clean up completed for: ${randomToken}`);
+  } 
+
+};
+
+export const cleanUpAndExit = async (
+  exitCode: number,
+  randomToken?: string,
+  isError: boolean = false,
+): Promise<void> => {
+  if (__shuttingDown) {
+    consoleLogger.info('Cleanup already in progress; ignoring duplicate exit request.');
+    return;
+  }
+  __shuttingDown = true;
+
+  try {
+    await cleanUp(randomToken, isError);   // runs stopAll inside cleanUp
+  } catch (e: any) {
+    consoleLogger.warn(`Cleanup error: ${e?.message || e}`);
+  }
+
+  consoleLogger.info(`Exiting with code: ${exitCode}`);
+  process.exit(exitCode); // explicit exit after cleanup completes
+};
+
+export const listenForCleanUp = (randomToken: string): void => {
+  consoleLogger.info(`PID: ${process.pid}`);
+
+  process.on('SIGINT', async () => {   // ← keep handler installed
+    consoleLogger.info('SIGINT received. Cleaning up and exiting.');
+    await cleanUpAndExit(130, randomToken, true);
+  });
+
+  process.on('SIGTERM', async () => {  // ← keep handler installed
+    consoleLogger.info('SIGTERM received. Cleaning up and exiting.');
+    await cleanUpAndExit(143, randomToken, true);
+  });
 };
 
 export const getWcagPassPercentage = (
@@ -711,16 +998,18 @@ export const zipResults = (zipName: string, resultsPath: string): void => {
     fs.unlinkSync(zipName);
   }
 
+  // Check if user specified absolute or relative path
+  const zipFilePath = path.isAbsolute(zipName) ? zipName : path.join(process.cwd(), zipName);
+
+
   if (os.platform() === 'win32') {
     execSync(
-      `Get-ChildItem -Path "${resultsPath}\\*.*" -Recurse | Compress-Archive -DestinationPath "${zipName}"`,
-      { shell: 'powershell.exe' },
+      `tar.exe -a -c -f "${zipFilePath}" *`,
+      { cwd: resultsPath },
     );
   } else {
     // Get zip command in Mac and Linux
     const command = '/usr/bin/zip';
-    // Check if user specified absolute or relative path
-    const zipFilePath = path.isAbsolute(zipName) ? zipName : path.join(process.cwd(), zipName);
 
     // To zip up files recursively (-r) in the results folder path and write it to user's specified path
     const args = ['-r', zipFilePath, '.'];