@govtechsg/oobee 0.10.91 → 0.10.92

package/src/mergeAxeResults.ts

@@ -5,6 +5,7 @@ import printMessage from 'print-message';
 import path from 'path';
 import ejs from 'ejs';
 import { fileURLToPath } from 'url';
+import { Dataset, RequestQueue, Configuration } from 'crawlee';
 import constants, {
   BrowserTypes,
   ScannerTypes,
@@ -358,61 +359,97 @@ const writeSummaryPdf = async (
   browser: string,
   _userDataDirectory: string,
 ) => {
-  let browserInstance;
-  let context;
-  let page;
+  const renderPdfWithBrowser = async (browserToUse: string) => {
+    let browserInstance;
+    let context;
+    let page;
 
-  try {
-    const htmlFilePath = path.join(storagePath, `${filename}.html`);
-    const fileDestinationPath = path.join(storagePath, `${filename}.pdf`);
-    const htmlFileUrl = `file://${htmlFilePath}`;
+    try {
+      const htmlFilePath = path.join(storagePath, `${filename}.html`);
+      const fileDestinationPath = path.join(storagePath, `${filename}.pdf`);
+      const htmlFileUrl = `file://${htmlFilePath}`;
 
-    const launchOptions = getPlaywrightLaunchOptions(browser);
+      const launchOptions = getPlaywrightLaunchOptions(browserToUse);
 
-    browserInstance = await constants.launcher.launch({
-      ...launchOptions,
-      headless: true,
-    });
+      browserInstance = await constants.launcher.launch({
+        ...launchOptions,
+        headless: true,
+      });
 
-    register(browserInstance as unknown as { close: () => Promise<void> });
+      register(browserInstance as unknown as { close: () => Promise<void> });
 
-    context = await browserInstance.newContext();
-    page = await context.newPage();
+      context = await browserInstance.newContext();
+      page = await context.newPage();
 
-    await page.goto(htmlFileUrl, {
-      waitUntil: 'domcontentloaded',
-      timeout: 120000,
-    });
+      await page.goto(htmlFileUrl, {
+        waitUntil: 'domcontentloaded',
+        timeout: 120000,
+      });
 
-    await page.emulateMedia({ media: 'print' });
+      await page.emulateMedia({ media: 'print' });
 
-    await page.pdf({
-      margin: { bottom: '32px' },
-      path: fileDestinationPath,
-      format: 'A4',
-      displayHeaderFooter: true,
-      footerTemplate: `
+      await page.pdf({
+        margin: { bottom: '32px' },
+        path: fileDestinationPath,
+        format: 'A4',
+        displayHeaderFooter: true,
+        footerTemplate: `
     <div style="margin-top:50px;color:#26241b;font-family:Open Sans;text-align: center;width: 100%;font-weight:400">
       <span style="color:#26241b;font-size: 14px;font-weight:400">Page <span class="pageNumber"></span> of <span class="totalPages"></span></span>
     </div>
   `,
-    });
+      });
 
-    if (pagesScanned < 2000) {
       fs.unlinkSync(htmlFilePath);
+    } finally {
+      try {
+        await page?.close();
+      } catch (err) {
+        consoleLogger.info(`Error at page close writeSummaryPDF ${err}`);
+      }
+      try {
+        await context?.close();
+      } catch (err) {
+        consoleLogger.info(`Error at context close writeSummaryPDF ${err}`);
+      }
+      try {
+        await browserInstance?.close();
+      } catch (err) {
+        consoleLogger.info(`Error at browserInstance close writeSummaryPDF ${err}`);
+      }
+    }
+  };
+
+  const browserAttempts = [browser];
+
+  // Runtime fallback: if Chrome launch fails on Windows, try Edge once for PDF generation.
+  if (process.platform === 'win32' && browser === BrowserTypes.CHROME) {
+    browserAttempts.push(BrowserTypes.EDGE);
+  }
+
+  for (let i = 0; i < browserAttempts.length; i++) {
+    const currentBrowser = browserAttempts[i];
+    try {
+      await renderPdfWithBrowser(currentBrowser);
+      if (i > 0) {
+        consoleLogger.warn(
+          `writeSummaryPDF succeeded with fallback browser '${currentBrowser}' after '${browser}' failed.`,
+        );
+      }
+      return;
+    } catch (err) {
+      const isLastAttempt = i === browserAttempts.length - 1;
+      consoleLogger.info(
+        `Error at writeSummaryPDF using browser '${currentBrowser}': ${err instanceof Error ? err.stack : err}`,
+      );
+      if (isLastAttempt) {
+        return;
+      }
+      const nextBrowser = browserAttempts[i + 1];
+      consoleLogger.warn(
+        `writeSummaryPDF failed using browser '${currentBrowser}', retrying with '${nextBrowser}'.`,
+      );
     }
-  } catch (err) {
-    consoleLogger.info(`Error at writeSummaryPDF ${err instanceof Error ? err.stack : err}`);
-  } finally {
-    await page?.close().catch(err => {
-      consoleLogger.info(`Error at page close writeSummaryPDF ${err}`);
-    });
-    await context?.close().catch(err => {
-      consoleLogger.info(`Error at context close writeSummaryPDF ${err}`);
-    });
-    await browserInstance?.close().catch(err => {
-      consoleLogger.info(`Error at browserInstance close writeSummaryPDF ${err}`);
-    });
   }
 };
 
@@ -645,6 +682,8 @@ export const createRuleIdJson = async (allIssues, itemsStore?: ItemsStore) => {
   const compiledRuleJson = {};
 
   for (const category of ['mustFix', 'goodToFix', 'needsReview'] as const) {
+    compiledRuleJson[category] = {};
+
     for (const rule of allIssues.items[category].rules) {
       let allItems: any[] = [];
 
@@ -656,7 +695,7 @@ export const createRuleIdJson = async (allIssues, itemsStore?: ItemsStore) => {
         allItems = rule.pagesAffected.flatMap(page => page.items || []);
       }
 
-      compiledRuleJson[rule.rule] = extractRuleAiData(rule.rule, rule.totalItems, allItems);
+      compiledRuleJson[category][rule.rule] = extractRuleAiData(rule.rule, rule.totalItems, allItems);
     }
   }
 
@@ -668,10 +707,12 @@ export const createBasicFormHTMLSnippet = filteredResults => {
   const compiledRuleJson = {};
 
   ['mustFix', 'goodToFix', 'needsReview'].forEach(category => {
+    compiledRuleJson[category] = {};
+
     if (filteredResults[category] && filteredResults[category].rules) {
       Object.entries(filteredResults[category].rules).forEach(
         ([ruleId, ruleVal]: [string, any]) => {
-          compiledRuleJson[ruleId] = extractRuleAiData(ruleId, ruleVal.totalItems, ruleVal.items);
+          compiledRuleJson[category][ruleId] = extractRuleAiData(ruleId, ruleVal.totalItems, ruleVal.items);
         },
       );
     }
@@ -741,6 +782,7 @@ const generateArtifacts = async (
   },
   zip: string = undefined, // optional
   generateJsonFiles = false,
+  preferredBrowser?: string,
 ) => {
   consoleLogger.info('Generating report artifacts');
 
@@ -760,6 +802,8 @@ const generateArtifacts = async (
     endTime: scanDetails.endTime ? scanDetails.endTime : new Date(),
     urlScanned,
     scanType,
+    totalLinksFetchedFromSitemaps: constants.sitemapFetchedLinks?.totalLinksFetchedFromSitemaps ?? 0,
+    fetchedSitemaps: constants.sitemapFetchedLinks?.fetchedSitemaps ?? [],
     deviceChosen: scanDetails.deviceChosen || 'Desktop',
     formatAboutStartTime,
     isCustomFlow,
@@ -1005,7 +1049,11 @@ const generateArtifacts = async (
     ]);
   }
 
-  const browserChannel = getBrowserToRun(randomToken, BrowserTypes.CHROME, false).browserToRun;
+  const browserChannel = getBrowserToRun(
+    randomToken,
+    (preferredBrowser as BrowserTypes) || BrowserTypes.CHROME,
+    false,
+  ).browserToRun;
 
   // Should consider refactor constants.userDataDirectory to be a parameter in future
   await retryFunction(
@@ -1020,36 +1068,34 @@ const generateArtifacts = async (
     1,
   );
 
-  // Suppress uncaught EPERM errors from lingering Crawlee async lock-file operations
-  // (Windows holds mandatory file locks; Crawlee may still attempt mkdir on .json.lock
-  // files after the crawl has finished). Without this, Node crashes with uncaughtException.
-  const crawleeEpermHandler = (err: Error & { code?: string }) => {
-    if (err.code === 'EPERM' && err.message?.includes('crawlee')) {
-      consoleLogger.info(`Suppressed lingering Crawlee storage error: ${err.message}`);
-      return;
-    }
-    // Re-throw non-crawlee EPERM errors so they aren't silently swallowed
-    throw err;
-  };
-  process.on('uncaughtException', crawleeEpermHandler);
-  process.on('unhandledRejection', crawleeEpermHandler);
+  // Flush pending background storage operations (metadata writes, lock-file ops)
+  const storageClient = Configuration.getStorageClient();
+  if (storageClient.teardown) {
+    await storageClient.teardown();
+  }
 
-  // Brief delay to allow lingering async crawlee storage operations to flush
-  await new Promise(resolve => setTimeout(resolve, process.platform === 'win32' ? 5000 : 3000));
+  // Gracefully drop Dataset and RequestQueue — releases locks and removes files
+  const crawleeDir = path.join(storagePath, 'crawlee');
+  try {
+    const dataset = await Dataset.open(crawleeDir);
+    await dataset.drop();
+  } catch (error) {
+    consoleLogger.info(`Dataset drop: ${error.message}`);
+  }
+
+  try {
+    const requestQueue = await RequestQueue.open(crawleeDir);
+    await requestQueue.drop();
+  } catch (error) {
+    consoleLogger.info(`RequestQueue drop: ${error.message}`);
+  }
 
+  // Fallback rm for any leftover files not managed by Crawlee's storage API
   const crawleePath = path.join(storagePath, 'crawlee');
   try {
     await fs.promises.rm(crawleePath, { recursive: true, force: true });
-  } catch (error) {
-    // On Windows, retry once after a delay if the folder is still locked
-    if (process.platform === 'win32') {
-      await new Promise(resolve => setTimeout(resolve, 3000));
-      try {
-        await fs.promises.rm(crawleePath, { recursive: true, force: true });
-      } catch {
-        // Best-effort cleanup — leave the folder; report generation continues
-      }
-    }
+  } catch {
+    // Best-effort; storage was already dropped via API
   }
 
   try {
@@ -1058,6 +1104,31 @@ const generateArtifacts = async (
     consoleLogger.warn(`Unable to force remove pdfs folder: ${error.message}`);
   }
 
+  // Generate scrubbed HTML Code Snippets
+  const ruleIdJson = await createRuleIdJson(allIssues, itemsStore);
+
+  // Clean up intermediate items files before zipping
+  await itemsStore.cleanup();
+
+  try {
+    await sendWcagBreakdownToSentry(
+      oobeeAppVersion,
+      wcagOccurrencesMap,
+      ruleIdJson,
+      {
+        entryUrl: urlScanned,
+        scanType,
+        browser: scanDetails.deviceChosen,
+        email: scanDetails.nameEmail?.email,
+        name: scanDetails.nameEmail?.name,
+      },
+      allIssues,
+      pagesScanned.length,
+    );
+  } catch (error) {
+    console.error('Error sending WCAG data to Sentry:', error);
+  }
+
   // Take option if set
   if (typeof zip === 'string') {
     constants.cliZipFileName = zip;
@@ -1104,40 +1175,9 @@ const generateArtifacts = async (
     printMessage([`Error in zipping results: ${error}`]);
   }
 
-  // Generate scrubbed HTML Code Snippets
-  const ruleIdJson = await createRuleIdJson(allIssues, itemsStore);
-
-  // Clean up intermediate items files
-  await itemsStore.cleanup();
-
-  // At the end of the function where results are generated, add:
-  try {
-    // Always send WCAG breakdown to Sentry, even if no violations were found
-    // This ensures that all criteria are reported, including those with 0 occurrences
-    await sendWcagBreakdownToSentry(
-      oobeeAppVersion,
-      wcagOccurrencesMap,
-      ruleIdJson,
-      {
-        entryUrl: urlScanned,
-        scanType,
-        browser: scanDetails.deviceChosen,
-        email: scanDetails.nameEmail?.email,
-        name: scanDetails.nameEmail?.name,
-      },
-      allIssues,
-      pagesScanned.length,
-    );
-  } catch (error) {
-    console.error('Error sending WCAG data to Sentry:', error);
-  }
-
   if (process.env.RUNNING_FROM_PH_GUI || process.env.OOBEE_VERBOSE)
     console.log('Report generated successfully');
 
-  process.removeListener('uncaughtException', crawleeEpermHandler);
-  process.removeListener('unhandledRejection', crawleeEpermHandler);
-
   return ruleIdJson;
 };
 

package/src/npmIndex.ts

@@ -363,6 +363,7 @@ export const init = async ({
   const { mustFix: mustFixThreshold, goodToFix: goodToFixThreshold } = thresholds;
 
   process.env.CRAWLEE_STORAGE_DIR = randomToken;
+  constants.sitemapFetchedLinks = null;
 
   const scanDetails = {
     startTime: new Date(),

package/src/utils.ts

@@ -6,6 +6,7 @@ import axe, { Rule } from 'axe-core';
 import { v4 as uuidv4 } from 'uuid';
 import { getDomain } from 'tldts';
 import { normalizeUrl } from '@apify/utilities';
+import { Dataset, RequestQueue, Configuration } from 'crawlee';
 import constants, {
   BrowserTypes,
   destinationPath,
@@ -390,6 +391,19 @@ export const cleanUp = async (randomToken?: string, isError: boolean = false): P
   if (randomToken !== undefined) {
     const storagePath = getStoragePath(randomToken);
 
+    try {
+      const storageClient = Configuration.getStorageClient();
+      if (storageClient.teardown) {
+        await storageClient.teardown();
+      }
+      const crawleeDir = path.join(storagePath, 'crawlee');
+      const dataset = await Dataset.open(crawleeDir);
+      await dataset.drop();
+      const requestQueue = await RequestQueue.open(crawleeDir);
+      await requestQueue.drop();
+    } catch (error) {
+      consoleLogger.info(`Crawlee storage drop in cleanUp: ${error.message}`);
+    }
     try {
       fs.rmSync(path.join(storagePath, 'crawlee'), { recursive: true, force: true });
     } catch (error) {
@@ -402,37 +416,8 @@ export const cleanUp = async (randomToken?: string, isError: boolean = false): P
       consoleLogger.warn(`Unable to force remove pdfs folder: ${error.message}`);
     }
     
-    let deleteErrorLogFile = true;
-
-    if (isError) {
-      let logsPath = storagePath;
-
-      if (process.env.OOBEE_LOGS_PATH) {
-        logsPath = process.env.OOBEE_LOGS_PATH;
-      }
-
-      if (fs.existsSync(errorsTxtPath)) {
-        try {
-          const logFilePath = path.join(logsPath, `logs-${randomToken}.txt`);
-          fs.copyFileSync(errorsTxtPath, logFilePath);
-          console.log(`An error occured. Log file is located at: ${logFilePath}`);
-
-        } catch (copyError) {
-          consoleLogger.error(`Error copying errors file during cleanup: ${copyError.message}`);
-          console.log(`An error occured. Log file is located at: ${errorsTxtPath}`);
-          deleteErrorLogFile = false; // Do not delete the log file if copy failed
-        }
-    
-        if (deleteErrorLogFile && fs.existsSync(errorsTxtPath)) {
-          try {
-            fs.unlinkSync(errorsTxtPath);
-          } catch (error) {
-            consoleLogger.warn(`Unable to delete log file ${errorsTxtPath}: ${error.message}`);
-          }
-        }
-
-      }
-
+    if (isError && fs.existsSync(errorsTxtPath)) {
+      console.log(`An error occured. Log file is located at: ${errorsTxtPath}`);
     } 
     
     if (fs.existsSync(storagePath) && fs.readdirSync(storagePath).length === 0) {
@@ -1081,13 +1066,20 @@ export const randomThreeDigitNumberString = () => {
 
 export const normUrl = (u: string): string => (u ? normalizeUrl(u) || u : '');
 
+export const stripWwwPrefix = (hostname: string): string => hostname.replace(/^www\./, '');
+
+export const isSameHostname = (hostname1: string, hostname2: string): boolean =>
+  stripWwwPrefix(hostname1) === stripWwwPrefix(hostname2);
+
 export const isFollowStrategy = (link1: string, link2: string, rule: string): boolean => {
   if (rule === 'all') return true;
   try {
     const parsedLink1 = new URL(link1);
     const parsedLink2 = new URL(link2);
     if (rule === 'same-origin') {
-      return parsedLink1.origin === parsedLink2.origin;
+      return parsedLink1.protocol === parsedLink2.protocol &&
+        isSameHostname(parsedLink1.hostname, parsedLink2.hostname) &&
+        parsedLink1.port === parsedLink2.port;
     }
     if (rule === 'same-domain') {
       const link1Domain = getDomain(parsedLink1.hostname, { allowPrivateDomains: true }) || parsedLink1.hostname;
@@ -1095,7 +1087,7 @@ export const isFollowStrategy = (link1: string, link2: string, rule: string): bo
       return link1Domain.toLowerCase() === link2Domain.toLowerCase();
     }
     // default: same-hostname
-    return parsedLink1.hostname === parsedLink2.hostname;
+    return isSameHostname(parsedLink1.hostname, parsedLink2.hostname);
   } catch {
     return false;
   }