@govtechsg/oobee 0.10.89 → 0.10.91

package/src/mergeAxeResults/writeCsv.ts

@@ -1,18 +1,46 @@
 import { createWriteStream } from 'fs';
-import { AsyncParser, ParserOptions } from '@json2csv/node';
 import { a11yRuleShortDescriptionMap } from '../constants/constants.js';
 import type { AllIssues, RuleInfo } from './types.js';
+import type { ItemsStore } from './itemsStore.js';
 
-const writeCsv = async (allIssues: AllIssues, storagePath: string): Promise<void> => {
+function escapeCsvField(value: string): string {
+  if (value == null) return '';
+  const str = String(value);
+  return `"${str.replace(/"/g, '""')}"`;
+}
+
+const writeCsv = async (
+  allIssues: AllIssues,
+  storagePath: string,
+  itemsStore?: ItemsStore,
+): Promise<void> => {
   const csvOutput = createWriteStream(`${storagePath}/report.csv`, { encoding: 'utf8' });
+
   const formatPageViolation = (pageNum: number) => {
     if (pageNum < 0) return 'Document';
     return `Page ${pageNum}`;
   };
 
-  // transform allIssues into the form:
-  // [['mustFix', rule1], ['mustFix', rule2], ['goodToFix', rule3], ...]
-  const getRulesByCategory = (issues: AllIssues) => {
+  const fields = [
+    'customFlowLabel',
+    'deviceChosen',
+    'scanCompletedAt',
+    'severity',
+    'issueId',
+    'issueDescription',
+    'wcagConformance',
+    'url',
+    'pageTitle',
+    'context',
+    'howToFix',
+    'axeImpact',
+    'xpath',
+    'learnMore',
+  ];
+
+  csvOutput.write(fields.map(escapeCsvField).join(',') + '\n');
+
+  const getRulesByCategory = (issues: AllIssues): [string, RuleInfo][] => {
     return Object.entries(issues.items)
       .filter(([category]) => category !== 'passed')
       .reduce((prev: [string, RuleInfo][], [category, value]) => {
@@ -23,15 +51,14 @@ const writeCsv = async (allIssues: AllIssues, storagePath: string): Promise<void
         return prev;
       }, [])
       .sort((a, b) => {
-        // sort rules according to severity, then ruleId
         const compareCategory = -a[0].localeCompare(b[0]);
         return compareCategory === 0 ? a[1].rule.localeCompare(b[1].rule) : compareCategory;
       });
   };
 
-  const flattenRule = (catAndRule: [string, RuleInfo]) => {
-    const [severity, rule] = catAndRule;
-    const results = [];
+  const rulesByCategory = getRulesByCategory(allIssues);
+
+  for (const [severity, rule] of rulesByCategory) {
     const {
       rule: issueId,
       description: issueDescription,
@@ -41,104 +68,106 @@ const writeCsv = async (allIssues: AllIssues, storagePath: string): Promise<void
       helpUrl: learnMore,
     } = rule;
 
-    // format clauses as a string
     const wcagConformance = conformance.join(',');
 
-    pagesAffected.sort((a, b) => a.url.localeCompare(b.url));
-
-    pagesAffected.forEach(affectedPage => {
-      const { url, items } = affectedPage;
-      items.forEach(item => {
-        const { html, message, xpath } = item;
-        const page = (item as any).page;
-        const howToFix = message.replace(/(\r\n|\n|\r)/g, '\\n'); // preserve newlines as \n
-        const violation = html || formatPageViolation(page); // page is a number, not a string
-        const context = violation.replace(/(\r\n|\n|\r)/g, ''); // remove newlines
-
-        results.push({
-          customFlowLabel: allIssues.customFlowLabel || '',
-          deviceChosen: allIssues.deviceChosen || '',
-          scanCompletedAt: allIssues.endTime ? allIssues.endTime.toISOString() : '',
-          severity: severity || '',
-          issueId: issueId || '',
-          issueDescription: a11yRuleShortDescriptionMap[issueId] || issueDescription || '',
-          wcagConformance: wcagConformance || '',
-          url: url || '',
-          pageTitle: affectedPage.pageTitle || 'No page title',
-          context: context || '',
-          howToFix: howToFix || '',
-          axeImpact: axeImpact || '',
-          xpath: xpath || '',
-          learnMore: learnMore || '',
-        });
-      });
-    });
-    if (results.length === 0) return {};
-    return results;
-  };
+    if (itemsStore) {
+      const itemsMap = await itemsStore.readRuleItemsMap(severity, issueId);
+      const sortedPages = [...pagesAffected].sort((a, b) => (a.url || '').localeCompare(b.url || ''));
 
-  const opts: ParserOptions<any, any> = {
-    transforms: [getRulesByCategory, flattenRule],
-    fields: [
-      'customFlowLabel',
-      'deviceChosen',
-      'scanCompletedAt',
-      'severity',
-      'issueId',
-      'issueDescription',
-      'wcagConformance',
-      'url',
-      'pageTitle',
-      'context',
-      'howToFix',
-      'axeImpact',
-      'xpath',
-      'learnMore',
-    ],
-    includeEmptyRows: true,
-  };
+      for (const affectedPage of sortedPages) {
+        const key = affectedPage.pageIndex != null ? String(affectedPage.pageIndex) : affectedPage.url;
+        const entry = itemsMap.get(key);
+        if (!entry) continue;
 
-  // Create the parse stream (it's asynchronous)
-  const parser = new AsyncParser(opts);
-  const parseStream = parser.parse(allIssues);
-
-  // Pipe JSON2CSV output into the file, but don't end automatically
-  parseStream.pipe(csvOutput, { end: false });
-
-  // Once JSON2CSV is done writing all normal rows, append any "pagesNotScanned"
-  parseStream.on('end', () => {
-    if (allIssues.pagesNotScanned && allIssues.pagesNotScanned.length > 0) {
-      csvOutput.write('\n');
-      allIssues.pagesNotScanned.forEach(page => {
-        const skippedPage = {
-          customFlowLabel: allIssues.customFlowLabel || '',
-          deviceChosen: allIssues.deviceChosen || '',
-          scanCompletedAt: allIssues.endTime ? allIssues.endTime.toISOString() : '',
-          severity: 'error',
-          issueId: 'error-pages-skipped',
-          issueDescription: page.metadata
-            ? page.metadata
-            : 'An unknown error caused the page to be skipped',
-          wcagConformance: '',
-          url: page.url || page || '',
-          pageTitle: 'Error',
-          context: '',
-          howToFix: '',
-          axeImpact: '',
-          xpath: '',
-          learnMore: '',
-        };
-        csvOutput.write(`${Object.values(skippedPage).join(',')}\n`);
-      });
+        for (const item of entry.items) {
+          const { html, message, xpath } = item;
+          const page = (item as any).page;
+          const howToFix = (message || '').replace(/(\r\n|\n|\r)/g, '\\n');
+          const violation = html || formatPageViolation(page);
+          const context = violation.replace(/(\r\n|\n|\r)/g, '');
+
+          const row = [
+            allIssues.customFlowLabel || '',
+            allIssues.deviceChosen || '',
+            allIssues.endTime ? allIssues.endTime.toISOString() : '',
+            severity || '',
+            issueId || '',
+            a11yRuleShortDescriptionMap[issueId] || issueDescription || '',
+            wcagConformance || '',
+            affectedPage.url || '',
+            affectedPage.pageTitle || 'No page title',
+            context || '',
+            howToFix || '',
+            axeImpact || '',
+            xpath || '',
+            learnMore || '',
+          ].map(escapeCsvField);
+
+          csvOutput.write(row.join(',') + '\n');
+        }
+      }
+    } else {
+      const sortedPages = [...pagesAffected].sort((a, b) => (a.url || '').localeCompare(b.url || ''));
+
+      for (const affectedPage of sortedPages) {
+        const items = (affectedPage as any).items || [];
+        for (const item of items) {
+          const { html, message, xpath } = item;
+          const page = (item as any).page;
+          const howToFix = (message || '').replace(/(\r\n|\n|\r)/g, '\\n');
+          const violation = html || formatPageViolation(page);
+          const context = violation.replace(/(\r\n|\n|\r)/g, '');
+
+          const row = [
+            allIssues.customFlowLabel || '',
+            allIssues.deviceChosen || '',
+            allIssues.endTime ? allIssues.endTime.toISOString() : '',
+            severity || '',
+            issueId || '',
+            a11yRuleShortDescriptionMap[issueId] || issueDescription || '',
+            wcagConformance || '',
+            affectedPage.url || '',
+            affectedPage.pageTitle || 'No page title',
+            context || '',
+            howToFix || '',
+            axeImpact || '',
+            xpath || '',
+            learnMore || '',
+          ].map(escapeCsvField);
+
+          csvOutput.write(row.join(',') + '\n');
+        }
+      }
     }
+  }
 
-    // Now close the CSV file
-    csvOutput.end();
-  });
+  if (allIssues.pagesNotScanned && allIssues.pagesNotScanned.length > 0) {
+    allIssues.pagesNotScanned.forEach(page => {
+      const row = [
+        allIssues.customFlowLabel || '',
+        allIssues.deviceChosen || '',
+        allIssues.endTime ? allIssues.endTime.toISOString() : '',
+        'error',
+        'error-pages-skipped',
+        page.metadata ? page.metadata : 'An unknown error caused the page to be skipped',
+        '',
+        (page as any).url || page || '',
+        'Error',
+        '',
+        '',
+        '',
+        '',
+        '',
+      ].map(escapeCsvField);
+
+      csvOutput.write(row.join(',') + '\n');
+    });
+  }
 
-  parseStream.on('error', (err: unknown) => {
-    console.error('Error parsing CSV:', err);
-    csvOutput.end();
+  csvOutput.end();
+  await new Promise<void>((resolve, reject) => {
+    csvOutput.on('finish', resolve);
+    csvOutput.on('error', reject);
   });
 };
 

package/src/mergeAxeResults.ts

@@ -31,6 +31,7 @@ import { consoleLogger } from './logs.js';
 import itemTypeDescription from './constants/itemTypeDescription.js';
 import { oobeeAiHtmlETL, oobeeAiRules } from './constants/oobeeAi.js';
 import { buildHtmlGroups, convertItemsToReferences } from './mergeAxeResults/itemReferences.js';
+import { ItemsStore } from './mergeAxeResults/itemsStore.js';
 import {
   compressJsonFileStreaming,
   writeJsonAndBase64Files,
@@ -418,7 +419,7 @@ const writeSummaryPdf = async (
 // Tracking WCAG occurrences
 const wcagOccurrencesMap = new Map<string, number>();
 
-const pushResults = async (pageResults, allIssues, isCustomFlow) => {
+const pushResults = async (pageResults, allIssues, isCustomFlow, itemsStore: ItemsStore) => {
   const { url, pageTitle, filePath } = pageResults;
 
   const totalIssuesInPage = new Set();
@@ -433,15 +434,15 @@ const pushResults = async (pageResults, allIssues, isCustomFlow) => {
     totalOccurrences: 0,
   });
 
-  ['mustFix', 'goodToFix', 'needsReview', 'passed'].forEach(category => {
-    if (!pageResults[category]) return;
+  for (const category of ['mustFix', 'goodToFix', 'needsReview', 'passed'] as const) {
+    if (!pageResults[category]) continue;
 
     const { totalItems, rules } = pageResults[category];
     const currCategoryFromAllIssues = allIssues.items[category];
 
     currCategoryFromAllIssues.totalItems += totalItems;
 
-    Object.keys(rules).forEach(rule => {
+    for (const rule of Object.keys(rules)) {
       const {
         description,
         axeImpact,
@@ -457,7 +458,6 @@ const pushResults = async (pageResults, allIssues, isCustomFlow) => {
           helpUrl,
           conformance,
           totalItems: 0,
-          // numberOfPagesAffectedAfterRedirects: 0,
           pagesAffected: {},
         };
       }
@@ -470,7 +470,6 @@ const pushResults = async (pageResults, allIssues, isCustomFlow) => {
               allIssues.wcagViolations.push(c);
             }
 
-            // Track WCAG criteria occurrences for Sentry
             const currentCount = wcagOccurrencesMap.get(c) || 0;
             wcagOccurrencesMap.set(c, currentCount + count);
           });
@@ -480,9 +479,6 @@ const pushResults = async (pageResults, allIssues, isCustomFlow) => {
 
       currRuleFromAllIssues.totalItems += count;
 
-      // Build htmlGroups for pre-computed Group by HTML Element
-      buildHtmlGroups(currRuleFromAllIssues, items, url);
-
       if (isCustomFlow) {
         const { pageIndex, pageImagePath, metadata } = pageResults;
         currRuleFromAllIssues.pagesAffected[pageIndex] = {
@@ -490,17 +486,31 @@ const pushResults = async (pageResults, allIssues, isCustomFlow) => {
           pageTitle,
           pageImagePath,
           metadata,
-          items: [...items],
+          itemsCount: items.length,
         };
+        await itemsStore.appendPageItems(category, rule, {
+          url,
+          pageTitle,
+          items,
+          pageIndex,
+          pageImagePath,
+          metadata,
+        });
       } else if (!(url in currRuleFromAllIssues.pagesAffected)) {
         currRuleFromAllIssues.pagesAffected[url] = {
           pageTitle,
-          items: [...items],
+          itemsCount: items.length,
           ...(filePath && { filePath }),
         };
+        await itemsStore.appendPageItems(category, rule, {
+          url,
+          pageTitle,
+          items,
+          ...(filePath && { filePath }),
+        });
       }
-    });
-  });
+    }
+  }
 };
 
 const getTopTenIssues = allIssues => {
@@ -561,26 +571,26 @@ const flattenAndSortResults = (allIssues: AllIssues, isCustomFlow: boolean) => {
           .map(pageEntry => {
             if (isCustomFlow) {
               const [pageIndex, pageInfo] = pageEntry as unknown as [number, PageInfo];
-              // Only update the occurrences map if not passed.
               if (category !== 'passed') {
                 urlOccurrencesMap.set(
                   pageInfo.url!,
-                  (urlOccurrencesMap.get(pageInfo.url!) || 0) + pageInfo.items.length,
+                  (urlOccurrencesMap.get(pageInfo.url!) || 0) + (pageInfo.itemsCount || 0),
                 );
               }
               return { pageIndex, ...pageInfo };
             }
             const [url, pageInfo] = pageEntry as unknown as [string, PageInfo];
             if (category !== 'passed') {
-              urlOccurrencesMap.set(url, (urlOccurrencesMap.get(url) || 0) + pageInfo.items.length);
+              urlOccurrencesMap.set(
+                url,
+                (urlOccurrencesMap.get(url) || 0) + (pageInfo.itemsCount || 0),
+              );
             }
             return { url, ...pageInfo };
           })
-          // Sort pages so that those with the most items come first
-          .sort((page1, page2) => page2.items.length - page1.items.length);
+          .sort((page1, page2) => (page2.itemsCount || 0) - (page1.itemsCount || 0));
         return { rule, ...ruleInfo };
       })
-      // Sort the rules by totalItems (descending)
       .sort((rule1, rule2) => rule2.totalItems - rule1.totalItems);
   });
 
@@ -631,19 +641,24 @@ const extractRuleAiData = (
 };
 
 // This is for telemetry purposes called within mergeAxeResults.ts
-export const createRuleIdJson = allIssues => {
+export const createRuleIdJson = async (allIssues, itemsStore?: ItemsStore) => {
   const compiledRuleJson = {};
 
-  ['mustFix', 'goodToFix', 'needsReview'].forEach(category => {
-    allIssues.items[category].rules.forEach(rule => {
-      const allItems = rule.pagesAffected.flatMap(page => page.items || []);
-      compiledRuleJson[rule.rule] = extractRuleAiData(rule.rule, rule.totalItems, allItems, () => {
-        rule.pagesAffected.forEach(p => {
-          delete p.items;
-        });
-      });
-    });
-  });
+  for (const category of ['mustFix', 'goodToFix', 'needsReview'] as const) {
+    for (const rule of allIssues.items[category].rules) {
+      let allItems: any[] = [];
+
+      if (itemsStore) {
+        for await (const entry of itemsStore.readRuleItems(category, rule.rule)) {
+          allItems.push(...entry.items);
+        }
+      } else {
+        allItems = rule.pagesAffected.flatMap(page => page.items || []);
+      }
+
+      compiledRuleJson[rule.rule] = extractRuleAiData(rule.rule, rule.totalItems, allItems);
+    }
+  }
 
   return compiledRuleJson;
 };
@@ -815,20 +830,20 @@ const generateArtifacts = async (
   };
 
   const allFiles = await extractFileNames(intermediateDatasetsPath);
+  const itemsStore = new ItemsStore(storagePath);
 
-  const jsonArray = await Promise.all(
-    allFiles.map(async file => parseContentToJson(`${intermediateDatasetsPath}/${file}`)),
-  );
-
-  await Promise.all(
-    jsonArray.map(async pageResults => {
-      await pushResults(pageResults, allIssues, isCustomFlow);
-    }),
-  ).catch(flattenIssuesError => {
-    consoleLogger.error(
-      `[generateArtifacts] Error flattening issues: ${flattenIssuesError?.stack || flattenIssuesError}`,
-    );
-  });
+  for (const file of allFiles) {
+    try {
+      const pageResults = await parseContentToJson(`${intermediateDatasetsPath}/${file}`);
+      if (pageResults) {
+        await pushResults(pageResults, allIssues, isCustomFlow, itemsStore);
+      }
+    } catch (flattenIssuesError: any) {
+      consoleLogger.error(
+        `[generateArtifacts] Error processing ${file}: ${flattenIssuesError?.stack || flattenIssuesError}`,
+      );
+    }
+  }
 
   flattenAndSortResults(allIssues, isCustomFlow);
 
@@ -864,6 +879,15 @@ const generateArtifacts = async (
 
   populateScanPagesDetail(allIssues);
 
+  // Build htmlGroups in a second pass from disk-backed items
+  for (const category of ['mustFix', 'goodToFix', 'needsReview', 'passed'] as const) {
+    for (const rule of allIssues.items[category].rules) {
+      for await (const entry of itemsStore.readRuleItems(category, rule.rule)) {
+        buildHtmlGroups(rule, entry.items, entry.url);
+      }
+    }
+  }
+
   allIssues.wcagPassPercentage = getWcagPassPercentage(
     allIssues.wcagViolations,
     allIssues.advancedScanOptionsSummaryItems.showEnableWcagAaa,
@@ -928,7 +952,7 @@ const generateArtifacts = async (
     rest.minor = axeImpactCount.minor;
   }
 
-  await writeCsv(allIssues, storagePath);
+  await writeCsv(allIssues, storagePath, itemsStore);
   await writeSitemap(pagesScanned, storagePath);
   const {
     scanDataJsonFilePath,
@@ -945,7 +969,7 @@ const generateArtifacts = async (
     scanPagesSummaryBase64FilePath,
     scanDataJsonFileSize,
     scanItemsJsonFileSize,
-  } = await writeJsonAndBase64Files(allIssues, storagePath);
+  } = await writeJsonAndBase64Files(allIssues, storagePath, itemsStore);
   // Removed BIG_RESULTS_THRESHOLD check - always use full scanItems
 
   await writeScanDetailsCsv(
@@ -996,13 +1020,36 @@ const generateArtifacts = async (
     1,
   );
 
+  // Suppress uncaught EPERM errors from lingering Crawlee async lock-file operations
+  // (Windows holds mandatory file locks; Crawlee may still attempt mkdir on .json.lock
+  // files after the crawl has finished). Without this, Node crashes with uncaughtException.
+  const crawleeEpermHandler = (err: Error & { code?: string }) => {
+    if (err.code === 'EPERM' && err.message?.includes('crawlee')) {
+      consoleLogger.info(`Suppressed lingering Crawlee storage error: ${err.message}`);
+      return;
+    }
+    // Re-throw non-crawlee EPERM errors so they aren't silently swallowed
+    throw err;
+  };
+  process.on('uncaughtException', crawleeEpermHandler);
+  process.on('unhandledRejection', crawleeEpermHandler);
+
   // Brief delay to allow lingering async crawlee storage operations to flush
-  await new Promise(resolve => setTimeout(resolve, 3000));
+  await new Promise(resolve => setTimeout(resolve, process.platform === 'win32' ? 5000 : 3000));
 
+  const crawleePath = path.join(storagePath, 'crawlee');
   try {
-    await fs.promises.rm(path.join(storagePath, 'crawlee'), { recursive: true, force: true });
+    await fs.promises.rm(crawleePath, { recursive: true, force: true });
   } catch (error) {
-    // Silently ignore — folder may already be gone or still locked
+    // On Windows, retry once after a delay if the folder is still locked
+    if (process.platform === 'win32') {
+      await new Promise(resolve => setTimeout(resolve, 3000));
+      try {
+        await fs.promises.rm(crawleePath, { recursive: true, force: true });
+      } catch {
+        // Best-effort cleanup — leave the folder; report generation continues
+      }
+    }
   }
 
   try {
@@ -1058,7 +1105,10 @@ const generateArtifacts = async (
   }
 
   // Generate scrubbed HTML Code Snippets
-  const ruleIdJson = createRuleIdJson(allIssues);
+  const ruleIdJson = await createRuleIdJson(allIssues, itemsStore);
+
+  // Clean up intermediate items files
+  await itemsStore.cleanup();
 
   // At the end of the function where results are generated, add:
   try {
@@ -1085,6 +1135,9 @@ const generateArtifacts = async (
   if (process.env.RUNNING_FROM_PH_GUI || process.env.OOBEE_VERBOSE)
     console.log('Report generated successfully');
 
+  process.removeListener('uncaughtException', crawleeEpermHandler);
+  process.removeListener('unhandledRejection', crawleeEpermHandler);
+
   return ruleIdJson;
 };