@govtechsg/oobee 0.10.89 → 0.10.91

package/dist/mergeAxeResults/writeCsv.js

@@ -1,15 +1,35 @@
 import { createWriteStream } from 'fs';
-import { AsyncParser } from '@json2csv/node';
 import { a11yRuleShortDescriptionMap } from '../constants/constants.js';
-const writeCsv = async (allIssues, storagePath) => {
+function escapeCsvField(value) {
+    if (value == null)
+        return '';
+    const str = String(value);
+    return `"${str.replace(/"/g, '""')}"`;
+}
+const writeCsv = async (allIssues, storagePath, itemsStore) => {
     const csvOutput = createWriteStream(`${storagePath}/report.csv`, { encoding: 'utf8' });
     const formatPageViolation = (pageNum) => {
         if (pageNum < 0)
             return 'Document';
         return `Page ${pageNum}`;
     };
-    // transform allIssues into the form:
-    // [['mustFix', rule1], ['mustFix', rule2], ['goodToFix', rule3], ...]
+    const fields = [
+        'customFlowLabel',
+        'deviceChosen',
+        'scanCompletedAt',
+        'severity',
+        'issueId',
+        'issueDescription',
+        'wcagConformance',
+        'url',
+        'pageTitle',
+        'context',
+        'howToFix',
+        'axeImpact',
+        'xpath',
+        'learnMore',
+    ];
+    csvOutput.write(fields.map(escapeCsvField).join(',') + '\n');
     const getRulesByCategory = (issues) => {
         return Object.entries(issues.items)
             .filter(([category]) => category !== 'passed')
@@ -21,105 +41,104 @@ const writeCsv = async (allIssues, storagePath) => {
             return prev;
         }, [])
             .sort((a, b) => {
-            // sort rules according to severity, then ruleId
             const compareCategory = -a[0].localeCompare(b[0]);
             return compareCategory === 0 ? a[1].rule.localeCompare(b[1].rule) : compareCategory;
         });
     };
-    const flattenRule = (catAndRule) => {
-        const [severity, rule] = catAndRule;
-        const results = [];
+    const rulesByCategory = getRulesByCategory(allIssues);
+    for (const [severity, rule] of rulesByCategory) {
         const { rule: issueId, description: issueDescription, axeImpact, conformance, pagesAffected, helpUrl: learnMore, } = rule;
-        // format clauses as a string
         const wcagConformance = conformance.join(',');
-        pagesAffected.sort((a, b) => a.url.localeCompare(b.url));
-        pagesAffected.forEach(affectedPage => {
-            const { url, items } = affectedPage;
-            items.forEach(item => {
-                const { html, message, xpath } = item;
-                const page = item.page;
-                const howToFix = message.replace(/(\r\n|\n|\r)/g, '\\n'); // preserve newlines as \n
-                const violation = html || formatPageViolation(page); // page is a number, not a string
-                const context = violation.replace(/(\r\n|\n|\r)/g, ''); // remove newlines
-                results.push({
-                    customFlowLabel: allIssues.customFlowLabel || '',
-                    deviceChosen: allIssues.deviceChosen || '',
-                    scanCompletedAt: allIssues.endTime ? allIssues.endTime.toISOString() : '',
-                    severity: severity || '',
-                    issueId: issueId || '',
-                    issueDescription: a11yRuleShortDescriptionMap[issueId] || issueDescription || '',
-                    wcagConformance: wcagConformance || '',
-                    url: url || '',
-                    pageTitle: affectedPage.pageTitle || 'No page title',
-                    context: context || '',
-                    howToFix: howToFix || '',
-                    axeImpact: axeImpact || '',
-                    xpath: xpath || '',
-                    learnMore: learnMore || '',
-                });
-            });
-        });
-        if (results.length === 0)
-            return {};
-        return results;
-    };
-    const opts = {
-        transforms: [getRulesByCategory, flattenRule],
-        fields: [
-            'customFlowLabel',
-            'deviceChosen',
-            'scanCompletedAt',
-            'severity',
-            'issueId',
-            'issueDescription',
-            'wcagConformance',
-            'url',
-            'pageTitle',
-            'context',
-            'howToFix',
-            'axeImpact',
-            'xpath',
-            'learnMore',
-        ],
-        includeEmptyRows: true,
-    };
-    // Create the parse stream (it's asynchronous)
-    const parser = new AsyncParser(opts);
-    const parseStream = parser.parse(allIssues);
-    // Pipe JSON2CSV output into the file, but don't end automatically
-    parseStream.pipe(csvOutput, { end: false });
-    // Once JSON2CSV is done writing all normal rows, append any "pagesNotScanned"
-    parseStream.on('end', () => {
-        if (allIssues.pagesNotScanned && allIssues.pagesNotScanned.length > 0) {
-            csvOutput.write('\n');
-            allIssues.pagesNotScanned.forEach(page => {
-                const skippedPage = {
-                    customFlowLabel: allIssues.customFlowLabel || '',
-                    deviceChosen: allIssues.deviceChosen || '',
-                    scanCompletedAt: allIssues.endTime ? allIssues.endTime.toISOString() : '',
-                    severity: 'error',
-                    issueId: 'error-pages-skipped',
-                    issueDescription: page.metadata
-                        ? page.metadata
-                        : 'An unknown error caused the page to be skipped',
-                    wcagConformance: '',
-                    url: page.url || page || '',
-                    pageTitle: 'Error',
-                    context: '',
-                    howToFix: '',
-                    axeImpact: '',
-                    xpath: '',
-                    learnMore: '',
-                };
-                csvOutput.write(`${Object.values(skippedPage).join(',')}\n`);
-            });
+        if (itemsStore) {
+            const itemsMap = await itemsStore.readRuleItemsMap(severity, issueId);
+            const sortedPages = [...pagesAffected].sort((a, b) => (a.url || '').localeCompare(b.url || ''));
+            for (const affectedPage of sortedPages) {
+                const key = affectedPage.pageIndex != null ? String(affectedPage.pageIndex) : affectedPage.url;
+                const entry = itemsMap.get(key);
+                if (!entry)
+                    continue;
+                for (const item of entry.items) {
+                    const { html, message, xpath } = item;
+                    const page = item.page;
+                    const howToFix = (message || '').replace(/(\r\n|\n|\r)/g, '\\n');
+                    const violation = html || formatPageViolation(page);
+                    const context = violation.replace(/(\r\n|\n|\r)/g, '');
+                    const row = [
+                        allIssues.customFlowLabel || '',
+                        allIssues.deviceChosen || '',
+                        allIssues.endTime ? allIssues.endTime.toISOString() : '',
+                        severity || '',
+                        issueId || '',
+                        a11yRuleShortDescriptionMap[issueId] || issueDescription || '',
+                        wcagConformance || '',
+                        affectedPage.url || '',
+                        affectedPage.pageTitle || 'No page title',
+                        context || '',
+                        howToFix || '',
+                        axeImpact || '',
+                        xpath || '',
+                        learnMore || '',
+                    ].map(escapeCsvField);
+                    csvOutput.write(row.join(',') + '\n');
+                }
+            }
         }
-        // Now close the CSV file
-        csvOutput.end();
-    });
-    parseStream.on('error', (err) => {
-        console.error('Error parsing CSV:', err);
-        csvOutput.end();
+        else {
+            const sortedPages = [...pagesAffected].sort((a, b) => (a.url || '').localeCompare(b.url || ''));
+            for (const affectedPage of sortedPages) {
+                const items = affectedPage.items || [];
+                for (const item of items) {
+                    const { html, message, xpath } = item;
+                    const page = item.page;
+                    const howToFix = (message || '').replace(/(\r\n|\n|\r)/g, '\\n');
+                    const violation = html || formatPageViolation(page);
+                    const context = violation.replace(/(\r\n|\n|\r)/g, '');
+                    const row = [
+                        allIssues.customFlowLabel || '',
+                        allIssues.deviceChosen || '',
+                        allIssues.endTime ? allIssues.endTime.toISOString() : '',
+                        severity || '',
+                        issueId || '',
+                        a11yRuleShortDescriptionMap[issueId] || issueDescription || '',
+                        wcagConformance || '',
+                        affectedPage.url || '',
+                        affectedPage.pageTitle || 'No page title',
+                        context || '',
+                        howToFix || '',
+                        axeImpact || '',
+                        xpath || '',
+                        learnMore || '',
+                    ].map(escapeCsvField);
+                    csvOutput.write(row.join(',') + '\n');
+                }
+            }
+        }
+    }
+    if (allIssues.pagesNotScanned && allIssues.pagesNotScanned.length > 0) {
+        allIssues.pagesNotScanned.forEach(page => {
+            const row = [
+                allIssues.customFlowLabel || '',
+                allIssues.deviceChosen || '',
+                allIssues.endTime ? allIssues.endTime.toISOString() : '',
+                'error',
+                'error-pages-skipped',
+                page.metadata ? page.metadata : 'An unknown error caused the page to be skipped',
+                '',
+                page.url || page || '',
+                'Error',
+                '',
+                '',
+                '',
+                '',
+                '',
+            ].map(escapeCsvField);
+            csvOutput.write(row.join(',') + '\n');
+        });
+    }
+    csvOutput.end();
+    await new Promise((resolve, reject) => {
+        csvOutput.on('finish', resolve);
+        csvOutput.on('error', reject);
     });
 };
 export default writeCsv;

package/dist/mergeAxeResults.js

@@ -12,6 +12,7 @@ import { consoleLogger } from './logs.js';
 import itemTypeDescription from './constants/itemTypeDescription.js';
 import { oobeeAiHtmlETL, oobeeAiRules } from './constants/oobeeAi.js';
 import { buildHtmlGroups, convertItemsToReferences } from './mergeAxeResults/itemReferences.js';
+import { ItemsStore } from './mergeAxeResults/itemsStore.js';
 import { compressJsonFileStreaming, writeJsonAndBase64Files, writeJsonFileAndCompressedJsonFile, } from './mergeAxeResults/jsonArtifacts.js';
 import writeCsv from './mergeAxeResults/writeCsv.js';
 import writeScanDetailsCsv from './mergeAxeResults/writeScanDetailsCsv.js';
@@ -306,7 +307,7 @@ const writeSummaryPdf = async (storagePath, pagesScanned, filename = 'summary',
 };
 // Tracking WCAG occurrences
 const wcagOccurrencesMap = new Map();
-const pushResults = async (pageResults, allIssues, isCustomFlow) => {
+const pushResults = async (pageResults, allIssues, isCustomFlow, itemsStore) => {
     const { url, pageTitle, filePath } = pageResults;
     const totalIssuesInPage = new Set();
     Object.keys(pageResults.mustFix?.rules ?? {}).forEach(k => totalIssuesInPage.add(k));
@@ -318,13 +319,13 @@ const pushResults = async (pageResults, allIssues, isCustomFlow) => {
         totalIssues: totalIssuesInPage.size,
         totalOccurrences: 0,
     });
-    ['mustFix', 'goodToFix', 'needsReview', 'passed'].forEach(category => {
+    for (const category of ['mustFix', 'goodToFix', 'needsReview', 'passed']) {
         if (!pageResults[category])
-            return;
+            continue;
         const { totalItems, rules } = pageResults[category];
         const currCategoryFromAllIssues = allIssues.items[category];
         currCategoryFromAllIssues.totalItems += totalItems;
-        Object.keys(rules).forEach(rule => {
+        for (const rule of Object.keys(rules)) {
             const { description, axeImpact, helpUrl, conformance, totalItems: count, items, } = rules[rule];
             if (!(rule in currCategoryFromAllIssues.rules)) {
                 currCategoryFromAllIssues.rules[rule] = {
@@ -333,7 +334,6 @@ const pushResults = async (pageResults, allIssues, isCustomFlow) => {
                     helpUrl,
                     conformance,
                     totalItems: 0,
-                    // numberOfPagesAffectedAfterRedirects: 0,
                     pagesAffected: {},
                 };
             }
@@ -344,15 +344,12 @@ const pushResults = async (pageResults, allIssues, isCustomFlow) => {
                     if (!allIssues.wcagViolations.includes(c)) {
                         allIssues.wcagViolations.push(c);
                     }
-                    // Track WCAG criteria occurrences for Sentry
                     const currentCount = wcagOccurrencesMap.get(c) || 0;
                     wcagOccurrencesMap.set(c, currentCount + count);
                 });
             }
             const currRuleFromAllIssues = currCategoryFromAllIssues.rules[rule];
             currRuleFromAllIssues.totalItems += count;
-            // Build htmlGroups for pre-computed Group by HTML Element
-            buildHtmlGroups(currRuleFromAllIssues, items, url);
             if (isCustomFlow) {
                 const { pageIndex, pageImagePath, metadata } = pageResults;
                 currRuleFromAllIssues.pagesAffected[pageIndex] = {
@@ -360,18 +357,32 @@ const pushResults = async (pageResults, allIssues, isCustomFlow) => {
                     pageTitle,
                     pageImagePath,
                     metadata,
-                    items: [...items],
+                    itemsCount: items.length,
                 };
+                await itemsStore.appendPageItems(category, rule, {
+                    url,
+                    pageTitle,
+                    items,
+                    pageIndex,
+                    pageImagePath,
+                    metadata,
+                });
             }
             else if (!(url in currRuleFromAllIssues.pagesAffected)) {
                 currRuleFromAllIssues.pagesAffected[url] = {
                     pageTitle,
-                    items: [...items],
+                    itemsCount: items.length,
                     ...(filePath && { filePath }),
                 };
+                await itemsStore.appendPageItems(category, rule, {
+                    url,
+                    pageTitle,
+                    items,
+                    ...(filePath && { filePath }),
+                });
             }
-        });
-    });
+        }
+    }
 };
 const getTopTenIssues = allIssues => {
     const categories = ['mustFix', 'goodToFix'];
@@ -422,23 +433,20 @@ const flattenAndSortResults = (allIssues, isCustomFlow) => {
                 .map(pageEntry => {
                 if (isCustomFlow) {
                     const [pageIndex, pageInfo] = pageEntry;
-                    // Only update the occurrences map if not passed.
                     if (category !== 'passed') {
-                        urlOccurrencesMap.set(pageInfo.url, (urlOccurrencesMap.get(pageInfo.url) || 0) + pageInfo.items.length);
+                        urlOccurrencesMap.set(pageInfo.url, (urlOccurrencesMap.get(pageInfo.url) || 0) + (pageInfo.itemsCount || 0));
                     }
                     return { pageIndex, ...pageInfo };
                 }
                 const [url, pageInfo] = pageEntry;
                 if (category !== 'passed') {
-                    urlOccurrencesMap.set(url, (urlOccurrencesMap.get(url) || 0) + pageInfo.items.length);
+                    urlOccurrencesMap.set(url, (urlOccurrencesMap.get(url) || 0) + (pageInfo.itemsCount || 0));
                 }
                 return { url, ...pageInfo };
             })
-                // Sort pages so that those with the most items come first
-                .sort((page1, page2) => page2.items.length - page1.items.length);
+                .sort((page1, page2) => (page2.itemsCount || 0) - (page1.itemsCount || 0));
             return { rule, ...ruleInfo };
         })
-            // Sort the rules by totalItems (descending)
             .sort((rule1, rule2) => rule2.totalItems - rule1.totalItems);
     });
     // Sort top pages (assumes topFiveMostIssues is already populated)
@@ -478,18 +486,22 @@ const extractRuleAiData = (ruleId, totalItems, items, callback) => {
     };
 };
 // This is for telemetry purposes called within mergeAxeResults.ts
-export const createRuleIdJson = allIssues => {
+export const createRuleIdJson = async (allIssues, itemsStore) => {
     const compiledRuleJson = {};
-    ['mustFix', 'goodToFix', 'needsReview'].forEach(category => {
-        allIssues.items[category].rules.forEach(rule => {
-            const allItems = rule.pagesAffected.flatMap(page => page.items || []);
-            compiledRuleJson[rule.rule] = extractRuleAiData(rule.rule, rule.totalItems, allItems, () => {
-                rule.pagesAffected.forEach(p => {
-                    delete p.items;
-                });
-            });
-        });
-    });
+    for (const category of ['mustFix', 'goodToFix', 'needsReview']) {
+        for (const rule of allIssues.items[category].rules) {
+            let allItems = [];
+            if (itemsStore) {
+                for await (const entry of itemsStore.readRuleItems(category, rule.rule)) {
+                    allItems.push(...entry.items);
+                }
+            }
+            else {
+                allItems = rule.pagesAffected.flatMap(page => page.items || []);
+            }
+            compiledRuleJson[rule.rule] = extractRuleAiData(rule.rule, rule.totalItems, allItems);
+        }
+    }
     return compiledRuleJson;
 };
 // This is for telemetry purposes called from npmIndex (scanPage and scanHTML) where report is not generated
@@ -619,12 +631,18 @@ generateJsonFiles = false) => {
         },
     };
     const allFiles = await extractFileNames(intermediateDatasetsPath);
-    const jsonArray = await Promise.all(allFiles.map(async (file) => parseContentToJson(`${intermediateDatasetsPath}/${file}`)));
-    await Promise.all(jsonArray.map(async (pageResults) => {
-        await pushResults(pageResults, allIssues, isCustomFlow);
-    })).catch(flattenIssuesError => {
-        consoleLogger.error(`[generateArtifacts] Error flattening issues: ${flattenIssuesError?.stack || flattenIssuesError}`);
-    });
+    const itemsStore = new ItemsStore(storagePath);
+    for (const file of allFiles) {
+        try {
+            const pageResults = await parseContentToJson(`${intermediateDatasetsPath}/${file}`);
+            if (pageResults) {
+                await pushResults(pageResults, allIssues, isCustomFlow, itemsStore);
+            }
+        }
+        catch (flattenIssuesError) {
+            consoleLogger.error(`[generateArtifacts] Error processing ${file}: ${flattenIssuesError?.stack || flattenIssuesError}`);
+        }
+    }
     flattenAndSortResults(allIssues, isCustomFlow);
     const labelKey = scanType.toLowerCase() === 'custom' ? 'CustomFlowLabel' : 'Label';
     const labelValue = allIssues.customFlowLabel || 'N/A';
@@ -654,6 +672,14 @@ generateJsonFiles = false) => {
         createScreenshotsFolder(randomToken);
     }
     populateScanPagesDetail(allIssues);
+    // Build htmlGroups in a second pass from disk-backed items
+    for (const category of ['mustFix', 'goodToFix', 'needsReview', 'passed']) {
+        for (const rule of allIssues.items[category].rules) {
+            for await (const entry of itemsStore.readRuleItems(category, rule.rule)) {
+                buildHtmlGroups(rule, entry.items, entry.url);
+            }
+        }
+    }
     allIssues.wcagPassPercentage = getWcagPassPercentage(allIssues.wcagViolations, allIssues.advancedScanOptionsSummaryItems.showEnableWcagAaa);
     allIssues.progressPercentage = getProgressPercentage(allIssues.scanPagesDetail, allIssues.advancedScanOptionsSummaryItems.showEnableWcagAaa);
     allIssues.issuesPercentage = await getIssuesPercentage(allIssues.scanPagesDetail, allIssues.advancedScanOptionsSummaryItems.showEnableWcagAaa, allIssues.advancedScanOptionsSummaryItems.disableOobee);
@@ -703,9 +729,9 @@ generateJsonFiles = false) => {
         rest.moderate = axeImpactCount.moderate;
         rest.minor = axeImpactCount.minor;
     }
-    await writeCsv(allIssues, storagePath);
+    await writeCsv(allIssues, storagePath, itemsStore);
     await writeSitemap(pagesScanned, storagePath);
-    const { scanDataJsonFilePath, scanDataBase64FilePath, scanItemsJsonFilePath, scanItemsBase64FilePath, scanItemsSummaryJsonFilePath, scanItemsSummaryBase64FilePath, scanIssuesSummaryJsonFilePath, scanIssuesSummaryBase64FilePath, scanPagesDetailJsonFilePath, scanPagesDetailBase64FilePath, scanPagesSummaryJsonFilePath, scanPagesSummaryBase64FilePath, scanDataJsonFileSize, scanItemsJsonFileSize, } = await writeJsonAndBase64Files(allIssues, storagePath);
+    const { scanDataJsonFilePath, scanDataBase64FilePath, scanItemsJsonFilePath, scanItemsBase64FilePath, scanItemsSummaryJsonFilePath, scanItemsSummaryBase64FilePath, scanIssuesSummaryJsonFilePath, scanIssuesSummaryBase64FilePath, scanPagesDetailJsonFilePath, scanPagesDetailBase64FilePath, scanPagesSummaryJsonFilePath, scanPagesSummaryBase64FilePath, scanDataJsonFileSize, scanItemsJsonFileSize, } = await writeJsonAndBase64Files(allIssues, storagePath, itemsStore);
     // Removed BIG_RESULTS_THRESHOLD check - always use full scanItems
     await writeScanDetailsCsv(scanDataBase64FilePath, scanItemsBase64FilePath, scanItemsSummaryBase64FilePath, storagePath);
     await writeSummaryHTML(allIssues, storagePath);
@@ -729,13 +755,36 @@ generateJsonFiles = false) => {
     const browserChannel = getBrowserToRun(randomToken, BrowserTypes.CHROME, false).browserToRun;
     // Should consider refactor constants.userDataDirectory to be a parameter in future
     await retryFunction(() => writeSummaryPdf(storagePath, pagesScanned.length, 'summary', browserChannel, constants.userDataDirectory), 1);
+    // Suppress uncaught EPERM errors from lingering Crawlee async lock-file operations
+    // (Windows holds mandatory file locks; Crawlee may still attempt mkdir on .json.lock
+    // files after the crawl has finished). Without this, Node crashes with uncaughtException.
+    const crawleeEpermHandler = (err) => {
+        if (err.code === 'EPERM' && err.message?.includes('crawlee')) {
+            consoleLogger.info(`Suppressed lingering Crawlee storage error: ${err.message}`);
+            return;
+        }
+        // Re-throw non-crawlee EPERM errors so they aren't silently swallowed
+        throw err;
+    };
+    process.on('uncaughtException', crawleeEpermHandler);
+    process.on('unhandledRejection', crawleeEpermHandler);
     // Brief delay to allow lingering async crawlee storage operations to flush
-    await new Promise(resolve => setTimeout(resolve, 3000));
+    await new Promise(resolve => setTimeout(resolve, process.platform === 'win32' ? 5000 : 3000));
+    const crawleePath = path.join(storagePath, 'crawlee');
     try {
-        await fs.promises.rm(path.join(storagePath, 'crawlee'), { recursive: true, force: true });
+        await fs.promises.rm(crawleePath, { recursive: true, force: true });
     }
     catch (error) {
-        // Silently ignore — folder may already be gone or still locked
+        // On Windows, retry once after a delay if the folder is still locked
+        if (process.platform === 'win32') {
+            await new Promise(resolve => setTimeout(resolve, 3000));
+            try {
+                await fs.promises.rm(crawleePath, { recursive: true, force: true });
+            }
+            catch {
+                // Best-effort cleanup — leave the folder; report generation continues
+            }
+        }
     }
     try {
         await fs.promises.rm(path.join(storagePath, 'pdfs'), { recursive: true, force: true });
@@ -779,7 +828,9 @@ generateJsonFiles = false) => {
         printMessage([`Error in zipping results: ${error}`]);
     }
     // Generate scrubbed HTML Code Snippets
-    const ruleIdJson = createRuleIdJson(allIssues);
+    const ruleIdJson = await createRuleIdJson(allIssues, itemsStore);
+    // Clean up intermediate items files
+    await itemsStore.cleanup();
     // At the end of the function where results are generated, add:
     try {
         // Always send WCAG breakdown to Sentry, even if no violations were found
@@ -797,6 +848,8 @@ generateJsonFiles = false) => {
     }
     if (process.env.RUNNING_FROM_PH_GUI || process.env.OOBEE_VERBOSE)
         console.log('Report generated successfully');
+    process.removeListener('uncaughtException', crawleeEpermHandler);
+    process.removeListener('unhandledRejection', crawleeEpermHandler);
     return ruleIdJson;
 };
 export { writeHTML, compressJsonFileStreaming, convertItemsToReferences, flattenAndSortResults, populateScanPagesDetail, sendWcagBreakdownToSentry, getWcagPassPercentage, getProgressPercentage, getIssuesPercentage, itemTypeDescription, oobeeAiHtmlETL, oobeeAiRules, formatAboutStartTime, };