@govtechsg/oobee 0.10.83 → 0.10.84

package/dist/mergeAxeResults.js

@@ -1,20 +1,23 @@
+/* eslint-disable consistent-return */
+/* eslint-disable no-console */
 import fs, { ensureDirSync } from 'fs-extra';
 import printMessage from 'print-message';
 import path from 'path';
 import ejs from 'ejs';
 import { fileURLToPath } from 'url';
-import { createWriteStream } from 'fs';
-import { AsyncParser } from '@json2csv/node';
-import zlib from 'zlib';
-import { Base64Encode } from 'base64-stream';
-import { pipeline } from 'stream/promises';
-import * as Sentry from '@sentry/node';
-import constants, { BrowserTypes, ScannerTypes, sentryConfig, setSentryUser, WCAGclauses, a11yRuleShortDescriptionMap, disabilityBadgesMap, a11yRuleLongDescriptionMap, } from './constants/constants.js';
+import constants, { BrowserTypes, ScannerTypes, WCAGclauses, a11yRuleShortDescriptionMap, disabilityBadgesMap, a11yRuleLongDescriptionMap, } from './constants/constants.js';
 import { getBrowserToRun, getPlaywrightLaunchOptions } from './constants/common.js';
-import { createScreenshotsFolder, getStoragePath, getVersion, getWcagPassPercentage, getProgressPercentage, retryFunction, zipResults, getIssuesPercentage, getWcagCriteriaMap, categorizeWcagCriteria, getUserDataTxt, register, } from './utils.js';
+import { createScreenshotsFolder, getStoragePath, getVersion, getWcagPassPercentage, getProgressPercentage, retryFunction, zipResults, getIssuesPercentage, register, } from './utils.js';
 import { consoleLogger } from './logs.js';
 import itemTypeDescription from './constants/itemTypeDescription.js';
 import { oobeeAiHtmlETL, oobeeAiRules } from './constants/oobeeAi.js';
+import { buildHtmlGroups, convertItemsToReferences } from './mergeAxeResults/itemReferences.js';
+import { compressJsonFileStreaming, writeJsonAndBase64Files, writeJsonFileAndCompressedJsonFile, } from './mergeAxeResults/jsonArtifacts.js';
+import writeCsv from './mergeAxeResults/writeCsv.js';
+import writeScanDetailsCsv from './mergeAxeResults/writeScanDetailsCsv.js';
+import writeSitemap from './mergeAxeResults/writeSitemap.js';
+import populateScanPagesDetail from './mergeAxeResults/scanPages.js';
+import sendWcagBreakdownToSentry from './mergeAxeResults/sentryTelemetry.js';
 const filename = fileURLToPath(import.meta.url);
 const dirname = path.dirname(filename);
 const BUFFER_LIMIT = 100 * 1024 * 1024; // 100MB size
@@ -64,126 +67,6 @@ const parseContentToJson = async (rPath) => {
         return undefined;
     }
 };
-const writeCsv = async (allIssues, storagePath) => {
-    const csvOutput = createWriteStream(`${storagePath}/report.csv`, { encoding: 'utf8' });
-    const formatPageViolation = pageNum => {
-        if (pageNum < 0)
-            return 'Document';
-        return `Page ${pageNum}`;
-    };
-    // transform allIssues into the form:
-    // [['mustFix', rule1], ['mustFix', rule2], ['goodToFix', rule3], ...]
-    const getRulesByCategory = (allIssues) => {
-        return Object.entries(allIssues.items)
-            .filter(([category]) => category !== 'passed')
-            .reduce((prev, [category, value]) => {
-            const rulesEntries = Object.entries(value.rules);
-            rulesEntries.forEach(([, ruleInfo]) => {
-                prev.push([category, ruleInfo]);
-            });
-            return prev;
-        }, [])
-            .sort((a, b) => {
-            // sort rules according to severity, then ruleId
-            const compareCategory = -a[0].localeCompare(b[0]);
-            return compareCategory === 0 ? a[1].rule.localeCompare(b[1].rule) : compareCategory;
-        });
-    };
-    const flattenRule = catAndRule => {
-        const [severity, rule] = catAndRule;
-        const results = [];
-        const { rule: issueId, description: issueDescription, axeImpact, conformance, pagesAffected, helpUrl: learnMore, } = rule;
-        // format clauses as a string
-        const wcagConformance = conformance.join(',');
-        pagesAffected.sort((a, b) => a.url.localeCompare(b.url));
-        pagesAffected.forEach(affectedPage => {
-            const { url, items } = affectedPage;
-            items.forEach(item => {
-                const { html, page, message, xpath } = item;
-                const howToFix = message.replace(/(\r\n|\n|\r)/g, '\\n'); // preserve newlines as \n
-                const violation = html || formatPageViolation(page); // page is a number, not a string
-                const context = violation.replace(/(\r\n|\n|\r)/g, ''); // remove newlines
-                results.push({
-                    customFlowLabel: allIssues.customFlowLabel || '',
-                    deviceChosen: allIssues.deviceChosen || '',
-                    scanCompletedAt: allIssues.endTime ? allIssues.endTime.toISOString() : '',
-                    severity: severity || '',
-                    issueId: issueId || '',
-                    issueDescription: a11yRuleShortDescriptionMap[issueId] || issueDescription || '',
-                    wcagConformance: wcagConformance || '',
-                    url: url || '',
-                    pageTitle: affectedPage.pageTitle || 'No page title',
-                    context: context || '',
-                    howToFix: howToFix || '',
-                    axeImpact: axeImpact || '',
-                    xpath: xpath || '',
-                    learnMore: learnMore || '',
-                });
-            });
-        });
-        if (results.length === 0)
-            return {};
-        return results;
-    };
-    const opts = {
-        transforms: [getRulesByCategory, flattenRule],
-        fields: [
-            'customFlowLabel',
-            'deviceChosen',
-            'scanCompletedAt',
-            'severity',
-            'issueId',
-            'issueDescription',
-            'wcagConformance',
-            'url',
-            'pageTitle',
-            'context',
-            'howToFix',
-            'axeImpact',
-            'xpath',
-            'learnMore',
-        ],
-        includeEmptyRows: true,
-    };
-    // Create the parse stream (it's asynchronous)
-    const parser = new AsyncParser(opts);
-    const parseStream = parser.parse(allIssues);
-    // Pipe JSON2CSV output into the file, but don't end automatically
-    parseStream.pipe(csvOutput, { end: false });
-    // Once JSON2CSV is done writing all normal rows, append any "pagesNotScanned"
-    parseStream.on('end', () => {
-        if (allIssues.pagesNotScanned && allIssues.pagesNotScanned.length > 0) {
-            csvOutput.write('\n');
-            allIssues.pagesNotScanned.forEach(page => {
-                const skippedPage = {
-                    customFlowLabel: allIssues.customFlowLabel || '',
-                    deviceChosen: allIssues.deviceChosen || '',
-                    scanCompletedAt: allIssues.endTime ? allIssues.endTime.toISOString() : '',
-                    severity: 'error',
-                    issueId: 'error-pages-skipped',
-                    issueDescription: page.metadata
-                        ? page.metadata
-                        : 'An unknown error caused the page to be skipped',
-                    wcagConformance: '',
-                    url: page.url || page || '',
-                    pageTitle: 'Error',
-                    context: '',
-                    howToFix: '',
-                    axeImpact: '',
-                    xpath: '',
-                    learnMore: '',
-                };
-                csvOutput.write(`${Object.values(skippedPage).join(',')}\n`);
-            });
-        }
-        // Now close the CSV file
-        csvOutput.end();
-    });
-    parseStream.on('error', err => {
-        console.error('Error parsing CSV:', err);
-        csvOutput.end();
-    });
-};
 const compileHtmlWithEJS = async (allIssues, storagePath, htmlFilename = 'report') => {
     const htmlFilePath = `${path.join(storagePath, htmlFilename)}.html`;
     const ejsString = fs.readFileSync(path.join(dirname, './static/ejs/report.ejs'), 'utf-8');
@@ -239,7 +122,7 @@ const writeHTML = async (allIssues, storagePath, htmlFilename = 'report', scanDe
     // Create lighter version with item references for embedding in HTML
     const scanItemsWithHtmlGroupRefs = convertItemsToReferences(allIssues);
     // Write the lighter items to a file and get the base64 path
-    const { jsonFilePath: scanItemsWithHtmlGroupRefsJsonFilePath, base64FilePath: scanItemsWithHtmlGroupRefsBase64FilePath } = await writeJsonFileAndCompressedJsonFile(scanItemsWithHtmlGroupRefs.items, storagePath, 'scanItems-light');
+    const { jsonFilePath: scanItemsWithHtmlGroupRefsJsonFilePath, base64FilePath: scanItemsWithHtmlGroupRefsBase64FilePath, } = await writeJsonFileAndCompressedJsonFile(scanItemsWithHtmlGroupRefs.items, storagePath, 'scanItems-light');
     return new Promise((resolve, reject) => {
         const scanDetailsReadStream = fs.createReadStream(scanDetailsFilePath, {
             encoding: 'utf8',
@@ -262,7 +145,7 @@ const writeHTML = async (allIssues, storagePath, htmlFilename = 'report', scanDe
         };
         outputStream.write(prefixData);
         // For Proxied AI environments only
-        outputStream.write(`let proxyUrl = "${process.env.PROXY_API_BASE_URL || ""}"\n`);
+        outputStream.write(`let proxyUrl = "${process.env.PROXY_API_BASE_URL || ''}"\n`);
         // Initialize GenAI feature flag
         outputStream.write(`
   // Fetch GenAI feature flag from backend
@@ -290,12 +173,12 @@ const writeHTML = async (allIssues, storagePath, htmlFilename = 'report', scanDe
     console.warn('Skipping fetch GenAI feature as it is local report');
   }
   \n`);
-        outputStream.write("</script>\n<script type=\"text/plain\" id=\"scanDataRaw\">");
+        outputStream.write('</script>\n<script type="text/plain" id="scanDataRaw">');
         scanDetailsReadStream.pipe(outputStream, { end: false });
         scanDetailsReadStream.on('end', async () => {
-            outputStream.write("</script>\n<script>\n");
+            outputStream.write('</script>\n<script>\n');
             outputStream.write("var scanDataPromise = (async () => { console.log('Loading scanData...'); scanData = await decodeUnzipParse(document.getElementById('scanDataRaw').textContent); })();\n");
-            outputStream.write("</script>\n");
+            outputStream.write('</script>\n');
             // Write scanItems in 2MB chunks using a stream to avoid loading entire file into memory
             try {
                 let chunkIndex = 1;
@@ -307,7 +190,7 @@ const writeHTML = async (allIssues, storagePath, htmlFilename = 'report', scanDe
                     outputStream.write(`<script type="text/plain" id="scanItemsRaw${chunkIndex}">${chunk}</script>\n`);
                     chunkIndex++;
                 }
-                outputStream.write("<script>\n");
+                outputStream.write('<script>\n');
                 outputStream.write(`
 var scanItemsPromise = (async () => {
   console.log('Loading scanItems...');
@@ -356,12 +239,6 @@ const writeSummaryHTML = async (allIssues, storagePath, htmlFilename = 'summary'
     const html = template(allIssues);
     fs.writeFileSync(`${storagePath}/${htmlFilename}.html`, html);
 };
-const writeSitemap = async (pagesScanned, storagePath) => {
-    const sitemapPath = path.join(storagePath, 'sitemap.txt');
-    const content = pagesScanned.map(p => p.url).join('\n');
-    await fs.writeFile(sitemapPath, content, { encoding: 'utf-8' });
-    consoleLogger.info(`Sitemap written to ${sitemapPath}`);
-};
 const cleanUpJsonFiles = async (filesToDelete) => {
     consoleLogger.info('Cleaning up JSON files...');
     filesToDelete.forEach(file => {
@@ -369,379 +246,6 @@ const cleanUpJsonFiles = async (filesToDelete) => {
         consoleLogger.info(`Deleted ${file}`);
     });
 };
-function* serializeObject(obj, depth = 0, indent = '  ') {
-    const currentIndent = indent.repeat(depth);
-    const nextIndent = indent.repeat(depth + 1);
-    if (obj instanceof Date) {
-        yield JSON.stringify(obj.toISOString());
-        return;
-    }
-    if (Array.isArray(obj)) {
-        yield '[\n';
-        for (let i = 0; i < obj.length; i++) {
-            if (i > 0)
-                yield ',\n';
-            yield nextIndent;
-            yield* serializeObject(obj[i], depth + 1, indent);
-        }
-        yield `\n${currentIndent}]`;
-        return;
-    }
-    if (obj !== null && typeof obj === 'object') {
-        yield '{\n';
-        const keys = Object.keys(obj);
-        for (let i = 0; i < keys.length; i++) {
-            const key = keys[i];
-            if (i > 0)
-                yield ',\n';
-            yield `${nextIndent}${JSON.stringify(key)}: `;
-            yield* serializeObject(obj[key], depth + 1, indent);
-        }
-        yield `\n${currentIndent}}`;
-        return;
-    }
-    if (obj === null || typeof obj === 'function' || typeof obj === 'undefined') {
-        yield 'null';
-        return;
-    }
-    yield JSON.stringify(obj);
-}
-function writeLargeJsonToFile(obj, filePath) {
-    return new Promise((resolve, reject) => {
-        const writeStream = fs.createWriteStream(filePath, { encoding: 'utf8' });
-        writeStream.on('error', error => {
-            consoleLogger.error('Stream error:', error);
-            reject(error);
-        });
-        writeStream.on('finish', () => {
-            consoleLogger.info(`JSON file written successfully: ${filePath}`);
-            resolve(true);
-        });
-        const generator = serializeObject(obj);
-        function write() {
-            let next;
-            while (!(next = generator.next()).done) {
-                if (!writeStream.write(next.value)) {
-                    writeStream.once('drain', write);
-                    return;
-                }
-            }
-            writeStream.end();
-        }
-        write();
-    });
-}
-const writeLargeScanItemsJsonToFile = async (obj, filePath) => {
-    return new Promise((resolve, reject) => {
-        const writeStream = fs.createWriteStream(filePath, { flags: 'a', encoding: 'utf8' });
-        const writeQueue = [];
-        let isWriting = false;
-        const processNextWrite = async () => {
-            if (isWriting || writeQueue.length === 0)
-                return;
-            isWriting = true;
-            const data = writeQueue.shift();
-            try {
-                if (!writeStream.write(data)) {
-                    await new Promise(resolve => {
-                        writeStream.once('drain', () => {
-                            resolve();
-                        });
-                    });
-                }
-            }
-            catch (error) {
-                writeStream.destroy(error);
-                return;
-            }
-            isWriting = false;
-            processNextWrite();
-        };
-        const queueWrite = (data) => {
-            writeQueue.push(data);
-            processNextWrite();
-        };
-        writeStream.on('error', error => {
-            consoleLogger.error(`Error writing object to JSON file: ${error}`);
-            reject(error);
-        });
-        writeStream.on('finish', () => {
-            consoleLogger.info(`JSON file written successfully: ${filePath}`);
-            resolve(true);
-        });
-        try {
-            queueWrite('{\n');
-            const keys = Object.keys(obj);
-            keys.forEach((key, i) => {
-                const value = obj[key];
-                if (value === null || typeof value !== 'object' || Array.isArray(value)) {
-                    queueWrite(`  "${key}": ${JSON.stringify(value)}`);
-                }
-                else {
-                    queueWrite(`  "${key}": {\n`);
-                    const { rules, ...otherProperties } = value;
-                    // Write other properties
-                    Object.entries(otherProperties).forEach(([propKey, propValue], j) => {
-                        const propValueString = propValue === null ||
-                            typeof propValue === 'function' ||
-                            typeof propValue === 'undefined'
-                            ? 'null'
-                            : JSON.stringify(propValue);
-                        queueWrite(`    "${propKey}": ${propValueString}`);
-                        if (j < Object.keys(otherProperties).length - 1 || (rules && rules.length >= 0)) {
-                            queueWrite(',\n');
-                        }
-                        else {
-                            queueWrite('\n');
-                        }
-                    });
-                    if (rules && Array.isArray(rules)) {
-                        queueWrite('    "rules": [\n');
-                        rules.forEach((rule, j) => {
-                            queueWrite('      {\n');
-                            const { pagesAffected, ...otherRuleProperties } = rule;
-                            Object.entries(otherRuleProperties).forEach(([ruleKey, ruleValue], k) => {
-                                const ruleValueString = ruleValue === null ||
-                                    typeof ruleValue === 'function' ||
-                                    typeof ruleValue === 'undefined'
-                                    ? 'null'
-                                    : JSON.stringify(ruleValue);
-                                queueWrite(`        "${ruleKey}": ${ruleValueString}`);
-                                if (k < Object.keys(otherRuleProperties).length - 1 || pagesAffected) {
-                                    queueWrite(',\n');
-                                }
-                                else {
-                                    queueWrite('\n');
-                                }
-                            });
-                            if (pagesAffected && Array.isArray(pagesAffected)) {
-                                queueWrite('        "pagesAffected": [\n');
-                                pagesAffected.forEach((page, p) => {
-                                    const pageJson = JSON.stringify(page, null, 2)
-                                        .split('\n')
-                                        .map((line, idx) => (idx === 0 ? `          ${line}` : `          ${line}`))
-                                        .join('\n');
-                                    queueWrite(pageJson);
-                                    if (p < pagesAffected.length - 1) {
-                                        queueWrite(',\n');
-                                    }
-                                    else {
-                                        queueWrite('\n');
-                                    }
-                                });
-                                queueWrite('        ]');
-                            }
-                            queueWrite('\n      }');
-                            if (j < rules.length - 1) {
-                                queueWrite(',\n');
-                            }
-                            else {
-                                queueWrite('\n');
-                            }
-                        });
-                        queueWrite('    ]');
-                    }
-                    queueWrite('\n  }');
-                }
-                if (i < keys.length - 1) {
-                    queueWrite(',\n');
-                }
-                else {
-                    queueWrite('\n');
-                }
-            });
-            queueWrite('}\n');
-            // Ensure all queued writes are processed before ending
-            const checkQueueAndEnd = () => {
-                if (writeQueue.length === 0 && !isWriting) {
-                    writeStream.end();
-                }
-                else {
-                    setTimeout(checkQueueAndEnd, 100);
-                }
-            };
-            checkQueueAndEnd();
-        }
-        catch (err) {
-            writeStream.destroy(err);
-            reject(err);
-        }
-    });
-};
-async function compressJsonFileStreaming(inputPath, outputPath) {
-    // Create the read and write streams
-    const readStream = fs.createReadStream(inputPath);
-    const writeStream = fs.createWriteStream(outputPath);
-    // Create a gzip transform stream
-    const gzip = zlib.createGzip();
-    // Create a Base64 transform stream
-    const base64Encode = new Base64Encode();
-    // Pipe the streams:
-    //   read -> gzip -> base64 -> write
-    await pipeline(readStream, gzip, base64Encode, writeStream);
-    consoleLogger.info(`File successfully compressed and saved to ${outputPath}`);
-}
-const writeJsonFileAndCompressedJsonFile = async (data, storagePath, filename) => {
-    try {
-        consoleLogger.info(`Writing JSON to ${filename}.json`);
-        const jsonFilePath = path.join(storagePath, `${filename}.json`);
-        if (filename === 'scanItems') {
-            await writeLargeScanItemsJsonToFile(data, jsonFilePath);
-        }
-        else {
-            await writeLargeJsonToFile(data, jsonFilePath);
-        }
-        consoleLogger.info(`Reading ${filename}.json, gzipping and base64 encoding it into ${filename}.json.gz.b64`);
-        const base64FilePath = path.join(storagePath, `${filename}.json.gz.b64`);
-        await compressJsonFileStreaming(jsonFilePath, base64FilePath);
-        consoleLogger.info(`Finished compression and base64 encoding for ${filename}`);
-        return {
-            jsonFilePath,
-            base64FilePath,
-        };
-    }
-    catch (error) {
-        consoleLogger.error(`Error compressing and encoding ${filename}`);
-        throw error;
-    }
-};
-const streamEncodedDataToFile = async (inputFilePath, writeStream, appendComma) => {
-    const readStream = fs.createReadStream(inputFilePath, { encoding: 'utf8' });
-    let isFirstChunk = true;
-    for await (const chunk of readStream) {
-        if (isFirstChunk) {
-            isFirstChunk = false;
-            writeStream.write(chunk);
-        }
-        else {
-            writeStream.write(chunk);
-        }
-    }
-    if (appendComma) {
-        writeStream.write(',');
-    }
-};
-const writeJsonAndBase64Files = async (allIssues, storagePath) => {
-    const { items, ...rest } = allIssues;
-    const { jsonFilePath: scanDataJsonFilePath, base64FilePath: scanDataBase64FilePath } = await writeJsonFileAndCompressedJsonFile(rest, storagePath, 'scanData');
-    const { jsonFilePath: scanItemsJsonFilePath, base64FilePath: scanItemsBase64FilePath } = await writeJsonFileAndCompressedJsonFile({ oobeeAppVersion: allIssues.oobeeAppVersion, ...items }, storagePath, 'scanItems');
-    // Add pagesAffectedCount to each rule in items and sort them in descending order of pagesAffectedCount
-    ['mustFix', 'goodToFix', 'needsReview', 'passed'].forEach(category => {
-        if (items[category].rules && Array.isArray(items[category].rules)) {
-            items[category].rules.forEach(rule => {
-                rule.pagesAffectedCount = Array.isArray(rule.pagesAffected) ? rule.pagesAffected.length : 0;
-            });
-            // Sort in descending order of pagesAffectedCount
-            items[category].rules.sort((a, b) => (b.pagesAffectedCount || 0) - (a.pagesAffectedCount || 0));
-        }
-    });
-    // Refactor scanIssuesSummary to reuse the structure by stripping out pagesAffected
-    const scanIssuesSummary = {
-        // Replace rule descriptions with short descriptions from the map
-        mustFix: items.mustFix.rules.map(({ pagesAffected, ...ruleInfo }) => ({
-            ...ruleInfo,
-            description: a11yRuleShortDescriptionMap[ruleInfo.rule] || ruleInfo.description,
-        })),
-        goodToFix: items.goodToFix.rules.map(({ pagesAffected, ...ruleInfo }) => ({
-            ...ruleInfo,
-            description: a11yRuleShortDescriptionMap[ruleInfo.rule] || ruleInfo.description,
-        })),
-        needsReview: items.needsReview.rules.map(({ pagesAffected, ...ruleInfo }) => ({
-            ...ruleInfo,
-            description: a11yRuleShortDescriptionMap[ruleInfo.rule] || ruleInfo.description,
-        })),
-        passed: items.passed.rules.map(({ pagesAffected, ...ruleInfo }) => ({
-            ...ruleInfo,
-            description: a11yRuleShortDescriptionMap[ruleInfo.rule] || ruleInfo.description,
-        })),
-    };
-    // Write out the scanIssuesSummary JSON using the new structure
-    const { jsonFilePath: scanIssuesSummaryJsonFilePath, base64FilePath: scanIssuesSummaryBase64FilePath, } = await writeJsonFileAndCompressedJsonFile({ oobeeAppVersion: allIssues.oobeeAppVersion, ...scanIssuesSummary }, storagePath, 'scanIssuesSummary');
-    // scanItemsSummary
-    // the below mutates the original items object, since it is expensive to clone
-    items.mustFix.rules.forEach(rule => {
-        rule.pagesAffected.forEach(page => {
-            page.itemsCount = page.items.length;
-        });
-    });
-    items.goodToFix.rules.forEach(rule => {
-        rule.pagesAffected.forEach(page => {
-            page.itemsCount = page.items.length;
-        });
-    });
-    items.needsReview.rules.forEach(rule => {
-        rule.pagesAffected.forEach(page => {
-            page.itemsCount = page.items.length;
-        });
-    });
-    items.passed.rules.forEach(rule => {
-        rule.pagesAffected.forEach(page => {
-            page.itemsCount = page.items.length;
-        });
-    });
-    items.mustFix.totalRuleIssues = items.mustFix.rules.length;
-    items.goodToFix.totalRuleIssues = items.goodToFix.rules.length;
-    items.needsReview.totalRuleIssues = items.needsReview.rules.length;
-    items.passed.totalRuleIssues = items.passed.rules.length;
-    const { pagesScanned, topTenPagesWithMostIssues, pagesNotScanned, wcagLinks, wcagPassPercentage, progressPercentage, issuesPercentage, totalPagesScanned, totalPagesNotScanned, topTenIssues, } = rest;
-    const summaryItems = {
-        mustFix: {
-            totalItems: items.mustFix?.totalItems || 0,
-            totalRuleIssues: items.mustFix?.totalRuleIssues || 0,
-        },
-        goodToFix: {
-            totalItems: items.goodToFix?.totalItems || 0,
-            totalRuleIssues: items.goodToFix?.totalRuleIssues || 0,
-        },
-        needsReview: {
-            totalItems: items.needsReview?.totalItems || 0,
-            totalRuleIssues: items.needsReview?.totalRuleIssues || 0,
-        },
-        topTenPagesWithMostIssues,
-        wcagLinks,
-        wcagPassPercentage,
-        progressPercentage,
-        issuesPercentage,
-        totalPagesScanned,
-        totalPagesNotScanned,
-        topTenIssues,
-    };
-    const { jsonFilePath: scanItemsSummaryJsonFilePath, base64FilePath: scanItemsSummaryBase64FilePath, } = await writeJsonFileAndCompressedJsonFile({ oobeeAppVersion: allIssues.oobeeAppVersion, ...summaryItems }, storagePath, 'scanItemsSummary');
-    const { jsonFilePath: scanPagesDetailJsonFilePath, base64FilePath: scanPagesDetailBase64FilePath, } = await writeJsonFileAndCompressedJsonFile({ oobeeAppVersion: allIssues.oobeeAppVersion, ...allIssues.scanPagesDetail }, storagePath, 'scanPagesDetail');
-    const { jsonFilePath: scanPagesSummaryJsonFilePath, base64FilePath: scanPagesSummaryBase64FilePath, } = await writeJsonFileAndCompressedJsonFile({ oobeeAppVersion: allIssues.oobeeAppVersion, ...allIssues.scanPagesSummary }, storagePath, 'scanPagesSummary');
-    return {
-        scanDataJsonFilePath,
-        scanDataBase64FilePath,
-        scanItemsJsonFilePath,
-        scanItemsBase64FilePath,
-        scanItemsSummaryJsonFilePath,
-        scanItemsSummaryBase64FilePath,
-        scanIssuesSummaryJsonFilePath,
-        scanIssuesSummaryBase64FilePath,
-        scanPagesDetailJsonFilePath,
-        scanPagesDetailBase64FilePath,
-        scanPagesSummaryJsonFilePath,
-        scanPagesSummaryBase64FilePath,
-        scanDataJsonFileSize: fs.statSync(scanDataJsonFilePath).size,
-        scanItemsJsonFileSize: fs.statSync(scanItemsJsonFilePath).size,
-    };
-};
-const writeScanDetailsCsv = async (scanDataFilePath, scanItemsFilePath, scanItemsSummaryFilePath, storagePath) => {
-    const filePath = path.join(storagePath, 'scanDetails.csv');
-    const csvWriteStream = fs.createWriteStream(filePath, { encoding: 'utf8' });
-    const directoryPath = path.dirname(filePath);
-    if (!fs.existsSync(directoryPath)) {
-        fs.mkdirSync(directoryPath, { recursive: true });
-    }
-    csvWriteStream.write('scanData_base64,scanItems_base64,scanItemsSummary_base64\n');
-    await streamEncodedDataToFile(scanDataFilePath, csvWriteStream, true);
-    await streamEncodedDataToFile(scanItemsFilePath, csvWriteStream, true);
-    await streamEncodedDataToFile(scanItemsSummaryFilePath, csvWriteStream, false);
-    await new Promise((resolve, reject) => {
-        csvWriteStream.end(resolve);
-        csvWriteStream.on('error', reject);
-    });
-};
 const writeSummaryPdf = async (storagePath, pagesScanned, filename = 'summary', browser, _userDataDirectory) => {
     const htmlFilePath = `${storagePath}/${filename}.html`;
     const fileDestinationPath = `${storagePath}/${filename}.pdf`;
@@ -776,16 +280,6 @@ const writeSummaryPdf = async (storagePath, pagesScanned, filename = 'summary',
 };
 // Tracking WCAG occurrences
 const wcagOccurrencesMap = new Map();
-// Format WCAG tag in requested format: wcag111a_Occurrences
-const formatWcagTag = async (wcagId) => {
-    // Get dynamic WCAG criteria map
-    const wcagCriteriaMap = await getWcagCriteriaMap();
-    if (wcagCriteriaMap[wcagId]) {
-        const { level } = wcagCriteriaMap[wcagId];
-        return `${wcagId}${level}_Occurrences`;
-    }
-    return null;
-};
 const pushResults = async (pageResults, allIssues, isCustomFlow) => {
     const { url, pageTitle, filePath } = pageResults;
     const totalIssuesInPage = new Set();
@@ -843,73 +337,16 @@ const pushResults = async (pageResults, allIssues, isCustomFlow) => {
                     items: [...items],
                 };
             }
-            else {
-                if (!(url in currRuleFromAllIssues.pagesAffected)) {
-                    currRuleFromAllIssues.pagesAffected[url] = {
-                        pageTitle,
-                        items: [...items],
-                        ...(filePath && { filePath }),
-                    };
-                }
+            else if (!(url in currRuleFromAllIssues.pagesAffected)) {
+                currRuleFromAllIssues.pagesAffected[url] = {
+                    pageTitle,
+                    items: [...items],
+                    ...(filePath && { filePath }),
+                };
             }
         });
     });
 };
-/**
- * Builds pre-computed HTML groups to optimize Group by HTML Element functionality.
- * Keys are composite "html\x00xpath" strings to ensure unique matching per element instance.
- */
-const buildHtmlGroups = (rule, items, pageUrl) => {
-    if (!rule.htmlGroups) {
-        rule.htmlGroups = {};
-    }
-    items.forEach(item => {
-        // Use composite key of html + xpath for precise matching
-        const htmlKey = `${item.html || 'No HTML element'}\x00${item.xpath || ''}`;
-        if (!rule.htmlGroups[htmlKey]) {
-            // Create new group with the first occurrence
-            rule.htmlGroups[htmlKey] = {
-                html: item.html || '',
-                xpath: item.xpath || '',
-                message: item.message || '',
-                screenshotPath: item.screenshotPath || '',
-                displayNeedsReview: item.displayNeedsReview,
-                pageUrls: [],
-            };
-        }
-        if (!rule.htmlGroups[htmlKey].pageUrls.includes(pageUrl)) {
-            rule.htmlGroups[htmlKey].pageUrls.push(pageUrl);
-        }
-    });
-};
-/**
- * Converts items in pagesAffected to references (html\x00xpath composite keys) for embedding in HTML report.
- * Additionally, it deep-clones allIssues, replaces page.items objects with composite reference keys.
- * Those refs are specifically for htmlGroups lookup (html + xpath).
- */
-export const convertItemsToReferences = (allIssues) => {
-    const cloned = JSON.parse(JSON.stringify(allIssues));
-    ['mustFix', 'goodToFix', 'needsReview', 'passed'].forEach(category => {
-        if (!cloned.items[category]?.rules)
-            return;
-        cloned.items[category].rules.forEach((rule) => {
-            if (!rule.pagesAffected || !rule.htmlGroups)
-                return;
-            rule.pagesAffected.forEach((page) => {
-                if (!page.items)
-                    return;
-                page.items = page.items.map((item) => {
-                    if (typeof item === 'string')
-                        return item; // Already a reference
-                    // Use composite key matching buildHtmlGroups
-                    const htmlKey = `${item.html || 'No HTML element'}\x00${item.xpath || ''}`;
-                    return htmlKey;
-                });
-            });
-        });
-    });
-    return cloned;
-};
 const getTopTenIssues = allIssues => {
     const categories = ['mustFix', 'goodToFix'];
     const rulesWithCounts = [];
@@ -1048,348 +485,6 @@ const moveElemScreenshots = (randomToken, storagePath) => {
         fs.moveSync(currentScreenshotsPath, resultsScreenshotsPath);
     }
 };
-/**
- * Build allIssues.scanPagesDetail and allIssues.scanPagesSummary
- * by analyzing pagesScanned (including mustFix/goodToFix/etc.).
- */
-function populateScanPagesDetail(allIssues) {
-    // --------------------------------------------
-    // 1) Gather your "scanned" pages from allIssues
-    // --------------------------------------------
-    const allScannedPages = Array.isArray(allIssues.pagesScanned) ? allIssues.pagesScanned : [];
-    // --------------------------------------------
-    // 2) Define category constants (optional, just for clarity)
-    // --------------------------------------------
-    const mustFixCategory = 'mustFix';
-    const goodToFixCategory = 'goodToFix';
-    const needsReviewCategory = 'needsReview';
-    const passedCategory = 'passed';
-    // --------------------------------------------
-    // 4) We'll accumulate pages in a map keyed by URL
-    // --------------------------------------------
-    const pagesMap = {};
-    // --------------------------------------------
-    // 5) Build pagesMap by iterating over each category in allIssues.items
-    // --------------------------------------------
-    Object.entries(allIssues.items).forEach(([categoryName, categoryData]) => {
-        if (!categoryData?.rules)
-            return; // no rules in this category? skip
-        categoryData.rules.forEach(rule => {
-            const { rule: ruleId, conformance = [] } = rule;
-            rule.pagesAffected.forEach(p => {
-                const { url, pageTitle, items = [] } = p;
-                const itemsCount = items.length;
-                // Ensure the page is in pagesMap
-                if (!pagesMap[url]) {
-                    pagesMap[url] = {
-                        pageTitle,
-                        url,
-                        totalOccurrencesFailedIncludingNeedsReview: 0,
-                        totalOccurrencesFailedExcludingNeedsReview: 0,
-                        totalOccurrencesNeedsReview: 0,
-                        totalOccurrencesPassed: 0,
-                        typesOfIssues: {},
-                    };
-                }
-                // Ensure the rule is present for this page
-                if (!pagesMap[url].typesOfIssues[ruleId]) {
-                    pagesMap[url].typesOfIssues[ruleId] = {
-                        ruleId,
-                        wcagConformance: conformance,
-                        occurrencesMustFix: 0,
-                        occurrencesGoodToFix: 0,
-                        occurrencesNeedsReview: 0,
-                        occurrencesPassed: 0,
-                    };
-                }
-                // Depending on the category, increment the relevant occurrence counts
-                if (categoryName === mustFixCategory) {
-                    pagesMap[url].typesOfIssues[ruleId].occurrencesMustFix += itemsCount;
-                    pagesMap[url].totalOccurrencesFailedIncludingNeedsReview += itemsCount;
-                    pagesMap[url].totalOccurrencesFailedExcludingNeedsReview += itemsCount;
-                }
-                else if (categoryName === goodToFixCategory) {
-                    pagesMap[url].typesOfIssues[ruleId].occurrencesGoodToFix += itemsCount;
-                    pagesMap[url].totalOccurrencesFailedIncludingNeedsReview += itemsCount;
-                    pagesMap[url].totalOccurrencesFailedExcludingNeedsReview += itemsCount;
-                }
-                else if (categoryName === needsReviewCategory) {
-                    pagesMap[url].typesOfIssues[ruleId].occurrencesNeedsReview += itemsCount;
-                    pagesMap[url].totalOccurrencesFailedIncludingNeedsReview += itemsCount;
-                    pagesMap[url].totalOccurrencesNeedsReview += itemsCount;
-                }
-                else if (categoryName === passedCategory) {
-                    pagesMap[url].typesOfIssues[ruleId].occurrencesPassed += itemsCount;
-                    pagesMap[url].totalOccurrencesPassed += itemsCount;
-                }
-            });
-        });
-    });
-    // --------------------------------------------
-    // 6) Separate scanned pages into “affected” vs. “notAffected”
-    // --------------------------------------------
-    const pagesInMap = Object.values(pagesMap); // All pages that have some record in pagesMap
-    const pagesInMapUrls = new Set(Object.keys(pagesMap));
-    // (a) Pages with only passed (no mustFix/goodToFix/needsReview)
-    const pagesAllPassed = pagesInMap.filter(p => p.totalOccurrencesFailedIncludingNeedsReview === 0);
-    // (b) Pages that do NOT appear in pagesMap at all => scanned but no items found
-    const pagesNoEntries = allScannedPages
-        .filter(sp => !pagesInMapUrls.has(sp.url))
-        .map(sp => ({
-        pageTitle: sp.pageTitle,
-        url: sp.url,
-        totalOccurrencesFailedIncludingNeedsReview: 0,
-        totalOccurrencesFailedExcludingNeedsReview: 0,
-        totalOccurrencesNeedsReview: 0,
-        totalOccurrencesPassed: 0,
-        typesOfIssues: {},
-    }));
-    // Combine these into "notAffected"
-    const pagesNotAffectedRaw = [...pagesAllPassed, ...pagesNoEntries];
-    // "affected" pages => have at least 1 mustFix/goodToFix/needsReview
-    const pagesAffectedRaw = pagesInMap.filter(p => p.totalOccurrencesFailedIncludingNeedsReview > 0);
-    // --------------------------------------------
-    // 7) Transform both arrays to the final shape
-    // --------------------------------------------
-    function transformPageData(page) {
-        const typesOfIssuesArray = Object.values(page.typesOfIssues);
-        // Compute sums for each failing category
-        const mustFixSum = typesOfIssuesArray.reduce((acc, r) => acc + r.occurrencesMustFix, 0);
-        const goodToFixSum = typesOfIssuesArray.reduce((acc, r) => acc + r.occurrencesGoodToFix, 0);
-        const needsReviewSum = typesOfIssuesArray.reduce((acc, r) => acc + r.occurrencesNeedsReview, 0);
-        // Build categoriesPresent based on nonzero failing counts
-        const categoriesPresent = [];
-        if (mustFixSum > 0)
-            categoriesPresent.push('mustFix');
-        if (goodToFixSum > 0)
-            categoriesPresent.push('goodToFix');
-        if (needsReviewSum > 0)
-            categoriesPresent.push('needsReview');
-        // Count how many rules have failing issues
-        const failedRuleIds = new Set();
-        typesOfIssuesArray.forEach(r => {
-            if ((r.occurrencesMustFix || 0) > 0 ||
-                (r.occurrencesGoodToFix || 0) > 0 ||
-                (r.occurrencesNeedsReview || 0) > 0) {
-                failedRuleIds.add(r.ruleId); // Ensure ruleId is unique
-            }
-        });
-        const failedRuleCount = failedRuleIds.size;
-        // Possibly these two for future convenience
-        const typesOfIssuesExcludingNeedsReviewCount = typesOfIssuesArray.filter(r => (r.occurrencesMustFix || 0) + (r.occurrencesGoodToFix || 0) > 0).length;
-        const typesOfIssuesExclusiveToNeedsReviewCount = typesOfIssuesArray.filter(r => (r.occurrencesNeedsReview || 0) > 0 &&
-            (r.occurrencesMustFix || 0) === 0 &&
-            (r.occurrencesGoodToFix || 0) === 0).length;
-        // Aggregate wcagConformance for rules that actually fail
-        const allConformance = typesOfIssuesArray.reduce((acc, curr) => {
-            const nonPassedCount = (curr.occurrencesMustFix || 0) +
-                (curr.occurrencesGoodToFix || 0) +
-                (curr.occurrencesNeedsReview || 0);
-            if (nonPassedCount > 0) {
-                return acc.concat(curr.wcagConformance || []);
-            }
-            return acc;
-        }, []);
-        // Remove duplicates
-        const conformance = Array.from(new Set(allConformance));
-        return {
-            pageTitle: page.pageTitle,
-            url: page.url,
-            totalOccurrencesFailedIncludingNeedsReview: page.totalOccurrencesFailedIncludingNeedsReview,
-            totalOccurrencesFailedExcludingNeedsReview: page.totalOccurrencesFailedExcludingNeedsReview,
-            totalOccurrencesMustFix: mustFixSum,
-            totalOccurrencesGoodToFix: goodToFixSum,
-            totalOccurrencesNeedsReview: needsReviewSum,
-            totalOccurrencesPassed: page.totalOccurrencesPassed,
-            typesOfIssuesExclusiveToNeedsReviewCount,
-            typesOfIssuesCount: failedRuleCount,
-            typesOfIssuesExcludingNeedsReviewCount,
-            categoriesPresent,
-            conformance,
-            // Keep full detail for "scanPagesDetail"
-            typesOfIssues: typesOfIssuesArray,
-        };
-    }
-    // Transform raw pages
-    const pagesAffected = pagesAffectedRaw.map(transformPageData);
-    const pagesNotAffected = pagesNotAffectedRaw.map(transformPageData);
-    // --------------------------------------------
-    // 8) Sort pages by typesOfIssuesCount (descending) for both arrays
-    // --------------------------------------------
-    pagesAffected.sort((a, b) => b.typesOfIssuesCount - a.typesOfIssuesCount);
-    pagesNotAffected.sort((a, b) => b.typesOfIssuesCount - a.typesOfIssuesCount);
-    // --------------------------------------------
-    // 9) Compute scanned/ skipped counts
-    // --------------------------------------------
-    const scannedPagesCount = pagesAffected.length + pagesNotAffected.length;
-    const pagesNotScannedCount = Array.isArray(allIssues.pagesNotScanned)
-        ? allIssues.pagesNotScanned.length
-        : 0;
-    // --------------------------------------------
-    // 10) Build scanPagesDetail (with full "typesOfIssues")
-    // --------------------------------------------
-    allIssues.scanPagesDetail = {
-        pagesAffected,
-        pagesNotAffected,
-        scannedPagesCount,
-        pagesNotScanned: Array.isArray(allIssues.pagesNotScanned) ? allIssues.pagesNotScanned : [],
-        pagesNotScannedCount,
-    };
-    // --------------------------------------------
-    // 11) Build scanPagesSummary (strip out "typesOfIssues")
-    // --------------------------------------------
-    function stripTypesOfIssues(page) {
-        const { typesOfIssues, ...rest } = page;
-        return rest;
-    }
-    const summaryPagesAffected = pagesAffected.map(stripTypesOfIssues);
-    const summaryPagesNotAffected = pagesNotAffected.map(stripTypesOfIssues);
-    allIssues.scanPagesSummary = {
-        pagesAffected: summaryPagesAffected,
-        pagesNotAffected: summaryPagesNotAffected,
-        scannedPagesCount,
-        pagesNotScanned: Array.isArray(allIssues.pagesNotScanned) ? allIssues.pagesNotScanned : [],
-        pagesNotScannedCount,
-    };
-}
-// Send WCAG criteria breakdown to Sentry
-export const sendWcagBreakdownToSentry = async (appVersion, wcagBreakdown, ruleIdJson, scanInfo, allIssues, pagesScannedCount = 0) => {
-    try {
-        // Initialize Sentry
-        Sentry.init(sentryConfig);
-        // Set user ID for Sentry tracking
-        const userData = getUserDataTxt();
-        if (userData && userData.userId) {
-            setSentryUser(userData.userId);
-        }
-        // Prepare tags for the event
-        const tags = {};
-        const wcagCriteriaBreakdown = {};
-        // Tag app version
-        tags.version = appVersion;
-        // Get dynamic WCAG criteria map once
-        const wcagCriteriaMap = await getWcagCriteriaMap();
-        // Categorize all WCAG criteria for reporting
-        const wcagIds = Array.from(new Set([...Object.keys(wcagCriteriaMap), ...Array.from(wcagBreakdown.keys())]));
-        const categorizedWcag = await categorizeWcagCriteria(wcagIds);
-        // First ensure all WCAG criteria are included in the tags with a value of 0
-        // This ensures criteria with no violations are still reported
-        for (const [wcagId, info] of Object.entries(wcagCriteriaMap)) {
-            const formattedTag = await formatWcagTag(wcagId);
-            if (formattedTag) {
-                // Initialize with zero
-                tags[formattedTag] = '0';
-                // Store in breakdown object with category information
-                wcagCriteriaBreakdown[formattedTag] = {
-                    count: 0,
-                    category: categorizedWcag[wcagId] || 'mustFix', // Default to mustFix if not found
-                };
-            }
-        }
-        // Now override with actual counts from the scan
-        for (const [wcagId, count] of wcagBreakdown.entries()) {
-            const formattedTag = await formatWcagTag(wcagId);
-            if (formattedTag) {
-                // Add as a tag with the count as value
-                tags[formattedTag] = String(count);
-                // Update count in breakdown object
-                if (wcagCriteriaBreakdown[formattedTag]) {
-                    wcagCriteriaBreakdown[formattedTag].count = count;
-                }
-                else {
-                    // If somehow this wasn't in our initial map
-                    wcagCriteriaBreakdown[formattedTag] = {
-                        count,
-                        category: categorizedWcag[wcagId] || 'mustFix',
-                    };
-                }
-            }
-        }
-        // Calculate category counts based on actual issue counts from the report
-        // rather than occurrence counts from wcagBreakdown
-        const categoryCounts = {
-            mustFix: 0,
-            goodToFix: 0,
-            needsReview: 0,
-        };
-        if (allIssues) {
-            // Use the actual report data for the counts
-            categoryCounts.mustFix = allIssues.items.mustFix.rules.length;
-            categoryCounts.goodToFix = allIssues.items.goodToFix.rules.length;
-            categoryCounts.needsReview = allIssues.items.needsReview.rules.length;
-        }
-        else {
-            // Fallback to the old way if allIssues not provided
-            Object.values(wcagCriteriaBreakdown).forEach(item => {
-                if (item.count > 0 && categoryCounts[item.category] !== undefined) {
-                    categoryCounts[item.category] += 1; // Count rules, not occurrences
-                }
-            });
-        }
-        // Add category counts as tags
-        tags['WCAG-MustFix-Count'] = String(categoryCounts.mustFix);
-        tags['WCAG-GoodToFix-Count'] = String(categoryCounts.goodToFix);
-        tags['WCAG-NeedsReview-Count'] = String(categoryCounts.needsReview);
-        // Also add occurrence counts for reference
-        if (allIssues) {
-            tags['WCAG-MustFix-Occurrences'] = String(allIssues.items.mustFix.totalItems);
-            tags['WCAG-GoodToFix-Occurrences'] = String(allIssues.items.goodToFix.totalItems);
-            tags['WCAG-NeedsReview-Occurrences'] = String(allIssues.items.needsReview.totalItems);
-            // Add number of pages scanned tag
-            tags['Pages-Scanned-Count'] = String(allIssues.totalPagesScanned);
-        }
-        else if (pagesScannedCount > 0) {
-            // Still add the pages scanned count even if we don't have allIssues
-            tags['Pages-Scanned-Count'] = String(pagesScannedCount);
-        }
-        // Send the event to Sentry
-        await Sentry.captureEvent({
-            message: 'Accessibility Scan Completed',
-            level: 'info',
-            tags: {
-                ...tags,
-                event_type: 'accessibility_scan',
-                scanType: scanInfo.scanType,
-                browser: scanInfo.browser,
-                entryUrl: scanInfo.entryUrl,
-            },
-            user: {
-                ...(scanInfo.email && scanInfo.name
-                    ? {
-                        email: scanInfo.email,
-                        username: scanInfo.name,
-                    }
-                    : {}),
-                ...(userData && userData.userId ? { id: userData.userId } : {}),
-            },
-            extra: {
-                additionalScanMetadata: ruleIdJson != null ? JSON.stringify(ruleIdJson) : '{}',
-                wcagBreakdown: wcagCriteriaBreakdown,
-                reportCounts: allIssues
-                    ? {
-                        mustFix: {
-                            issues: allIssues.items.mustFix.rules?.length ?? 0,
-                            occurrences: allIssues.items.mustFix.totalItems ?? 0,
-                        },
-                        goodToFix: {
-                            issues: allIssues.items.goodToFix.rules?.length ?? 0,
-                            occurrences: allIssues.items.goodToFix.totalItems ?? 0,
-                        },
-                        needsReview: {
-                            issues: allIssues.items.needsReview.rules?.length ?? 0,
-                            occurrences: allIssues.items.needsReview.totalItems ?? 0,
-                        },
-                    }
-                    : undefined,
-            },
-        });
-        // Wait for events to be sent
-        await Sentry.flush(2000);
-    }
-    catch (error) {
-        console.error('Error sending WCAG breakdown to Sentry:', error);
-    }
-};
 const formatAboutStartTime = (dateString) => {
     const utcStartTimeDate = new Date(dateString);
     const formattedStartTime = utcStartTimeDate.toLocaleTimeString('en-GB', {
@@ -1675,5 +770,5 @@ generateJsonFiles = false) => {
         console.log('Report generated successfully');
     return ruleIdJson;
 };
-export { writeHTML, compressJsonFileStreaming, flattenAndSortResults, populateScanPagesDetail, getWcagPassPercentage, getProgressPercentage, getIssuesPercentage, itemTypeDescription, oobeeAiHtmlETL, oobeeAiRules, formatAboutStartTime, };
+export { writeHTML, compressJsonFileStreaming, convertItemsToReferences, flattenAndSortResults, populateScanPagesDetail, sendWcagBreakdownToSentry, getWcagPassPercentage, getProgressPercentage, getIssuesPercentage, itemTypeDescription, oobeeAiHtmlETL, oobeeAiRules, formatAboutStartTime, };
 export default generateArtifacts;