@govtechsg/oobee 0.10.20 → 0.10.28

package/src/mergeAxeResults.ts

@@ -9,9 +9,11 @@ import { fileURLToPath } from 'url';
 import { chromium } from 'playwright';
 import { createWriteStream } from 'fs';
 import { AsyncParser, ParserOptions } from '@json2csv/node';
-import { v4 as uuidv4 } from 'uuid';
+import zlib from 'zlib';
+import { Base64Encode } from 'base64-stream';
+import { pipeline } from 'stream/promises';
 import constants, { ScannerTypes } from './constants/constants.js';
-import { urlWithoutAuth } from './constants/common.js';
+import { urlWithoutAuth, prepareData } from './constants/common.js';
 import {
   createScreenshotsFolder,
   getStoragePath,
@@ -24,8 +26,6 @@ import { consoleLogger, silentLogger } from './logs.js';
 import itemTypeDescription from './constants/itemTypeDescription.js';
 import { oobeeAiHtmlETL, oobeeAiRules } from './constants/oobeeAi.js';
 
-const cwd = process.cwd();
-
 export type ItemsInfo = {
   html: string;
   message: string;
@@ -36,6 +36,7 @@ export type ItemsInfo = {
 
 type PageInfo = {
   items: ItemsInfo[];
+  itemsCount?: number;
   pageTitle: string;
   url?: string;
   pageImagePath?: string;
@@ -53,6 +54,13 @@ export type RuleInfo = {
   helpUrl: string;
 };
 
+type Category = {
+  description: string;
+  totalItems: number;
+  totalRuleIssues: number;
+  rules: RuleInfo[];
+};
+
 type AllIssues = {
   storagePath: string;
   oobeeAi: {
@@ -63,6 +71,7 @@ type AllIssues = {
   endTime: Date;
   urlScanned: string;
   scanType: string;
+  deviceChosen: string;
   formatAboutStartTime: (dateString: any) => string;
   isCustomFlow: boolean;
   viewport: string;
@@ -72,22 +81,26 @@ type AllIssues = {
   totalPagesNotScanned: number;
   totalItems: number;
   topFiveMostIssues: Array<any>;
+  topTenPagesWithMostIssues: Array<any>;
+  topTenIssues: Array<any>;
   wcagViolations: string[];
   customFlowLabel: string;
   phAppVersion: string;
   items: {
-    mustFix: { description: string; totalItems: number; rules: RuleInfo[] };
-    goodToFix: { description: string; totalItems: number; rules: RuleInfo[] };
-    needsReview: { description: string; totalItems: number; rules: RuleInfo[] };
-    passed: { description: string; totalItems: number; rules: RuleInfo[] };
+    mustFix: Category;
+    goodToFix: Category;
+    needsReview: Category;
+    passed: Category;
   };
   cypressScanAboutMetadata: string;
   wcagLinks: { [key: string]: string };
   [key: string]: any;
+  advancedScanOptionsSummaryItems: { [key: string]: boolean };
 };
 
 const filename = fileURLToPath(import.meta.url);
 const dirname = path.dirname(filename);
+const BUFFER_LIMIT = 100 * 1024 * 1024; // 100MB size
 
 const extractFileNames = async (directory: string): Promise<string[]> => {
   ensureDirSync(directory);
@@ -135,7 +148,7 @@ const writeCsv = async (allIssues, storagePath) => {
         return compareCategory === 0 ? a[1].rule.localeCompare(b[1].rule) : compareCategory;
       });
   };
-  // seems to go into
+
   const flattenRule = catAndRule => {
     const [severity, rule] = catAndRule;
     const results = [];
@@ -154,39 +167,49 @@ const writeCsv = async (allIssues, storagePath) => {
     pagesAffected.sort((a, b) => a.url.localeCompare(b.url));
     // format clauses as a string
     const wcagConformance = clausesArr.join(',');
+
     pagesAffected.forEach(affectedPage => {
       const { url, items } = affectedPage;
       items.forEach(item => {
         const { html, page, message, xpath } = item;
-        const howToFix = message.replace(/(\r\n|\n|\r)/g, ' '); // remove newlines
+        const howToFix = message.replace(/(\r\n|\n|\r)/g, '\\n'); // preserve newlines as \n
         const violation = html || formatPageViolation(page); // page is a number, not a string
         const context = violation.replace(/(\r\n|\n|\r)/g, ''); // remove newlines
 
         results.push({
-          severity,
-          issueId,
-          issueDescription,
-          wcagConformance,
-          url,
-          context,
-          howToFix,
-          axeImpact,
-          xpath,
-          learnMore,
+          customFlowLabel: allIssues.customFlowLabel || '',
+          deviceChosen: allIssues.deviceChosen || '',
+          scanCompletedAt: allIssues.endTime ? allIssues.endTime.toISOString() : '',
+          severity: severity || '',
+          issueId: issueId || '',
+          issueDescription: issueDescription || '',
+          wcagConformance: wcagConformance || '',
+          url: url || '',
+          pageTitle: affectedPage.pageTitle || 'No page title',
+          context: context || '',
+          howToFix: howToFix || '',
+          axeImpact: axeImpact || '',
+          xpath: xpath || '',
+          learnMore: learnMore || '',
         });
       });
     });
     if (results.length === 0) return {};
     return results;
   };
+
   const opts: ParserOptions<any, any> = {
     transforms: [getRulesByCategory, flattenRule],
     fields: [
+      'customFlowLabel',
+      'deviceChosen',
+      'scanCompletedAt',
       'severity',
       'issueId',
       'issueDescription',
       'wcagConformance',
       'url',
+      'pageTitle',
       'context',
       'howToFix',
       'axeImpact',
@@ -195,17 +218,23 @@ const writeCsv = async (allIssues, storagePath) => {
     ],
     includeEmptyRows: true,
   };
+
   const parser = new AsyncParser(opts);
   parser.parse(allIssues).pipe(csvOutput);
 };
 
-const compileHtmlWithEJS = async (allIssues, storagePath, htmlFilename = 'report') => {
+const compileHtmlWithEJS = async (
+  allIssues: AllIssues,
+  storagePath: string,
+  htmlFilename = 'report',
+) => {
   const htmlFilePath = `${path.join(storagePath, htmlFilename)}.html`;
   const ejsString = fs.readFileSync(path.join(dirname, './static/ejs/report.ejs'), 'utf-8');
   const template = ejs.compile(ejsString, {
     filename: path.join(dirname, './static/ejs/report.ejs'),
   });
-  const html = template(allIssues);
+
+  const html = template({...allIssues, storagePath: JSON.stringify(storagePath)});
   await fs.writeFile(htmlFilePath, html);
 
   let htmlContent = await fs.readFile(htmlFilePath, { encoding: 'utf8' });
@@ -213,28 +242,8 @@ const compileHtmlWithEJS = async (allIssues, storagePath, htmlFilename = 'report
   const headIndex = htmlContent.indexOf('</head>');
   const injectScript = `
   <script>
-    try {
-      const base64DecodeChunkedWithDecoder = (data, chunkSize = 1024 * 1024) => {
-      const encodedChunks = data.split('.');
-      const decoder = new TextDecoder();
-      const jsonParts = [];
-
-      encodedChunks.forEach(chunk => {
-          for (let i = 0; i < chunk.length; i += chunkSize) {
-              const chunkPart = chunk.slice(i, i + chunkSize);
-              const decodedBytes = Uint8Array.from(atob(chunkPart), c => c.charCodeAt(0));
-              jsonParts.push(decoder.decode(decodedBytes, { stream: true }));
-          }
-      });
-
-      return JSON.parse(jsonParts.join(''));
-
-    };
-
     // IMPORTANT! DO NOT REMOVE ME: Decode the encoded data
-    } catch (error) {
-      console.error("Error decoding base64 data:", error);
-    }
+
   </script>
   `;
 
@@ -259,7 +268,7 @@ const splitHtmlAndCreateFiles = async (htmlFilePath, storagePath) => {
       throw new Error('Marker comment not found in the HTML file.');
     }
 
-    const topContent = htmlContent.slice(0, splitIndex + splitMarker.length) + '\n\n';
+    const topContent = `${htmlContent.slice(0, splitIndex + splitMarker.length)}\n\n`;
     const bottomContent = htmlContent.slice(splitIndex + splitMarker.length);
 
     const topFilePath = path.join(storagePath, 'report-partial-top.htm.txt');
@@ -276,42 +285,32 @@ const splitHtmlAndCreateFiles = async (htmlFilePath, storagePath) => {
   }
 };
 
-const writeHTML = async (allIssues, storagePath, htmlFilename = 'report') => {
+const writeHTML = async (
+  allIssues: AllIssues,
+  storagePath: string,
+  htmlFilename = 'report',
+  scanDetailsFilePath: string,
+  scanItemsFilePath: string,
+) => {
   const htmlFilePath = await compileHtmlWithEJS(allIssues, storagePath, htmlFilename);
-  const inputFilePath = path.resolve(storagePath, 'scanDetails.csv');
-  const outputFilePath = `${storagePath}/${htmlFilename}.html`;
-
   const { topFilePath, bottomFilePath } = await splitHtmlAndCreateFiles(htmlFilePath, storagePath);
-
   const prefixData = fs.readFileSync(path.join(storagePath, 'report-partial-top.htm.txt'), 'utf-8');
   const suffixData = fs.readFileSync(
     path.join(storagePath, 'report-partial-bottom.htm.txt'),
     'utf-8',
   );
 
-  const outputStream = fs.createWriteStream(outputFilePath, { flags: 'a' });
-
-  outputStream.write(prefixData);
-
-  // Create a readable stream for the input file with a highWaterMark set to 10MB
-  const BUFFER_LIMIT = 10 * 1024 * 1024; // 10 MB
-  const inputStream = fs.createReadStream(inputFilePath, {
-    encoding: 'utf-8',
+  const scanDetailsReadStream = fs.createReadStream(scanDetailsFilePath, {
+    encoding: 'utf8',
+    highWaterMark: BUFFER_LIMIT,
+  });
+  const scanItemsReadStream = fs.createReadStream(scanItemsFilePath, {
+    encoding: 'utf8',
     highWaterMark: BUFFER_LIMIT,
   });
 
-  let isFirstLine = true;
-  let lineEndingDetected = false;
-  let isFirstField = true;
-  let isWritingFirstDataLine = true;
-  let buffer = '';
-
-  function flushBuffer() {
-    if (buffer.length > 0) {
-      outputStream.write(buffer);
-      buffer = '';
-    }
-  }
+  const outputFilePath = `${storagePath}/${htmlFilename}.html`;
+  const outputStream = fs.createWriteStream(outputFilePath, { flags: 'a' });
 
   const cleanupFiles = async () => {
     try {
@@ -321,75 +320,54 @@ const writeHTML = async (allIssues, storagePath, htmlFilename = 'report') => {
     }
   };
 
-  inputStream.on('data', chunk => {
-    let chunkIndex = 0;
-
-    while (chunkIndex < chunk.length) {
-      const char = chunk[chunkIndex];
-
-      if (isFirstLine) {
-        if (char === '\n' || char === '\r') {
-          lineEndingDetected = true;
-        } else if (lineEndingDetected) {
-          if (char !== '\n' && char !== '\r') {
-            isFirstLine = false;
-
-            if (isWritingFirstDataLine) {
-              buffer += "scanData = base64DecodeChunkedWithDecoder('";
-              isWritingFirstDataLine = false;
-            }
-            buffer += char;
-          }
-          lineEndingDetected = false;
-        }
-      } else {
-        if (char === ',') {
-          buffer += "')\n\n";
-          buffer += "scanItems = base64DecodeChunkedWithDecoder('";
-          isFirstField = false;
-        } else if (char === '\n' || char === '\r') {
-          if (!isFirstField) {
-            buffer += "')\n";
-          }
-        } else {
-          buffer += char;
-        }
-
-        if (buffer.length >= BUFFER_LIMIT) {
-          flushBuffer();
-        }
-      }
+  outputStream.write(prefixData);
 
-      chunkIndex++;
-    }
+  // outputStream.write("scanData = decompressJsonObject('");
+  outputStream.write(
+    "let scanDataPromise = (async () => { console.log('Loading scanData...'); scanData = await decodeUnzipParse('",
+  );
+  scanDetailsReadStream.pipe(outputStream, { end: false });
+
+  scanDetailsReadStream.on('end', () => {
+    // outputStream.write("')\n\n");
+    outputStream.write("'); })();\n\n");
+    // outputStream.write("(scanItems = decompressJsonObject('");
+    outputStream.write(
+      "let scanItemsPromise = (async () => { console.log('Loading scanItems...'); scanItems = await decodeUnzipParse('",
+    );
+    scanItemsReadStream.pipe(outputStream, { end: false });
   });
 
-  inputStream.on('end', async () => {
-    if (!isFirstField) {
-      buffer += "')\n";
-    }
-    flushBuffer();
+  scanDetailsReadStream.on('error', err => {
+    console.error('Read stream error:', err);
+    outputStream.end();
+  });
 
+  scanItemsReadStream.on('end', () => {
+    // outputStream.write("')\n\n");
+    outputStream.write("'); })();\n\n");
     outputStream.write(suffixData);
     outputStream.end();
-    console.log('Content appended successfully.');
-
-    await cleanupFiles();
   });
 
-  inputStream.on('error', async err => {
-    console.error('Error reading input file:', err);
+  scanItemsReadStream.on('error', err => {
+    console.error('Read stream error:', err);
     outputStream.end();
-
-    await cleanupFiles();
   });
 
+  consoleLogger.info('Content appended successfully.');
+  await cleanupFiles();
+
   outputStream.on('error', err => {
-    console.error('Error writing to output file:', err);
+    consoleLogger.error('Error writing to output file:', err);
   });
 };
 
-const writeSummaryHTML = async (allIssues, storagePath, htmlFilename = 'summary') => {
+const writeSummaryHTML = async (
+  allIssues: AllIssues,
+  storagePath: string,
+  htmlFilename = 'summary',
+) => {
   const ejsString = fs.readFileSync(path.join(dirname, './static/ejs/summary.ejs'), 'utf-8');
   const template = ejs.compile(ejsString, {
     filename: path.join(dirname, './static/ejs/summary.ejs'),
@@ -398,47 +376,56 @@ const writeSummaryHTML = async (allIssues, storagePath, htmlFilename = 'summary'
   fs.writeFileSync(`${storagePath}/${htmlFilename}.html`, html);
 };
 
-function writeFormattedValue(value, writeStream) {
-  if (typeof value === 'function') {
-    writeStream.write('null');
-  } else if (value === undefined) {
-    writeStream.write('null');
-  } else if (typeof value === 'string' || typeof value === 'boolean' || typeof value === 'number') {
-    writeStream.write(JSON.stringify(value));
-  } else if (value === null) {
-    writeStream.write('null');
-  }
-}
+const cleanUpJsonFiles = async (filesToDelete: string[]) => {
+  consoleLogger.info('Cleaning up JSON files...');
+  filesToDelete.forEach(file => {
+    fs.unlinkSync(file);
+    consoleLogger.info(`Deleted ${file}`);
+  });
+};
 
-function serializeObject(obj, writeStream, depth = 0, indent = '  ') {
+function* serializeObject(obj: any, depth = 0, indent = '  ') {
   const currentIndent = indent.repeat(depth);
   const nextIndent = indent.repeat(depth + 1);
 
   if (obj instanceof Date) {
-    writeStream.write(JSON.stringify(obj.toISOString()));
-  } else if (Array.isArray(obj)) {
-    writeStream.write('[\n');
-    obj.forEach((item, index) => {
-      if (index > 0) writeStream.write(',\n');
-      writeStream.write(nextIndent);
-      serializeObject(item, writeStream, depth + 1, indent);
-    });
-    writeStream.write(`\n${currentIndent}]`);
-  } else if (typeof obj === 'object' && obj !== null) {
-    writeStream.write('{\n');
+    yield JSON.stringify(obj.toISOString());
+    return;
+  }
+
+  if (Array.isArray(obj)) {
+    yield '[\n';
+    for (let i = 0; i < obj.length; i++) {
+      if (i > 0) yield ',\n';
+      yield nextIndent;
+      yield* serializeObject(obj[i], depth + 1, indent);
+    }
+    yield `\n${currentIndent}]`;
+    return;
+  }
+
+  if (obj !== null && typeof obj === 'object') {
+    yield '{\n';
     const keys = Object.keys(obj);
-    keys.forEach((key, index) => {
-      if (index > 0) writeStream.write(',\n');
-      writeStream.write(`${nextIndent}${JSON.stringify(key)}: `);
-      serializeObject(obj[key], writeStream, depth + 1, indent);
-    });
-    writeStream.write(`\n${currentIndent}}`);
-  } else {
-    writeFormattedValue(obj, writeStream);
+    for (let i = 0; i < keys.length; i++) {
+      const key = keys[i];
+      if (i > 0) yield ',\n';
+      yield `${nextIndent}${JSON.stringify(key)}: `;
+      yield* serializeObject(obj[key], depth + 1, indent);
+    }
+    yield `\n${currentIndent}}`;
+    return;
   }
+
+  if (obj === null || typeof obj === 'function' || typeof obj === 'undefined') {
+    yield 'null';
+    return;
+  }
+
+  yield JSON.stringify(obj);
 }
 
-function writeLargeJsonToFile(obj, filePath) {
+function writeLargeJsonToFile(obj: object, filePath: string) {
   return new Promise((resolve, reject) => {
     const writeStream = fs.createWriteStream(filePath, { encoding: 'utf8' });
 
@@ -448,61 +435,231 @@ function writeLargeJsonToFile(obj, filePath) {
     });
 
     writeStream.on('finish', () => {
-      consoleLogger.info('Temporary file written successfully:', filePath);
+      consoleLogger.info(`JSON file written successfully: ${filePath}`);
       resolve(true);
     });
 
-    serializeObject(obj, writeStream);
-    writeStream.end();
+    const generator = serializeObject(obj);
+
+    function write() {
+      let next: any;
+      while (!(next = generator.next()).done) {
+        if (!writeStream.write(next.value)) {
+          writeStream.once('drain', write);
+          return;
+        }
+      }
+      writeStream.end();
+    }
+
+    write();
   });
 }
 
-const base64Encode = async (data, num) => {
-  try {
-    const tempFilename =
-      num === 1
-        ? `scanItems_${uuidv4()}.json`
-        : num === 2
-          ? `scanData_${uuidv4()}.json`
-          : `${uuidv4()}.json`;
-    const tempFilePath = path.join(process.cwd(), tempFilename);
+const writeLargeScanItemsJsonToFile = async (obj: object, filePath: string) => {
+  return new Promise((resolve, reject) => {
+    const writeStream = fs.createWriteStream(filePath, { flags: 'a', encoding: 'utf8' });
+    const writeQueue: string[] = [];
+    let isWriting = false;
+
+    const processNextWrite = async () => {
+      if (isWriting || writeQueue.length === 0) return;
+
+      isWriting = true;
+      const data = writeQueue.shift()!;
+
+      try {
+        if (!writeStream.write(data)) {
+          await new Promise<void>(resolve => {
+            writeStream.once('drain', () => {
+              resolve();
+            });
+          });
+        }
+      } catch (error) {
+        writeStream.destroy(error as Error);
+        return;
+      }
+
+      isWriting = false;
+      processNextWrite();
+    };
+
+    const queueWrite = (data: string) => {
+      writeQueue.push(data);
+      processNextWrite();
+    };
 
-    await writeLargeJsonToFile(data, tempFilePath);
+    writeStream.on('error', error => {
+      consoleLogger.error(`Error writing object to JSON file: ${error}`);
+      reject(error);
+    });
 
-    const outputFilename = `encoded_${uuidv4()}.txt`;
-    const outputFilePath = path.join(process.cwd(), outputFilename);
+    writeStream.on('finish', () => {
+      consoleLogger.info(`JSON file written successfully: ${filePath}`);
+      resolve(true);
+    });
 
     try {
-      const CHUNK_SIZE = 1024 * 1024; // 1MB chunks
-      const readStream = fs.createReadStream(tempFilePath, {
-        encoding: 'utf8',
-        highWaterMark: CHUNK_SIZE,
-      });
-      const writeStream = fs.createWriteStream(outputFilePath, { encoding: 'utf8' });
+      queueWrite('{\n');
+      const keys = Object.keys(obj);
+
+      keys.forEach((key, i) => {
+        const value = obj[key];
+        queueWrite(`  "${key}": {\n`);
+
+        const { rules, ...otherProperties } = value;
+
+        // Write other properties
+        Object.entries(otherProperties).forEach(([propKey, propValue], j) => {
+          const propValueString =
+            propValue === null ||
+              typeof propValue === 'function' ||
+              typeof propValue === 'undefined'
+              ? 'null'
+              : JSON.stringify(propValue);
+          queueWrite(`    "${propKey}": ${propValueString}`);
+          if (j < Object.keys(otherProperties).length - 1 || (rules && rules.length >= 0)) {
+            queueWrite(',\n');
+          } else {
+            queueWrite('\n');
+          }
+        });
 
-      for await (const chunk of readStream) {
-        const encodedChunk = Buffer.from(chunk).toString('base64');
-        writeStream.write(`${encodedChunk}.`);
-      }
+        if (rules && Array.isArray(rules)) {
+          queueWrite('    "rules": [\n');
+
+          rules.forEach((rule, j) => {
+            queueWrite('      {\n');
+            const { pagesAffected, ...otherRuleProperties } = rule;
+
+            Object.entries(otherRuleProperties).forEach(([ruleKey, ruleValue], k) => {
+              const ruleValueString =
+                ruleValue === null ||
+                  typeof ruleValue === 'function' ||
+                  typeof ruleValue === 'undefined'
+                  ? 'null'
+                  : JSON.stringify(ruleValue);
+              queueWrite(`        "${ruleKey}": ${ruleValueString}`);
+              if (k < Object.keys(otherRuleProperties).length - 1 || pagesAffected) {
+                queueWrite(',\n');
+              } else {
+                queueWrite('\n');
+              }
+            });
+
+            if (pagesAffected && Array.isArray(pagesAffected)) {
+              queueWrite('        "pagesAffected": [\n');
+
+              pagesAffected.forEach((page, p) => {
+                const pageJson = JSON.stringify(page, null, 2)
+                  .split('\n')
+                  .map((line, idx) => (idx === 0 ? `          ${line}` : `          ${line}`))
+                  .join('\n');
+
+                queueWrite(pageJson);
+
+                if (p < pagesAffected.length - 1) {
+                  queueWrite(',\n');
+                } else {
+                  queueWrite('\n');
+                }
+              });
+
+              queueWrite('        ]');
+            }
 
-      await new Promise((resolve, reject) => {
-        writeStream.end(resolve);
-        writeStream.on('error', reject);
+            queueWrite('\n      }');
+            if (j < rules.length - 1) {
+              queueWrite(',\n');
+            } else {
+              queueWrite('\n');
+            }
+          });
+
+          queueWrite('    ]');
+        }
+
+        queueWrite('\n  }');
+        if (i < keys.length - 1) {
+          queueWrite(',\n');
+        } else {
+          queueWrite('\n');
+        }
       });
 
-      return outputFilePath;
-    } finally {
-      await fs.promises
-        .unlink(tempFilePath)
-        .catch(err => console.error('Temp file delete error:', err));
+      queueWrite('}\n');
+
+      // Ensure all queued writes are processed before ending
+      const checkQueueAndEnd = () => {
+        if (writeQueue.length === 0 && !isWriting) {
+          writeStream.end();
+        } else {
+          setTimeout(checkQueueAndEnd, 100);
+        }
+      };
+
+      checkQueueAndEnd();
+    } catch (err) {
+      writeStream.destroy(err as Error);
+      reject(err);
+    }
+  });
+};
+
+async function compressJsonFileStreaming(inputPath: string, outputPath: string) {
+  // Create the read and write streams
+  const readStream = fs.createReadStream(inputPath);
+  const writeStream = fs.createWriteStream(outputPath);
+
+  // Create a gzip transform stream
+  const gzip = zlib.createGzip();
+
+  // Create a Base64 transform stream
+  const base64Encode = new Base64Encode();
+
+  // Pipe the streams:
+  //   read -> gzip -> base64 -> write
+  await pipeline(readStream, gzip, base64Encode, writeStream);
+  console.log(`File successfully compressed and saved to ${outputPath}`);
+}
+
+const writeJsonFileAndCompressedJsonFile = async (
+  data: object,
+  storagePath: string,
+  filename: string,
+): Promise<{ jsonFilePath: string; base64FilePath: string }> => {
+  try {
+    consoleLogger.info(`Writing JSON to ${filename}.json`);
+    const jsonFilePath = path.join(storagePath, `${filename}.json`);
+    if (filename === 'scanItems') {
+      await writeLargeScanItemsJsonToFile(data, jsonFilePath);
+    } else {
+      await writeLargeJsonToFile(data, jsonFilePath);
     }
+
+    consoleLogger.info(
+      `Reading ${filename}.json, gzipping and base64 encoding it into ${filename}.json.gz.b64`,
+    );
+    const base64FilePath = path.join(storagePath, `${filename}.json.gz.b64`);
+    await compressJsonFileStreaming(jsonFilePath, base64FilePath);
+
+    consoleLogger.info(`Finished compression and base64 encoding for ${filename}`);
+    return {
+      jsonFilePath,
+      base64FilePath,
+    };
   } catch (error) {
-    console.error('Error encoding data to Base64:', error);
+    consoleLogger.error(`Error compressing and encoding ${filename}`);
     throw error;
   }
 };
 
-const streamEncodedDataToFile = async (inputFilePath, writeStream, appendComma) => {
+const streamEncodedDataToFile = async (
+  inputFilePath: string,
+  writeStream: fs.WriteStream,
+  appendComma: boolean,
+) => {
   const readStream = fs.createReadStream(inputFilePath, { encoding: 'utf8' });
   let isFirstChunk = true;
 
@@ -520,35 +677,120 @@ const streamEncodedDataToFile = async (inputFilePath, writeStream, appendComma)
   }
 };
 
-const writeBase64 = async (allIssues, storagePath) => {
+const writeJsonAndBase64Files = async (
+  allIssues: AllIssues,
+  storagePath: string,
+): Promise<{
+  scanDataJsonFilePath: string;
+  scanDataBase64FilePath: string;
+  scanItemsJsonFilePath: string;
+  scanItemsBase64FilePath: string;
+  scanItemsSummaryJsonFilePath: string;
+  scanItemsSummaryBase64FilePath: string;
+  scanDataJsonFileSize: number;
+  scanItemsJsonFileSize: number;
+}> => {
   const { items, ...rest } = allIssues;
-  const encodedScanItemsPath = await base64Encode(items, 1);
-  const encodedScanDataPath = await base64Encode(rest, 2);
+  const { jsonFilePath: scanDataJsonFilePath, base64FilePath: scanDataBase64FilePath } =
+    await writeJsonFileAndCompressedJsonFile(rest, storagePath, 'scanData');
+  const { jsonFilePath: scanItemsJsonFilePath, base64FilePath: scanItemsBase64FilePath } =
+    await writeJsonFileAndCompressedJsonFile(items, storagePath, 'scanItems');
+
+  // scanItemsSummary
+  // the below mutates the original items object, since it is expensive to clone
+  items.mustFix.rules.forEach(rule => {
+    rule.pagesAffected.forEach(page => {
+      page.itemsCount = page.items.length;
+      page.items = [];
+    });
+  });
+  items.goodToFix.rules.forEach(rule => {
+    rule.pagesAffected.forEach(page => {
+      page.itemsCount = page.items.length;
+      page.items = [];
+    });
+  });
+  items.needsReview.rules.forEach(rule => {
+    rule.pagesAffected.forEach(page => {
+      page.itemsCount = page.items.length;
+      page.items = [];
+    });
+  });
+  items.passed.rules.forEach(rule => {
+    rule.pagesAffected.forEach(page => {
+      page.itemsCount = page.items.length;
+      page.items = [];
+    });
+  });
+
+  items.mustFix.totalRuleIssues = items.mustFix.rules.length;
+  items.goodToFix.totalRuleIssues = items.goodToFix.rules.length;
+  items.needsReview.totalRuleIssues = items.needsReview.rules.length;
+  items.passed.totalRuleIssues = items.passed.rules.length;
+
+  const {
+    pagesScanned,
+    topTenPagesWithMostIssues,
+    pagesNotScanned,
+    wcagLinks,
+    wcagPassPercentage,
+    totalPagesScanned,
+    totalPagesNotScanned,
+    topTenIssues,
+  } = rest;
+
+  const summaryItems = {
+    ...items,
+    pagesScanned,
+    topTenPagesWithMostIssues,
+    pagesNotScanned,
+    wcagLinks,
+    wcagPassPercentage,
+    totalPagesScanned,
+    totalPagesNotScanned,
+    topTenIssues,
+  };
+
+  const {
+    jsonFilePath: scanItemsSummaryJsonFilePath,
+    base64FilePath: scanItemsSummaryBase64FilePath,
+  } = await writeJsonFileAndCompressedJsonFile(summaryItems, storagePath, 'scanItemsSummary');
+
+  return {
+    scanDataJsonFilePath,
+    scanDataBase64FilePath,
+    scanItemsJsonFilePath,
+    scanItemsBase64FilePath,
+    scanItemsSummaryJsonFilePath,
+    scanItemsSummaryBase64FilePath,
+    scanDataJsonFileSize: fs.statSync(scanDataJsonFilePath).size,
+    scanItemsJsonFileSize: fs.statSync(scanItemsJsonFilePath).size,
+  };
+};
 
+const writeScanDetailsCsv = async (
+  scanDataFilePath: string,
+  scanItemsFilePath: string,
+  scanItemsSummaryFilePath: string,
+  storagePath: string,
+) => {
   const filePath = path.join(storagePath, 'scanDetails.csv');
+  const csvWriteStream = fs.createWriteStream(filePath, { encoding: 'utf8' });
   const directoryPath = path.dirname(filePath);
 
   if (!fs.existsSync(directoryPath)) {
     fs.mkdirSync(directoryPath, { recursive: true });
   }
 
-  const csvWriteStream = fs.createWriteStream(filePath, { encoding: 'utf8' });
-
-  csvWriteStream.write('scanData_base64,scanItems_base64\n');
-  await streamEncodedDataToFile(encodedScanDataPath, csvWriteStream, true);
-  await streamEncodedDataToFile(encodedScanItemsPath, csvWriteStream, false);
+  csvWriteStream.write('scanData_base64,scanItems_base64,scanItemsSummary_base64\n');
+  await streamEncodedDataToFile(scanDataFilePath, csvWriteStream, true);
+  await streamEncodedDataToFile(scanItemsFilePath, csvWriteStream, true);
+  await streamEncodedDataToFile(scanItemsSummaryFilePath, csvWriteStream, false);
 
   await new Promise((resolve, reject) => {
     csvWriteStream.end(resolve);
     csvWriteStream.on('error', reject);
   });
-
-  await fs.promises
-    .unlink(encodedScanDataPath)
-    .catch(err => console.error('Encoded file delete error:', err));
-  await fs.promises
-    .unlink(encodedScanItemsPath)
-    .catch(err => console.error('Encoded file delete error:', err));
 };
 
 let browserChannel = 'chrome';
@@ -561,12 +803,13 @@ if (os.platform() === 'linux') {
   browserChannel = 'chromium';
 }
 
-const writeSummaryPdf = async (storagePath, pagesScanned, filename = 'summary') => {
+const writeSummaryPdf = async (storagePath: string, pagesScanned: number, filename = 'summary') => {
   const htmlFilePath = `${storagePath}/${filename}.html`;
   const fileDestinationPath = `${storagePath}/${filename}.pdf`;
   const browser = await chromium.launch({
-    headless: true,
+    headless: false,
     channel: browserChannel,
+    args: ['--headless=new', '--no-sandbox'],
   });
 
   const context = await browser.newContext({
@@ -613,7 +856,12 @@ const pushResults = async (pageResults, allIssues, isCustomFlow) => {
   Object.keys(pageResults.goodToFix.rules).forEach(k => totalIssuesInPage.add(k));
   Object.keys(pageResults.needsReview.rules).forEach(k => totalIssuesInPage.add(k));
 
-  allIssues.topFiveMostIssues.push({ url, pageTitle, totalIssues: totalIssuesInPage.size });
+  allIssues.topFiveMostIssues.push({
+    url,
+    pageTitle,
+    totalIssues: totalIssuesInPage.size,
+    totalOccurrences: 0,
+  });
 
   ['mustFix', 'goodToFix', 'needsReview', 'passed'].forEach(category => {
     if (!pageResults[category]) return;
@@ -695,9 +943,47 @@ const pushResults = async (pageResults, allIssues, isCustomFlow) => {
   });
 };
 
+const getTopTenIssues = allIssues => {
+  const categories = ['mustFix', 'goodToFix'];
+  const rulesWithCounts = [];
+
+  const conformanceLevels = {
+    wcag2a: 'A',
+    wcag2aa: 'AA',
+    wcag21aa: 'AA',
+    wcag22aa: 'AA',
+    wcag2aaa: 'AAA',
+  };
+
+  categories.forEach(category => {
+    const rules = allIssues.items[category]?.rules || [];
+
+    rules.forEach(rule => {
+      const wcagLevel = rule.conformance[0];
+      const aLevel = conformanceLevels[wcagLevel] || wcagLevel;
+
+      rulesWithCounts.push({
+        category,
+        ruleId: rule.rule,
+        description: rule.description,
+        axeImpact: rule.axeImpact,
+        conformance: aLevel,
+        totalItems: rule.totalItems,
+      });
+    });
+  });
+
+  rulesWithCounts.sort((a, b) => b.totalItems - a.totalItems);
+
+  return rulesWithCounts.slice(0, 10);
+};
+
 const flattenAndSortResults = (allIssues: AllIssues, isCustomFlow: boolean) => {
+  const urlOccurrencesMap = new Map<string, number>();
+
   ['mustFix', 'goodToFix', 'needsReview', 'passed'].forEach(category => {
     allIssues.totalItems += allIssues.items[category].totalItems;
+
     allIssues.items[category].rules = Object.entries(allIssues.items[category].rules)
       .map(ruleEntry => {
         const [rule, ruleInfo] = ruleEntry as [string, RuleInfo];
@@ -705,9 +991,14 @@ const flattenAndSortResults = (allIssues: AllIssues, isCustomFlow: boolean) => {
           .map(pageEntry => {
             if (isCustomFlow) {
               const [pageIndex, pageInfo] = pageEntry as unknown as [number, PageInfo];
+              urlOccurrencesMap.set(
+                pageInfo.url!,
+                (urlOccurrencesMap.get(pageInfo.url!) || 0) + pageInfo.items.length,
+              );
               return { pageIndex, ...pageInfo };
             }
             const [url, pageInfo] = pageEntry as unknown as [string, PageInfo];
+            urlOccurrencesMap.set(url, (urlOccurrencesMap.get(url) || 0) + pageInfo.items.length);
             return { url, ...pageInfo };
           })
           .sort((page1, page2) => page2.items.length - page1.items.length);
@@ -715,8 +1006,19 @@ const flattenAndSortResults = (allIssues: AllIssues, isCustomFlow: boolean) => {
       })
       .sort((rule1, rule2) => rule2.totalItems - rule1.totalItems);
   });
+
+  const updateIssuesWithOccurrences = (issuesList: Array<any>) => {
+    issuesList.forEach(issue => {
+      issue.totalOccurrences = urlOccurrencesMap.get(issue.url) || 0;
+    });
+  };
+
   allIssues.topFiveMostIssues.sort((page1, page2) => page2.totalIssues - page1.totalIssues);
   allIssues.topFiveMostIssues = allIssues.topFiveMostIssues.slice(0, 5);
+  allIssues.topTenPagesWithMostIssues = allIssues.topFiveMostIssues.slice(0, 10);
+  updateIssuesWithOccurrences(allIssues.topTenPagesWithMostIssues);
+  const topTenIssues = getTopTenIssues(allIssues);
+  allIssues.topTenIssues = topTenIssues;
 };
 
 const createRuleIdJson = allIssues => {
@@ -766,6 +1068,7 @@ const generateArtifacts = async (
   cypressScanAboutMetadata,
   scanDetails,
   zip = undefined, // optional
+  generateJsonFiles = false,
 ) => {
   const intermediateDatasetsPath = `${randomToken}/datasets/${randomToken}`;
   const phAppVersion = getVersion();
@@ -815,6 +1118,7 @@ const generateArtifacts = async (
     endTime: scanDetails.endTime ? scanDetails.endTime : new Date(),
     urlScanned,
     scanType,
+    deviceChosen: scanDetails.deviceChosen || 'Desktop',
     formatAboutStartTime,
     isCustomFlow,
     viewport,
@@ -824,17 +1128,50 @@ const generateArtifacts = async (
     totalPagesNotScanned: pagesNotScanned.length,
     totalItems: 0,
     topFiveMostIssues: [],
+    topTenPagesWithMostIssues: [],
+    topTenIssues: [],
     wcagViolations: [],
     customFlowLabel,
     phAppVersion,
     items: {
-      mustFix: { description: itemTypeDescription.mustFix, totalItems: 0, rules: [] },
-      goodToFix: { description: itemTypeDescription.goodToFix, totalItems: 0, rules: [] },
-      needsReview: { description: itemTypeDescription.needsReview, totalItems: 0, rules: [] },
-      passed: { description: itemTypeDescription.passed, totalItems: 0, rules: [] },
+      mustFix: {
+        description: itemTypeDescription.mustFix,
+        totalItems: 0,
+        totalRuleIssues: 0,
+        rules: [],
+      },
+      goodToFix: {
+        description: itemTypeDescription.goodToFix,
+        totalItems: 0,
+        totalRuleIssues: 0,
+        rules: [],
+      },
+      needsReview: {
+        description: itemTypeDescription.needsReview,
+        totalItems: 0,
+        totalRuleIssues: 0,
+        rules: [],
+      },
+      passed: {
+        description: itemTypeDescription.passed,
+        totalItems: 0,
+        totalRuleIssues: 0,
+        rules: [],
+      },
     },
     cypressScanAboutMetadata,
     wcagLinks: constants.wcagLinks,
+    // Populate boolean values for id="advancedScanOptionsSummary"
+    advancedScanOptionsSummaryItems: {
+      showIncludeScreenshots: [true].includes(scanDetails.isIncludeScreenshots),
+      showAllowSubdomains: ['same-domain'].includes(scanDetails.isAllowSubdomains),
+      showEnableCustomChecks: ['default', 'enable-wcag-aaa'].includes(
+        scanDetails.isEnableCustomChecks?.[0],
+      ),
+      showEnableWcagAaa: (scanDetails.isEnableWcagAaa || []).includes('enable-wcag-aaa'),
+      showSlowScanMode: [1].includes(scanDetails.isSlowScanMode),
+      showAdhereRobots: [true].includes(scanDetails.isAdhereRobots),
+    },
   };
 
   const allFiles = await extractFileNames(intermediateDatasetsPath);
@@ -870,6 +1207,9 @@ const generateArtifacts = async (
   }
 
   allIssues.wcagPassPercentage = getWcagPassPercentage(allIssues.wcagViolations);
+  consoleLogger.info(
+    `advancedScanOptionsSummaryItems is ${allIssues.advancedScanOptionsSummaryItems}`,
+  );
 
   const getAxeImpactCount = (allIssues: AllIssues) => {
     const impactCount = {
@@ -908,9 +1248,45 @@ const generateArtifacts = async (
   }
 
   await writeCsv(allIssues, storagePath);
-  await writeBase64(allIssues, storagePath);
+  const {
+    scanDataJsonFilePath,
+    scanDataBase64FilePath,
+    scanItemsJsonFilePath,
+    scanItemsBase64FilePath,
+    scanItemsSummaryJsonFilePath,
+    scanItemsSummaryBase64FilePath,
+    scanDataJsonFileSize,
+    scanItemsJsonFileSize,
+  } = await writeJsonAndBase64Files(allIssues, storagePath);
+  const BIG_RESULTS_THRESHOLD = 500 * 1024 * 1024; // 500 MB
+  const resultsTooBig = scanDataJsonFileSize + scanItemsJsonFileSize > BIG_RESULTS_THRESHOLD;
+
+  await writeScanDetailsCsv(
+    scanDataBase64FilePath,
+    scanItemsBase64FilePath,
+    scanItemsSummaryBase64FilePath,
+    storagePath,
+  );
   await writeSummaryHTML(allIssues, storagePath);
-  await writeHTML(allIssues, storagePath);
+  await writeHTML(
+    allIssues,
+    storagePath,
+    'report',
+    scanDataBase64FilePath,
+    resultsTooBig ? scanItemsSummaryBase64FilePath : scanItemsBase64FilePath,
+  );
+
+  if (!generateJsonFiles) {
+    await cleanUpJsonFiles([
+      scanDataJsonFilePath,
+      scanDataBase64FilePath,
+      scanItemsJsonFilePath,
+      scanItemsBase64FilePath,
+      scanItemsSummaryJsonFilePath,
+      scanItemsSummaryBase64FilePath,
+    ]);
+  }
+
   await retryFunction(() => writeSummaryPdf(storagePath, pagesScanned.length), 1);
 
   // Take option if set