@govtechsg/oobee 0.10.36 → 0.10.42

package/src/mergeAxeResults.ts

@@ -13,14 +13,16 @@ import zlib from 'zlib';
 import { Base64Encode } from 'base64-stream';
 import { pipeline } from 'stream/promises';
 import constants, { ScannerTypes } from './constants/constants.js';
-import { urlWithoutAuth, prepareData } from './constants/common.js';
+import { urlWithoutAuth } from './constants/common.js';
 import {
   createScreenshotsFolder,
   getStoragePath,
   getVersion,
   getWcagPassPercentage,
+  getProgressPercentage,
   retryFunction,
   zipResults,
+  getIssuesPercentage,
 } from './utils.js';
 import { consoleLogger, silentLogger } from './logs.js';
 import itemTypeDescription from './constants/itemTypeDescription.js';
@@ -34,19 +36,21 @@ export type ItemsInfo = {
   displayNeedsReview?: boolean;
 };
 
-type PageInfo = {
-  items: ItemsInfo[];
+export type PageInfo = {
+  items?: ItemsInfo[];
   itemsCount?: number;
   pageTitle: string;
-  url?: string;
+  url: string;
+  actualUrl: string;
   pageImagePath?: string;
   pageIndex?: number;
-  metadata: string;
+  metadata?: string;
 };
 
 export type RuleInfo = {
   totalItems: number;
   pagesAffected: PageInfo[];
+  pagesAffectedCount: number;
   rule: string;
   description: string;
   axeImpact: string;
@@ -74,7 +78,6 @@ type AllIssues = {
   deviceChosen: string;
   formatAboutStartTime: (dateString: any) => string;
   isCustomFlow: boolean;
-  viewport: string;
   pagesScanned: PageInfo[];
   pagesNotScanned: PageInfo[];
   totalPagesScanned: number;
@@ -85,17 +88,27 @@ type AllIssues = {
   topTenIssues: Array<any>;
   wcagViolations: string[];
   customFlowLabel: string;
-  phAppVersion: string;
+  oobeeAppVersion: string;
   items: {
     mustFix: Category;
     goodToFix: Category;
     needsReview: Category;
     passed: Category;
   };
-  cypressScanAboutMetadata: string;
+  cypressScanAboutMetadata: {
+    browser?: string;
+    viewport?: { width: number; height: number };
+  };
   wcagLinks: { [key: string]: string };
   [key: string]: any;
   advancedScanOptionsSummaryItems: { [key: string]: boolean };
+  scanPagesDetail: {
+    pagesAffected: any[];
+    pagesNotAffected: any[];
+    scannedPagesCount: number;
+    pagesNotScanned: any[];
+    pagesNotScannedCount: number;
+  };
 };
 
 const filename = fileURLToPath(import.meta.url);
@@ -160,13 +173,11 @@ const writeCsv = async (allIssues, storagePath) => {
       pagesAffected,
       helpUrl: learnMore,
     } = rule;
-    // we filter out the below as it represents the A/AA/AAA level, not the clause itself
-    const clausesArr = conformance.filter(
-      clause => !['wcag2a', 'wcag2aa', 'wcag2aaa'].includes(clause),
-    );
-    pagesAffected.sort((a, b) => a.url.localeCompare(b.url));
+
     // format clauses as a string
-    const wcagConformance = clausesArr.join(',');
+    const wcagConformance = conformance.join(',');
+
+    pagesAffected.sort((a, b) => a.url.localeCompare(b.url));
 
     pagesAffected.forEach(affectedPage => {
       const { url, items } = affectedPage;
@@ -239,8 +250,8 @@ const writeCsv = async (allIssues, storagePath) => {
           issueId: 'error-pages-skipped',
           issueDescription: 'Page was skipped during the scan',
           wcagConformance: '',
-          url: page.url || '',
-          pageTitle: '',
+          url: page.url || page || '',
+          pageTitle: 'Error',
           context: '',
           howToFix: '',
           axeImpact: '',
@@ -544,86 +555,93 @@ const writeLargeScanItemsJsonToFile = async (obj: object, filePath: string) => {
 
       keys.forEach((key, i) => {
         const value = obj[key];
-        queueWrite(`  "${key}": {\n`);
-
-        const { rules, ...otherProperties } = value;
-
-        // Write other properties
-        Object.entries(otherProperties).forEach(([propKey, propValue], j) => {
-          const propValueString =
-            propValue === null ||
-              typeof propValue === 'function' ||
-              typeof propValue === 'undefined'
-              ? 'null'
-              : JSON.stringify(propValue);
-          queueWrite(`    "${propKey}": ${propValueString}`);
-          if (j < Object.keys(otherProperties).length - 1 || (rules && rules.length >= 0)) {
-            queueWrite(',\n');
-          } else {
-            queueWrite('\n');
-          }
-        });
-
-        if (rules && Array.isArray(rules)) {
-          queueWrite('    "rules": [\n');
-
-          rules.forEach((rule, j) => {
-            queueWrite('      {\n');
-            const { pagesAffected, ...otherRuleProperties } = rule;
-
-            Object.entries(otherRuleProperties).forEach(([ruleKey, ruleValue], k) => {
-              const ruleValueString =
-                ruleValue === null ||
-                  typeof ruleValue === 'function' ||
-                  typeof ruleValue === 'undefined'
-                  ? 'null'
-                  : JSON.stringify(ruleValue);
-              queueWrite(`        "${ruleKey}": ${ruleValueString}`);
-              if (k < Object.keys(otherRuleProperties).length - 1 || pagesAffected) {
-                queueWrite(',\n');
-              } else {
-                queueWrite('\n');
-              }
-            });
 
-            if (pagesAffected && Array.isArray(pagesAffected)) {
-              queueWrite('        "pagesAffected": [\n');
-
-              pagesAffected.forEach((page, p) => {
-                const pageJson = JSON.stringify(page, null, 2)
-                  .split('\n')
-                  .map((line, idx) => (idx === 0 ? `          ${line}` : `          ${line}`))
-                  .join('\n');
-
-                queueWrite(pageJson);
+        if (value === null || typeof value !== 'object' || Array.isArray(value)) {
+          queueWrite(`  "${key}": ${JSON.stringify(value)}`);
+        } else {
+          queueWrite(`  "${key}": {\n`);
+        
+          const { rules, ...otherProperties } = value;
+        
+          // Write other properties
+          Object.entries(otherProperties).forEach(([propKey, propValue], j) => {
+            const propValueString =
+              propValue === null ||
+                typeof propValue === 'function' ||
+                typeof propValue === 'undefined'
+                ? 'null'
+                : JSON.stringify(propValue);
+            queueWrite(`    "${propKey}": ${propValueString}`);
+            if (j < Object.keys(otherProperties).length - 1 || (rules && rules.length >= 0)) {
+              queueWrite(',\n');
+            } else {
+              queueWrite('\n');
+            }
+          });
 
-                if (p < pagesAffected.length - 1) {
+          if (rules && Array.isArray(rules)) {
+            queueWrite('    "rules": [\n');
+
+            rules.forEach((rule, j) => {
+              queueWrite('      {\n');
+              const { pagesAffected, ...otherRuleProperties } = rule;
+
+              Object.entries(otherRuleProperties).forEach(([ruleKey, ruleValue], k) => {
+                const ruleValueString =
+                  ruleValue === null ||
+                    typeof ruleValue === 'function' ||
+                    typeof ruleValue === 'undefined'
+                    ? 'null'
+                    : JSON.stringify(ruleValue);
+                queueWrite(`        "${ruleKey}": ${ruleValueString}`);
+                if (k < Object.keys(otherRuleProperties).length - 1 || pagesAffected) {
                   queueWrite(',\n');
                 } else {
                   queueWrite('\n');
                 }
               });
 
-              queueWrite('        ]');
-            }
+              if (pagesAffected && Array.isArray(pagesAffected)) {
+                queueWrite('        "pagesAffected": [\n');
 
-            queueWrite('\n      }');
-            if (j < rules.length - 1) {
-              queueWrite(',\n');
-            } else {
-              queueWrite('\n');
-            }
-          });
+                pagesAffected.forEach((page, p) => {
+                  const pageJson = JSON.stringify(page, null, 2)
+                    .split('\n')
+                    .map((line, idx) => (idx === 0 ? `          ${line}` : `          ${line}`))
+                    .join('\n');
+
+                  queueWrite(pageJson);
+
+                  if (p < pagesAffected.length - 1) {
+                    queueWrite(',\n');
+                  } else {
+                    queueWrite('\n');
+                  }
+                });
+
+                queueWrite('        ]');
+              }
+
+              queueWrite('\n      }');
+              if (j < rules.length - 1) {
+                queueWrite(',\n');
+              } else {
+                queueWrite('\n');
+              }
+            });
+
+            queueWrite('    ]');
+          }
+          queueWrite('\n  }');
+      }
+
+      if (i < keys.length - 1) {
+        queueWrite(',\n');
+      } else {
+        queueWrite('\n');
+      }
 
-          queueWrite('    ]');
-        }
 
-        queueWrite('\n  }');
-        if (i < keys.length - 1) {
-          queueWrite(',\n');
-        } else {
-          queueWrite('\n');
-        }
       });
 
       queueWrite('}\n');
@@ -727,6 +745,12 @@ const writeJsonAndBase64Files = async (
   scanItemsSummaryBase64FilePath: string;
   scanItemsMiniReportJsonFilePath: string;
   scanItemsMiniReportBase64FilePath: string;
+  scanIssuesSummaryJsonFilePath: string;
+  scanIssuesSummaryBase64FilePath: string;
+  scanPagesDetailJsonFilePath: string;
+  scanPagesDetailBase64FilePath: string;
+  scanPagesSummaryJsonFilePath: string;
+  scanPagesSummaryBase64FilePath: string;
   scanDataJsonFileSize: number;
   scanItemsJsonFileSize: number;
 }> => {
@@ -734,7 +758,33 @@ const writeJsonAndBase64Files = async (
   const { jsonFilePath: scanDataJsonFilePath, base64FilePath: scanDataBase64FilePath } =
     await writeJsonFileAndCompressedJsonFile(rest, storagePath, 'scanData');
   const { jsonFilePath: scanItemsJsonFilePath, base64FilePath: scanItemsBase64FilePath } =
-    await writeJsonFileAndCompressedJsonFile(items, storagePath, 'scanItems');
+    await writeJsonFileAndCompressedJsonFile({ oobeeAppVersion: allIssues.oobeeAppVersion, ...items }, storagePath, 'scanItems');
+
+    // Add pagesAffectedCount to each rule in scanItemsMiniReport (items) and sort them in descending order of pagesAffectedCount
+    ['mustFix', 'goodToFix', 'needsReview', 'passed'].forEach((category) => {
+      if (items[category].rules && Array.isArray(items[category].rules)) {
+        items[category].rules.forEach((rule) => {
+          rule.pagesAffectedCount = Array.isArray(rule.pagesAffected)
+            ? rule.pagesAffected.length
+            : 0;
+        });
+
+        // Sort in descending order of pagesAffectedCount
+        items[category].rules.sort((a, b) => (b.pagesAffectedCount || 0) - (a.pagesAffectedCount || 0));
+      }
+    });
+
+    // Refactor scanIssuesSummary to reuse the scanItemsMiniReport structure by stripping out pagesAffected
+    const scanIssuesSummary = {
+      mustFix: items.mustFix.rules.map(({ pagesAffected, ...ruleInfo }) => ruleInfo),
+      goodToFix: items.goodToFix.rules.map(({ pagesAffected, ...ruleInfo }) => ruleInfo),
+      needsReview: items.needsReview.rules.map(({ pagesAffected, ...ruleInfo }) => ruleInfo),
+      passed: items.passed.rules.map(({ pagesAffected, ...ruleInfo }) => ruleInfo),
+    };
+
+    // Write out the scanIssuesSummary JSON using the new structure
+    const { jsonFilePath: scanIssuesSummaryJsonFilePath, base64FilePath: scanIssuesSummaryBase64FilePath } =
+    await writeJsonFileAndCompressedJsonFile({ oobeeAppVersion: allIssues.oobeeAppVersion, ...scanIssuesSummary }, storagePath, 'scanIssuesSummary');
 
   // scanItemsSummary
   // the below mutates the original items object, since it is expensive to clone
@@ -774,6 +824,8 @@ const writeJsonAndBase64Files = async (
     pagesNotScanned,
     wcagLinks,
     wcagPassPercentage,
+    progressPercentage,
+    issuesPercentage,
     totalPagesScanned,
     totalPagesNotScanned,
     topTenIssues,
@@ -786,6 +838,8 @@ const writeJsonAndBase64Files = async (
     pagesNotScanned,
     wcagLinks,
     wcagPassPercentage,
+    progressPercentage,
+    issuesPercentage,
     totalPagesScanned,
     totalPagesNotScanned,
     topTenIssues,
@@ -794,8 +848,8 @@ const writeJsonAndBase64Files = async (
   const {
     jsonFilePath: scanItemsMiniReportJsonFilePath,
     base64FilePath: scanItemsMiniReportBase64FilePath,
-  } = await writeJsonFileAndCompressedJsonFile(summaryItemsMini, storagePath, 'scanItemsSummaryMiniReport');
-
+  } = await writeJsonFileAndCompressedJsonFile({ oobeeAppVersion: allIssues.oobeeAppVersion, ...summaryItemsMini }, storagePath, 'scanItemsSummaryMiniReport');
+  
   const summaryItems = {
     mustFix: {
       totalItems: items.mustFix?.totalItems || 0,
@@ -812,6 +866,8 @@ const writeJsonAndBase64Files = async (
     topTenPagesWithMostIssues,
     wcagLinks,
     wcagPassPercentage,
+    progressPercentage,
+    issuesPercentage,
     totalPagesScanned,
     totalPagesNotScanned,
     topTenIssues,
@@ -820,7 +876,17 @@ const writeJsonAndBase64Files = async (
   const {
     jsonFilePath: scanItemsSummaryJsonFilePath,
     base64FilePath: scanItemsSummaryBase64FilePath,
-  } = await writeJsonFileAndCompressedJsonFile(summaryItems, storagePath, 'scanItemsSummary');
+  } = await writeJsonFileAndCompressedJsonFile({ oobeeAppVersion: allIssues.oobeeAppVersion, ...summaryItems }, storagePath, 'scanItemsSummary');
+
+  const {
+    jsonFilePath: scanPagesDetailJsonFilePath,
+    base64FilePath: scanPagesDetailBase64FilePath
+  } = await writeJsonFileAndCompressedJsonFile({ oobeeAppVersion: allIssues.oobeeAppVersion, ...allIssues.scanPagesDetail }, storagePath, 'scanPagesDetail');
+
+  const {
+    jsonFilePath: scanPagesSummaryJsonFilePath,
+    base64FilePath: scanPagesSummaryBase64FilePath
+  } = await writeJsonFileAndCompressedJsonFile({ oobeeAppVersion: allIssues.oobeeAppVersion, ...allIssues.scanPagesSummary }, storagePath, 'scanPagesSummary');
 
   return {
     scanDataJsonFilePath,
@@ -831,6 +897,12 @@ const writeJsonAndBase64Files = async (
     scanItemsSummaryBase64FilePath,
     scanItemsMiniReportJsonFilePath,
     scanItemsMiniReportBase64FilePath,
+    scanIssuesSummaryJsonFilePath,
+    scanIssuesSummaryBase64FilePath,
+    scanPagesDetailJsonFilePath,
+    scanPagesDetailBase64FilePath,
+    scanPagesSummaryJsonFilePath,
+    scanPagesSummaryBase64FilePath,
     scanDataJsonFileSize: fs.statSync(scanDataJsonFilePath).size,
     scanItemsJsonFileSize: fs.statSync(scanItemsJsonFilePath).size,
   };
@@ -1015,6 +1087,8 @@ const getTopTenIssues = allIssues => {
   const categories = ['mustFix', 'goodToFix'];
   const rulesWithCounts = [];
 
+  // This is no longer required and shall not be maintained in future
+  /*
   const conformanceLevels = {
     wcag2a: 'A',
     wcag2aa: 'AA',
@@ -1022,20 +1096,24 @@ const getTopTenIssues = allIssues => {
     wcag22aa: 'AA',
     wcag2aaa: 'AAA',
   };
+  */
 
   categories.forEach(category => {
     const rules = allIssues.items[category]?.rules || [];
 
     rules.forEach(rule => {
+      // This is not needed anymore since we want to have the clause number too
+      /*
       const wcagLevel = rule.conformance[0];
       const aLevel = conformanceLevels[wcagLevel] || wcagLevel;
+      */
 
       rulesWithCounts.push({
         category,
         ruleId: rule.rule,
         description: rule.description,
         axeImpact: rule.axeImpact,
-        conformance: aLevel,
+        conformance: rule.conformance,
         totalItems: rule.totalItems,
       });
     });
@@ -1047,48 +1125,71 @@ const getTopTenIssues = allIssues => {
 };
 
 const flattenAndSortResults = (allIssues: AllIssues, isCustomFlow: boolean) => {
+  // Create a map that will sum items only from mustFix, goodToFix, and needsReview.
   const urlOccurrencesMap = new Map<string, number>();
 
-  ['mustFix', 'goodToFix', 'needsReview', 'passed'].forEach(category => {
+  // Iterate over all categories; update the map only if the category is not "passed"
+  ['mustFix', 'goodToFix', 'needsReview', 'passed'].forEach((category) => {
+    // Accumulate totalItems regardless of category.
     allIssues.totalItems += allIssues.items[category].totalItems;
 
     allIssues.items[category].rules = Object.entries(allIssues.items[category].rules)
-      .map(ruleEntry => {
+      .map((ruleEntry) => {
         const [rule, ruleInfo] = ruleEntry as [string, RuleInfo];
         ruleInfo.pagesAffected = Object.entries(ruleInfo.pagesAffected)
-          .map(pageEntry => {
+          .map((pageEntry) => {
             if (isCustomFlow) {
               const [pageIndex, pageInfo] = pageEntry as unknown as [number, PageInfo];
-              urlOccurrencesMap.set(
-                pageInfo.url!,
-                (urlOccurrencesMap.get(pageInfo.url!) || 0) + pageInfo.items.length,
-              );
+              // Only update the occurrences map if not passed.
+              if (category !== 'passed') {
+                urlOccurrencesMap.set(
+                  pageInfo.url!,
+                  (urlOccurrencesMap.get(pageInfo.url!) || 0) + pageInfo.items.length
+                );
+              }
               return { pageIndex, ...pageInfo };
+            } else {
+              const [url, pageInfo] = pageEntry as unknown as [string, PageInfo];
+              if (category !== 'passed') {
+                urlOccurrencesMap.set(
+                  url,
+                  (urlOccurrencesMap.get(url) || 0) + pageInfo.items.length
+                );
+              }
+              return { url, ...pageInfo };
             }
-            const [url, pageInfo] = pageEntry as unknown as [string, PageInfo];
-            urlOccurrencesMap.set(url, (urlOccurrencesMap.get(url) || 0) + pageInfo.items.length);
-            return { url, ...pageInfo };
           })
+          // Sort pages so that those with the most items come first
           .sort((page1, page2) => page2.items.length - page1.items.length);
         return { rule, ...ruleInfo };
       })
+      // Sort the rules by totalItems (descending)
       .sort((rule1, rule2) => rule2.totalItems - rule1.totalItems);
   });
 
-  const updateIssuesWithOccurrences = (issuesList: Array<any>) => {
-    issuesList.forEach(issue => {
-      issue.totalOccurrences = urlOccurrencesMap.get(issue.url) || 0;
-    });
-  };
-
-  allIssues.topFiveMostIssues.sort((page1, page2) => page2.totalIssues - page1.totalIssues);
+  // Sort top pages (assumes topFiveMostIssues is already populated)
+  allIssues.topFiveMostIssues.sort((p1, p2) => p2.totalIssues - p1.totalIssues);
   allIssues.topTenPagesWithMostIssues = allIssues.topFiveMostIssues.slice(0, 10);
   allIssues.topFiveMostIssues = allIssues.topFiveMostIssues.slice(0, 5);
-  updateIssuesWithOccurrences(allIssues.topTenPagesWithMostIssues);
+
+  // Update each issue in topTenPagesWithMostIssues with the computed occurrences,
+  // excluding passed items.
+  updateIssuesWithOccurrences(allIssues.topTenPagesWithMostIssues, urlOccurrencesMap);
+
+  // Get and assign the topTenIssues (using your existing helper)
   const topTenIssues = getTopTenIssues(allIssues);
   allIssues.topTenIssues = topTenIssues;
 };
 
+// Helper: Update totalOccurrences for each issue using our urlOccurrencesMap.
+// For pages that have only passed items, the map will return undefined, so default to 0.
+function updateIssuesWithOccurrences(issuesList: any[], urlOccurrencesMap: Map<string, number>) {
+  issuesList.forEach((issue) => {
+    issue.totalOccurrences = urlOccurrencesMap.get(issue.url) || 0;
+  });
+}
+
+
 const createRuleIdJson = allIssues => {
   const compiledRuleJson = {};
 
@@ -1117,7 +1218,7 @@ const createRuleIdJson = allIssues => {
   return compiledRuleJson;
 };
 
-const moveElemScreenshots = (randomToken, storagePath) => {
+const moveElemScreenshots = (randomToken: string, storagePath: string) => {
   const currentScreenshotsPath = `${randomToken}/elemScreenshots`;
   const resultsScreenshotsPath = `${storagePath}/elemScreenshots`;
   if (fs.existsSync(currentScreenshotsPath)) {
@@ -1125,21 +1226,300 @@ const moveElemScreenshots = (randomToken, storagePath) => {
   }
 };
 
+/**
+ * Build allIssues.scanPagesDetail and allIssues.scanPagesSummary
+ * by analyzing pagesScanned (including mustFix/goodToFix/etc.).
+ */
+function populateScanPagesDetail(allIssues: AllIssues): void {
+  // --------------------------------------------
+  // 1) Gather your "scanned" pages from allIssues
+  // --------------------------------------------
+  const allScannedPages = Array.isArray(allIssues.pagesScanned)
+    ? allIssues.pagesScanned
+    : [];
+
+  // --------------------------------------------
+  // 2) Define category constants (optional, just for clarity)
+  // --------------------------------------------
+  const mustFixCategory = "mustFix";      
+  const goodToFixCategory = "goodToFix";  
+  const needsReviewCategory = "needsReview"; 
+  const passedCategory = "passed";           
+
+  // --------------------------------------------
+  // 3) Set up type declarations (if you want them local to this function)
+  // --------------------------------------------
+  type RuleData = {
+    ruleId: string;
+    wcagConformance: string[];
+    occurrencesMustFix: number;
+    occurrencesGoodToFix: number;
+    occurrencesNeedsReview: number;
+    occurrencesPassed: number;
+  };
+
+  type PageData = {
+    pageTitle: string;
+    url: string;
+    // Summaries
+    totalOccurrencesFailedIncludingNeedsReview: number; // mustFix + goodToFix + needsReview
+    totalOccurrencesFailedExcludingNeedsReview: number; // mustFix + goodToFix
+    totalOccurrencesNeedsReview: number;                // needsReview
+    totalOccurrencesPassed: number;                     // passed only
+    typesOfIssues: Record<string, RuleData>;
+  };
+
+  // --------------------------------------------
+  // 4) We'll accumulate pages in a map keyed by URL
+  // --------------------------------------------
+  const pagesMap: Record<string, PageData> = {};
+
+  // --------------------------------------------
+  // 5) Build pagesMap by iterating over each category in allIssues.items
+  // --------------------------------------------
+  Object.entries(allIssues.items).forEach(([categoryName, categoryData]) => {
+    if (!categoryData?.rules) return; // no rules in this category? skip
+
+    categoryData.rules.forEach(rule => {
+      const { rule: ruleId, conformance = [] } = rule;
+
+      rule.pagesAffected.forEach(p => {
+        const { url, pageTitle, items = [] } = p;
+        const itemsCount = items.length;
+
+        // Ensure the page is in pagesMap
+        if (!pagesMap[url]) {
+          pagesMap[url] = {
+            pageTitle,
+            url,
+            totalOccurrencesFailedIncludingNeedsReview: 0,
+            totalOccurrencesFailedExcludingNeedsReview: 0,
+            totalOccurrencesNeedsReview: 0,
+            totalOccurrencesPassed: 0,
+            typesOfIssues: {},
+          };
+        }
+
+        // Ensure the rule is present for this page
+        if (!pagesMap[url].typesOfIssues[ruleId]) {
+          pagesMap[url].typesOfIssues[ruleId] = {
+            ruleId,
+            wcagConformance: conformance,
+            occurrencesMustFix: 0,
+            occurrencesGoodToFix: 0,
+            occurrencesNeedsReview: 0,
+            occurrencesPassed: 0,
+          };
+        }
+
+        // Depending on the category, increment the relevant occurrence counts
+        if (categoryName === mustFixCategory) {
+          pagesMap[url].typesOfIssues[ruleId].occurrencesMustFix += itemsCount;
+          pagesMap[url].totalOccurrencesFailedIncludingNeedsReview += itemsCount;
+          pagesMap[url].totalOccurrencesFailedExcludingNeedsReview += itemsCount;
+        } else if (categoryName === goodToFixCategory) {
+          pagesMap[url].typesOfIssues[ruleId].occurrencesGoodToFix += itemsCount;
+          pagesMap[url].totalOccurrencesFailedIncludingNeedsReview += itemsCount;
+          pagesMap[url].totalOccurrencesFailedExcludingNeedsReview += itemsCount;
+        } else if (categoryName === needsReviewCategory) {
+          pagesMap[url].typesOfIssues[ruleId].occurrencesNeedsReview += itemsCount;
+          pagesMap[url].totalOccurrencesFailedIncludingNeedsReview += itemsCount;
+          pagesMap[url].totalOccurrencesNeedsReview += itemsCount;
+        } else if (categoryName === passedCategory) {
+          pagesMap[url].typesOfIssues[ruleId].occurrencesPassed += itemsCount;
+          pagesMap[url].totalOccurrencesPassed += itemsCount;
+        }
+      });
+    });
+  });
+
+  // --------------------------------------------
+  // 6) Separate scanned pages into “affected” vs. “notAffected”
+  // --------------------------------------------
+  const pagesInMap = Object.values(pagesMap); // All pages that have some record in pagesMap
+  const pagesInMapUrls = new Set(Object.keys(pagesMap));
+
+  // (a) Pages with only passed (no mustFix/goodToFix/needsReview)
+  const pagesAllPassed = pagesInMap.filter(
+    p => p.totalOccurrencesFailedIncludingNeedsReview === 0
+  );
+
+  // (b) Pages that do NOT appear in pagesMap at all => scanned but no items found
+  const pagesNoEntries = allScannedPages
+    .filter(sp => !pagesInMapUrls.has(sp.url))
+    .map(sp => ({
+      pageTitle: sp.pageTitle,
+      url: sp.url,
+      totalOccurrencesFailedIncludingNeedsReview: 0,
+      totalOccurrencesFailedExcludingNeedsReview: 0,
+      totalOccurrencesNeedsReview: 0,
+      totalOccurrencesPassed: 0,
+      typesOfIssues: {},
+    }));
+
+  // Combine these into "notAffected"
+  const pagesNotAffectedRaw = [...pagesAllPassed, ...pagesNoEntries];
+
+  // "affected" pages => have at least 1 mustFix/goodToFix/needsReview
+  const pagesAffectedRaw = pagesInMap.filter(
+    p => p.totalOccurrencesFailedIncludingNeedsReview > 0
+  );
+
+  // --------------------------------------------
+  // 7) Transform both arrays to the final shape
+  // --------------------------------------------
+  function transformPageData(page: PageData) {
+    const typesOfIssuesArray = Object.values(page.typesOfIssues);
+
+    // Compute sums for each failing category
+    const mustFixSum = typesOfIssuesArray.reduce((acc, r) => acc + r.occurrencesMustFix, 0);
+    const goodToFixSum = typesOfIssuesArray.reduce((acc, r) => acc + r.occurrencesGoodToFix, 0);
+    const needsReviewSum = typesOfIssuesArray.reduce((acc, r) => acc + r.occurrencesNeedsReview, 0);
+
+    // Build categoriesPresent based on nonzero failing counts
+    const categoriesPresent: string[] = [];
+    if (mustFixSum > 0) categoriesPresent.push("mustFix");
+    if (goodToFixSum > 0) categoriesPresent.push("goodToFix");
+    if (needsReviewSum > 0) categoriesPresent.push("needsReview");
+
+    // Count how many rules have failing issues
+    const failedRuleIds = new Set<string>();
+    typesOfIssuesArray.forEach(r => {
+      if ((r.occurrencesMustFix || 0) > 0 ||
+          (r.occurrencesGoodToFix || 0) > 0 ||
+          (r.occurrencesNeedsReview || 0) > 0) {
+        failedRuleIds.add(r.ruleId); // Ensure ruleId is unique
+      }
+    });
+    const failedRuleCount = failedRuleIds.size;
+
+    // Possibly these two for future convenience
+    const typesOfIssuesExcludingNeedsReviewCount = typesOfIssuesArray.filter(
+      r => (r.occurrencesMustFix || 0) + (r.occurrencesGoodToFix || 0) > 0
+    ).length;
+
+    const typesOfIssuesExclusiveToNeedsReviewCount = typesOfIssuesArray.filter(
+      r =>
+        (r.occurrencesNeedsReview || 0) > 0 &&
+        (r.occurrencesMustFix || 0) === 0 &&
+        (r.occurrencesGoodToFix || 0) === 0
+    ).length;
+    
+    // Aggregate wcagConformance for rules that actually fail
+    const allConformance = typesOfIssuesArray.reduce((acc, curr) => {
+      const nonPassedCount =
+        (curr.occurrencesMustFix || 0) +
+        (curr.occurrencesGoodToFix || 0) +
+        (curr.occurrencesNeedsReview || 0);
+
+      if (nonPassedCount > 0) {
+        return acc.concat(curr.wcagConformance || []);
+      }
+      return acc;
+    }, [] as string[]);
+    // Remove duplicates
+    const conformance = Array.from(new Set(allConformance));
+
+    return {
+      pageTitle: page.pageTitle,
+      url: page.url,
+      totalOccurrencesFailedIncludingNeedsReview: page.totalOccurrencesFailedIncludingNeedsReview,
+      totalOccurrencesFailedExcludingNeedsReview: page.totalOccurrencesFailedExcludingNeedsReview,
+      totalOccurrencesMustFix: mustFixSum,
+      totalOccurrencesGoodToFix: goodToFixSum,
+      totalOccurrencesNeedsReview: needsReviewSum,
+      totalOccurrencesPassed: page.totalOccurrencesPassed,
+      typesOfIssuesExclusiveToNeedsReviewCount,
+      typesOfIssuesCount: failedRuleCount,
+      typesOfIssuesExcludingNeedsReviewCount,
+      categoriesPresent,
+      conformance,
+      // Keep full detail for "scanPagesDetail"
+      typesOfIssues: typesOfIssuesArray,
+    };
+  }
+
+  // Transform raw pages
+  const pagesAffected = pagesAffectedRaw.map(transformPageData);
+  const pagesNotAffected = pagesNotAffectedRaw.map(transformPageData);
+
+  // --------------------------------------------
+  // 8) Sort pages by typesOfIssuesCount (descending) for both arrays
+  // --------------------------------------------
+  pagesAffected.sort((a, b) => b.typesOfIssuesCount - a.typesOfIssuesCount);
+  pagesNotAffected.sort((a, b) => b.typesOfIssuesCount - a.typesOfIssuesCount);
+
+  // --------------------------------------------
+  // 9) Compute scanned/ skipped counts
+  // --------------------------------------------
+  const scannedPagesCount = pagesAffected.length + pagesNotAffected.length;
+  const pagesNotScannedCount = Array.isArray(allIssues.pagesNotScanned)
+    ? allIssues.pagesNotScanned.length
+    : 0;
+
+  // --------------------------------------------
+  // 10) Build scanPagesDetail (with full "typesOfIssues")
+  // --------------------------------------------
+  allIssues.scanPagesDetail = {
+    pagesAffected,
+    pagesNotAffected,
+    scannedPagesCount,
+    pagesNotScanned: Array.isArray(allIssues.pagesNotScanned)
+      ? allIssues.pagesNotScanned
+      : [],
+    pagesNotScannedCount,
+  };
+
+  // --------------------------------------------
+  // 11) Build scanPagesSummary (strip out "typesOfIssues")
+  // --------------------------------------------
+  function stripTypesOfIssues(page: ReturnType<typeof transformPageData>) {
+    const { typesOfIssues, ...rest } = page;
+    return rest;
+  }
+
+  const summaryPagesAffected = pagesAffected.map(stripTypesOfIssues);
+  const summaryPagesNotAffected = pagesNotAffected.map(stripTypesOfIssues);
+
+  allIssues.scanPagesSummary = {
+    pagesAffected: summaryPagesAffected,
+    pagesNotAffected: summaryPagesNotAffected,
+    scannedPagesCount,
+    pagesNotScanned: Array.isArray(allIssues.pagesNotScanned)
+      ? allIssues.pagesNotScanned
+      : [],
+    pagesNotScannedCount,
+  };
+}
+
 const generateArtifacts = async (
-  randomToken,
-  urlScanned,
-  scanType,
-  viewport,
-  pagesScanned,
-  pagesNotScanned,
-  customFlowLabel,
-  cypressScanAboutMetadata,
-  scanDetails,
-  zip = undefined, // optional
+  randomToken: string,
+  urlScanned: string,
+  scanType: ScannerTypes,
+  viewport: string,
+  pagesScanned: PageInfo[],
+  pagesNotScanned: PageInfo[],
+  customFlowLabel: string,
+  cypressScanAboutMetadata: {
+    browser?: string;
+    viewport: { width: number; height: number };
+  },
+  scanDetails: {
+    startTime: Date;
+    endTime: Date;
+    deviceChosen: string;
+    isIncludeScreenshots: boolean;
+    isAllowSubdomains: string;
+    isEnableCustomChecks: string[];
+    isEnableWcagAaa: string[];
+    isSlowScanMode: number;
+    isAdhereRobots: boolean;
+  },
+  zip: string = undefined, // optional
   generateJsonFiles = false,
 ) => {
   const intermediateDatasetsPath = `${randomToken}/datasets/${randomToken}`;
-  const phAppVersion = getVersion();
+  const oobeeAppVersion = getVersion();
   const storagePath = getStoragePath(randomToken);
 
   urlScanned =
@@ -1147,7 +1527,7 @@ const generateArtifacts = async (
       ? urlScanned
       : urlWithoutAuth(urlScanned);
 
-  const formatAboutStartTime = dateString => {
+  const formatAboutStartTime = (dateString: string) => {
     const utcStartTimeDate = new Date(dateString);
     const formattedStartTime = utcStartTimeDate.toLocaleTimeString('en-GB', {
       year: 'numeric',
@@ -1200,7 +1580,7 @@ const generateArtifacts = async (
     topTenIssues: [],
     wcagViolations: [],
     customFlowLabel,
-    phAppVersion,
+    oobeeAppVersion,
     items: {
       mustFix: {
         description: itemTypeDescription.mustFix,
@@ -1229,6 +1609,13 @@ const generateArtifacts = async (
     },
     cypressScanAboutMetadata,
     wcagLinks: constants.wcagLinks,
+    scanPagesDetail: {
+      pagesAffected: [],
+      pagesNotAffected: [],
+      scannedPagesCount: 0,
+      pagesNotScanned: [],
+      pagesNotScannedCount: 0,
+    },
     // Populate boolean values for id="advancedScanOptionsSummary"
     advancedScanOptionsSummaryItems: {
       showIncludeScreenshots: [true].includes(scanDetails.isIncludeScreenshots),
@@ -1274,10 +1661,18 @@ const generateArtifacts = async (
     createScreenshotsFolder(randomToken);
   }
 
+  populateScanPagesDetail(allIssues);
+
   allIssues.wcagPassPercentage = getWcagPassPercentage(allIssues.wcagViolations, allIssues.advancedScanOptionsSummaryItems.showEnableWcagAaa);
-  consoleLogger.info(
-    `advancedScanOptionsSummaryItems is ${allIssues.advancedScanOptionsSummaryItems}`,
-  );
+  allIssues.progressPercentage = getProgressPercentage(allIssues.scanPagesDetail, allIssues.advancedScanOptionsSummaryItems.showEnableWcagAaa);
+  
+  allIssues.issuesPercentage = await getIssuesPercentage(
+    allIssues.scanPagesDetail, 
+    allIssues.advancedScanOptionsSummaryItems.showEnableWcagAaa, 
+    allIssues.advancedScanOptionsSummaryItems.disableOobee);
+
+  // console.log(allIssues.progressPercentage);
+  // console.log(allIssues.issuesPercentage);
 
   const getAxeImpactCount = (allIssues: AllIssues) => {
     const impactCount = {
@@ -1325,6 +1720,12 @@ const generateArtifacts = async (
     scanItemsSummaryBase64FilePath,
     scanItemsMiniReportJsonFilePath,
     scanItemsMiniReportBase64FilePath,
+    scanIssuesSummaryJsonFilePath,
+    scanIssuesSummaryBase64FilePath,
+    scanPagesDetailJsonFilePath,
+    scanPagesDetailBase64FilePath,
+    scanPagesSummaryJsonFilePath,
+    scanPagesSummaryBase64FilePath,
     scanDataJsonFileSize,
     scanItemsJsonFileSize,
   } = await writeJsonAndBase64Files(allIssues, storagePath);
@@ -1355,6 +1756,14 @@ const generateArtifacts = async (
       scanItemsBase64FilePath,
       scanItemsSummaryJsonFilePath,
       scanItemsSummaryBase64FilePath,
+      scanItemsMiniReportJsonFilePath,
+      scanItemsMiniReportBase64FilePath,
+      scanIssuesSummaryJsonFilePath,
+      scanIssuesSummaryBase64FilePath,
+      scanPagesDetailJsonFilePath,
+      scanPagesDetailBase64FilePath,
+      scanPagesSummaryJsonFilePath,
+      scanPagesSummaryBase64FilePath,
     ]);
   }
 
@@ -1378,13 +1787,7 @@ const generateArtifacts = async (
         `Results directory is at ${storagePath}`,
       ];
 
-      if (process.env.REPORT_BREAKDOWN === '1') {
-        messageToDisplay.push(
-          'Reports have been further broken down according to their respective impact level.',
-        );
-      }
-
-      if (process.send && process.env.OOBEE_VERBOSE && process.env.REPORT_BREAKDOWN != '1') {
+      if (process.send && process.env.OOBEE_VERBOSE) {
         const zipFileNameMessage = {
           type: 'zipFileName',
           payload: `${constants.cliZipFileName}`,