@govtechsg/oobee 0.10.42 → 0.10.45

package/src/mergeAxeResults.ts

@@ -12,8 +12,10 @@ import { AsyncParser, ParserOptions } from '@json2csv/node';
 import zlib from 'zlib';
 import { Base64Encode } from 'base64-stream';
 import { pipeline } from 'stream/promises';
-import constants, { ScannerTypes } from './constants/constants.js';
+import constants, { ScannerTypes, sentryConfig } from './constants/constants.js';
 import { urlWithoutAuth } from './constants/common.js';
+// @ts-ignore
+import * as Sentry from '@sentry/node';
 import {
   createScreenshotsFolder,
   getStoragePath,
@@ -23,6 +25,7 @@ import {
   retryFunction,
   zipResults,
   getIssuesPercentage,
+  getWcagCriteriaMap,
 } from './utils.js';
 import { consoleLogger, silentLogger } from './logs.js';
 import itemTypeDescription from './constants/itemTypeDescription.js';
@@ -45,6 +48,7 @@ export type PageInfo = {
   pageImagePath?: string;
   pageIndex?: number;
   metadata?: string;
+  httpStatusCode?: number;
 };
 
 export type RuleInfo = {
@@ -230,7 +234,7 @@ const writeCsv = async (allIssues, storagePath) => {
     includeEmptyRows: true,
   };
 
-  // Create the parse stream (it’s asynchronous)
+  // Create the parse stream (it's asynchronous)
   const parser = new AsyncParser(opts);
   const parseStream = parser.parse(allIssues);
 
@@ -248,7 +252,7 @@ const writeCsv = async (allIssues, storagePath) => {
           scanCompletedAt: allIssues.endTime ? allIssues.endTime.toISOString() : '',
           severity: 'error',
           issueId: 'error-pages-skipped',
-          issueDescription: 'Page was skipped during the scan',
+          issueDescription: page.metadata ? page.metadata : 'An unknown error caused the page to be skipped',
           wcagConformance: '',
           url: page.url || page || '',
           pageTitle: 'Error',
@@ -791,25 +795,21 @@ const writeJsonAndBase64Files = async (
   items.mustFix.rules.forEach(rule => {
     rule.pagesAffected.forEach(page => {
       page.itemsCount = page.items.length;
-      page.items = [];
     });
   });
   items.goodToFix.rules.forEach(rule => {
     rule.pagesAffected.forEach(page => {
       page.itemsCount = page.items.length;
-      page.items = [];
     });
   });
   items.needsReview.rules.forEach(rule => {
     rule.pagesAffected.forEach(page => {
       page.itemsCount = page.items.length;
-      page.items = [];
     });
   });
   items.passed.rules.forEach(rule => {
     rule.pagesAffected.forEach(page => {
       page.itemsCount = page.items.length;
-      page.items = [];
     });
   });
 
@@ -988,6 +988,21 @@ const writeSummaryPdf = async (storagePath: string, pagesScanned: number, filena
   }
 };
 
+// Tracking WCAG occurrences 
+const wcagOccurrencesMap = new Map<string, number>();
+
+// Format WCAG tag in requested format: wcag111a_Occurrences
+const formatWcagTag = async (wcagId: string): Promise<string | null> => {
+  // Get dynamic WCAG criteria map
+  const wcagCriteriaMap = await getWcagCriteriaMap();
+  
+  if (wcagCriteriaMap[wcagId]) {
+    const { level } = wcagCriteriaMap[wcagId];
+    return `${wcagId}${level}_Occurrences`;
+  }
+  return null;
+};
+
 const pushResults = async (pageResults, allIssues, isCustomFlow) => {
   const { url, pageTitle, filePath } = pageResults;
 
@@ -1039,6 +1054,10 @@ const pushResults = async (pageResults, allIssues, isCustomFlow) => {
             if (!allIssues.wcagViolations.includes(c)) {
               allIssues.wcagViolations.push(c);
             }
+            
+            // Track WCAG criteria occurrences for Sentry
+            const currentCount = wcagOccurrencesMap.get(c) || 0;
+            wcagOccurrencesMap.set(c, currentCount + count);
           });
       }
 
@@ -1205,6 +1224,7 @@ const createRuleIdJson = allIssues => {
         });
       });
       snippets = [...snippetsSet];
+      rule.pagesAffected.forEach(p => { delete p.items; });
     }
     compiledRuleJson[ruleId] = {
       snippets,
@@ -1492,6 +1512,87 @@ function populateScanPagesDetail(allIssues: AllIssues): void {
   };
 }
 
+// Send WCAG criteria breakdown to Sentry
+const sendWcagBreakdownToSentry = async (
+  wcagBreakdown: Map<string, number>,
+  scanInfo: {
+    entryUrl: string;
+    scanType: string;
+    browser: string;
+    email?: string;
+    name?: string;
+  }
+) => {
+  try {
+    // Initialize Sentry
+    Sentry.init(sentryConfig);
+
+    // Prepare tags for the event
+    const tags: Record<string, string> = {};
+    const wcagCriteriaBreakdown: Record<string, number> = {};
+    
+    // Get dynamic WCAG criteria map once
+    const wcagCriteriaMap = await getWcagCriteriaMap();
+    
+    // First ensure all WCAG criteria are included in the tags with a value of 0
+    // This ensures criteria with no violations are still reported
+    for (const [wcagId, info] of Object.entries(wcagCriteriaMap)) {
+      const formattedTag = await formatWcagTag(wcagId);
+      if (formattedTag) {
+        // Initialize with zero
+        tags[formattedTag] = '0';
+        wcagCriteriaBreakdown[formattedTag] = 0;
+      }
+    }
+    
+    // Now override with actual counts from the scan
+    for (const [wcagId, count] of wcagBreakdown.entries()) {
+      const formattedTag = await formatWcagTag(wcagId);
+      if (formattedTag) {
+        // Add as a tag with the count as value
+        tags[formattedTag] = String(count);
+        
+        // Store in breakdown object for the extra data
+        wcagCriteriaBreakdown[formattedTag] = count;
+      }
+    }
+    
+    // Calculate the WCAG passing percentage
+    const totalCriteria = Object.keys(wcagCriteriaMap).length;
+    const violatedCriteria = wcagBreakdown.size;
+    const passingPercentage = Math.round(((totalCriteria - violatedCriteria) / totalCriteria) * 100);
+    
+    // Add the percentage as a tag
+    tags['WCAG-Percentage-Passed'] = String(passingPercentage);
+    
+    // Send the event to Sentry
+    await Sentry.captureEvent({
+      message: `WCAG Accessibility Scan Results for ${scanInfo.entryUrl}`,
+      level: 'info',
+      tags: {
+        ...tags,
+        event_type: 'accessibility_scan',
+        scanType: scanInfo.scanType,
+        browser: scanInfo.browser,
+      },
+      user: scanInfo.email && scanInfo.name ? {
+        email: scanInfo.email,
+        username: scanInfo.name
+      } : undefined,
+      extra: {
+        entryUrl: scanInfo.entryUrl,
+        wcagBreakdown: wcagCriteriaBreakdown,
+        wcagPassPercentage: passingPercentage
+      }
+    });
+    
+    // Wait for events to be sent
+    await Sentry.flush(2000);
+  } catch (error) {
+    console.error('Error sending WCAG breakdown to Sentry:', error);
+  }
+};
+
 const generateArtifacts = async (
   randomToken: string,
   urlScanned: string,
@@ -1514,6 +1615,7 @@ const generateArtifacts = async (
     isEnableWcagAaa: string[];
     isSlowScanMode: number;
     isAdhereRobots: boolean;
+    nameEmail?: { name: string; email: string };
   },
   zip: string = undefined, // optional
   generateJsonFiles = false,
@@ -1808,6 +1910,23 @@ const generateArtifacts = async (
       printMessage([`Error in zipping results: ${error}`]);
     });
 
+  // At the end of the function where results are generated, add:
+  try {
+    // Always send WCAG breakdown to Sentry, even if no violations were found
+    // This ensures that all criteria are reported, including those with 0 occurrences
+    await sendWcagBreakdownToSentry(wcagOccurrencesMap, {
+      entryUrl: urlScanned,
+      scanType: scanType,
+      browser: scanDetails.deviceChosen,
+      ...(scanDetails.nameEmail && {
+        email: scanDetails.nameEmail.email,
+        name: scanDetails.nameEmail.name
+      })
+    });
+  } catch (error) {
+    console.error('Error sending WCAG data to Sentry:', error);
+  }
+
   return createRuleIdJson(allIssues);
 };
 

package/src/npmIndex.ts

@@ -1,7 +1,7 @@
 import fs from 'fs';
 import path from 'path';
 import printMessage from 'print-message';
-import axe, { ImpactValue } from 'axe-core';
+import axe, { AxeResults, ImpactValue } from 'axe-core';
 import { fileURLToPath } from 'url';
 import { EnqueueStrategy } from 'crawlee';
 import constants, { BrowserTypes, RuleFlags, ScannerTypes } from './constants/constants.js';
@@ -16,7 +16,7 @@ import { createCrawleeSubFolders, filterAxeResults } from './crawlers/commonCraw
 import { createAndUpdateResultsFolders, createDetailsAndLogs } from './utils.js';
 import generateArtifacts from './mergeAxeResults.js';
 import { takeScreenshotForHTMLElements } from './screenshotFunc/htmlScreenshotFunc.js';
-import { silentLogger } from './logs.js';
+import { consoleLogger, silentLogger } from './logs.js';
 import { alertMessageOptions } from './constants/cliFunctions.js';
 import { evaluateAltText } from './crawlers/custom/evaluateAltText.js';
 import { escapeCssSelector } from './crawlers/custom/escapeCssSelector.js';
@@ -24,7 +24,7 @@ import { framesCheck } from './crawlers/custom/framesCheck.js';
 import { findElementByCssSelector } from './crawlers/custom/findElementByCssSelector.js';
 import { getAxeConfiguration } from './crawlers/custom/getAxeConfiguration.js';
 import { flagUnlabelledClickableElements } from './crawlers/custom/flagUnlabelledClickableElements.js';
-import { xPathToCss } from './crawlers/custom/xPathToCss.js';
+import xPathToCss from './crawlers/custom/xPathToCss.js';
 import { extractText } from './crawlers/custom/extractText.js';
 import { gradeReadability } from './crawlers/custom/gradeReadability.js';
 
@@ -65,7 +65,7 @@ export const init = async ({
   specifiedMaxConcurrency?: number;
   followRobots?: boolean;
 }) => {
-  console.log('Starting Oobee');
+  consoleLogger.info('Starting Oobee');
 
   const [date, time] = new Date().toLocaleString('sv').replaceAll(/-|:/g, '').split(' ');
   const domain = new URL(entryUrl).hostname;
@@ -126,7 +126,7 @@ export const init = async ({
             const cssSelector = xPathToCss(xpath);
             return cssSelector;
           } catch (e) {
-            console.error('Error converting XPath to CSS: ', xpath, e);
+            consoleLogger.error(`Error converting XPath to CSS: ${xpath} - ${e}`);
             return '';
           }
         })
@@ -197,7 +197,11 @@ export const init = async ({
     `;
   };
 
-  const pushScanResults = async (res, metadata, elementsToClick) => {
+  const pushScanResults = async (
+    res: { pageUrl: string; pageTitle: string; axeScanResults: AxeResults },
+    metadata: string,
+    elementsToClick: string[],
+  ) => {
     throwErrorIfTerminated();
     if (includeScreenshots) {
       // use chrome by default
@@ -211,7 +215,7 @@ export const init = async ({
       await page.waitForLoadState('networkidle');
 
       // click on elements to reveal hidden elements so screenshots can be taken
-      elementsToClick?.forEach(async elem => {
+      elementsToClick?.forEach(async (elem: string) => {
         try {
           await page.locator(elem).click();
         } catch (e) {
@@ -259,7 +263,7 @@ export const init = async ({
 
   const terminate = async () => {
     throwErrorIfTerminated();
-    console.log('Stopping Oobee');
+    consoleLogger.info('Stopping Oobee');
     isInstanceTerminated = true;
     scanDetails.endTime = new Date();
     scanDetails.urlsCrawled = urlsCrawled;

package/src/screenshotFunc/htmlScreenshotFunc.ts

@@ -7,7 +7,7 @@ import { Result } from 'axe-core';
 import { Page } from 'playwright';
 import { NodeResultWithScreenshot, ResultWithScreenshot } from '../crawlers/commonCrawlerFunc.js';
 
-const screenshotMap = {}; // Map of screenshot hashkey to its buffer value and screenshot path
+const screenshotMap: Record<string, string> = {}; // Map of screenshot hashkey to its buffer value and screenshot path
 
 export const takeScreenshotForHTMLElements = async (
   violations: Result[],
@@ -32,7 +32,7 @@ export const takeScreenshotForHTMLElements = async (
 
     // Check if rule ID is 'oobee-grading-text-contents' and skip screenshot logic
     if (rule === 'oobee-grading-text-contents') {
-      consoleLogger.info('Skipping screenshot for rule oobee-grading-text-contents');
+      // consoleLogger.info('Skipping screenshot for rule oobee-grading-text-contents');
       newViolations.push(violation); // Make sure it gets added
       continue;
     }
@@ -75,30 +75,18 @@ export const takeScreenshotForHTMLElements = async (
   return newViolations;
 };
 
-const generateBufferHash = (buffer: Buffer) => {
+const generateBufferHash = (buffer: Buffer): string => {
   const hash = createHash('sha256');
   hash.update(buffer);
   return hash.digest('hex');
 };
 
-const isSameBufferHash = (buffer: Buffer, hash: string) => {
-  const bufferHash = generateBufferHash(buffer);
-  return hash === bufferHash;
-};
-
-const getIdenticalScreenshotKey = (buffer: Buffer) => {
-  for (const hashKey in screenshotMap) {
-    const isIdentical = isSameBufferHash(buffer, hashKey);
-    if (isIdentical) return hashKey;
-  }
-  return undefined;
-};
-
-const getScreenshotPath = (buffer: Buffer, randomToken: string) => {
-  let hashKey = getIdenticalScreenshotKey(buffer);
+const getScreenshotPath = (buffer: Buffer, randomToken: string): string => {
+  let hashKey = generateBufferHash(buffer);
+  const existingPath = screenshotMap[hashKey];
   // If exists identical entry in screenshot map, get its filepath
-  if (hashKey) {
-    return screenshotMap[hashKey];
+  if (existingPath) {
+    return existingPath;
   }
   // Create new entry in screenshot map
   hashKey = generateBufferHash(buffer);

package/src/types/text-readability.d.ts

@@ -0,0 +1,3 @@
+declare module 'text-readability' {
+  function fleschReadingEase(text: string): number;
+}

package/src/types/types.ts

@@ -21,7 +21,7 @@ export type StructureTree = {
   pageIndex?: number;
 };
 
-type DeviceDescriptor = {
+export type DeviceDescriptor = {
   viewport: ViewportSize;
   userAgent: string;
   deviceScaleFactor: number;