@govtechsg/oobee 0.10.43 → 0.10.45

package/package.json

@@ -1,12 +1,13 @@
 {
   "name": "@govtechsg/oobee",
   "main": "dist/npmIndex.js",
-  "version": "0.10.43",
+  "version": "0.10.45",
   "type": "module",
   "author": "Government Technology Agency <info@tech.gov.sg>",
   "dependencies": {
     "@json2csv/node": "^7.0.3",
     "@napi-rs/canvas": "^0.1.53",
+    "@sentry/node": "^9.13.0",
     "axe-core": "^4.10.2",
     "axios": "^1.8.2",
     "base64-stream": "^1.0.0",

package/src/constants/common.ts

@@ -18,12 +18,17 @@ import { minimatch } from 'minimatch';
 import { globSync, GlobOptionsWithFileTypesFalse } from 'glob';
 import { LaunchOptions, Locator, Page, devices, webkit } from 'playwright';
 import printMessage from 'print-message';
+// @ts-ignore
+import * as Sentry from '@sentry/node';
 import constants, {
   getDefaultChromeDataDir,
   getDefaultEdgeDataDir,
   getDefaultChromiumDataDir,
   proxy,
+  sentryConfig,
+    // Legacy code start - Google Sheets submission
   formDataFields,
+    // Legacy code end - Google Sheets submission
   ScannerTypes,
   BrowserTypes,
 } from './constants.js';
@@ -1753,41 +1758,160 @@ export const submitForm = async (
   numberOfPagesNotScanned: number,
   metadata: string,
 ) => {
-  const additionalPageDataJson = JSON.stringify({
-    redirectsScanned: numberOfRedirectsScanned,
+  // Initialize Sentry
+  Sentry.init(sentryConfig);
+
+  // Format the data as you want it to appear in Sentry
+  const additionalPageData = {
     pagesNotScanned: numberOfPagesNotScanned,
-  });
+    redirectsScanned: numberOfRedirectsScanned
+  };
 
-  let finalUrl =
-    `${formDataFields.formUrl}?` +
-    `${formDataFields.entryUrlField}=${entryUrl}&` +
-    `${formDataFields.scanTypeField}=${scanType}&` +
-    `${formDataFields.emailField}=${email}&` +
-    `${formDataFields.nameField}=${name}&` +
-    `${formDataFields.resultsField}=${encodeURIComponent(scanResultsJson)}&` +
-    `${formDataFields.numberOfPagesScannedField}=${numberOfPagesScanned}&` +
-    `${formDataFields.additionalPageDataField}=${encodeURIComponent(additionalPageDataJson)}&` +
-    `${formDataFields.metadataField}=${encodeURIComponent(metadata)}`;
+  // Extract issue occurrences from scan results if possible
+  const issueOccurrences = extractIssueOccurrences(scanResultsJson);
+  
+  // Determine if it's a government website
+  const isGov = entryUrl.includes('.gov');
+  
+  // Get email domain/tag
+  const emailTag = email.split('@')[1] || '';
+  
+  // Format timestamp 
+  const timestamp = new Date().toISOString();
+  
+  // Prepare redirect URL if different from entry URL
+  const redirectUrl = scannedUrl !== entryUrl ? scannedUrl : null;
+  
+  try {
+    // Capture the scan data as a Sentry event with each field as a separate entry
+    Sentry.captureEvent({
+      message: `Accessibility scan completed for ${entryUrl}`,
+      level: 'info',
+      tags: {
+        scanType: scanType,
+        browser: browserToRun,
+        isGov: isGov,
+        emailDomain: emailTag,
+      },
+      user: {
+        email: email,
+        username: name,
+      },
+      extra: {
+        // Top-level fields as shown in your screenshot
+        entryUrl: entryUrl,
+        websiteUrl: scannedUrl,
+        scanType: scanType,
+        numberOfPagesScanned: numberOfPagesScanned,
+        metadata: metadata ? JSON.parse(metadata) : {},
+        scanResults: scanResultsJson.length > 8000 ? 
+          scanResultsJson.substring(0, 8000) + '...[truncated]' : 
+          scanResultsJson,
+        
+        // Additional fields you requested
+        additionalPageData: additionalPageData,
+        additionalScan: additionalPageData,
+        additionalPagesData: additionalPageData,
+        
+        // Individual fields as requested
+        timestamp: timestamp,
+        redirectUrl: redirectUrl,
+        isGov: isGov,
+        emailTag: emailTag,
+        consolidatedScanType: scanType.toLowerCase(),
+        email: email,
+        name: name,
+        filledNoPagesScanned: numberOfPagesScanned > 0,
+        redirectsScanned: numberOfRedirectsScanned,
+        pagesNotScanned: numberOfPagesNotScanned,
+        issueOccurrences: issueOccurrences
+      }
+    });
 
-  if (scannedUrl !== entryUrl) {
-    finalUrl += `&${formDataFields.redirectUrlField}=${scannedUrl}`;
+    // IMPORTANT: Wait for the event to be sent
+    await Sentry.flush(2000); // Wait up to 2 seconds for the event to be sent
+    
+  } catch (error) {
+    console.error('Error sending data to Sentry:', error);
   }
 
-  if (proxy) {
-    await submitFormViaPlaywright(browserToRun, userDataDirectory, finalUrl);
-  } else {
-    try {
-      await axios.get(finalUrl, { timeout: 2000 });
-    } catch (error) {
-      if (error.code === 'ECONNABORTED') {
-        if (browserToRun || constants.launcher === webkit) {
-          await submitFormViaPlaywright(browserToRun, userDataDirectory, finalUrl);
+  // Legacy code start - Google Sheets submission
+  try {
+    const additionalPageDataJson = JSON.stringify({
+      redirectsScanned: numberOfRedirectsScanned,
+      pagesNotScanned: numberOfPagesNotScanned,
+    });
+
+    let finalUrl =
+      `${formDataFields.formUrl}?` +
+      `${formDataFields.entryUrlField}=${entryUrl}&` +
+      `${formDataFields.scanTypeField}=${scanType}&` +
+      `${formDataFields.emailField}=${email}&` +
+      `${formDataFields.nameField}=${name}&` +
+      `${formDataFields.resultsField}=${encodeURIComponent(scanResultsJson)}&` +
+      `${formDataFields.numberOfPagesScannedField}=${numberOfPagesScanned}&` +
+      `${formDataFields.additionalPageDataField}=${encodeURIComponent(additionalPageDataJson)}&` +
+      `${formDataFields.metadataField}=${encodeURIComponent(metadata)}`;
+
+    if (scannedUrl !== entryUrl) {
+      finalUrl += `&${formDataFields.redirectUrlField}=${scannedUrl}`;
+    }
+
+    if (proxy) {
+      await submitFormViaPlaywright(browserToRun, userDataDirectory, finalUrl);
+    } else {
+      try {
+        await axios.get(finalUrl, { timeout: 2000 });
+      } catch (error) {
+        if (error.code === 'ECONNABORTED') {
+          if (browserToRun || constants.launcher === webkit) {
+            await submitFormViaPlaywright(browserToRun, userDataDirectory, finalUrl);
+          }
         }
       }
     }
+    console.log('Legacy Google Sheets form submitted successfully');
+  } catch (legacyError) {
+    console.error('Error submitting legacy Google Sheets form:', legacyError);
   }
+  // Legacy code end - Google Sheets submission
 };
 
+// Helper function to extract issue occurrences from scan results
+function extractIssueOccurrences(scanResultsJson: string): number {
+  try {
+    const results = JSON.parse(scanResultsJson);
+    // Count total occurrences from all issues in the scan results
+    // This may need adjustment based on your specific JSON structure
+    let totalOccurrences = 0;
+    
+    // Try to parse the format shown in your screenshot
+    if (typeof results === 'object') {
+      // Loop through all keys that have "occurrences" properties
+      Object.keys(results).forEach(key => {
+        if (results[key] && typeof results[key] === 'object' && 'occurrences' in results[key]) {
+          totalOccurrences += parseInt(results[key].occurrences, 10) || 0;
+        }
+      });
+      
+      // If we found any occurrences, return the total
+      if (totalOccurrences > 0) {
+        return totalOccurrences;
+      }
+    }
+    
+    // Fallback to direct occurrences property if available
+    if (results && results.occurrences) {
+      return parseInt(results.occurrences, 10) || 0;
+    }
+    
+    return 0;
+  } catch (e) {
+    console.error('Error extracting issue occurrences:', e);
+    return 0;
+  }
+}
+
 export async function initModifiedUserAgent(
   browser?: string,
   playwrightDeviceDetailsObject?: object,

package/src/constants/constants.ts

@@ -6,6 +6,7 @@ import which from 'which';
 import os from 'os';
 import { spawnSync, execSync } from 'child_process';
 import { chromium } from 'playwright';
+import * as Sentry from '@sentry/node';
 import { silentLogger } from '../logs.js';
 import { PageInfo } from '../mergeAxeResults.js';
 
@@ -274,6 +275,12 @@ export const impactOrder = {
   critical: 3,
 };
 
+export const sentryConfig = {
+  dsn: "https://e4ab99e457c531e7bde4a8dc3dd2b1ab@o4509047624761344.ingest.us.sentry.io/4509192349548544",
+  tracesSampleRate: 1.0, // Capture 100% of transactions for performance monitoring
+  profilesSampleRate: 1.0, // Capture 100% of profiles
+};
+  // Legacy code start - Google Sheets submission
 export const formDataFields = {
   formUrl: `https://docs.google.com/forms/d/e/1FAIpQLSem5C8fyNs5TiU5Vv2Y63-SH7CHN86f-LEPxeN_1u_ldUbgUA/formResponse`, // prod
   entryUrlField: 'entry.1562345227',
@@ -286,6 +293,7 @@ export const formDataFields = {
   additionalPageDataField: 'entry.2090887881',
   metadataField: 'entry.1027769131',
 };
+  // Legacy code end - Google Sheets submission
 
 export const sitemapPaths = [
   '/sitemap.xml',

package/src/mergeAxeResults.ts

@@ -12,8 +12,10 @@ import { AsyncParser, ParserOptions } from '@json2csv/node';
 import zlib from 'zlib';
 import { Base64Encode } from 'base64-stream';
 import { pipeline } from 'stream/promises';
-import constants, { ScannerTypes } from './constants/constants.js';
+import constants, { ScannerTypes, sentryConfig } from './constants/constants.js';
 import { urlWithoutAuth } from './constants/common.js';
+// @ts-ignore
+import * as Sentry from '@sentry/node';
 import {
   createScreenshotsFolder,
   getStoragePath,
@@ -23,6 +25,7 @@ import {
   retryFunction,
   zipResults,
   getIssuesPercentage,
+  getWcagCriteriaMap,
 } from './utils.js';
 import { consoleLogger, silentLogger } from './logs.js';
 import itemTypeDescription from './constants/itemTypeDescription.js';
@@ -231,7 +234,7 @@ const writeCsv = async (allIssues, storagePath) => {
     includeEmptyRows: true,
   };
 
-  // Create the parse stream (it’s asynchronous)
+  // Create the parse stream (it's asynchronous)
   const parser = new AsyncParser(opts);
   const parseStream = parser.parse(allIssues);
 
@@ -985,6 +988,21 @@ const writeSummaryPdf = async (storagePath: string, pagesScanned: number, filena
   }
 };
 
+// Tracking WCAG occurrences 
+const wcagOccurrencesMap = new Map<string, number>();
+
+// Format WCAG tag in requested format: wcag111a_Occurrences
+const formatWcagTag = async (wcagId: string): Promise<string | null> => {
+  // Get dynamic WCAG criteria map
+  const wcagCriteriaMap = await getWcagCriteriaMap();
+  
+  if (wcagCriteriaMap[wcagId]) {
+    const { level } = wcagCriteriaMap[wcagId];
+    return `${wcagId}${level}_Occurrences`;
+  }
+  return null;
+};
+
 const pushResults = async (pageResults, allIssues, isCustomFlow) => {
   const { url, pageTitle, filePath } = pageResults;
 
@@ -1036,6 +1054,10 @@ const pushResults = async (pageResults, allIssues, isCustomFlow) => {
             if (!allIssues.wcagViolations.includes(c)) {
               allIssues.wcagViolations.push(c);
             }
+            
+            // Track WCAG criteria occurrences for Sentry
+            const currentCount = wcagOccurrencesMap.get(c) || 0;
+            wcagOccurrencesMap.set(c, currentCount + count);
           });
       }
 
@@ -1490,6 +1512,87 @@ function populateScanPagesDetail(allIssues: AllIssues): void {
   };
 }
 
+// Send WCAG criteria breakdown to Sentry
+const sendWcagBreakdownToSentry = async (
+  wcagBreakdown: Map<string, number>,
+  scanInfo: {
+    entryUrl: string;
+    scanType: string;
+    browser: string;
+    email?: string;
+    name?: string;
+  }
+) => {
+  try {
+    // Initialize Sentry
+    Sentry.init(sentryConfig);
+
+    // Prepare tags for the event
+    const tags: Record<string, string> = {};
+    const wcagCriteriaBreakdown: Record<string, number> = {};
+    
+    // Get dynamic WCAG criteria map once
+    const wcagCriteriaMap = await getWcagCriteriaMap();
+    
+    // First ensure all WCAG criteria are included in the tags with a value of 0
+    // This ensures criteria with no violations are still reported
+    for (const [wcagId, info] of Object.entries(wcagCriteriaMap)) {
+      const formattedTag = await formatWcagTag(wcagId);
+      if (formattedTag) {
+        // Initialize with zero
+        tags[formattedTag] = '0';
+        wcagCriteriaBreakdown[formattedTag] = 0;
+      }
+    }
+    
+    // Now override with actual counts from the scan
+    for (const [wcagId, count] of wcagBreakdown.entries()) {
+      const formattedTag = await formatWcagTag(wcagId);
+      if (formattedTag) {
+        // Add as a tag with the count as value
+        tags[formattedTag] = String(count);
+        
+        // Store in breakdown object for the extra data
+        wcagCriteriaBreakdown[formattedTag] = count;
+      }
+    }
+    
+    // Calculate the WCAG passing percentage
+    const totalCriteria = Object.keys(wcagCriteriaMap).length;
+    const violatedCriteria = wcagBreakdown.size;
+    const passingPercentage = Math.round(((totalCriteria - violatedCriteria) / totalCriteria) * 100);
+    
+    // Add the percentage as a tag
+    tags['WCAG-Percentage-Passed'] = String(passingPercentage);
+    
+    // Send the event to Sentry
+    await Sentry.captureEvent({
+      message: `WCAG Accessibility Scan Results for ${scanInfo.entryUrl}`,
+      level: 'info',
+      tags: {
+        ...tags,
+        event_type: 'accessibility_scan',
+        scanType: scanInfo.scanType,
+        browser: scanInfo.browser,
+      },
+      user: scanInfo.email && scanInfo.name ? {
+        email: scanInfo.email,
+        username: scanInfo.name
+      } : undefined,
+      extra: {
+        entryUrl: scanInfo.entryUrl,
+        wcagBreakdown: wcagCriteriaBreakdown,
+        wcagPassPercentage: passingPercentage
+      }
+    });
+    
+    // Wait for events to be sent
+    await Sentry.flush(2000);
+  } catch (error) {
+    console.error('Error sending WCAG breakdown to Sentry:', error);
+  }
+};
+
 const generateArtifacts = async (
   randomToken: string,
   urlScanned: string,
@@ -1512,6 +1615,7 @@ const generateArtifacts = async (
     isEnableWcagAaa: string[];
     isSlowScanMode: number;
     isAdhereRobots: boolean;
+    nameEmail?: { name: string; email: string };
   },
   zip: string = undefined, // optional
   generateJsonFiles = false,
@@ -1806,6 +1910,23 @@ const generateArtifacts = async (
       printMessage([`Error in zipping results: ${error}`]);
     });
 
+  // At the end of the function where results are generated, add:
+  try {
+    // Always send WCAG breakdown to Sentry, even if no violations were found
+    // This ensures that all criteria are reported, including those with 0 occurrences
+    await sendWcagBreakdownToSentry(wcagOccurrencesMap, {
+      entryUrl: urlScanned,
+      scanType: scanType,
+      browser: scanDetails.deviceChosen,
+      ...(scanDetails.nameEmail && {
+        email: scanDetails.nameEmail.email,
+        name: scanDetails.nameEmail.name
+      })
+    });
+  } catch (error) {
+    console.error('Error sending WCAG data to Sentry:', error);
+  }
+
   return createRuleIdJson(allIssues);
 };
 

package/src/screenshotFunc/htmlScreenshotFunc.ts

@@ -32,7 +32,7 @@ export const takeScreenshotForHTMLElements = async (
 
     // Check if rule ID is 'oobee-grading-text-contents' and skip screenshot logic
     if (rule === 'oobee-grading-text-contents') {
-      consoleLogger.info('Skipping screenshot for rule oobee-grading-text-contents');
+      // consoleLogger.info('Skipping screenshot for rule oobee-grading-text-contents');
       newViolations.push(violation); // Make sure it gets added
       continue;
     }

package/src/utils.ts

@@ -420,6 +420,132 @@ export const getTotalRulesCount = async (
   };
 };
 
+/**
+ * Dynamically generates a map of WCAG criteria IDs to their details (name and level)
+ * Reuses the rule processing logic from getTotalRulesCount
+ */
+export const getWcagCriteriaMap = async (
+  enableWcagAaa: boolean = true,
+  disableOobee: boolean = false
+): Promise<Record<string, { name: string; level: string }>> => {
+  // Reuse the configuration setup from getTotalRulesCount
+  const axeConfig = getAxeConfiguration({
+    enableWcagAaa,
+    gradingReadabilityFlag: '',
+    disableOobee,
+  });
+
+  // Get default rules from axe-core
+  const defaultRules = axe.getRules();
+
+  // Merge custom rules with default rules
+  const mergedRules: Rule[] = defaultRules.map(defaultRule => {
+    const customRule = axeConfig.rules.find(r => r.id === defaultRule.ruleId);
+    if (customRule) {
+      return {
+        id: defaultRule.ruleId,
+        enabled: customRule.enabled,
+        selector: customRule.selector,
+        any: customRule.any,
+        tags: defaultRule.tags,
+        metadata: customRule.metadata,
+      };
+    }
+    return {
+      id: defaultRule.ruleId,
+      enabled: true,
+      tags: defaultRule.tags,
+    };
+  });
+
+  // Add custom rules that don't override default rules
+  axeConfig.rules.forEach(customRule => {
+    if (!mergedRules.some(rule => rule.id === customRule.id)) {
+      mergedRules.push({
+        id: customRule.id,
+        enabled: customRule.enabled,
+        selector: customRule.selector,
+        any: customRule.any,
+        tags: customRule.tags,
+        metadata: customRule.metadata,
+      });
+    }
+  });
+
+  // Apply configuration
+  axe.configure({ ...axeConfig, rules: mergedRules });
+
+  // Build WCAG criteria map
+  const wcagCriteriaMap: Record<string, { name: string; level: string }> = {};
+  
+  // Process rules to extract WCAG information
+  mergedRules.forEach(rule => {
+    if (!rule.enabled) return;
+    if (rule.id === 'frame-tested') return;
+    
+    const tags = rule.tags || [];
+    if (tags.includes('experimental') || tags.includes('deprecated')) return;
+    
+    // Look for WCAG criteria tags (format: wcag111, wcag143, etc.)
+    tags.forEach(tag => {
+      const wcagMatch = tag.match(/^wcag(\d+)$/);
+      if (wcagMatch) {
+        const wcagId = tag;
+        
+        // Default values
+        let level = 'a';
+        let name = '';
+        
+        // Try to extract better info from metadata if available
+        const metadata = rule.metadata as any;
+        if (metadata && metadata.wcag) {
+          const wcagInfo = metadata.wcag as any;
+          
+          // Find matching criterion in metadata
+          for (const key in wcagInfo) {
+            const criterion = wcagInfo[key];
+            if (criterion && 
+                criterion.num && 
+                `wcag${criterion.num.replace(/\./g, '')}` === wcagId) {
+              
+              // Extract level
+              if (criterion.level) {
+                level = String(criterion.level).toLowerCase();
+              }
+              
+              // Extract name
+              if (criterion.handle) {
+                name = String(criterion.handle);
+              } else if (criterion.id) {
+                name = String(criterion.id);
+              } else if (criterion.num) {
+                name = `wcag-${String(criterion.num).replace(/\./g, '-')}`;
+              }
+              
+              break;
+            }
+          }
+        }
+        
+        // Generate fallback name if none found
+        if (!name) {
+          const numStr = wcagMatch[1];
+          const formattedNum = numStr.replace(/(\d)(\d)(\d+)?/, '$1.$2.$3');
+          name = `wcag-${formattedNum.replace(/\./g, '-')}`;
+        }
+        
+        // Store in map
+        wcagCriteriaMap[wcagId] = { 
+          name: name.toLowerCase().replace(/_/g, '-'),
+          level
+        };
+      }
+    });
+  });
+  
+  return wcagCriteriaMap;
+};
+
 export const getIssuesPercentage = async (
   scanPagesDetail: ScanPagesDetail,
   enableWcagAaa: boolean,