@govtechsg/oobee 0.10.76 → 0.10.78-alpha1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (137) hide show
  1. package/.github/workflows/publish.yml +8 -1
  2. package/INTEGRATION.md +50 -3
  3. package/dist/cli.js +252 -0
  4. package/dist/combine.js +221 -0
  5. package/dist/constants/cliFunctions.js +306 -0
  6. package/dist/constants/common.js +1669 -0
  7. package/dist/constants/constants.js +913 -0
  8. package/dist/constants/errorMeta.json +319 -0
  9. package/dist/constants/itemTypeDescription.js +7 -0
  10. package/dist/constants/oobeeAi.js +121 -0
  11. package/dist/constants/questions.js +151 -0
  12. package/dist/constants/sampleData.js +176 -0
  13. package/dist/crawlers/commonCrawlerFunc.js +428 -0
  14. package/dist/crawlers/crawlDomain.js +613 -0
  15. package/dist/crawlers/crawlIntelligentSitemap.js +135 -0
  16. package/dist/crawlers/crawlLocalFile.js +151 -0
  17. package/dist/crawlers/crawlSitemap.js +303 -0
  18. package/dist/crawlers/custom/escapeCssSelector.js +10 -0
  19. package/dist/crawlers/custom/evaluateAltText.js +11 -0
  20. package/dist/crawlers/custom/extractAndGradeText.js +44 -0
  21. package/dist/crawlers/custom/extractText.js +27 -0
  22. package/dist/crawlers/custom/findElementByCssSelector.js +36 -0
  23. package/dist/crawlers/custom/flagUnlabelledClickableElements.js +963 -0
  24. package/dist/crawlers/custom/framesCheck.js +37 -0
  25. package/dist/crawlers/custom/getAxeConfiguration.js +111 -0
  26. package/dist/crawlers/custom/gradeReadability.js +23 -0
  27. package/dist/crawlers/custom/utils.js +1024 -0
  28. package/dist/crawlers/custom/xPathToCss.js +147 -0
  29. package/dist/crawlers/guards/urlGuard.js +71 -0
  30. package/dist/crawlers/pdfScanFunc.js +276 -0
  31. package/dist/crawlers/runCustom.js +89 -0
  32. package/dist/exclusions.txt +7 -0
  33. package/dist/generateHtmlReport.js +144 -0
  34. package/dist/index.js +62 -0
  35. package/dist/logs.js +84 -0
  36. package/dist/mergeAxeResults.js +1588 -0
  37. package/dist/npmIndex.js +640 -0
  38. package/dist/proxyService.js +360 -0
  39. package/dist/runGenerateJustHtmlReport.js +16 -0
  40. package/dist/screenshotFunc/htmlScreenshotFunc.js +355 -0
  41. package/dist/screenshotFunc/pdfScreenshotFunc.js +645 -0
  42. package/dist/services/s3Uploader.js +127 -0
  43. package/dist/static/ejs/partials/components/allIssues/AllIssues.ejs +9 -0
  44. package/dist/static/ejs/partials/components/allIssues/CategoryBadges.ejs +82 -0
  45. package/dist/static/ejs/partials/components/allIssues/FilterBar.ejs +33 -0
  46. package/dist/static/ejs/partials/components/allIssues/IssuesTable.ejs +41 -0
  47. package/dist/static/ejs/partials/components/header/SiteInfo.ejs +119 -0
  48. package/dist/static/ejs/partials/components/header/aboutScanModal/AboutScanModal.ejs +15 -0
  49. package/dist/static/ejs/partials/components/header/aboutScanModal/ScanConfiguration.ejs +44 -0
  50. package/dist/static/ejs/partials/components/header/aboutScanModal/ScanDetails.ejs +142 -0
  51. package/dist/static/ejs/partials/components/prioritiseIssues/IssueDetailCard.ejs +36 -0
  52. package/dist/static/ejs/partials/components/prioritiseIssues/PrioritiseIssues.ejs +47 -0
  53. package/dist/static/ejs/partials/components/ruleModal/ruleOffcanvas.ejs +196 -0
  54. package/dist/static/ejs/partials/components/scannedPagesSegmentedTabs.ejs +48 -0
  55. package/dist/static/ejs/partials/components/screenshotLightbox.ejs +13 -0
  56. package/dist/static/ejs/partials/components/shared/InfoAlert.ejs +3 -0
  57. package/dist/static/ejs/partials/components/summaryScanAbout.ejs +141 -0
  58. package/dist/static/ejs/partials/components/summaryScanResults.ejs +16 -0
  59. package/dist/static/ejs/partials/components/summaryTable.ejs +20 -0
  60. package/dist/static/ejs/partials/components/summaryWcagCompliance.ejs +94 -0
  61. package/dist/static/ejs/partials/components/topTen.ejs +6 -0
  62. package/dist/static/ejs/partials/components/wcagCompliance/FailedCriteria.ejs +47 -0
  63. package/dist/static/ejs/partials/components/wcagCompliance/WcagCompliance.ejs +16 -0
  64. package/dist/static/ejs/partials/components/wcagCompliance/WcagGaugeBar.ejs +16 -0
  65. package/dist/static/ejs/partials/components/wcagCoverageDetails.ejs +18 -0
  66. package/dist/static/ejs/partials/footer.ejs +24 -0
  67. package/dist/static/ejs/partials/header.ejs +14 -0
  68. package/dist/static/ejs/partials/main.ejs +29 -0
  69. package/dist/static/ejs/partials/scripts/allIssues/AllIssues.ejs +376 -0
  70. package/dist/static/ejs/partials/scripts/bootstrap.ejs +8 -0
  71. package/dist/static/ejs/partials/scripts/categorySummary.ejs +141 -0
  72. package/dist/static/ejs/partials/scripts/decodeUnzipParse.ejs +3 -0
  73. package/dist/static/ejs/partials/scripts/header/SiteInfo.ejs +44 -0
  74. package/dist/static/ejs/partials/scripts/header/aboutScanModal/AboutScanModal.ejs +51 -0
  75. package/dist/static/ejs/partials/scripts/header/aboutScanModal/ScanConfiguration.ejs +127 -0
  76. package/dist/static/ejs/partials/scripts/header/aboutScanModal/ScanDetails.ejs +60 -0
  77. package/dist/static/ejs/partials/scripts/highlightjs.ejs +335 -0
  78. package/dist/static/ejs/partials/scripts/popper.ejs +7 -0
  79. package/dist/static/ejs/partials/scripts/prioritiseIssues/IssueDetailCard.ejs +137 -0
  80. package/dist/static/ejs/partials/scripts/prioritiseIssues/PrioritiseIssues.ejs +214 -0
  81. package/dist/static/ejs/partials/scripts/prioritiseIssues/wcagSvgMap.ejs +861 -0
  82. package/dist/static/ejs/partials/scripts/ruleModal/constants.ejs +957 -0
  83. package/dist/static/ejs/partials/scripts/ruleModal/itemCardRenderer.ejs +353 -0
  84. package/dist/static/ejs/partials/scripts/ruleModal/pageAccordionBuilder.ejs +468 -0
  85. package/dist/static/ejs/partials/scripts/ruleModal/ruleOffcanvas.ejs +306 -0
  86. package/dist/static/ejs/partials/scripts/ruleModal/utilities.ejs +483 -0
  87. package/dist/static/ejs/partials/scripts/scannedPagesSegmentedTabs.ejs +35 -0
  88. package/dist/static/ejs/partials/scripts/screenshotLightbox.ejs +75 -0
  89. package/dist/static/ejs/partials/scripts/summaryScanResults.ejs +14 -0
  90. package/dist/static/ejs/partials/scripts/summaryTable.ejs +78 -0
  91. package/dist/static/ejs/partials/scripts/topTen.ejs +61 -0
  92. package/dist/static/ejs/partials/scripts/utils.ejs +453 -0
  93. package/dist/static/ejs/partials/scripts/wcagCompliance/FailedCriteria.ejs +103 -0
  94. package/dist/static/ejs/partials/scripts/wcagCompliance/WcagGaugeBar.ejs +47 -0
  95. package/dist/static/ejs/partials/scripts/wcagCompliance.ejs +15 -0
  96. package/dist/static/ejs/partials/scripts/wcagCoverageDetails.ejs +75 -0
  97. package/dist/static/ejs/partials/styles/allIssues/AllIssues.ejs +384 -0
  98. package/dist/static/ejs/partials/styles/bootstrap.ejs +12391 -0
  99. package/dist/static/ejs/partials/styles/header/SiteInfo.ejs +121 -0
  100. package/dist/static/ejs/partials/styles/header/aboutScanModal/AboutScanModal.ejs +82 -0
  101. package/dist/static/ejs/partials/styles/header/aboutScanModal/ScanConfiguration.ejs +50 -0
  102. package/dist/static/ejs/partials/styles/header/aboutScanModal/ScanDetails.ejs +149 -0
  103. package/dist/static/ejs/partials/styles/header.ejs +7 -0
  104. package/dist/static/ejs/partials/styles/highlightjs.ejs +54 -0
  105. package/dist/static/ejs/partials/styles/prioritiseIssues/IssueDetailCard.ejs +141 -0
  106. package/dist/static/ejs/partials/styles/prioritiseIssues/PrioritiseIssues.ejs +204 -0
  107. package/dist/static/ejs/partials/styles/ruleModal/ruleOffcanvas.ejs +456 -0
  108. package/dist/static/ejs/partials/styles/scannedPagesSegmentedTabs.ejs +46 -0
  109. package/dist/static/ejs/partials/styles/shared/InfoAlert.ejs +12 -0
  110. package/dist/static/ejs/partials/styles/styles.ejs +1607 -0
  111. package/dist/static/ejs/partials/styles/summaryBootstrap.ejs +12458 -0
  112. package/dist/static/ejs/partials/styles/topTenCard.ejs +44 -0
  113. package/dist/static/ejs/partials/styles/wcagCompliance/FailedCriteria.ejs +59 -0
  114. package/dist/static/ejs/partials/styles/wcagCompliance/WcagGaugeBar.ejs +62 -0
  115. package/dist/static/ejs/partials/styles/wcagCompliance.ejs +36 -0
  116. package/dist/static/ejs/partials/styles/wcagCoverageDetails.ejs +33 -0
  117. package/dist/static/ejs/partials/summaryHeader.ejs +70 -0
  118. package/dist/static/ejs/partials/summaryMain.ejs +49 -0
  119. package/dist/static/ejs/report.ejs +226 -0
  120. package/dist/static/ejs/summary.ejs +47 -0
  121. package/dist/types/types.js +1 -0
  122. package/dist/utils.js +1070 -0
  123. package/examples/oobee-cypress-integration-js/cypress/support/e2e.js +36 -6
  124. package/examples/oobee-cypress-integration-js/cypress.config.js +45 -1
  125. package/examples/oobee-cypress-integration-ts/cypress.config.ts +47 -1
  126. package/examples/oobee-cypress-integration-ts/src/cypress/support/e2e.ts +36 -6
  127. package/examples/oobee-playwright-integration-js/oobee-playwright-demo.js +2 -1
  128. package/examples/oobee-playwright-integration-ts/src/oobee-playwright-demo.ts +2 -1
  129. package/examples/oobee-scan-html-demo.js +51 -0
  130. package/examples/oobee-scan-page-demo.js +40 -0
  131. package/package.json +9 -3
  132. package/src/constants/common.ts +2 -2
  133. package/src/constants/constants.ts +3 -1
  134. package/src/crawlers/crawlDomain.ts +1 -0
  135. package/src/crawlers/runCustom.ts +0 -1
  136. package/src/mergeAxeResults.ts +43 -22
  137. package/src/npmIndex.ts +500 -131
@@ -0,0 +1,1588 @@
1
+ import fs, { ensureDirSync } from 'fs-extra';
2
+ import printMessage from 'print-message';
3
+ import path from 'path';
4
+ import ejs from 'ejs';
5
+ import { fileURLToPath } from 'url';
6
+ import { createWriteStream } from 'fs';
7
+ import { AsyncParser } from '@json2csv/node';
8
+ import zlib from 'zlib';
9
+ import { Base64Encode } from 'base64-stream';
10
+ import { pipeline } from 'stream/promises';
11
+ import * as Sentry from '@sentry/node';
12
+ import constants, { BrowserTypes, ScannerTypes, sentryConfig, setSentryUser, WCAGclauses, a11yRuleShortDescriptionMap, disabilityBadgesMap, a11yRuleLongDescriptionMap, } from './constants/constants.js';
13
+ import { getBrowserToRun, getPlaywrightLaunchOptions } from './constants/common.js';
14
+ import { createScreenshotsFolder, getStoragePath, getVersion, getWcagPassPercentage, getProgressPercentage, retryFunction, zipResults, getIssuesPercentage, getWcagCriteriaMap, categorizeWcagCriteria, getUserDataTxt, register, } from './utils.js';
15
+ import { consoleLogger } from './logs.js';
16
+ import itemTypeDescription from './constants/itemTypeDescription.js';
17
+ import { oobeeAiHtmlETL, oobeeAiRules } from './constants/oobeeAi.js';
18
+ const filename = fileURLToPath(import.meta.url);
19
+ const dirname = path.dirname(filename);
20
+ const BUFFER_LIMIT = 100 * 1024 * 1024; // 100MB size
21
+ const extractFileNames = async (directory) => {
22
+ ensureDirSync(directory);
23
+ return fs
24
+ .readdir(directory)
25
+ .then(allFiles => allFiles.filter(file => path.extname(file).toLowerCase() === '.json'))
26
+ .catch(readdirError => {
27
+ consoleLogger.info('An error has occurred when retrieving files, please try again.');
28
+ throw readdirError;
29
+ });
30
+ };
31
+ const parseContentToJson = async (rPath) => fs
32
+ .readFile(rPath, 'utf8')
33
+ .then(content => JSON.parse(content))
34
+ .catch(parseError => {
35
+ consoleLogger.info('An error has occurred when parsing the content, please try again.');
36
+ });
37
+ const writeCsv = async (allIssues, storagePath) => {
38
+ const csvOutput = createWriteStream(`${storagePath}/report.csv`, { encoding: 'utf8' });
39
+ const formatPageViolation = pageNum => {
40
+ if (pageNum < 0)
41
+ return 'Document';
42
+ return `Page ${pageNum}`;
43
+ };
44
+ // transform allIssues into the form:
45
+ // [['mustFix', rule1], ['mustFix', rule2], ['goodToFix', rule3], ...]
46
+ const getRulesByCategory = (allIssues) => {
47
+ return Object.entries(allIssues.items)
48
+ .filter(([category]) => category !== 'passed')
49
+ .reduce((prev, [category, value]) => {
50
+ const rulesEntries = Object.entries(value.rules);
51
+ rulesEntries.forEach(([, ruleInfo]) => {
52
+ prev.push([category, ruleInfo]);
53
+ });
54
+ return prev;
55
+ }, [])
56
+ .sort((a, b) => {
57
+ // sort rules according to severity, then ruleId
58
+ const compareCategory = -a[0].localeCompare(b[0]);
59
+ return compareCategory === 0 ? a[1].rule.localeCompare(b[1].rule) : compareCategory;
60
+ });
61
+ };
62
+ const flattenRule = catAndRule => {
63
+ const [severity, rule] = catAndRule;
64
+ const results = [];
65
+ const { rule: issueId, description: issueDescription, axeImpact, conformance, pagesAffected, helpUrl: learnMore, } = rule;
66
+ // format clauses as a string
67
+ const wcagConformance = conformance.join(',');
68
+ pagesAffected.sort((a, b) => a.url.localeCompare(b.url));
69
+ pagesAffected.forEach(affectedPage => {
70
+ const { url, items } = affectedPage;
71
+ items.forEach(item => {
72
+ const { html, page, message, xpath } = item;
73
+ const howToFix = message.replace(/(\r\n|\n|\r)/g, '\\n'); // preserve newlines as \n
74
+ const violation = html || formatPageViolation(page); // page is a number, not a string
75
+ const context = violation.replace(/(\r\n|\n|\r)/g, ''); // remove newlines
76
+ results.push({
77
+ customFlowLabel: allIssues.customFlowLabel || '',
78
+ deviceChosen: allIssues.deviceChosen || '',
79
+ scanCompletedAt: allIssues.endTime ? allIssues.endTime.toISOString() : '',
80
+ severity: severity || '',
81
+ issueId: issueId || '',
82
+ issueDescription: a11yRuleShortDescriptionMap[issueId] || issueDescription || '',
83
+ wcagConformance: wcagConformance || '',
84
+ url: url || '',
85
+ pageTitle: affectedPage.pageTitle || 'No page title',
86
+ context: context || '',
87
+ howToFix: howToFix || '',
88
+ axeImpact: axeImpact || '',
89
+ xpath: xpath || '',
90
+ learnMore: learnMore || '',
91
+ });
92
+ });
93
+ });
94
+ if (results.length === 0)
95
+ return {};
96
+ return results;
97
+ };
98
+ const opts = {
99
+ transforms: [getRulesByCategory, flattenRule],
100
+ fields: [
101
+ 'customFlowLabel',
102
+ 'deviceChosen',
103
+ 'scanCompletedAt',
104
+ 'severity',
105
+ 'issueId',
106
+ 'issueDescription',
107
+ 'wcagConformance',
108
+ 'url',
109
+ 'pageTitle',
110
+ 'context',
111
+ 'howToFix',
112
+ 'axeImpact',
113
+ 'xpath',
114
+ 'learnMore',
115
+ ],
116
+ includeEmptyRows: true,
117
+ };
118
+ // Create the parse stream (it's asynchronous)
119
+ const parser = new AsyncParser(opts);
120
+ const parseStream = parser.parse(allIssues);
121
+ // Pipe JSON2CSV output into the file, but don't end automatically
122
+ parseStream.pipe(csvOutput, { end: false });
123
+ // Once JSON2CSV is done writing all normal rows, append any "pagesNotScanned"
124
+ parseStream.on('end', () => {
125
+ if (allIssues.pagesNotScanned && allIssues.pagesNotScanned.length > 0) {
126
+ csvOutput.write('\n');
127
+ allIssues.pagesNotScanned.forEach(page => {
128
+ const skippedPage = {
129
+ customFlowLabel: allIssues.customFlowLabel || '',
130
+ deviceChosen: allIssues.deviceChosen || '',
131
+ scanCompletedAt: allIssues.endTime ? allIssues.endTime.toISOString() : '',
132
+ severity: 'error',
133
+ issueId: 'error-pages-skipped',
134
+ issueDescription: page.metadata
135
+ ? page.metadata
136
+ : 'An unknown error caused the page to be skipped',
137
+ wcagConformance: '',
138
+ url: page.url || page || '',
139
+ pageTitle: 'Error',
140
+ context: '',
141
+ howToFix: '',
142
+ axeImpact: '',
143
+ xpath: '',
144
+ learnMore: '',
145
+ };
146
+ csvOutput.write(`${Object.values(skippedPage).join(',')}\n`);
147
+ });
148
+ }
149
+ // Now close the CSV file
150
+ csvOutput.end();
151
+ });
152
+ parseStream.on('error', err => {
153
+ console.error('Error parsing CSV:', err);
154
+ csvOutput.end();
155
+ });
156
+ };
157
+ const compileHtmlWithEJS = async (allIssues, storagePath, htmlFilename = 'report') => {
158
+ const htmlFilePath = `${path.join(storagePath, htmlFilename)}.html`;
159
+ const ejsString = fs.readFileSync(path.join(dirname, './static/ejs/report.ejs'), 'utf-8');
160
+ const template = ejs.compile(ejsString, {
161
+ filename: path.join(dirname, './static/ejs/report.ejs'),
162
+ });
163
+ const html = template({ ...allIssues, storagePath: JSON.stringify(storagePath) });
164
+ await fs.writeFile(htmlFilePath, html);
165
+ let htmlContent = await fs.readFile(htmlFilePath, { encoding: 'utf8' });
166
+ const headIndex = htmlContent.indexOf('</head>');
167
+ const injectScript = `
168
+ <script>
169
+ // IMPORTANT! DO NOT REMOVE ME: Decode the encoded data
170
+
171
+ </script>
172
+ `;
173
+ if (headIndex !== -1) {
174
+ htmlContent = htmlContent.slice(0, headIndex) + injectScript + htmlContent.slice(headIndex);
175
+ }
176
+ else {
177
+ htmlContent += injectScript;
178
+ }
179
+ await fs.writeFile(htmlFilePath, htmlContent);
180
+ return htmlFilePath;
181
+ };
182
+ const splitHtmlAndCreateFiles = async (htmlFilePath, storagePath) => {
183
+ try {
184
+ const htmlContent = await fs.readFile(htmlFilePath, { encoding: 'utf8' });
185
+ const splitMarker = '// IMPORTANT! DO NOT REMOVE ME: Decode the encoded data';
186
+ const splitIndex = htmlContent.indexOf(splitMarker);
187
+ if (splitIndex === -1) {
188
+ throw new Error('Marker comment not found in the HTML file.');
189
+ }
190
+ const topContent = `${htmlContent.slice(0, splitIndex + splitMarker.length)}\n\n`;
191
+ const bottomContent = htmlContent.slice(splitIndex + splitMarker.length);
192
+ const topFilePath = path.join(storagePath, 'report-partial-top.htm.txt');
193
+ const bottomFilePath = path.join(storagePath, 'report-partial-bottom.htm.txt');
194
+ await fs.writeFile(topFilePath, topContent, { encoding: 'utf8' });
195
+ await fs.writeFile(bottomFilePath, bottomContent, { encoding: 'utf8' });
196
+ await fs.unlink(htmlFilePath);
197
+ return { topFilePath, bottomFilePath };
198
+ }
199
+ catch (error) {
200
+ console.error('Error splitting HTML and creating files:', error);
201
+ }
202
+ };
203
+ const writeHTML = async (allIssues, storagePath, htmlFilename = 'report', scanDetailsFilePath, scanItemsFilePath) => {
204
+ const htmlFilePath = await compileHtmlWithEJS(allIssues, storagePath, htmlFilename);
205
+ const { topFilePath, bottomFilePath } = await splitHtmlAndCreateFiles(htmlFilePath, storagePath);
206
+ const prefixData = fs.readFileSync(path.join(storagePath, 'report-partial-top.htm.txt'), 'utf-8');
207
+ const suffixData = fs.readFileSync(path.join(storagePath, 'report-partial-bottom.htm.txt'), 'utf-8');
208
+ const scanDetailsReadStream = fs.createReadStream(scanDetailsFilePath, {
209
+ encoding: 'utf8',
210
+ highWaterMark: BUFFER_LIMIT,
211
+ });
212
+ const scanItemsReadStream = fs.createReadStream(scanItemsFilePath, {
213
+ encoding: 'utf8',
214
+ highWaterMark: BUFFER_LIMIT,
215
+ });
216
+ const outputFilePath = `${storagePath}/${htmlFilename}.html`;
217
+ const outputStream = fs.createWriteStream(outputFilePath, { flags: 'a' });
218
+ const cleanupFiles = async () => {
219
+ try {
220
+ await Promise.all([fs.promises.unlink(topFilePath), fs.promises.unlink(bottomFilePath)]);
221
+ }
222
+ catch (err) {
223
+ console.error('Error cleaning up temporary files:', err);
224
+ }
225
+ };
226
+ outputStream.write(prefixData);
227
+ // For Proxied AI environments only
228
+ outputStream.write(`let proxyUrl = "${process.env.PROXY_API_BASE_URL || ""}"\n`);
229
+ // Initialize GenAI feature flag
230
+ outputStream.write(`
231
+ // Fetch GenAI feature flag from backend
232
+ window.oobeeGenAiFeatureEnabled = false;
233
+ if (proxyUrl !== "" && proxyUrl !== undefined && proxyUrl !== null) {
234
+ (async () => {
235
+ try {
236
+ const featuresUrl = proxyUrl + '/api/ai/features';
237
+ const response = await fetch(featuresUrl, {
238
+ method: 'GET',
239
+ headers: { 'Accept': 'application/json' }
240
+ });
241
+ if (response.ok) {
242
+ const features = await response.json();
243
+ window.oobeeGenAiFeatureEnabled = features.genai_ui_enabled || false;
244
+ console.log('GenAI UI feature flag:', window.oobeeGenAiFeatureEnabled);
245
+ } else {
246
+ console.warn('Failed to fetch GenAI feature flag:', response.status);
247
+ }
248
+ } catch (error) {
249
+ console.warn('Error fetching GenAI feature flag:', error);
250
+ }
251
+ })();
252
+ } else {
253
+ console.warn('Skipping fetch GenAI feature as it is local report');
254
+ }
255
+ \n`);
256
+ // outputStream.write("scanData = decompressJsonObject('");
257
+ outputStream.write("let scanDataPromise = (async () => { console.log('Loading scanData...'); scanData = await decodeUnzipParse('");
258
+ scanDetailsReadStream.pipe(outputStream, { end: false });
259
+ scanDetailsReadStream.on('end', () => {
260
+ // outputStream.write("')\n\n");
261
+ outputStream.write("'); })();\n\n");
262
+ // outputStream.write("(scanItems = decompressJsonObject('");
263
+ outputStream.write("let scanItemsPromise = (async () => { console.log('Loading scanItems...'); scanItems = await decodeUnzipParse('");
264
+ scanItemsReadStream.pipe(outputStream, { end: false });
265
+ });
266
+ scanDetailsReadStream.on('error', err => {
267
+ console.error('Read stream error:', err);
268
+ outputStream.end();
269
+ });
270
+ scanItemsReadStream.on('end', () => {
271
+ // outputStream.write("')\n\n");
272
+ outputStream.write("'); })();\n\n");
273
+ outputStream.write(suffixData);
274
+ outputStream.end();
275
+ });
276
+ scanItemsReadStream.on('error', err => {
277
+ console.error('Read stream error:', err);
278
+ outputStream.end();
279
+ });
280
+ consoleLogger.info('Content appended successfully.');
281
+ await cleanupFiles();
282
+ outputStream.on('error', err => {
283
+ consoleLogger.error('Error writing to output file:', err);
284
+ });
285
+ };
286
+ const writeSummaryHTML = async (allIssues, storagePath, htmlFilename = 'summary') => {
287
+ const ejsString = fs.readFileSync(path.join(dirname, './static/ejs/summary.ejs'), 'utf-8');
288
+ const template = ejs.compile(ejsString, {
289
+ filename: path.join(dirname, './static/ejs/summary.ejs'),
290
+ });
291
+ const html = template(allIssues);
292
+ fs.writeFileSync(`${storagePath}/${htmlFilename}.html`, html);
293
+ };
294
+ const writeSitemap = async (pagesScanned, storagePath) => {
295
+ const sitemapPath = path.join(storagePath, 'sitemap.txt');
296
+ const content = pagesScanned.map(p => p.url).join('\n');
297
+ await fs.writeFile(sitemapPath, content, { encoding: 'utf-8' });
298
+ consoleLogger.info(`Sitemap written to ${sitemapPath}`);
299
+ };
300
+ const cleanUpJsonFiles = async (filesToDelete) => {
301
+ consoleLogger.info('Cleaning up JSON files...');
302
+ filesToDelete.forEach(file => {
303
+ fs.unlinkSync(file);
304
+ consoleLogger.info(`Deleted ${file}`);
305
+ });
306
+ };
307
+ function* serializeObject(obj, depth = 0, indent = ' ') {
308
+ const currentIndent = indent.repeat(depth);
309
+ const nextIndent = indent.repeat(depth + 1);
310
+ if (obj instanceof Date) {
311
+ yield JSON.stringify(obj.toISOString());
312
+ return;
313
+ }
314
+ if (Array.isArray(obj)) {
315
+ yield '[\n';
316
+ for (let i = 0; i < obj.length; i++) {
317
+ if (i > 0)
318
+ yield ',\n';
319
+ yield nextIndent;
320
+ yield* serializeObject(obj[i], depth + 1, indent);
321
+ }
322
+ yield `\n${currentIndent}]`;
323
+ return;
324
+ }
325
+ if (obj !== null && typeof obj === 'object') {
326
+ yield '{\n';
327
+ const keys = Object.keys(obj);
328
+ for (let i = 0; i < keys.length; i++) {
329
+ const key = keys[i];
330
+ if (i > 0)
331
+ yield ',\n';
332
+ yield `${nextIndent}${JSON.stringify(key)}: `;
333
+ yield* serializeObject(obj[key], depth + 1, indent);
334
+ }
335
+ yield `\n${currentIndent}}`;
336
+ return;
337
+ }
338
+ if (obj === null || typeof obj === 'function' || typeof obj === 'undefined') {
339
+ yield 'null';
340
+ return;
341
+ }
342
+ yield JSON.stringify(obj);
343
+ }
344
+ function writeLargeJsonToFile(obj, filePath) {
345
+ return new Promise((resolve, reject) => {
346
+ const writeStream = fs.createWriteStream(filePath, { encoding: 'utf8' });
347
+ writeStream.on('error', error => {
348
+ consoleLogger.error('Stream error:', error);
349
+ reject(error);
350
+ });
351
+ writeStream.on('finish', () => {
352
+ consoleLogger.info(`JSON file written successfully: ${filePath}`);
353
+ resolve(true);
354
+ });
355
+ const generator = serializeObject(obj);
356
+ function write() {
357
+ let next;
358
+ while (!(next = generator.next()).done) {
359
+ if (!writeStream.write(next.value)) {
360
+ writeStream.once('drain', write);
361
+ return;
362
+ }
363
+ }
364
+ writeStream.end();
365
+ }
366
+ write();
367
+ });
368
+ }
369
+ const writeLargeScanItemsJsonToFile = async (obj, filePath) => {
370
+ return new Promise((resolve, reject) => {
371
+ const writeStream = fs.createWriteStream(filePath, { flags: 'a', encoding: 'utf8' });
372
+ const writeQueue = [];
373
+ let isWriting = false;
374
+ const processNextWrite = async () => {
375
+ if (isWriting || writeQueue.length === 0)
376
+ return;
377
+ isWriting = true;
378
+ const data = writeQueue.shift();
379
+ try {
380
+ if (!writeStream.write(data)) {
381
+ await new Promise(resolve => {
382
+ writeStream.once('drain', () => {
383
+ resolve();
384
+ });
385
+ });
386
+ }
387
+ }
388
+ catch (error) {
389
+ writeStream.destroy(error);
390
+ return;
391
+ }
392
+ isWriting = false;
393
+ processNextWrite();
394
+ };
395
+ const queueWrite = (data) => {
396
+ writeQueue.push(data);
397
+ processNextWrite();
398
+ };
399
+ writeStream.on('error', error => {
400
+ consoleLogger.error(`Error writing object to JSON file: ${error}`);
401
+ reject(error);
402
+ });
403
+ writeStream.on('finish', () => {
404
+ consoleLogger.info(`JSON file written successfully: ${filePath}`);
405
+ resolve(true);
406
+ });
407
+ try {
408
+ queueWrite('{\n');
409
+ const keys = Object.keys(obj);
410
+ keys.forEach((key, i) => {
411
+ const value = obj[key];
412
+ if (value === null || typeof value !== 'object' || Array.isArray(value)) {
413
+ queueWrite(` "${key}": ${JSON.stringify(value)}`);
414
+ }
415
+ else {
416
+ queueWrite(` "${key}": {\n`);
417
+ const { rules, ...otherProperties } = value;
418
+ // Write other properties
419
+ Object.entries(otherProperties).forEach(([propKey, propValue], j) => {
420
+ const propValueString = propValue === null ||
421
+ typeof propValue === 'function' ||
422
+ typeof propValue === 'undefined'
423
+ ? 'null'
424
+ : JSON.stringify(propValue);
425
+ queueWrite(` "${propKey}": ${propValueString}`);
426
+ if (j < Object.keys(otherProperties).length - 1 || (rules && rules.length >= 0)) {
427
+ queueWrite(',\n');
428
+ }
429
+ else {
430
+ queueWrite('\n');
431
+ }
432
+ });
433
+ if (rules && Array.isArray(rules)) {
434
+ queueWrite(' "rules": [\n');
435
+ rules.forEach((rule, j) => {
436
+ queueWrite(' {\n');
437
+ const { pagesAffected, ...otherRuleProperties } = rule;
438
+ Object.entries(otherRuleProperties).forEach(([ruleKey, ruleValue], k) => {
439
+ const ruleValueString = ruleValue === null ||
440
+ typeof ruleValue === 'function' ||
441
+ typeof ruleValue === 'undefined'
442
+ ? 'null'
443
+ : JSON.stringify(ruleValue);
444
+ queueWrite(` "${ruleKey}": ${ruleValueString}`);
445
+ if (k < Object.keys(otherRuleProperties).length - 1 || pagesAffected) {
446
+ queueWrite(',\n');
447
+ }
448
+ else {
449
+ queueWrite('\n');
450
+ }
451
+ });
452
+ if (pagesAffected && Array.isArray(pagesAffected)) {
453
+ queueWrite(' "pagesAffected": [\n');
454
+ pagesAffected.forEach((page, p) => {
455
+ const pageJson = JSON.stringify(page, null, 2)
456
+ .split('\n')
457
+ .map((line, idx) => (idx === 0 ? ` ${line}` : ` ${line}`))
458
+ .join('\n');
459
+ queueWrite(pageJson);
460
+ if (p < pagesAffected.length - 1) {
461
+ queueWrite(',\n');
462
+ }
463
+ else {
464
+ queueWrite('\n');
465
+ }
466
+ });
467
+ queueWrite(' ]');
468
+ }
469
+ queueWrite('\n }');
470
+ if (j < rules.length - 1) {
471
+ queueWrite(',\n');
472
+ }
473
+ else {
474
+ queueWrite('\n');
475
+ }
476
+ });
477
+ queueWrite(' ]');
478
+ }
479
+ queueWrite('\n }');
480
+ }
481
+ if (i < keys.length - 1) {
482
+ queueWrite(',\n');
483
+ }
484
+ else {
485
+ queueWrite('\n');
486
+ }
487
+ });
488
+ queueWrite('}\n');
489
+ // Ensure all queued writes are processed before ending
490
+ const checkQueueAndEnd = () => {
491
+ if (writeQueue.length === 0 && !isWriting) {
492
+ writeStream.end();
493
+ }
494
+ else {
495
+ setTimeout(checkQueueAndEnd, 100);
496
+ }
497
+ };
498
+ checkQueueAndEnd();
499
+ }
500
+ catch (err) {
501
+ writeStream.destroy(err);
502
+ reject(err);
503
+ }
504
+ });
505
+ };
506
+ async function compressJsonFileStreaming(inputPath, outputPath) {
507
+ // Create the read and write streams
508
+ const readStream = fs.createReadStream(inputPath);
509
+ const writeStream = fs.createWriteStream(outputPath);
510
+ // Create a gzip transform stream
511
+ const gzip = zlib.createGzip();
512
+ // Create a Base64 transform stream
513
+ const base64Encode = new Base64Encode();
514
+ // Pipe the streams:
515
+ // read -> gzip -> base64 -> write
516
+ await pipeline(readStream, gzip, base64Encode, writeStream);
517
+ consoleLogger.info(`File successfully compressed and saved to ${outputPath}`);
518
+ }
519
+ const writeJsonFileAndCompressedJsonFile = async (data, storagePath, filename) => {
520
+ try {
521
+ consoleLogger.info(`Writing JSON to ${filename}.json`);
522
+ const jsonFilePath = path.join(storagePath, `${filename}.json`);
523
+ if (filename === 'scanItems') {
524
+ await writeLargeScanItemsJsonToFile(data, jsonFilePath);
525
+ }
526
+ else {
527
+ await writeLargeJsonToFile(data, jsonFilePath);
528
+ }
529
+ consoleLogger.info(`Reading ${filename}.json, gzipping and base64 encoding it into ${filename}.json.gz.b64`);
530
+ const base64FilePath = path.join(storagePath, `${filename}.json.gz.b64`);
531
+ await compressJsonFileStreaming(jsonFilePath, base64FilePath);
532
+ consoleLogger.info(`Finished compression and base64 encoding for ${filename}`);
533
+ return {
534
+ jsonFilePath,
535
+ base64FilePath,
536
+ };
537
+ }
538
+ catch (error) {
539
+ consoleLogger.error(`Error compressing and encoding ${filename}`);
540
+ throw error;
541
+ }
542
+ };
543
+ const streamEncodedDataToFile = async (inputFilePath, writeStream, appendComma) => {
544
+ const readStream = fs.createReadStream(inputFilePath, { encoding: 'utf8' });
545
+ let isFirstChunk = true;
546
+ for await (const chunk of readStream) {
547
+ if (isFirstChunk) {
548
+ isFirstChunk = false;
549
+ writeStream.write(chunk);
550
+ }
551
+ else {
552
+ writeStream.write(chunk);
553
+ }
554
+ }
555
+ if (appendComma) {
556
+ writeStream.write(',');
557
+ }
558
+ };
559
+ const writeJsonAndBase64Files = async (allIssues, storagePath) => {
560
+ const { items, ...rest } = allIssues;
561
+ const { jsonFilePath: scanDataJsonFilePath, base64FilePath: scanDataBase64FilePath } = await writeJsonFileAndCompressedJsonFile(rest, storagePath, 'scanData');
562
+ const { jsonFilePath: scanItemsJsonFilePath, base64FilePath: scanItemsBase64FilePath } = await writeJsonFileAndCompressedJsonFile({ oobeeAppVersion: allIssues.oobeeAppVersion, ...items }, storagePath, 'scanItems');
563
+ // Add pagesAffectedCount to each rule in scanItemsMiniReport (items) and sort them in descending order of pagesAffectedCount
564
+ ['mustFix', 'goodToFix', 'needsReview', 'passed'].forEach(category => {
565
+ if (items[category].rules && Array.isArray(items[category].rules)) {
566
+ items[category].rules.forEach(rule => {
567
+ rule.pagesAffectedCount = Array.isArray(rule.pagesAffected) ? rule.pagesAffected.length : 0;
568
+ });
569
+ // Sort in descending order of pagesAffectedCount
570
+ items[category].rules.sort((a, b) => (b.pagesAffectedCount || 0) - (a.pagesAffectedCount || 0));
571
+ }
572
+ });
573
+ // Refactor scanIssuesSummary to reuse the scanItemsMiniReport structure by stripping out pagesAffected
574
+ const scanIssuesSummary = {
575
+ // Replace rule descriptions with short descriptions from the map
576
+ mustFix: items.mustFix.rules.map(({ pagesAffected, ...ruleInfo }) => ({
577
+ ...ruleInfo,
578
+ description: a11yRuleShortDescriptionMap[ruleInfo.rule] || ruleInfo.description,
579
+ })),
580
+ goodToFix: items.goodToFix.rules.map(({ pagesAffected, ...ruleInfo }) => ({
581
+ ...ruleInfo,
582
+ description: a11yRuleShortDescriptionMap[ruleInfo.rule] || ruleInfo.description,
583
+ })),
584
+ needsReview: items.needsReview.rules.map(({ pagesAffected, ...ruleInfo }) => ({
585
+ ...ruleInfo,
586
+ description: a11yRuleShortDescriptionMap[ruleInfo.rule] || ruleInfo.description,
587
+ })),
588
+ passed: items.passed.rules.map(({ pagesAffected, ...ruleInfo }) => ({
589
+ ...ruleInfo,
590
+ description: a11yRuleShortDescriptionMap[ruleInfo.rule] || ruleInfo.description,
591
+ })),
592
+ };
593
+ // Write out the scanIssuesSummary JSON using the new structure
594
+ const { jsonFilePath: scanIssuesSummaryJsonFilePath, base64FilePath: scanIssuesSummaryBase64FilePath, } = await writeJsonFileAndCompressedJsonFile({ oobeeAppVersion: allIssues.oobeeAppVersion, ...scanIssuesSummary }, storagePath, 'scanIssuesSummary');
595
+ // scanItemsSummary
596
+ // the below mutates the original items object, since it is expensive to clone
597
+ items.mustFix.rules.forEach(rule => {
598
+ rule.pagesAffected.forEach(page => {
599
+ page.itemsCount = page.items.length;
600
+ });
601
+ });
602
+ items.goodToFix.rules.forEach(rule => {
603
+ rule.pagesAffected.forEach(page => {
604
+ page.itemsCount = page.items.length;
605
+ });
606
+ });
607
+ items.needsReview.rules.forEach(rule => {
608
+ rule.pagesAffected.forEach(page => {
609
+ page.itemsCount = page.items.length;
610
+ });
611
+ });
612
+ items.passed.rules.forEach(rule => {
613
+ rule.pagesAffected.forEach(page => {
614
+ page.itemsCount = page.items.length;
615
+ });
616
+ });
617
+ items.mustFix.totalRuleIssues = items.mustFix.rules.length;
618
+ items.goodToFix.totalRuleIssues = items.goodToFix.rules.length;
619
+ items.needsReview.totalRuleIssues = items.needsReview.rules.length;
620
+ items.passed.totalRuleIssues = items.passed.rules.length;
621
+ const { pagesScanned, topTenPagesWithMostIssues, pagesNotScanned, wcagLinks, wcagPassPercentage, progressPercentage, issuesPercentage, totalPagesScanned, totalPagesNotScanned, topTenIssues, } = rest;
622
+ const summaryItemsMini = {
623
+ ...items,
624
+ pagesScanned,
625
+ topTenPagesWithMostIssues,
626
+ pagesNotScanned,
627
+ wcagLinks,
628
+ wcagPassPercentage,
629
+ progressPercentage,
630
+ issuesPercentage,
631
+ totalPagesScanned,
632
+ totalPagesNotScanned,
633
+ topTenIssues,
634
+ };
635
+ const { jsonFilePath: scanItemsMiniReportJsonFilePath, base64FilePath: scanItemsMiniReportBase64FilePath, } = await writeJsonFileAndCompressedJsonFile({ oobeeAppVersion: allIssues.oobeeAppVersion, ...summaryItemsMini }, storagePath, 'scanItemsSummaryMiniReport');
636
+ const summaryItems = {
637
+ mustFix: {
638
+ totalItems: items.mustFix?.totalItems || 0,
639
+ totalRuleIssues: items.mustFix?.totalRuleIssues || 0,
640
+ },
641
+ goodToFix: {
642
+ totalItems: items.goodToFix?.totalItems || 0,
643
+ totalRuleIssues: items.goodToFix?.totalRuleIssues || 0,
644
+ },
645
+ needsReview: {
646
+ totalItems: items.needsReview?.totalItems || 0,
647
+ totalRuleIssues: items.needsReview?.totalRuleIssues || 0,
648
+ },
649
+ topTenPagesWithMostIssues,
650
+ wcagLinks,
651
+ wcagPassPercentage,
652
+ progressPercentage,
653
+ issuesPercentage,
654
+ totalPagesScanned,
655
+ totalPagesNotScanned,
656
+ topTenIssues,
657
+ };
658
+ const { jsonFilePath: scanItemsSummaryJsonFilePath, base64FilePath: scanItemsSummaryBase64FilePath, } = await writeJsonFileAndCompressedJsonFile({ oobeeAppVersion: allIssues.oobeeAppVersion, ...summaryItems }, storagePath, 'scanItemsSummary');
659
+ const { jsonFilePath: scanPagesDetailJsonFilePath, base64FilePath: scanPagesDetailBase64FilePath, } = await writeJsonFileAndCompressedJsonFile({ oobeeAppVersion: allIssues.oobeeAppVersion, ...allIssues.scanPagesDetail }, storagePath, 'scanPagesDetail');
660
+ const { jsonFilePath: scanPagesSummaryJsonFilePath, base64FilePath: scanPagesSummaryBase64FilePath, } = await writeJsonFileAndCompressedJsonFile({ oobeeAppVersion: allIssues.oobeeAppVersion, ...allIssues.scanPagesSummary }, storagePath, 'scanPagesSummary');
661
+ return {
662
+ scanDataJsonFilePath,
663
+ scanDataBase64FilePath,
664
+ scanItemsJsonFilePath,
665
+ scanItemsBase64FilePath,
666
+ scanItemsSummaryJsonFilePath,
667
+ scanItemsSummaryBase64FilePath,
668
+ scanItemsMiniReportJsonFilePath,
669
+ scanItemsMiniReportBase64FilePath,
670
+ scanIssuesSummaryJsonFilePath,
671
+ scanIssuesSummaryBase64FilePath,
672
+ scanPagesDetailJsonFilePath,
673
+ scanPagesDetailBase64FilePath,
674
+ scanPagesSummaryJsonFilePath,
675
+ scanPagesSummaryBase64FilePath,
676
+ scanDataJsonFileSize: fs.statSync(scanDataJsonFilePath).size,
677
+ scanItemsJsonFileSize: fs.statSync(scanItemsJsonFilePath).size,
678
+ };
679
+ };
680
+ const writeScanDetailsCsv = async (scanDataFilePath, scanItemsFilePath, scanItemsSummaryFilePath, storagePath) => {
681
+ const filePath = path.join(storagePath, 'scanDetails.csv');
682
+ const csvWriteStream = fs.createWriteStream(filePath, { encoding: 'utf8' });
683
+ const directoryPath = path.dirname(filePath);
684
+ if (!fs.existsSync(directoryPath)) {
685
+ fs.mkdirSync(directoryPath, { recursive: true });
686
+ }
687
+ csvWriteStream.write('scanData_base64,scanItems_base64,scanItemsSummary_base64\n');
688
+ await streamEncodedDataToFile(scanDataFilePath, csvWriteStream, true);
689
+ await streamEncodedDataToFile(scanItemsFilePath, csvWriteStream, true);
690
+ await streamEncodedDataToFile(scanItemsSummaryFilePath, csvWriteStream, false);
691
+ await new Promise((resolve, reject) => {
692
+ csvWriteStream.end(resolve);
693
+ csvWriteStream.on('error', reject);
694
+ });
695
+ };
696
+ const writeSummaryPdf = async (storagePath, pagesScanned, filename = 'summary', browser, userDataDirectory) => {
697
+ const htmlFilePath = `${storagePath}/${filename}.html`;
698
+ const fileDestinationPath = `${storagePath}/${filename}.pdf`;
699
+ const effectiveUserDataDirectory = process.env.CRAWLEE_HEADLESS === '1' ? userDataDirectory : '';
700
+ const context = await constants.launcher.launchPersistentContext(effectiveUserDataDirectory, {
701
+ headless: true,
702
+ ...getPlaywrightLaunchOptions(browser),
703
+ });
704
+ register(context);
705
+ const page = await context.newPage();
706
+ const data = fs.readFileSync(htmlFilePath, { encoding: 'utf-8' });
707
+ await page.setContent(data);
708
+ await page.waitForLoadState('networkidle', { timeout: 30000 });
709
+ await page.emulateMedia({ media: 'print' });
710
+ await page.pdf({
711
+ margin: { bottom: '32px' },
712
+ path: fileDestinationPath,
713
+ format: 'A4',
714
+ displayHeaderFooter: true,
715
+ footerTemplate: `
716
+ <div style="margin-top:50px;color:#26241b;font-family:Open Sans;text-align: center;width: 100%;font-weight:400">
717
+ <span style="color:#26241b;font-size: 14px;font-weight:400">Page <span class="pageNumber"></span> of <span class="totalPages"></span></span>
718
+ </div>
719
+ `,
720
+ });
721
+ await page.close();
722
+ await context.close().catch(() => { });
723
+ if (pagesScanned < 2000) {
724
+ fs.unlinkSync(htmlFilePath);
725
+ }
726
+ };
727
+ // Tracking WCAG occurrences
728
+ const wcagOccurrencesMap = new Map();
729
+ // Format WCAG tag in requested format: wcag111a_Occurrences
730
+ const formatWcagTag = async (wcagId) => {
731
+ // Get dynamic WCAG criteria map
732
+ const wcagCriteriaMap = await getWcagCriteriaMap();
733
+ if (wcagCriteriaMap[wcagId]) {
734
+ const { level } = wcagCriteriaMap[wcagId];
735
+ return `${wcagId}${level}_Occurrences`;
736
+ }
737
+ return null;
738
+ };
739
+ const pushResults = async (pageResults, allIssues, isCustomFlow) => {
740
+ const { url, pageTitle, filePath } = pageResults;
741
+ const totalIssuesInPage = new Set();
742
+ Object.keys(pageResults.mustFix.rules).forEach(k => totalIssuesInPage.add(k));
743
+ Object.keys(pageResults.goodToFix.rules).forEach(k => totalIssuesInPage.add(k));
744
+ Object.keys(pageResults.needsReview.rules).forEach(k => totalIssuesInPage.add(k));
745
+ allIssues.topFiveMostIssues.push({
746
+ url,
747
+ pageTitle,
748
+ totalIssues: totalIssuesInPage.size,
749
+ totalOccurrences: 0,
750
+ });
751
+ ['mustFix', 'goodToFix', 'needsReview', 'passed'].forEach(category => {
752
+ if (!pageResults[category])
753
+ return;
754
+ const { totalItems, rules } = pageResults[category];
755
+ const currCategoryFromAllIssues = allIssues.items[category];
756
+ currCategoryFromAllIssues.totalItems += totalItems;
757
+ Object.keys(rules).forEach(rule => {
758
+ const { description, axeImpact, helpUrl, conformance, totalItems: count, items, } = rules[rule];
759
+ if (!(rule in currCategoryFromAllIssues.rules)) {
760
+ currCategoryFromAllIssues.rules[rule] = {
761
+ description,
762
+ axeImpact,
763
+ helpUrl,
764
+ conformance,
765
+ totalItems: 0,
766
+ // numberOfPagesAffectedAfterRedirects: 0,
767
+ pagesAffected: {},
768
+ };
769
+ }
770
+ if (category !== 'passed' && category !== 'needsReview') {
771
+ conformance
772
+ .filter(c => /wcag[0-9]{3,4}/.test(c))
773
+ .forEach(c => {
774
+ if (!allIssues.wcagViolations.includes(c)) {
775
+ allIssues.wcagViolations.push(c);
776
+ }
777
+ // Track WCAG criteria occurrences for Sentry
778
+ const currentCount = wcagOccurrencesMap.get(c) || 0;
779
+ wcagOccurrencesMap.set(c, currentCount + count);
780
+ });
781
+ }
782
+ const currRuleFromAllIssues = currCategoryFromAllIssues.rules[rule];
783
+ currRuleFromAllIssues.totalItems += count;
784
+ if (isCustomFlow) {
785
+ const { pageIndex, pageImagePath, metadata } = pageResults;
786
+ currRuleFromAllIssues.pagesAffected[pageIndex] = {
787
+ url,
788
+ pageTitle,
789
+ pageImagePath,
790
+ metadata,
791
+ items: [],
792
+ };
793
+ currRuleFromAllIssues.pagesAffected[pageIndex].items.push(...items);
794
+ }
795
+ else {
796
+ if (!(url in currRuleFromAllIssues.pagesAffected)) {
797
+ currRuleFromAllIssues.pagesAffected[url] = {
798
+ pageTitle,
799
+ items: [],
800
+ ...(filePath && { filePath }),
801
+ };
802
+ /* if (actualUrl) {
803
+ currRuleFromAllIssues.pagesAffected[url].actualUrl = actualUrl;
804
+ // Deduct duplication count from totalItems
805
+ currRuleFromAllIssues.totalItems -= 1;
806
+ // Previously using pagesAffected.length to display no. of pages affected
807
+ // However, since pagesAffected array contains duplicates, we need to deduct the duplicates
808
+ // Hence, start with negative offset, will add pagesAffected.length later
809
+ currRuleFromAllIssues.numberOfPagesAffectedAfterRedirects -= 1;
810
+ currCategoryFromAllIssues.totalItems -= 1;
811
+ } */
812
+ }
813
+ currRuleFromAllIssues.pagesAffected[url].items.push(...items);
814
+ // currRuleFromAllIssues.numberOfPagesAffectedAfterRedirects +=
815
+ // currRuleFromAllIssues.pagesAffected.length;
816
+ }
817
+ });
818
+ });
819
+ };
820
+ const getTopTenIssues = allIssues => {
821
+ const categories = ['mustFix', 'goodToFix'];
822
+ const rulesWithCounts = [];
823
+ // This is no longer required and shall not be maintained in future
824
+ /*
825
+ const conformanceLevels = {
826
+ wcag2a: 'A',
827
+ wcag2aa: 'AA',
828
+ wcag21aa: 'AA',
829
+ wcag22aa: 'AA',
830
+ wcag2aaa: 'AAA',
831
+ };
832
+ */
833
+ categories.forEach(category => {
834
+ const rules = allIssues.items[category]?.rules || [];
835
+ rules.forEach(rule => {
836
+ // This is not needed anymore since we want to have the clause number too
837
+ /*
838
+ const wcagLevel = rule.conformance[0];
839
+ const aLevel = conformanceLevels[wcagLevel] || wcagLevel;
840
+ */
841
+ rulesWithCounts.push({
842
+ category,
843
+ ruleId: rule.rule,
844
+ // Replace description with new Oobee short description if available
845
+ description: a11yRuleShortDescriptionMap[rule.rule] || rule.description,
846
+ axeImpact: rule.axeImpact,
847
+ conformance: rule.conformance,
848
+ totalItems: rule.totalItems,
849
+ });
850
+ });
851
+ });
852
+ rulesWithCounts.sort((a, b) => b.totalItems - a.totalItems);
853
+ return rulesWithCounts.slice(0, 10);
854
+ };
855
+ const flattenAndSortResults = (allIssues, isCustomFlow) => {
856
+ // Create a map that will sum items only from mustFix, goodToFix, and needsReview.
857
+ const urlOccurrencesMap = new Map();
858
+ // Iterate over all categories; update the map only if the category is not "passed"
859
+ ['mustFix', 'goodToFix', 'needsReview', 'passed'].forEach(category => {
860
+ // Accumulate totalItems regardless of category.
861
+ allIssues.totalItems += allIssues.items[category].totalItems;
862
+ allIssues.items[category].rules = Object.entries(allIssues.items[category].rules)
863
+ .map(ruleEntry => {
864
+ const [rule, ruleInfo] = ruleEntry;
865
+ ruleInfo.pagesAffected = Object.entries(ruleInfo.pagesAffected)
866
+ .map(pageEntry => {
867
+ if (isCustomFlow) {
868
+ const [pageIndex, pageInfo] = pageEntry;
869
+ // Only update the occurrences map if not passed.
870
+ if (category !== 'passed') {
871
+ urlOccurrencesMap.set(pageInfo.url, (urlOccurrencesMap.get(pageInfo.url) || 0) + pageInfo.items.length);
872
+ }
873
+ return { pageIndex, ...pageInfo };
874
+ }
875
+ const [url, pageInfo] = pageEntry;
876
+ if (category !== 'passed') {
877
+ urlOccurrencesMap.set(url, (urlOccurrencesMap.get(url) || 0) + pageInfo.items.length);
878
+ }
879
+ return { url, ...pageInfo };
880
+ })
881
+ // Sort pages so that those with the most items come first
882
+ .sort((page1, page2) => page2.items.length - page1.items.length);
883
+ return { rule, ...ruleInfo };
884
+ })
885
+ // Sort the rules by totalItems (descending)
886
+ .sort((rule1, rule2) => rule2.totalItems - rule1.totalItems);
887
+ });
888
+ // Sort top pages (assumes topFiveMostIssues is already populated)
889
+ allIssues.topFiveMostIssues.sort((p1, p2) => p2.totalIssues - p1.totalIssues);
890
+ allIssues.topTenPagesWithMostIssues = allIssues.topFiveMostIssues.slice(0, 10);
891
+ allIssues.topFiveMostIssues = allIssues.topFiveMostIssues.slice(0, 5);
892
+ // Update each issue in topTenPagesWithMostIssues with the computed occurrences,
893
+ // excluding passed items.
894
+ updateIssuesWithOccurrences(allIssues.topTenPagesWithMostIssues, urlOccurrencesMap);
895
+ // Get and assign the topTenIssues (using your existing helper)
896
+ const topTenIssues = getTopTenIssues(allIssues);
897
+ allIssues.topTenIssues = topTenIssues;
898
+ };
899
+ // Helper: Update totalOccurrences for each issue using our urlOccurrencesMap.
900
+ // For pages that have only passed items, the map will return undefined, so default to 0.
901
+ function updateIssuesWithOccurrences(issuesList, urlOccurrencesMap) {
902
+ issuesList.forEach(issue => {
903
+ issue.totalOccurrences = urlOccurrencesMap.get(issue.url) || 0;
904
+ });
905
+ }
906
+ const extractRuleAiData = (ruleId, totalItems, items, callback) => {
907
+ let snippets = [];
908
+ if (oobeeAiRules.includes(ruleId)) {
909
+ const snippetsSet = new Set();
910
+ if (items) {
911
+ items.forEach(item => {
912
+ snippetsSet.add(oobeeAiHtmlETL(item.html));
913
+ });
914
+ }
915
+ snippets = [...snippetsSet];
916
+ if (callback)
917
+ callback();
918
+ }
919
+ return {
920
+ snippets,
921
+ occurrences: totalItems,
922
+ };
923
+ };
924
+ // This is for telemetry purposes called within mergeAxeResults.ts
925
+ export const createRuleIdJson = allIssues => {
926
+ const compiledRuleJson = {};
927
+ ['mustFix', 'goodToFix', 'needsReview'].forEach(category => {
928
+ allIssues.items[category].rules.forEach(rule => {
929
+ const allItems = rule.pagesAffected.flatMap(page => page.items || []);
930
+ compiledRuleJson[rule.rule] = extractRuleAiData(rule.rule, rule.totalItems, allItems, () => {
931
+ rule.pagesAffected.forEach(p => {
932
+ delete p.items;
933
+ });
934
+ });
935
+ });
936
+ });
937
+ return compiledRuleJson;
938
+ };
939
+ // This is for telemetry purposes called from npmIndex (scanPage and scanHTML) where report is not generated
940
+ export const createBasicFormHTMLSnippet = filteredResults => {
941
+ const compiledRuleJson = {};
942
+ ['mustFix', 'goodToFix', 'needsReview'].forEach(category => {
943
+ if (filteredResults[category] && filteredResults[category].rules) {
944
+ Object.entries(filteredResults[category].rules).forEach(([ruleId, ruleVal]) => {
945
+ compiledRuleJson[ruleId] = extractRuleAiData(ruleId, ruleVal.totalItems, ruleVal.items);
946
+ });
947
+ }
948
+ });
949
+ return compiledRuleJson;
950
+ };
951
+ const moveElemScreenshots = (randomToken, storagePath) => {
952
+ const currentScreenshotsPath = `${randomToken}/elemScreenshots`;
953
+ const resultsScreenshotsPath = `${storagePath}/elemScreenshots`;
954
+ if (fs.existsSync(currentScreenshotsPath)) {
955
+ fs.moveSync(currentScreenshotsPath, resultsScreenshotsPath);
956
+ }
957
+ };
958
+ /**
959
+ * Build allIssues.scanPagesDetail and allIssues.scanPagesSummary
960
+ * by analyzing pagesScanned (including mustFix/goodToFix/etc.).
961
+ */
962
+ function populateScanPagesDetail(allIssues) {
963
+ // --------------------------------------------
964
+ // 1) Gather your "scanned" pages from allIssues
965
+ // --------------------------------------------
966
+ const allScannedPages = Array.isArray(allIssues.pagesScanned) ? allIssues.pagesScanned : [];
967
+ // --------------------------------------------
968
+ // 2) Define category constants (optional, just for clarity)
969
+ // --------------------------------------------
970
+ const mustFixCategory = 'mustFix';
971
+ const goodToFixCategory = 'goodToFix';
972
+ const needsReviewCategory = 'needsReview';
973
+ const passedCategory = 'passed';
974
+ // --------------------------------------------
975
+ // 4) We'll accumulate pages in a map keyed by URL
976
+ // --------------------------------------------
977
+ const pagesMap = {};
978
+ // --------------------------------------------
979
+ // 5) Build pagesMap by iterating over each category in allIssues.items
980
+ // --------------------------------------------
981
+ Object.entries(allIssues.items).forEach(([categoryName, categoryData]) => {
982
+ if (!categoryData?.rules)
983
+ return; // no rules in this category? skip
984
+ categoryData.rules.forEach(rule => {
985
+ const { rule: ruleId, conformance = [] } = rule;
986
+ rule.pagesAffected.forEach(p => {
987
+ const { url, pageTitle, items = [] } = p;
988
+ const itemsCount = items.length;
989
+ // Ensure the page is in pagesMap
990
+ if (!pagesMap[url]) {
991
+ pagesMap[url] = {
992
+ pageTitle,
993
+ url,
994
+ totalOccurrencesFailedIncludingNeedsReview: 0,
995
+ totalOccurrencesFailedExcludingNeedsReview: 0,
996
+ totalOccurrencesNeedsReview: 0,
997
+ totalOccurrencesPassed: 0,
998
+ typesOfIssues: {},
999
+ };
1000
+ }
1001
+ // Ensure the rule is present for this page
1002
+ if (!pagesMap[url].typesOfIssues[ruleId]) {
1003
+ pagesMap[url].typesOfIssues[ruleId] = {
1004
+ ruleId,
1005
+ wcagConformance: conformance,
1006
+ occurrencesMustFix: 0,
1007
+ occurrencesGoodToFix: 0,
1008
+ occurrencesNeedsReview: 0,
1009
+ occurrencesPassed: 0,
1010
+ };
1011
+ }
1012
+ // Depending on the category, increment the relevant occurrence counts
1013
+ if (categoryName === mustFixCategory) {
1014
+ pagesMap[url].typesOfIssues[ruleId].occurrencesMustFix += itemsCount;
1015
+ pagesMap[url].totalOccurrencesFailedIncludingNeedsReview += itemsCount;
1016
+ pagesMap[url].totalOccurrencesFailedExcludingNeedsReview += itemsCount;
1017
+ }
1018
+ else if (categoryName === goodToFixCategory) {
1019
+ pagesMap[url].typesOfIssues[ruleId].occurrencesGoodToFix += itemsCount;
1020
+ pagesMap[url].totalOccurrencesFailedIncludingNeedsReview += itemsCount;
1021
+ pagesMap[url].totalOccurrencesFailedExcludingNeedsReview += itemsCount;
1022
+ }
1023
+ else if (categoryName === needsReviewCategory) {
1024
+ pagesMap[url].typesOfIssues[ruleId].occurrencesNeedsReview += itemsCount;
1025
+ pagesMap[url].totalOccurrencesFailedIncludingNeedsReview += itemsCount;
1026
+ pagesMap[url].totalOccurrencesNeedsReview += itemsCount;
1027
+ }
1028
+ else if (categoryName === passedCategory) {
1029
+ pagesMap[url].typesOfIssues[ruleId].occurrencesPassed += itemsCount;
1030
+ pagesMap[url].totalOccurrencesPassed += itemsCount;
1031
+ }
1032
+ });
1033
+ });
1034
+ });
1035
+ // --------------------------------------------
1036
+ // 6) Separate scanned pages into “affected” vs. “notAffected”
1037
+ // --------------------------------------------
1038
+ const pagesInMap = Object.values(pagesMap); // All pages that have some record in pagesMap
1039
+ const pagesInMapUrls = new Set(Object.keys(pagesMap));
1040
+ // (a) Pages with only passed (no mustFix/goodToFix/needsReview)
1041
+ const pagesAllPassed = pagesInMap.filter(p => p.totalOccurrencesFailedIncludingNeedsReview === 0);
1042
+ // (b) Pages that do NOT appear in pagesMap at all => scanned but no items found
1043
+ const pagesNoEntries = allScannedPages
1044
+ .filter(sp => !pagesInMapUrls.has(sp.url))
1045
+ .map(sp => ({
1046
+ pageTitle: sp.pageTitle,
1047
+ url: sp.url,
1048
+ totalOccurrencesFailedIncludingNeedsReview: 0,
1049
+ totalOccurrencesFailedExcludingNeedsReview: 0,
1050
+ totalOccurrencesNeedsReview: 0,
1051
+ totalOccurrencesPassed: 0,
1052
+ typesOfIssues: {},
1053
+ }));
1054
+ // Combine these into "notAffected"
1055
+ const pagesNotAffectedRaw = [...pagesAllPassed, ...pagesNoEntries];
1056
+ // "affected" pages => have at least 1 mustFix/goodToFix/needsReview
1057
+ const pagesAffectedRaw = pagesInMap.filter(p => p.totalOccurrencesFailedIncludingNeedsReview > 0);
1058
+ // --------------------------------------------
1059
+ // 7) Transform both arrays to the final shape
1060
+ // --------------------------------------------
1061
+ function transformPageData(page) {
1062
+ const typesOfIssuesArray = Object.values(page.typesOfIssues);
1063
+ // Compute sums for each failing category
1064
+ const mustFixSum = typesOfIssuesArray.reduce((acc, r) => acc + r.occurrencesMustFix, 0);
1065
+ const goodToFixSum = typesOfIssuesArray.reduce((acc, r) => acc + r.occurrencesGoodToFix, 0);
1066
+ const needsReviewSum = typesOfIssuesArray.reduce((acc, r) => acc + r.occurrencesNeedsReview, 0);
1067
+ // Build categoriesPresent based on nonzero failing counts
1068
+ const categoriesPresent = [];
1069
+ if (mustFixSum > 0)
1070
+ categoriesPresent.push('mustFix');
1071
+ if (goodToFixSum > 0)
1072
+ categoriesPresent.push('goodToFix');
1073
+ if (needsReviewSum > 0)
1074
+ categoriesPresent.push('needsReview');
1075
+ // Count how many rules have failing issues
1076
+ const failedRuleIds = new Set();
1077
+ typesOfIssuesArray.forEach(r => {
1078
+ if ((r.occurrencesMustFix || 0) > 0 ||
1079
+ (r.occurrencesGoodToFix || 0) > 0 ||
1080
+ (r.occurrencesNeedsReview || 0) > 0) {
1081
+ failedRuleIds.add(r.ruleId); // Ensure ruleId is unique
1082
+ }
1083
+ });
1084
+ const failedRuleCount = failedRuleIds.size;
1085
+ // Possibly these two for future convenience
1086
+ const typesOfIssuesExcludingNeedsReviewCount = typesOfIssuesArray.filter(r => (r.occurrencesMustFix || 0) + (r.occurrencesGoodToFix || 0) > 0).length;
1087
+ const typesOfIssuesExclusiveToNeedsReviewCount = typesOfIssuesArray.filter(r => (r.occurrencesNeedsReview || 0) > 0 &&
1088
+ (r.occurrencesMustFix || 0) === 0 &&
1089
+ (r.occurrencesGoodToFix || 0) === 0).length;
1090
+ // Aggregate wcagConformance for rules that actually fail
1091
+ const allConformance = typesOfIssuesArray.reduce((acc, curr) => {
1092
+ const nonPassedCount = (curr.occurrencesMustFix || 0) +
1093
+ (curr.occurrencesGoodToFix || 0) +
1094
+ (curr.occurrencesNeedsReview || 0);
1095
+ if (nonPassedCount > 0) {
1096
+ return acc.concat(curr.wcagConformance || []);
1097
+ }
1098
+ return acc;
1099
+ }, []);
1100
+ // Remove duplicates
1101
+ const conformance = Array.from(new Set(allConformance));
1102
+ return {
1103
+ pageTitle: page.pageTitle,
1104
+ url: page.url,
1105
+ totalOccurrencesFailedIncludingNeedsReview: page.totalOccurrencesFailedIncludingNeedsReview,
1106
+ totalOccurrencesFailedExcludingNeedsReview: page.totalOccurrencesFailedExcludingNeedsReview,
1107
+ totalOccurrencesMustFix: mustFixSum,
1108
+ totalOccurrencesGoodToFix: goodToFixSum,
1109
+ totalOccurrencesNeedsReview: needsReviewSum,
1110
+ totalOccurrencesPassed: page.totalOccurrencesPassed,
1111
+ typesOfIssuesExclusiveToNeedsReviewCount,
1112
+ typesOfIssuesCount: failedRuleCount,
1113
+ typesOfIssuesExcludingNeedsReviewCount,
1114
+ categoriesPresent,
1115
+ conformance,
1116
+ // Keep full detail for "scanPagesDetail"
1117
+ typesOfIssues: typesOfIssuesArray,
1118
+ };
1119
+ }
1120
+ // Transform raw pages
1121
+ const pagesAffected = pagesAffectedRaw.map(transformPageData);
1122
+ const pagesNotAffected = pagesNotAffectedRaw.map(transformPageData);
1123
+ // --------------------------------------------
1124
+ // 8) Sort pages by typesOfIssuesCount (descending) for both arrays
1125
+ // --------------------------------------------
1126
+ pagesAffected.sort((a, b) => b.typesOfIssuesCount - a.typesOfIssuesCount);
1127
+ pagesNotAffected.sort((a, b) => b.typesOfIssuesCount - a.typesOfIssuesCount);
1128
+ // --------------------------------------------
1129
+ // 9) Compute scanned/ skipped counts
1130
+ // --------------------------------------------
1131
+ const scannedPagesCount = pagesAffected.length + pagesNotAffected.length;
1132
+ const pagesNotScannedCount = Array.isArray(allIssues.pagesNotScanned)
1133
+ ? allIssues.pagesNotScanned.length
1134
+ : 0;
1135
+ // --------------------------------------------
1136
+ // 10) Build scanPagesDetail (with full "typesOfIssues")
1137
+ // --------------------------------------------
1138
+ allIssues.scanPagesDetail = {
1139
+ pagesAffected,
1140
+ pagesNotAffected,
1141
+ scannedPagesCount,
1142
+ pagesNotScanned: Array.isArray(allIssues.pagesNotScanned) ? allIssues.pagesNotScanned : [],
1143
+ pagesNotScannedCount,
1144
+ };
1145
+ // --------------------------------------------
1146
+ // 11) Build scanPagesSummary (strip out "typesOfIssues")
1147
+ // --------------------------------------------
1148
+ function stripTypesOfIssues(page) {
1149
+ const { typesOfIssues, ...rest } = page;
1150
+ return rest;
1151
+ }
1152
+ const summaryPagesAffected = pagesAffected.map(stripTypesOfIssues);
1153
+ const summaryPagesNotAffected = pagesNotAffected.map(stripTypesOfIssues);
1154
+ allIssues.scanPagesSummary = {
1155
+ pagesAffected: summaryPagesAffected,
1156
+ pagesNotAffected: summaryPagesNotAffected,
1157
+ scannedPagesCount,
1158
+ pagesNotScanned: Array.isArray(allIssues.pagesNotScanned) ? allIssues.pagesNotScanned : [],
1159
+ pagesNotScannedCount,
1160
+ };
1161
+ }
1162
+ // Send WCAG criteria breakdown to Sentry
1163
+ export const sendWcagBreakdownToSentry = async (appVersion, wcagBreakdown, ruleIdJson, scanInfo, allIssues, pagesScannedCount = 0) => {
1164
+ try {
1165
+ // Initialize Sentry
1166
+ Sentry.init(sentryConfig);
1167
+ // Set user ID for Sentry tracking
1168
+ const userData = getUserDataTxt();
1169
+ if (userData && userData.userId) {
1170
+ setSentryUser(userData.userId);
1171
+ }
1172
+ // Prepare tags for the event
1173
+ const tags = {};
1174
+ const wcagCriteriaBreakdown = {};
1175
+ // Tag app version
1176
+ tags.version = appVersion;
1177
+ // Get dynamic WCAG criteria map once
1178
+ const wcagCriteriaMap = await getWcagCriteriaMap();
1179
+ // Categorize all WCAG criteria for reporting
1180
+ const wcagIds = Array.from(new Set([...Object.keys(wcagCriteriaMap), ...Array.from(wcagBreakdown.keys())]));
1181
+ const categorizedWcag = await categorizeWcagCriteria(wcagIds);
1182
+ // First ensure all WCAG criteria are included in the tags with a value of 0
1183
+ // This ensures criteria with no violations are still reported
1184
+ for (const [wcagId, info] of Object.entries(wcagCriteriaMap)) {
1185
+ const formattedTag = await formatWcagTag(wcagId);
1186
+ if (formattedTag) {
1187
+ // Initialize with zero
1188
+ tags[formattedTag] = '0';
1189
+ // Store in breakdown object with category information
1190
+ wcagCriteriaBreakdown[formattedTag] = {
1191
+ count: 0,
1192
+ category: categorizedWcag[wcagId] || 'mustFix', // Default to mustFix if not found
1193
+ };
1194
+ }
1195
+ }
1196
+ // Now override with actual counts from the scan
1197
+ for (const [wcagId, count] of wcagBreakdown.entries()) {
1198
+ const formattedTag = await formatWcagTag(wcagId);
1199
+ if (formattedTag) {
1200
+ // Add as a tag with the count as value
1201
+ tags[formattedTag] = String(count);
1202
+ // Update count in breakdown object
1203
+ if (wcagCriteriaBreakdown[formattedTag]) {
1204
+ wcagCriteriaBreakdown[formattedTag].count = count;
1205
+ }
1206
+ else {
1207
+ // If somehow this wasn't in our initial map
1208
+ wcagCriteriaBreakdown[formattedTag] = {
1209
+ count,
1210
+ category: categorizedWcag[wcagId] || 'mustFix',
1211
+ };
1212
+ }
1213
+ }
1214
+ }
1215
+ // Calculate category counts based on actual issue counts from the report
1216
+ // rather than occurrence counts from wcagBreakdown
1217
+ const categoryCounts = {
1218
+ mustFix: 0,
1219
+ goodToFix: 0,
1220
+ needsReview: 0,
1221
+ };
1222
+ if (allIssues) {
1223
+ // Use the actual report data for the counts
1224
+ categoryCounts.mustFix = allIssues.items.mustFix.rules.length;
1225
+ categoryCounts.goodToFix = allIssues.items.goodToFix.rules.length;
1226
+ categoryCounts.needsReview = allIssues.items.needsReview.rules.length;
1227
+ }
1228
+ else {
1229
+ // Fallback to the old way if allIssues not provided
1230
+ Object.values(wcagCriteriaBreakdown).forEach(item => {
1231
+ if (item.count > 0 && categoryCounts[item.category] !== undefined) {
1232
+ categoryCounts[item.category] += 1; // Count rules, not occurrences
1233
+ }
1234
+ });
1235
+ }
1236
+ // Add category counts as tags
1237
+ tags['WCAG-MustFix-Count'] = String(categoryCounts.mustFix);
1238
+ tags['WCAG-GoodToFix-Count'] = String(categoryCounts.goodToFix);
1239
+ tags['WCAG-NeedsReview-Count'] = String(categoryCounts.needsReview);
1240
+ // Also add occurrence counts for reference
1241
+ if (allIssues) {
1242
+ tags['WCAG-MustFix-Occurrences'] = String(allIssues.items.mustFix.totalItems);
1243
+ tags['WCAG-GoodToFix-Occurrences'] = String(allIssues.items.goodToFix.totalItems);
1244
+ tags['WCAG-NeedsReview-Occurrences'] = String(allIssues.items.needsReview.totalItems);
1245
+ // Add number of pages scanned tag
1246
+ tags['Pages-Scanned-Count'] = String(allIssues.totalPagesScanned);
1247
+ }
1248
+ else if (pagesScannedCount > 0) {
1249
+ // Still add the pages scanned count even if we don't have allIssues
1250
+ tags['Pages-Scanned-Count'] = String(pagesScannedCount);
1251
+ }
1252
+ // Send the event to Sentry
1253
+ await Sentry.captureEvent({
1254
+ message: 'Accessibility Scan Completed',
1255
+ level: 'info',
1256
+ tags: {
1257
+ ...tags,
1258
+ event_type: 'accessibility_scan',
1259
+ scanType: scanInfo.scanType,
1260
+ browser: scanInfo.browser,
1261
+ entryUrl: scanInfo.entryUrl,
1262
+ },
1263
+ user: {
1264
+ ...(scanInfo.email && scanInfo.name
1265
+ ? {
1266
+ email: scanInfo.email,
1267
+ username: scanInfo.name,
1268
+ }
1269
+ : {}),
1270
+ ...(userData && userData.userId ? { id: userData.userId } : {}),
1271
+ },
1272
+ extra: {
1273
+ additionalScanMetadata: ruleIdJson != null ? JSON.stringify(ruleIdJson) : '{}',
1274
+ wcagBreakdown: wcagCriteriaBreakdown,
1275
+ reportCounts: allIssues
1276
+ ? {
1277
+ mustFix: {
1278
+ issues: allIssues.items.mustFix.rules?.length ?? 0,
1279
+ occurrences: allIssues.items.mustFix.totalItems ?? 0,
1280
+ },
1281
+ goodToFix: {
1282
+ issues: allIssues.items.goodToFix.rules?.length ?? 0,
1283
+ occurrences: allIssues.items.goodToFix.totalItems ?? 0,
1284
+ },
1285
+ needsReview: {
1286
+ issues: allIssues.items.needsReview.rules?.length ?? 0,
1287
+ occurrences: allIssues.items.needsReview.totalItems ?? 0,
1288
+ },
1289
+ }
1290
+ : undefined,
1291
+ },
1292
+ });
1293
+ // Wait for events to be sent
1294
+ await Sentry.flush(2000);
1295
+ }
1296
+ catch (error) {
1297
+ console.error('Error sending WCAG breakdown to Sentry:', error);
1298
+ }
1299
+ };
1300
+ const formatAboutStartTime = (dateString) => {
1301
+ const utcStartTimeDate = new Date(dateString);
1302
+ const formattedStartTime = utcStartTimeDate.toLocaleTimeString('en-GB', {
1303
+ year: 'numeric',
1304
+ month: 'short',
1305
+ day: 'numeric',
1306
+ hour12: false,
1307
+ hour: 'numeric',
1308
+ minute: '2-digit',
1309
+ timeZoneName: 'shortGeneric',
1310
+ });
1311
+ const timezoneAbbreviation = new Intl.DateTimeFormat('en', {
1312
+ timeZoneName: 'shortOffset',
1313
+ })
1314
+ .formatToParts(utcStartTimeDate)
1315
+ .find(part => part.type === 'timeZoneName').value;
1316
+ // adding a breakline between the time and timezone so it looks neater on report
1317
+ const timeColonIndex = formattedStartTime.lastIndexOf(':');
1318
+ const timePart = formattedStartTime.slice(0, timeColonIndex + 3);
1319
+ const timeZonePart = formattedStartTime.slice(timeColonIndex + 4);
1320
+ const htmlFormattedStartTime = `${timePart}<br>${timeZonePart} ${timezoneAbbreviation}`;
1321
+ return htmlFormattedStartTime;
1322
+ };
1323
+ const generateArtifacts = async (randomToken, urlScanned, scanType, viewport, pagesScanned, pagesNotScanned, customFlowLabel, cypressScanAboutMetadata, scanDetails, zip = undefined, // optional
1324
+ generateJsonFiles = false) => {
1325
+ const storagePath = getStoragePath(randomToken);
1326
+ const intermediateDatasetsPath = `${storagePath}/crawlee`;
1327
+ const oobeeAppVersion = getVersion();
1328
+ const isCustomFlow = scanType === ScannerTypes.CUSTOM;
1329
+ const allIssues = {
1330
+ storagePath,
1331
+ oobeeAi: {
1332
+ htmlETL: oobeeAiHtmlETL,
1333
+ rules: oobeeAiRules,
1334
+ },
1335
+ siteName: (pagesScanned[0]?.pageTitle ?? '').replace(/^\d+\s*:\s*/, '').trim(),
1336
+ startTime: scanDetails.startTime ? scanDetails.startTime : new Date(),
1337
+ endTime: scanDetails.endTime ? scanDetails.endTime : new Date(),
1338
+ urlScanned,
1339
+ scanType,
1340
+ deviceChosen: scanDetails.deviceChosen || 'Desktop',
1341
+ formatAboutStartTime,
1342
+ isCustomFlow,
1343
+ viewport,
1344
+ pagesScanned,
1345
+ pagesNotScanned,
1346
+ totalPagesScanned: pagesScanned.length,
1347
+ totalPagesNotScanned: pagesNotScanned.length,
1348
+ totalItems: 0,
1349
+ topFiveMostIssues: [],
1350
+ topTenPagesWithMostIssues: [],
1351
+ topTenIssues: [],
1352
+ wcagViolations: [],
1353
+ customFlowLabel,
1354
+ oobeeAppVersion,
1355
+ items: {
1356
+ mustFix: {
1357
+ description: itemTypeDescription.mustFix,
1358
+ totalItems: 0,
1359
+ totalRuleIssues: 0,
1360
+ rules: [],
1361
+ },
1362
+ goodToFix: {
1363
+ description: itemTypeDescription.goodToFix,
1364
+ totalItems: 0,
1365
+ totalRuleIssues: 0,
1366
+ rules: [],
1367
+ },
1368
+ needsReview: {
1369
+ description: itemTypeDescription.needsReview,
1370
+ totalItems: 0,
1371
+ totalRuleIssues: 0,
1372
+ rules: [],
1373
+ },
1374
+ passed: {
1375
+ description: itemTypeDescription.passed,
1376
+ totalItems: 0,
1377
+ totalRuleIssues: 0,
1378
+ rules: [],
1379
+ },
1380
+ },
1381
+ cypressScanAboutMetadata,
1382
+ wcagLinks: constants.wcagLinks,
1383
+ wcagClauses: WCAGclauses,
1384
+ a11yRuleShortDescriptionMap,
1385
+ disabilityBadgesMap,
1386
+ a11yRuleLongDescriptionMap,
1387
+ wcagCriteriaLabels: constants.wcagCriteriaLabels,
1388
+ scanPagesDetail: {
1389
+ pagesAffected: [],
1390
+ pagesNotAffected: [],
1391
+ scannedPagesCount: 0,
1392
+ pagesNotScanned: [],
1393
+ pagesNotScannedCount: 0,
1394
+ },
1395
+ // Populate boolean values for id="advancedScanOptionsSummary"
1396
+ advancedScanOptionsSummaryItems: {
1397
+ showIncludeScreenshots: [true].includes(scanDetails.isIncludeScreenshots),
1398
+ showAllowSubdomains: ['same-domain'].includes(scanDetails.isAllowSubdomains),
1399
+ showEnableCustomChecks: ['default', 'enable-wcag-aaa'].includes(scanDetails.isEnableCustomChecks?.[0]),
1400
+ showEnableWcagAaa: (scanDetails.isEnableWcagAaa || []).includes('enable-wcag-aaa'),
1401
+ showSlowScanMode: [1].includes(scanDetails.isSlowScanMode),
1402
+ showAdhereRobots: [true].includes(scanDetails.isAdhereRobots),
1403
+ },
1404
+ };
1405
+ const allFiles = await extractFileNames(intermediateDatasetsPath);
1406
+ const jsonArray = await Promise.all(allFiles.map(async (file) => parseContentToJson(`${intermediateDatasetsPath}/${file}`)));
1407
+ await Promise.all(jsonArray.map(async (pageResults) => {
1408
+ await pushResults(pageResults, allIssues, isCustomFlow);
1409
+ })).catch(flattenIssuesError => {
1410
+ consoleLogger.info('An error has occurred when flattening the issues, please try again.');
1411
+ });
1412
+ flattenAndSortResults(allIssues, isCustomFlow);
1413
+ const labelKey = scanType.toLowerCase() === 'custom' ? 'CustomFlowLabel' : 'Label';
1414
+ const labelValue = allIssues.customFlowLabel || 'N/A';
1415
+ printMessage([
1416
+ 'Scan Summary',
1417
+ `Oobee App Version: ${allIssues.oobeeAppVersion}`,
1418
+ '',
1419
+ `Site Name: ${allIssues.siteName}`,
1420
+ `URL: ${allIssues.urlScanned}`,
1421
+ `Pages Scanned: ${allIssues.totalPagesScanned}`,
1422
+ `Start Time: ${allIssues.startTime}`,
1423
+ `End Time: ${allIssues.endTime}`,
1424
+ `Elapsed Time: ${(new Date(allIssues.endTime).getTime() - new Date(allIssues.startTime).getTime()) / 1000}s`,
1425
+ `Device: ${allIssues.deviceChosen}`,
1426
+ `Viewport: ${allIssues.viewport}`,
1427
+ `Scan Type: ${allIssues.scanType}`,
1428
+ `${labelKey}: ${labelValue}`,
1429
+ '',
1430
+ `Must Fix: ${allIssues.items.mustFix.rules.length} ${Object.keys(allIssues.items.mustFix.rules).length === 1 ? 'issue' : 'issues'} / ${allIssues.items.mustFix.totalItems} ${allIssues.items.mustFix.totalItems === 1 ? 'occurrence' : 'occurrences'}`,
1431
+ `Good to Fix: ${allIssues.items.goodToFix.rules.length} ${Object.keys(allIssues.items.goodToFix.rules).length === 1 ? 'issue' : 'issues'} / ${allIssues.items.goodToFix.totalItems} ${allIssues.items.goodToFix.totalItems === 1 ? 'occurrence' : 'occurrences'}`,
1432
+ `Manual Review Required: ${allIssues.items.needsReview.rules.length} ${Object.keys(allIssues.items.needsReview.rules).length === 1 ? 'issue' : 'issues'} / ${allIssues.items.needsReview.totalItems} ${allIssues.items.needsReview.totalItems === 1 ? 'occurrence' : 'occurrences'}`,
1433
+ `Passed: ${allIssues.items.passed.totalItems} ${allIssues.items.passed.totalItems === 1 ? 'occurrence' : 'occurrences'}`,
1434
+ ]);
1435
+ // move screenshots folder to report folders
1436
+ moveElemScreenshots(randomToken, storagePath);
1437
+ if (isCustomFlow) {
1438
+ createScreenshotsFolder(randomToken);
1439
+ }
1440
+ populateScanPagesDetail(allIssues);
1441
+ allIssues.wcagPassPercentage = getWcagPassPercentage(allIssues.wcagViolations, allIssues.advancedScanOptionsSummaryItems.showEnableWcagAaa);
1442
+ allIssues.progressPercentage = getProgressPercentage(allIssues.scanPagesDetail, allIssues.advancedScanOptionsSummaryItems.showEnableWcagAaa);
1443
+ allIssues.issuesPercentage = await getIssuesPercentage(allIssues.scanPagesDetail, allIssues.advancedScanOptionsSummaryItems.showEnableWcagAaa, allIssues.advancedScanOptionsSummaryItems.disableOobee);
1444
+ consoleLogger.info(`Site Name: ${allIssues.siteName}`);
1445
+ consoleLogger.info(`URL: ${allIssues.urlScanned}`);
1446
+ consoleLogger.info(`Pages Scanned: ${allIssues.totalPagesScanned}`);
1447
+ consoleLogger.info(`Start Time: ${allIssues.startTime}`);
1448
+ consoleLogger.info(`End Time: ${allIssues.endTime}`);
1449
+ const elapsedSeconds = (new Date(allIssues.endTime).getTime() - new Date(allIssues.startTime).getTime()) / 1000;
1450
+ consoleLogger.info(`Elapsed Time: ${elapsedSeconds}s`);
1451
+ consoleLogger.info(`Device: ${allIssues.deviceChosen}`);
1452
+ consoleLogger.info(`Viewport: ${allIssues.viewport}`);
1453
+ consoleLogger.info(`Scan Type: ${allIssues.scanType}`);
1454
+ consoleLogger.info(`Label: ${allIssues.customFlowLabel || 'N/A'}`);
1455
+ const getAxeImpactCount = (allIssues) => {
1456
+ const impactCount = {
1457
+ critical: 0,
1458
+ serious: 0,
1459
+ moderate: 0,
1460
+ minor: 0,
1461
+ };
1462
+ Object.values(allIssues.items).forEach(category => {
1463
+ if (category.totalItems > 0) {
1464
+ Object.values(category.rules).forEach(rule => {
1465
+ if (rule.axeImpact === 'critical') {
1466
+ impactCount.critical += rule.totalItems;
1467
+ }
1468
+ else if (rule.axeImpact === 'serious') {
1469
+ impactCount.serious += rule.totalItems;
1470
+ }
1471
+ else if (rule.axeImpact === 'moderate') {
1472
+ impactCount.moderate += rule.totalItems;
1473
+ }
1474
+ else if (rule.axeImpact === 'minor') {
1475
+ impactCount.minor += rule.totalItems;
1476
+ }
1477
+ });
1478
+ }
1479
+ });
1480
+ return impactCount;
1481
+ };
1482
+ if (process.env.OOBEE_VERBOSE) {
1483
+ const axeImpactCount = getAxeImpactCount(allIssues);
1484
+ const { items, startTime, endTime, ...rest } = allIssues;
1485
+ rest.critical = axeImpactCount.critical;
1486
+ rest.serious = axeImpactCount.serious;
1487
+ rest.moderate = axeImpactCount.moderate;
1488
+ rest.minor = axeImpactCount.minor;
1489
+ }
1490
+ await writeCsv(allIssues, storagePath);
1491
+ await writeSitemap(pagesScanned, storagePath);
1492
+ const { scanDataJsonFilePath, scanDataBase64FilePath, scanItemsJsonFilePath, scanItemsBase64FilePath, scanItemsSummaryJsonFilePath, scanItemsSummaryBase64FilePath, scanItemsMiniReportJsonFilePath, scanItemsMiniReportBase64FilePath, scanIssuesSummaryJsonFilePath, scanIssuesSummaryBase64FilePath, scanPagesDetailJsonFilePath, scanPagesDetailBase64FilePath, scanPagesSummaryJsonFilePath, scanPagesSummaryBase64FilePath, scanDataJsonFileSize, scanItemsJsonFileSize, } = await writeJsonAndBase64Files(allIssues, storagePath);
1493
+ const BIG_RESULTS_THRESHOLD = 500 * 1024 * 1024; // 500 MB
1494
+ const resultsTooBig = scanDataJsonFileSize + scanItemsJsonFileSize > BIG_RESULTS_THRESHOLD;
1495
+ await writeScanDetailsCsv(scanDataBase64FilePath, scanItemsBase64FilePath, scanItemsSummaryBase64FilePath, storagePath);
1496
+ await writeSummaryHTML(allIssues, storagePath);
1497
+ await writeHTML(allIssues, storagePath, 'report', scanDataBase64FilePath, resultsTooBig ? scanItemsMiniReportBase64FilePath : scanItemsBase64FilePath);
1498
+ if (!generateJsonFiles) {
1499
+ await cleanUpJsonFiles([
1500
+ scanDataJsonFilePath,
1501
+ scanDataBase64FilePath,
1502
+ scanItemsJsonFilePath,
1503
+ scanItemsBase64FilePath,
1504
+ scanItemsSummaryJsonFilePath,
1505
+ scanItemsSummaryBase64FilePath,
1506
+ scanItemsMiniReportJsonFilePath,
1507
+ scanItemsMiniReportBase64FilePath,
1508
+ scanIssuesSummaryJsonFilePath,
1509
+ scanIssuesSummaryBase64FilePath,
1510
+ scanPagesDetailJsonFilePath,
1511
+ scanPagesDetailBase64FilePath,
1512
+ scanPagesSummaryJsonFilePath,
1513
+ scanPagesSummaryBase64FilePath,
1514
+ ]);
1515
+ }
1516
+ const browserChannel = getBrowserToRun(randomToken, BrowserTypes.CHROME, false).browserToRun;
1517
+ // Should consider refactor constants.userDataDirectory to be a parameter in future
1518
+ await retryFunction(() => writeSummaryPdf(storagePath, pagesScanned.length, 'summary', browserChannel, constants.userDataDirectory), 1);
1519
+ try {
1520
+ fs.rmSync(path.join(storagePath, 'crawlee'), { recursive: true, force: true });
1521
+ }
1522
+ catch (error) {
1523
+ consoleLogger.warn(`Unable to force remove crawlee folder: ${error.message}`);
1524
+ }
1525
+ try {
1526
+ fs.rmSync(path.join(storagePath, 'pdfs'), { recursive: true, force: true });
1527
+ }
1528
+ catch (error) {
1529
+ consoleLogger.warn(`Unable to force remove pdfs folder: ${error.message}`);
1530
+ }
1531
+ // Take option if set
1532
+ if (typeof zip === 'string') {
1533
+ constants.cliZipFileName = zip;
1534
+ if (!zip.endsWith('.zip')) {
1535
+ constants.cliZipFileName += '.zip';
1536
+ }
1537
+ }
1538
+ if (!path.isAbsolute(constants.cliZipFileName) ||
1539
+ path.dirname(constants.cliZipFileName) === '.') {
1540
+ constants.cliZipFileName = path.join(storagePath, constants.cliZipFileName);
1541
+ }
1542
+ try {
1543
+ await fs.ensureDir(storagePath);
1544
+ await zipResults(constants.cliZipFileName, storagePath);
1545
+ const messageToDisplay = [
1546
+ `Report of this run is at ${constants.cliZipFileName}`,
1547
+ `Results directory is at ${storagePath}`,
1548
+ ];
1549
+ if (process.send && process.env.OOBEE_VERBOSE) {
1550
+ const zipFileNameMessage = {
1551
+ type: 'zipFileName',
1552
+ payload: `${constants.cliZipFileName}`,
1553
+ };
1554
+ const storagePathMessage = {
1555
+ type: 'storagePath',
1556
+ payload: `${storagePath}`,
1557
+ };
1558
+ process.send(JSON.stringify(storagePathMessage));
1559
+ process.send(JSON.stringify(zipFileNameMessage));
1560
+ }
1561
+ printMessage(messageToDisplay);
1562
+ }
1563
+ catch (error) {
1564
+ printMessage([`Error in zipping results: ${error}`]);
1565
+ }
1566
+ // Generate scrubbed HTML Code Snippets
1567
+ const ruleIdJson = createRuleIdJson(allIssues);
1568
+ // At the end of the function where results are generated, add:
1569
+ try {
1570
+ // Always send WCAG breakdown to Sentry, even if no violations were found
1571
+ // This ensures that all criteria are reported, including those with 0 occurrences
1572
+ await sendWcagBreakdownToSentry(oobeeAppVersion, wcagOccurrencesMap, ruleIdJson, {
1573
+ entryUrl: urlScanned,
1574
+ scanType,
1575
+ browser: scanDetails.deviceChosen,
1576
+ email: scanDetails.nameEmail?.email,
1577
+ name: scanDetails.nameEmail?.name,
1578
+ }, allIssues, pagesScanned.length);
1579
+ }
1580
+ catch (error) {
1581
+ console.error('Error sending WCAG data to Sentry:', error);
1582
+ }
1583
+ if (process.env.RUNNING_FROM_PH_GUI || process.env.OOBEE_VERBOSE)
1584
+ console.log('Report generated successfully');
1585
+ return ruleIdJson;
1586
+ };
1587
+ export { writeHTML, compressJsonFileStreaming, flattenAndSortResults, populateScanPagesDetail, getWcagPassPercentage, getProgressPercentage, getIssuesPercentage, itemTypeDescription, oobeeAiHtmlETL, oobeeAiRules, formatAboutStartTime, };
1588
+ export default generateArtifacts;