@govtechsg/oobee 0.10.83 → 0.10.84
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +6 -1
- package/dist/constants/common.js +13 -1
- package/dist/crawlers/crawlDomain.js +220 -120
- package/dist/crawlers/crawlIntelligentSitemap.js +22 -7
- package/dist/crawlers/runCustom.js +8 -2
- package/dist/mergeAxeResults/itemReferences.js +55 -0
- package/dist/mergeAxeResults/jsonArtifacts.js +335 -0
- package/dist/mergeAxeResults/scanPages.js +159 -0
- package/dist/mergeAxeResults/sentryTelemetry.js +152 -0
- package/dist/mergeAxeResults/types.js +1 -0
- package/dist/mergeAxeResults/writeCsv.js +125 -0
- package/dist/mergeAxeResults/writeScanDetailsCsv.js +35 -0
- package/dist/mergeAxeResults/writeSitemap.js +10 -0
- package/dist/mergeAxeResults.js +24 -929
- package/dist/proxyService.js +90 -5
- package/dist/utils.js +20 -7
- package/package.json +6 -6
- package/src/constants/common.ts +13 -1
- package/src/crawlers/crawlDomain.ts +248 -137
- package/src/crawlers/crawlIntelligentSitemap.ts +22 -8
- package/src/crawlers/runCustom.ts +10 -2
- package/src/mergeAxeResults/itemReferences.ts +62 -0
- package/src/mergeAxeResults/jsonArtifacts.ts +451 -0
- package/src/mergeAxeResults/scanPages.ts +207 -0
- package/src/mergeAxeResults/sentryTelemetry.ts +183 -0
- package/src/mergeAxeResults/types.ts +99 -0
- package/src/mergeAxeResults/writeCsv.ts +145 -0
- package/src/mergeAxeResults/writeScanDetailsCsv.ts +51 -0
- package/src/mergeAxeResults/writeSitemap.ts +13 -0
- package/src/mergeAxeResults.ts +82 -1318
- package/src/proxyService.ts +96 -4
- package/src/utils.ts +19 -7
|
@@ -0,0 +1,55 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Builds pre-computed HTML groups to optimize Group by HTML Element functionality.
|
|
3
|
+
* Keys are composite "html\x00xpath" strings to ensure unique matching per element instance.
|
|
4
|
+
*/
|
|
5
|
+
export const buildHtmlGroups = (rule, items, pageUrl) => {
|
|
6
|
+
if (!rule.htmlGroups) {
|
|
7
|
+
rule.htmlGroups = {};
|
|
8
|
+
}
|
|
9
|
+
items.forEach(item => {
|
|
10
|
+
// Use composite key of html + xpath for precise matching
|
|
11
|
+
const htmlKey = `${item.html || 'No HTML element'}\x00${item.xpath || ''}`;
|
|
12
|
+
if (!rule.htmlGroups[htmlKey]) {
|
|
13
|
+
// Create new group with the first occurrence
|
|
14
|
+
rule.htmlGroups[htmlKey] = {
|
|
15
|
+
html: item.html || '',
|
|
16
|
+
xpath: item.xpath || '',
|
|
17
|
+
message: item.message || '',
|
|
18
|
+
screenshotPath: item.screenshotPath || '',
|
|
19
|
+
displayNeedsReview: item.displayNeedsReview,
|
|
20
|
+
pageUrls: [],
|
|
21
|
+
};
|
|
22
|
+
}
|
|
23
|
+
if (!rule.htmlGroups[htmlKey].pageUrls.includes(pageUrl)) {
|
|
24
|
+
rule.htmlGroups[htmlKey].pageUrls.push(pageUrl);
|
|
25
|
+
}
|
|
26
|
+
});
|
|
27
|
+
};
|
|
28
|
+
/**
|
|
29
|
+
* Converts items in pagesAffected to references (html\x00xpath composite keys) for embedding in HTML report.
|
|
30
|
+
* Additionally, it deep-clones allIssues, replaces page.items objects with composite reference keys.
|
|
31
|
+
* Those refs are specifically for htmlGroups lookup (html + xpath).
|
|
32
|
+
*/
|
|
33
|
+
export const convertItemsToReferences = (allIssues) => {
|
|
34
|
+
const cloned = JSON.parse(JSON.stringify(allIssues));
|
|
35
|
+
['mustFix', 'goodToFix', 'needsReview', 'passed'].forEach(category => {
|
|
36
|
+
if (!cloned.items[category]?.rules)
|
|
37
|
+
return;
|
|
38
|
+
cloned.items[category].rules.forEach((rule) => {
|
|
39
|
+
if (!rule.pagesAffected || !rule.htmlGroups)
|
|
40
|
+
return;
|
|
41
|
+
rule.pagesAffected.forEach((page) => {
|
|
42
|
+
if (!page.items)
|
|
43
|
+
return;
|
|
44
|
+
page.items = page.items.map((item) => {
|
|
45
|
+
if (typeof item === 'string')
|
|
46
|
+
return item; // Already a reference
|
|
47
|
+
// Use composite key matching buildHtmlGroups
|
|
48
|
+
const htmlKey = `${item.html || 'No HTML element'}\x00${item.xpath || ''}`;
|
|
49
|
+
return htmlKey;
|
|
50
|
+
});
|
|
51
|
+
});
|
|
52
|
+
});
|
|
53
|
+
});
|
|
54
|
+
return cloned;
|
|
55
|
+
};
|
|
@@ -0,0 +1,335 @@
|
|
|
1
|
+
import fs from 'fs-extra';
|
|
2
|
+
import path from 'path';
|
|
3
|
+
import zlib from 'zlib';
|
|
4
|
+
import { Base64Encode } from 'base64-stream';
|
|
5
|
+
import { pipeline } from 'stream/promises';
|
|
6
|
+
import { a11yRuleShortDescriptionMap } from '../constants/constants.js';
|
|
7
|
+
import { consoleLogger } from '../logs.js';
|
|
8
|
+
function* serializeObject(obj, depth = 0, indent = ' ') {
|
|
9
|
+
const currentIndent = indent.repeat(depth);
|
|
10
|
+
const nextIndent = indent.repeat(depth + 1);
|
|
11
|
+
if (obj instanceof Date) {
|
|
12
|
+
yield JSON.stringify(obj.toISOString());
|
|
13
|
+
return;
|
|
14
|
+
}
|
|
15
|
+
if (Array.isArray(obj)) {
|
|
16
|
+
yield '[\n';
|
|
17
|
+
for (let i = 0; i < obj.length; i++) {
|
|
18
|
+
if (i > 0)
|
|
19
|
+
yield ',\n';
|
|
20
|
+
yield nextIndent;
|
|
21
|
+
yield* serializeObject(obj[i], depth + 1, indent);
|
|
22
|
+
}
|
|
23
|
+
yield `\n${currentIndent}]`;
|
|
24
|
+
return;
|
|
25
|
+
}
|
|
26
|
+
if (obj !== null && typeof obj === 'object') {
|
|
27
|
+
yield '{\n';
|
|
28
|
+
const keys = Object.keys(obj);
|
|
29
|
+
for (let i = 0; i < keys.length; i++) {
|
|
30
|
+
const key = keys[i];
|
|
31
|
+
if (i > 0)
|
|
32
|
+
yield ',\n';
|
|
33
|
+
yield `${nextIndent}${JSON.stringify(key)}: `;
|
|
34
|
+
yield* serializeObject(obj[key], depth + 1, indent);
|
|
35
|
+
}
|
|
36
|
+
yield `\n${currentIndent}}`;
|
|
37
|
+
return;
|
|
38
|
+
}
|
|
39
|
+
if (obj === null || typeof obj === 'function' || typeof obj === 'undefined') {
|
|
40
|
+
yield 'null';
|
|
41
|
+
return;
|
|
42
|
+
}
|
|
43
|
+
yield JSON.stringify(obj);
|
|
44
|
+
}
|
|
45
|
+
function writeLargeJsonToFile(obj, filePath) {
|
|
46
|
+
return new Promise((resolve, reject) => {
|
|
47
|
+
const writeStream = fs.createWriteStream(filePath, { encoding: 'utf8' });
|
|
48
|
+
writeStream.on('error', error => {
|
|
49
|
+
consoleLogger.error('Stream error:', error);
|
|
50
|
+
reject(error);
|
|
51
|
+
});
|
|
52
|
+
writeStream.on('finish', () => {
|
|
53
|
+
consoleLogger.info(`JSON file written successfully: ${filePath}`);
|
|
54
|
+
resolve(true);
|
|
55
|
+
});
|
|
56
|
+
const generator = serializeObject(obj);
|
|
57
|
+
function write() {
|
|
58
|
+
let next;
|
|
59
|
+
while (!(next = generator.next()).done) {
|
|
60
|
+
if (!writeStream.write(next.value)) {
|
|
61
|
+
writeStream.once('drain', write);
|
|
62
|
+
return;
|
|
63
|
+
}
|
|
64
|
+
}
|
|
65
|
+
writeStream.end();
|
|
66
|
+
}
|
|
67
|
+
write();
|
|
68
|
+
});
|
|
69
|
+
}
|
|
70
|
+
const writeLargeScanItemsJsonToFile = async (obj, filePath) => {
|
|
71
|
+
return new Promise((resolve, reject) => {
|
|
72
|
+
const writeStream = fs.createWriteStream(filePath, { flags: 'a', encoding: 'utf8' });
|
|
73
|
+
const writeQueue = [];
|
|
74
|
+
let isWriting = false;
|
|
75
|
+
const processNextWrite = async () => {
|
|
76
|
+
if (isWriting || writeQueue.length === 0)
|
|
77
|
+
return;
|
|
78
|
+
isWriting = true;
|
|
79
|
+
const data = writeQueue.shift();
|
|
80
|
+
try {
|
|
81
|
+
if (!writeStream.write(data)) {
|
|
82
|
+
await new Promise(resolve => {
|
|
83
|
+
writeStream.once('drain', () => {
|
|
84
|
+
resolve();
|
|
85
|
+
});
|
|
86
|
+
});
|
|
87
|
+
}
|
|
88
|
+
}
|
|
89
|
+
catch (error) {
|
|
90
|
+
writeStream.destroy(error);
|
|
91
|
+
return;
|
|
92
|
+
}
|
|
93
|
+
isWriting = false;
|
|
94
|
+
processNextWrite();
|
|
95
|
+
};
|
|
96
|
+
const queueWrite = (data) => {
|
|
97
|
+
writeQueue.push(data);
|
|
98
|
+
processNextWrite();
|
|
99
|
+
};
|
|
100
|
+
writeStream.on('error', error => {
|
|
101
|
+
consoleLogger.error(`Error writing object to JSON file: ${error}`);
|
|
102
|
+
reject(error);
|
|
103
|
+
});
|
|
104
|
+
writeStream.on('finish', () => {
|
|
105
|
+
consoleLogger.info(`JSON file written successfully: ${filePath}`);
|
|
106
|
+
resolve(true);
|
|
107
|
+
});
|
|
108
|
+
try {
|
|
109
|
+
queueWrite('{\n');
|
|
110
|
+
const keys = Object.keys(obj);
|
|
111
|
+
keys.forEach((key, i) => {
|
|
112
|
+
const value = obj[key];
|
|
113
|
+
if (value === null || typeof value !== 'object' || Array.isArray(value)) {
|
|
114
|
+
queueWrite(` "${key}": ${JSON.stringify(value)}`);
|
|
115
|
+
}
|
|
116
|
+
else {
|
|
117
|
+
queueWrite(` "${key}": {\n`);
|
|
118
|
+
const { rules, ...otherProperties } = value;
|
|
119
|
+
Object.entries(otherProperties).forEach(([propKey, propValue], j) => {
|
|
120
|
+
const propValueString = propValue === null ||
|
|
121
|
+
typeof propValue === 'function' ||
|
|
122
|
+
typeof propValue === 'undefined'
|
|
123
|
+
? 'null'
|
|
124
|
+
: JSON.stringify(propValue);
|
|
125
|
+
queueWrite(` "${propKey}": ${propValueString}`);
|
|
126
|
+
if (j < Object.keys(otherProperties).length - 1 || (rules && rules.length >= 0)) {
|
|
127
|
+
queueWrite(',\n');
|
|
128
|
+
}
|
|
129
|
+
else {
|
|
130
|
+
queueWrite('\n');
|
|
131
|
+
}
|
|
132
|
+
});
|
|
133
|
+
if (rules && Array.isArray(rules)) {
|
|
134
|
+
queueWrite(' "rules": [\n');
|
|
135
|
+
rules.forEach((rule, j) => {
|
|
136
|
+
queueWrite(' {\n');
|
|
137
|
+
const { pagesAffected, ...otherRuleProperties } = rule;
|
|
138
|
+
Object.entries(otherRuleProperties).forEach(([ruleKey, ruleValue], k) => {
|
|
139
|
+
const ruleValueString = ruleValue === null ||
|
|
140
|
+
typeof ruleValue === 'function' ||
|
|
141
|
+
typeof ruleValue === 'undefined'
|
|
142
|
+
? 'null'
|
|
143
|
+
: JSON.stringify(ruleValue);
|
|
144
|
+
queueWrite(` "${ruleKey}": ${ruleValueString}`);
|
|
145
|
+
if (k < Object.keys(otherRuleProperties).length - 1 || pagesAffected) {
|
|
146
|
+
queueWrite(',\n');
|
|
147
|
+
}
|
|
148
|
+
else {
|
|
149
|
+
queueWrite('\n');
|
|
150
|
+
}
|
|
151
|
+
});
|
|
152
|
+
if (pagesAffected && Array.isArray(pagesAffected)) {
|
|
153
|
+
queueWrite(' "pagesAffected": [\n');
|
|
154
|
+
pagesAffected.forEach((page, p) => {
|
|
155
|
+
const pageJson = JSON.stringify(page, null, 2)
|
|
156
|
+
.split('\n')
|
|
157
|
+
.map(line => ` ${line}`)
|
|
158
|
+
.join('\n');
|
|
159
|
+
queueWrite(pageJson);
|
|
160
|
+
if (p < pagesAffected.length - 1) {
|
|
161
|
+
queueWrite(',\n');
|
|
162
|
+
}
|
|
163
|
+
else {
|
|
164
|
+
queueWrite('\n');
|
|
165
|
+
}
|
|
166
|
+
});
|
|
167
|
+
queueWrite(' ]');
|
|
168
|
+
}
|
|
169
|
+
queueWrite('\n }');
|
|
170
|
+
if (j < rules.length - 1) {
|
|
171
|
+
queueWrite(',\n');
|
|
172
|
+
}
|
|
173
|
+
else {
|
|
174
|
+
queueWrite('\n');
|
|
175
|
+
}
|
|
176
|
+
});
|
|
177
|
+
queueWrite(' ]');
|
|
178
|
+
}
|
|
179
|
+
queueWrite('\n }');
|
|
180
|
+
}
|
|
181
|
+
if (i < keys.length - 1) {
|
|
182
|
+
queueWrite(',\n');
|
|
183
|
+
}
|
|
184
|
+
else {
|
|
185
|
+
queueWrite('\n');
|
|
186
|
+
}
|
|
187
|
+
});
|
|
188
|
+
queueWrite('}\n');
|
|
189
|
+
const checkQueueAndEnd = () => {
|
|
190
|
+
if (writeQueue.length === 0 && !isWriting) {
|
|
191
|
+
writeStream.end();
|
|
192
|
+
}
|
|
193
|
+
else {
|
|
194
|
+
setTimeout(checkQueueAndEnd, 100);
|
|
195
|
+
}
|
|
196
|
+
};
|
|
197
|
+
checkQueueAndEnd();
|
|
198
|
+
}
|
|
199
|
+
catch (err) {
|
|
200
|
+
writeStream.destroy(err);
|
|
201
|
+
reject(err);
|
|
202
|
+
}
|
|
203
|
+
});
|
|
204
|
+
};
|
|
205
|
+
async function compressJsonFileStreaming(inputPath, outputPath) {
|
|
206
|
+
const readStream = fs.createReadStream(inputPath);
|
|
207
|
+
const writeStream = fs.createWriteStream(outputPath);
|
|
208
|
+
const gzip = zlib.createGzip();
|
|
209
|
+
const base64Encode = new Base64Encode();
|
|
210
|
+
await pipeline(readStream, gzip, base64Encode, writeStream);
|
|
211
|
+
consoleLogger.info(`File successfully compressed and saved to ${outputPath}`);
|
|
212
|
+
}
|
|
213
|
+
const writeJsonFileAndCompressedJsonFile = async (data, storagePath, filename) => {
|
|
214
|
+
try {
|
|
215
|
+
consoleLogger.info(`Writing JSON to ${filename}.json`);
|
|
216
|
+
const jsonFilePath = path.join(storagePath, `${filename}.json`);
|
|
217
|
+
if (filename === 'scanItems') {
|
|
218
|
+
await writeLargeScanItemsJsonToFile(data, jsonFilePath);
|
|
219
|
+
}
|
|
220
|
+
else {
|
|
221
|
+
await writeLargeJsonToFile(data, jsonFilePath);
|
|
222
|
+
}
|
|
223
|
+
consoleLogger.info(`Reading ${filename}.json, gzipping and base64 encoding it into ${filename}.json.gz.b64`);
|
|
224
|
+
const base64FilePath = path.join(storagePath, `${filename}.json.gz.b64`);
|
|
225
|
+
await compressJsonFileStreaming(jsonFilePath, base64FilePath);
|
|
226
|
+
consoleLogger.info(`Finished compression and base64 encoding for ${filename}`);
|
|
227
|
+
return {
|
|
228
|
+
jsonFilePath,
|
|
229
|
+
base64FilePath,
|
|
230
|
+
};
|
|
231
|
+
}
|
|
232
|
+
catch (error) {
|
|
233
|
+
consoleLogger.error(`Error compressing and encoding ${filename}`);
|
|
234
|
+
throw error;
|
|
235
|
+
}
|
|
236
|
+
};
|
|
237
|
+
const writeJsonAndBase64Files = async (allIssues, storagePath) => {
|
|
238
|
+
const { items, ...rest } = allIssues;
|
|
239
|
+
const { jsonFilePath: scanDataJsonFilePath, base64FilePath: scanDataBase64FilePath } = await writeJsonFileAndCompressedJsonFile(rest, storagePath, 'scanData');
|
|
240
|
+
const { jsonFilePath: scanItemsJsonFilePath, base64FilePath: scanItemsBase64FilePath } = await writeJsonFileAndCompressedJsonFile({ oobeeAppVersion: allIssues.oobeeAppVersion, ...items }, storagePath, 'scanItems');
|
|
241
|
+
['mustFix', 'goodToFix', 'needsReview', 'passed'].forEach(category => {
|
|
242
|
+
if (items[category].rules && Array.isArray(items[category].rules)) {
|
|
243
|
+
items[category].rules.forEach(rule => {
|
|
244
|
+
rule.pagesAffectedCount = Array.isArray(rule.pagesAffected) ? rule.pagesAffected.length : 0;
|
|
245
|
+
});
|
|
246
|
+
items[category].rules.sort((a, b) => (b.pagesAffectedCount || 0) - (a.pagesAffectedCount || 0));
|
|
247
|
+
}
|
|
248
|
+
});
|
|
249
|
+
const scanIssuesSummary = {
|
|
250
|
+
mustFix: items.mustFix.rules.map(({ pagesAffected, ...ruleInfo }) => ({
|
|
251
|
+
...ruleInfo,
|
|
252
|
+
description: a11yRuleShortDescriptionMap[ruleInfo.rule] || ruleInfo.description,
|
|
253
|
+
})),
|
|
254
|
+
goodToFix: items.goodToFix.rules.map(({ pagesAffected, ...ruleInfo }) => ({
|
|
255
|
+
...ruleInfo,
|
|
256
|
+
description: a11yRuleShortDescriptionMap[ruleInfo.rule] || ruleInfo.description,
|
|
257
|
+
})),
|
|
258
|
+
needsReview: items.needsReview.rules.map(({ pagesAffected, ...ruleInfo }) => ({
|
|
259
|
+
...ruleInfo,
|
|
260
|
+
description: a11yRuleShortDescriptionMap[ruleInfo.rule] || ruleInfo.description,
|
|
261
|
+
})),
|
|
262
|
+
passed: items.passed.rules.map(({ pagesAffected, ...ruleInfo }) => ({
|
|
263
|
+
...ruleInfo,
|
|
264
|
+
description: a11yRuleShortDescriptionMap[ruleInfo.rule] || ruleInfo.description,
|
|
265
|
+
})),
|
|
266
|
+
};
|
|
267
|
+
const { jsonFilePath: scanIssuesSummaryJsonFilePath, base64FilePath: scanIssuesSummaryBase64FilePath, } = await writeJsonFileAndCompressedJsonFile({ oobeeAppVersion: allIssues.oobeeAppVersion, ...scanIssuesSummary }, storagePath, 'scanIssuesSummary');
|
|
268
|
+
items.mustFix.rules.forEach(rule => {
|
|
269
|
+
rule.pagesAffected.forEach(page => {
|
|
270
|
+
page.itemsCount = page.items.length;
|
|
271
|
+
});
|
|
272
|
+
});
|
|
273
|
+
items.goodToFix.rules.forEach(rule => {
|
|
274
|
+
rule.pagesAffected.forEach(page => {
|
|
275
|
+
page.itemsCount = page.items.length;
|
|
276
|
+
});
|
|
277
|
+
});
|
|
278
|
+
items.needsReview.rules.forEach(rule => {
|
|
279
|
+
rule.pagesAffected.forEach(page => {
|
|
280
|
+
page.itemsCount = page.items.length;
|
|
281
|
+
});
|
|
282
|
+
});
|
|
283
|
+
items.passed.rules.forEach(rule => {
|
|
284
|
+
rule.pagesAffected.forEach(page => {
|
|
285
|
+
page.itemsCount = page.items.length;
|
|
286
|
+
});
|
|
287
|
+
});
|
|
288
|
+
items.mustFix.totalRuleIssues = items.mustFix.rules.length;
|
|
289
|
+
items.goodToFix.totalRuleIssues = items.goodToFix.rules.length;
|
|
290
|
+
items.needsReview.totalRuleIssues = items.needsReview.rules.length;
|
|
291
|
+
items.passed.totalRuleIssues = items.passed.rules.length;
|
|
292
|
+
const { topTenPagesWithMostIssues, wcagLinks, wcagPassPercentage, progressPercentage, issuesPercentage, totalPagesScanned, totalPagesNotScanned, topTenIssues, } = rest;
|
|
293
|
+
const summaryItems = {
|
|
294
|
+
mustFix: {
|
|
295
|
+
totalItems: items.mustFix?.totalItems || 0,
|
|
296
|
+
totalRuleIssues: items.mustFix?.totalRuleIssues || 0,
|
|
297
|
+
},
|
|
298
|
+
goodToFix: {
|
|
299
|
+
totalItems: items.goodToFix?.totalItems || 0,
|
|
300
|
+
totalRuleIssues: items.goodToFix?.totalRuleIssues || 0,
|
|
301
|
+
},
|
|
302
|
+
needsReview: {
|
|
303
|
+
totalItems: items.needsReview?.totalItems || 0,
|
|
304
|
+
totalRuleIssues: items.needsReview?.totalRuleIssues || 0,
|
|
305
|
+
},
|
|
306
|
+
topTenPagesWithMostIssues,
|
|
307
|
+
wcagLinks,
|
|
308
|
+
wcagPassPercentage,
|
|
309
|
+
progressPercentage,
|
|
310
|
+
issuesPercentage,
|
|
311
|
+
totalPagesScanned,
|
|
312
|
+
totalPagesNotScanned,
|
|
313
|
+
topTenIssues,
|
|
314
|
+
};
|
|
315
|
+
const { jsonFilePath: scanItemsSummaryJsonFilePath, base64FilePath: scanItemsSummaryBase64FilePath, } = await writeJsonFileAndCompressedJsonFile({ oobeeAppVersion: allIssues.oobeeAppVersion, ...summaryItems }, storagePath, 'scanItemsSummary');
|
|
316
|
+
const { jsonFilePath: scanPagesDetailJsonFilePath, base64FilePath: scanPagesDetailBase64FilePath, } = await writeJsonFileAndCompressedJsonFile({ oobeeAppVersion: allIssues.oobeeAppVersion, ...allIssues.scanPagesDetail }, storagePath, 'scanPagesDetail');
|
|
317
|
+
const { jsonFilePath: scanPagesSummaryJsonFilePath, base64FilePath: scanPagesSummaryBase64FilePath, } = await writeJsonFileAndCompressedJsonFile({ oobeeAppVersion: allIssues.oobeeAppVersion, ...allIssues.scanPagesSummary }, storagePath, 'scanPagesSummary');
|
|
318
|
+
return {
|
|
319
|
+
scanDataJsonFilePath,
|
|
320
|
+
scanDataBase64FilePath,
|
|
321
|
+
scanItemsJsonFilePath,
|
|
322
|
+
scanItemsBase64FilePath,
|
|
323
|
+
scanItemsSummaryJsonFilePath,
|
|
324
|
+
scanItemsSummaryBase64FilePath,
|
|
325
|
+
scanIssuesSummaryJsonFilePath,
|
|
326
|
+
scanIssuesSummaryBase64FilePath,
|
|
327
|
+
scanPagesDetailJsonFilePath,
|
|
328
|
+
scanPagesDetailBase64FilePath,
|
|
329
|
+
scanPagesSummaryJsonFilePath,
|
|
330
|
+
scanPagesSummaryBase64FilePath,
|
|
331
|
+
scanDataJsonFileSize: fs.statSync(scanDataJsonFilePath).size,
|
|
332
|
+
scanItemsJsonFileSize: fs.statSync(scanItemsJsonFilePath).size,
|
|
333
|
+
};
|
|
334
|
+
};
|
|
335
|
+
export { compressJsonFileStreaming, writeJsonAndBase64Files, writeJsonFileAndCompressedJsonFile };
|
|
@@ -0,0 +1,159 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Build allIssues.scanPagesDetail and allIssues.scanPagesSummary
|
|
3
|
+
* by analyzing pagesScanned (including mustFix/goodToFix/etc.).
|
|
4
|
+
*/
|
|
5
|
+
export default function populateScanPagesDetail(allIssues) {
|
|
6
|
+
const allScannedPages = Array.isArray(allIssues.pagesScanned) ? allIssues.pagesScanned : [];
|
|
7
|
+
const mustFixCategory = 'mustFix';
|
|
8
|
+
const goodToFixCategory = 'goodToFix';
|
|
9
|
+
const needsReviewCategory = 'needsReview';
|
|
10
|
+
const passedCategory = 'passed';
|
|
11
|
+
const pagesMap = {};
|
|
12
|
+
Object.entries(allIssues.items).forEach(([categoryName, categoryData]) => {
|
|
13
|
+
if (!categoryData?.rules)
|
|
14
|
+
return;
|
|
15
|
+
categoryData.rules.forEach(rule => {
|
|
16
|
+
const { rule: ruleId, conformance = [] } = rule;
|
|
17
|
+
rule.pagesAffected.forEach(p => {
|
|
18
|
+
const { url, pageTitle, items = [] } = p;
|
|
19
|
+
const itemsCount = items.length;
|
|
20
|
+
if (!pagesMap[url]) {
|
|
21
|
+
pagesMap[url] = {
|
|
22
|
+
pageTitle,
|
|
23
|
+
url,
|
|
24
|
+
totalOccurrencesFailedIncludingNeedsReview: 0,
|
|
25
|
+
totalOccurrencesFailedExcludingNeedsReview: 0,
|
|
26
|
+
totalOccurrencesNeedsReview: 0,
|
|
27
|
+
totalOccurrencesPassed: 0,
|
|
28
|
+
typesOfIssues: {},
|
|
29
|
+
};
|
|
30
|
+
}
|
|
31
|
+
if (!pagesMap[url].typesOfIssues[ruleId]) {
|
|
32
|
+
pagesMap[url].typesOfIssues[ruleId] = {
|
|
33
|
+
ruleId,
|
|
34
|
+
wcagConformance: conformance,
|
|
35
|
+
occurrencesMustFix: 0,
|
|
36
|
+
occurrencesGoodToFix: 0,
|
|
37
|
+
occurrencesNeedsReview: 0,
|
|
38
|
+
occurrencesPassed: 0,
|
|
39
|
+
};
|
|
40
|
+
}
|
|
41
|
+
if (categoryName === mustFixCategory) {
|
|
42
|
+
pagesMap[url].typesOfIssues[ruleId].occurrencesMustFix += itemsCount;
|
|
43
|
+
pagesMap[url].totalOccurrencesFailedIncludingNeedsReview += itemsCount;
|
|
44
|
+
pagesMap[url].totalOccurrencesFailedExcludingNeedsReview += itemsCount;
|
|
45
|
+
}
|
|
46
|
+
else if (categoryName === goodToFixCategory) {
|
|
47
|
+
pagesMap[url].typesOfIssues[ruleId].occurrencesGoodToFix += itemsCount;
|
|
48
|
+
pagesMap[url].totalOccurrencesFailedIncludingNeedsReview += itemsCount;
|
|
49
|
+
pagesMap[url].totalOccurrencesFailedExcludingNeedsReview += itemsCount;
|
|
50
|
+
}
|
|
51
|
+
else if (categoryName === needsReviewCategory) {
|
|
52
|
+
pagesMap[url].typesOfIssues[ruleId].occurrencesNeedsReview += itemsCount;
|
|
53
|
+
pagesMap[url].totalOccurrencesFailedIncludingNeedsReview += itemsCount;
|
|
54
|
+
pagesMap[url].totalOccurrencesNeedsReview += itemsCount;
|
|
55
|
+
}
|
|
56
|
+
else if (categoryName === passedCategory) {
|
|
57
|
+
pagesMap[url].typesOfIssues[ruleId].occurrencesPassed += itemsCount;
|
|
58
|
+
pagesMap[url].totalOccurrencesPassed += itemsCount;
|
|
59
|
+
}
|
|
60
|
+
});
|
|
61
|
+
});
|
|
62
|
+
});
|
|
63
|
+
const pagesInMap = Object.values(pagesMap);
|
|
64
|
+
const pagesInMapUrls = new Set(Object.keys(pagesMap));
|
|
65
|
+
const pagesAllPassed = pagesInMap.filter(p => p.totalOccurrencesFailedIncludingNeedsReview === 0);
|
|
66
|
+
const pagesNoEntries = allScannedPages
|
|
67
|
+
.filter(sp => !pagesInMapUrls.has(sp.url))
|
|
68
|
+
.map(sp => ({
|
|
69
|
+
pageTitle: sp.pageTitle,
|
|
70
|
+
url: sp.url,
|
|
71
|
+
totalOccurrencesFailedIncludingNeedsReview: 0,
|
|
72
|
+
totalOccurrencesFailedExcludingNeedsReview: 0,
|
|
73
|
+
totalOccurrencesNeedsReview: 0,
|
|
74
|
+
totalOccurrencesPassed: 0,
|
|
75
|
+
typesOfIssues: {},
|
|
76
|
+
}));
|
|
77
|
+
const pagesNotAffectedRaw = [...pagesAllPassed, ...pagesNoEntries];
|
|
78
|
+
const pagesAffectedRaw = pagesInMap.filter(p => p.totalOccurrencesFailedIncludingNeedsReview > 0);
|
|
79
|
+
function transformPageData(page) {
|
|
80
|
+
const typesOfIssuesArray = Object.values(page.typesOfIssues);
|
|
81
|
+
const mustFixSum = typesOfIssuesArray.reduce((acc, r) => acc + r.occurrencesMustFix, 0);
|
|
82
|
+
const goodToFixSum = typesOfIssuesArray.reduce((acc, r) => acc + r.occurrencesGoodToFix, 0);
|
|
83
|
+
const needsReviewSum = typesOfIssuesArray.reduce((acc, r) => acc + r.occurrencesNeedsReview, 0);
|
|
84
|
+
const categoriesPresent = [];
|
|
85
|
+
if (mustFixSum > 0)
|
|
86
|
+
categoriesPresent.push('mustFix');
|
|
87
|
+
if (goodToFixSum > 0)
|
|
88
|
+
categoriesPresent.push('goodToFix');
|
|
89
|
+
if (needsReviewSum > 0)
|
|
90
|
+
categoriesPresent.push('needsReview');
|
|
91
|
+
const failedRuleIds = new Set();
|
|
92
|
+
typesOfIssuesArray.forEach(r => {
|
|
93
|
+
if ((r.occurrencesMustFix || 0) > 0 ||
|
|
94
|
+
(r.occurrencesGoodToFix || 0) > 0 ||
|
|
95
|
+
(r.occurrencesNeedsReview || 0) > 0) {
|
|
96
|
+
failedRuleIds.add(r.ruleId);
|
|
97
|
+
}
|
|
98
|
+
});
|
|
99
|
+
const failedRuleCount = failedRuleIds.size;
|
|
100
|
+
const typesOfIssuesExcludingNeedsReviewCount = typesOfIssuesArray.filter(r => (r.occurrencesMustFix || 0) + (r.occurrencesGoodToFix || 0) > 0).length;
|
|
101
|
+
const typesOfIssuesExclusiveToNeedsReviewCount = typesOfIssuesArray.filter(r => (r.occurrencesNeedsReview || 0) > 0 &&
|
|
102
|
+
(r.occurrencesMustFix || 0) === 0 &&
|
|
103
|
+
(r.occurrencesGoodToFix || 0) === 0).length;
|
|
104
|
+
const allConformance = typesOfIssuesArray.reduce((acc, curr) => {
|
|
105
|
+
const nonPassedCount = (curr.occurrencesMustFix || 0) +
|
|
106
|
+
(curr.occurrencesGoodToFix || 0) +
|
|
107
|
+
(curr.occurrencesNeedsReview || 0);
|
|
108
|
+
if (nonPassedCount > 0) {
|
|
109
|
+
return acc.concat(curr.wcagConformance || []);
|
|
110
|
+
}
|
|
111
|
+
return acc;
|
|
112
|
+
}, []);
|
|
113
|
+
const conformance = Array.from(new Set(allConformance));
|
|
114
|
+
return {
|
|
115
|
+
pageTitle: page.pageTitle,
|
|
116
|
+
url: page.url,
|
|
117
|
+
totalOccurrencesFailedIncludingNeedsReview: page.totalOccurrencesFailedIncludingNeedsReview,
|
|
118
|
+
totalOccurrencesFailedExcludingNeedsReview: page.totalOccurrencesFailedExcludingNeedsReview,
|
|
119
|
+
totalOccurrencesMustFix: mustFixSum,
|
|
120
|
+
totalOccurrencesGoodToFix: goodToFixSum,
|
|
121
|
+
totalOccurrencesNeedsReview: needsReviewSum,
|
|
122
|
+
totalOccurrencesPassed: page.totalOccurrencesPassed,
|
|
123
|
+
typesOfIssuesExclusiveToNeedsReviewCount,
|
|
124
|
+
typesOfIssuesCount: failedRuleCount,
|
|
125
|
+
typesOfIssuesExcludingNeedsReviewCount,
|
|
126
|
+
categoriesPresent,
|
|
127
|
+
conformance,
|
|
128
|
+
typesOfIssues: typesOfIssuesArray,
|
|
129
|
+
};
|
|
130
|
+
}
|
|
131
|
+
const pagesAffected = pagesAffectedRaw.map(transformPageData);
|
|
132
|
+
const pagesNotAffected = pagesNotAffectedRaw.map(transformPageData);
|
|
133
|
+
pagesAffected.sort((a, b) => b.typesOfIssuesCount - a.typesOfIssuesCount);
|
|
134
|
+
pagesNotAffected.sort((a, b) => b.typesOfIssuesCount - a.typesOfIssuesCount);
|
|
135
|
+
const scannedPagesCount = pagesAffected.length + pagesNotAffected.length;
|
|
136
|
+
const pagesNotScannedCount = Array.isArray(allIssues.pagesNotScanned)
|
|
137
|
+
? allIssues.pagesNotScanned.length
|
|
138
|
+
: 0;
|
|
139
|
+
allIssues.scanPagesDetail = {
|
|
140
|
+
pagesAffected,
|
|
141
|
+
pagesNotAffected,
|
|
142
|
+
scannedPagesCount,
|
|
143
|
+
pagesNotScanned: Array.isArray(allIssues.pagesNotScanned) ? allIssues.pagesNotScanned : [],
|
|
144
|
+
pagesNotScannedCount,
|
|
145
|
+
};
|
|
146
|
+
function stripTypesOfIssues(page) {
|
|
147
|
+
const { typesOfIssues, ...rest } = page;
|
|
148
|
+
return rest;
|
|
149
|
+
}
|
|
150
|
+
const summaryPagesAffected = pagesAffected.map(stripTypesOfIssues);
|
|
151
|
+
const summaryPagesNotAffected = pagesNotAffected.map(stripTypesOfIssues);
|
|
152
|
+
allIssues.scanPagesSummary = {
|
|
153
|
+
pagesAffected: summaryPagesAffected,
|
|
154
|
+
pagesNotAffected: summaryPagesNotAffected,
|
|
155
|
+
scannedPagesCount,
|
|
156
|
+
pagesNotScanned: Array.isArray(allIssues.pagesNotScanned) ? allIssues.pagesNotScanned : [],
|
|
157
|
+
pagesNotScannedCount,
|
|
158
|
+
};
|
|
159
|
+
}
|