@govtechsg/oobee 0.10.29 → 0.10.34
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/Dockerfile +1 -1
- package/exclusions.txt +2 -1
- package/package.json +1 -1
- package/src/combine.ts +3 -0
- package/src/constants/common.ts +62 -3
- package/src/constants/constants.ts +4 -4
- package/src/crawlers/crawlDomain.ts +53 -26
- package/src/crawlers/crawlLocalFile.ts +5 -3
- package/src/crawlers/crawlSitemap.ts +89 -24
- package/src/crawlers/custom/flagUnlabelledClickableElements.ts +0 -2
- package/src/crawlers/custom/utils.ts +7 -2
- package/src/crawlers/pdfScanFunc.ts +5 -1
- package/src/static/ejs/partials/summaryMain.ejs +3 -2
- package/src/static/ejs/report.ejs +2 -2
- package/src/utils.ts +9 -5
package/Dockerfile
CHANGED
@@ -1,6 +1,6 @@
|
|
1
1
|
# Use Microsoft Playwright image as base image
|
2
2
|
# Node version is v22
|
3
|
-
FROM mcr.microsoft.com/playwright:v1.50.
|
3
|
+
FROM mcr.microsoft.com/playwright:v1.50.1-noble
|
4
4
|
|
5
5
|
# Installation of packages for oobee and runner
|
6
6
|
RUN apt-get update && apt-get install -y zip git
|
package/exclusions.txt
CHANGED
package/package.json
CHANGED
package/src/combine.ts
CHANGED
@@ -210,6 +210,7 @@ const combineRun = async (details: Data, deviceToScan: string) => {
|
|
210
210
|
...urlsCrawledObj.error,
|
211
211
|
...urlsCrawledObj.invalid,
|
212
212
|
...urlsCrawledObj.forbidden,
|
213
|
+
...urlsCrawledObj.userExcluded,
|
213
214
|
];
|
214
215
|
const basicFormHTMLSnippet = await generateArtifacts(
|
215
216
|
randomToken,
|
@@ -240,6 +241,8 @@ const combineRun = async (details: Data, deviceToScan: string) => {
|
|
240
241
|
pagesNotScanned.length,
|
241
242
|
metadata,
|
242
243
|
);
|
244
|
+
} else {
|
245
|
+
printMessage([`No pages were scanned.`], alertMessageOptions);
|
243
246
|
}
|
244
247
|
} else {
|
245
248
|
printMessage([`No pages were scanned.`], alertMessageOptions);
|
package/src/constants/common.ts
CHANGED
@@ -1819,13 +1819,72 @@ export const urlWithoutAuth = (url: string): string => {
|
|
1819
1819
|
};
|
1820
1820
|
|
1821
1821
|
export const waitForPageLoaded = async (page, timeout = 10000) => {
|
1822
|
+
const OBSERVER_TIMEOUT = timeout; // Ensure observer timeout does not exceed the main timeout
|
1823
|
+
|
1822
1824
|
return Promise.race([
|
1823
|
-
page.waitForLoadState('load'),
|
1824
|
-
page.waitForLoadState('networkidle'),
|
1825
|
-
new Promise(resolve => setTimeout(resolve, timeout)),
|
1825
|
+
page.waitForLoadState('load'), // Ensure page load completes
|
1826
|
+
page.waitForLoadState('networkidle'), // Wait for network requests to settle
|
1827
|
+
new Promise(resolve => setTimeout(resolve, timeout)), // Hard timeout as a fallback
|
1828
|
+
page.evaluate((OBSERVER_TIMEOUT) => {
|
1829
|
+
return new Promise((resolve) => {
|
1830
|
+
// Skip mutation check for PDFs
|
1831
|
+
if (document.contentType === 'application/pdf') {
|
1832
|
+
resolve('Skipping DOM mutation check for PDF.');
|
1833
|
+
return;
|
1834
|
+
}
|
1835
|
+
|
1836
|
+
let timeout;
|
1837
|
+
let mutationCount = 0;
|
1838
|
+
const MAX_MUTATIONS = 250; // Limit max mutations
|
1839
|
+
const mutationHash = {};
|
1840
|
+
|
1841
|
+
const observer = new MutationObserver(mutationsList => {
|
1842
|
+
clearTimeout(timeout);
|
1843
|
+
|
1844
|
+
mutationCount++;
|
1845
|
+
if (mutationCount > MAX_MUTATIONS) {
|
1846
|
+
observer.disconnect();
|
1847
|
+
resolve('Too many mutations detected, exiting.');
|
1848
|
+
return;
|
1849
|
+
}
|
1850
|
+
|
1851
|
+
mutationsList.forEach(mutation => {
|
1852
|
+
if (mutation.target instanceof Element) {
|
1853
|
+
Array.from(mutation.target.attributes).forEach(attr => {
|
1854
|
+
const mutationKey = `${mutation.target.nodeName}-${attr.name}`;
|
1855
|
+
|
1856
|
+
if (mutationKey) {
|
1857
|
+
mutationHash[mutationKey] = (mutationHash[mutationKey] || 0) + 1;
|
1858
|
+
|
1859
|
+
if (mutationHash[mutationKey] >= 10) {
|
1860
|
+
observer.disconnect();
|
1861
|
+
resolve(`Repeated mutation detected for ${mutationKey}, exiting.`);
|
1862
|
+
}
|
1863
|
+
}
|
1864
|
+
});
|
1865
|
+
}
|
1866
|
+
});
|
1867
|
+
|
1868
|
+
// If no mutations occur for 1 second, resolve
|
1869
|
+
timeout = setTimeout(() => {
|
1870
|
+
observer.disconnect();
|
1871
|
+
resolve('DOM stabilized after mutations.');
|
1872
|
+
}, 1000);
|
1873
|
+
});
|
1874
|
+
|
1875
|
+
// Final timeout to avoid infinite waiting
|
1876
|
+
timeout = setTimeout(() => {
|
1877
|
+
observer.disconnect();
|
1878
|
+
resolve('Observer timeout reached, exiting.');
|
1879
|
+
}, OBSERVER_TIMEOUT);
|
1880
|
+
|
1881
|
+
observer.observe(document.documentElement, { childList: true, subtree: true, attributes: true });
|
1882
|
+
});
|
1883
|
+
}, OBSERVER_TIMEOUT), // Pass OBSERVER_TIMEOUT dynamically to the browser context
|
1826
1884
|
]);
|
1827
1885
|
};
|
1828
1886
|
|
1887
|
+
|
1829
1888
|
function isValidHttpUrl(urlString) {
|
1830
1889
|
const pattern = /^(http|https):\/\/[^ "]+$/;
|
1831
1890
|
return pattern.test(urlString);
|
@@ -178,15 +178,15 @@ export const axeScript = path.join(dirname, '../../node_modules/axe-core/axe.min
|
|
178
178
|
export class UrlsCrawled {
|
179
179
|
toScan: string[] = [];
|
180
180
|
scanned: { url: string; actualUrl: string; pageTitle: string }[] = [];
|
181
|
-
invalid: string[] = [];
|
181
|
+
invalid: { url: string; actualUrl: string; pageTitle: string }[] = [];
|
182
182
|
scannedRedirects: { fromUrl: string; toUrl: string }[] = [];
|
183
183
|
notScannedRedirects: { fromUrl: string; toUrl: string }[] = [];
|
184
184
|
outOfDomain: string[] = [];
|
185
|
-
blacklisted: string[] = [];
|
185
|
+
blacklisted: { url: string; actualUrl: string; pageTitle: string }[] = [];
|
186
186
|
error: { url: string }[] = [];
|
187
187
|
exceededRequests: string[] = [];
|
188
|
-
forbidden: string[] = [];
|
189
|
-
userExcluded: string[] = [];
|
188
|
+
forbidden: { url: string; actualUrl: string; pageTitle: string }[] = [];
|
189
|
+
userExcluded: { url: string; actualUrl: string; pageTitle: string }[] = [];
|
190
190
|
everything: string[] = [];
|
191
191
|
|
192
192
|
constructor(urlsCrawled?: Partial<UrlsCrawled>) {
|
@@ -125,14 +125,6 @@ const crawlDomain = async ({
|
|
125
125
|
|
126
126
|
const httpsAgent = new https.Agent({ rejectUnauthorized: false });
|
127
127
|
|
128
|
-
if (isBlacklistedUrl) {
|
129
|
-
guiInfoLog(guiInfoStatusTypes.SKIPPED, {
|
130
|
-
numScanned: urlsCrawled.scanned.length,
|
131
|
-
urlScanned: url,
|
132
|
-
});
|
133
|
-
return;
|
134
|
-
}
|
135
|
-
|
136
128
|
// Boolean to omit axe scan for basic auth URL
|
137
129
|
let isBasicAuth = false;
|
138
130
|
let authHeader = '';
|
@@ -608,13 +600,13 @@ const crawlDomain = async ({
|
|
608
600
|
}
|
609
601
|
|
610
602
|
await waitForPageLoaded(page, 10000);
|
611
|
-
let actualUrl = request.url;
|
603
|
+
let actualUrl = page.url() || request.loadedUrl || request.url;
|
612
604
|
|
613
605
|
if (page.url() !== 'about:blank') {
|
614
606
|
actualUrl = page.url();
|
615
607
|
}
|
616
608
|
|
617
|
-
if (isBlacklisted(actualUrl, blacklistedPatterns) || (isUrlPdf(actualUrl) && !isScanPdfs)) {
|
609
|
+
if (!isFollowStrategy(url, actualUrl, strategy) && (isBlacklisted(actualUrl, blacklistedPatterns) || (isUrlPdf(actualUrl) && !isScanPdfs))) {
|
618
610
|
guiInfoLog(guiInfoStatusTypes.SKIPPED, {
|
619
611
|
numScanned: urlsCrawled.scanned.length,
|
620
612
|
urlScanned: actualUrl,
|
@@ -646,7 +638,12 @@ const crawlDomain = async ({
|
|
646
638
|
numScanned: urlsCrawled.scanned.length,
|
647
639
|
urlScanned: request.url,
|
648
640
|
});
|
649
|
-
urlsCrawled.blacklisted.push(
|
641
|
+
urlsCrawled.blacklisted.push({
|
642
|
+
url: request.url,
|
643
|
+
pageTitle: request.url,
|
644
|
+
actualUrl: actualUrl, // i.e. actualUrl
|
645
|
+
});
|
646
|
+
|
650
647
|
return;
|
651
648
|
}
|
652
649
|
const { pdfFileName, url } = handlePdfDownload(
|
@@ -670,7 +667,12 @@ const crawlDomain = async ({
|
|
670
667
|
numScanned: urlsCrawled.scanned.length,
|
671
668
|
urlScanned: request.url,
|
672
669
|
});
|
673
|
-
urlsCrawled.blacklisted.push(
|
670
|
+
urlsCrawled.blacklisted.push({
|
671
|
+
url: request.url,
|
672
|
+
pageTitle: request.url,
|
673
|
+
actualUrl: actualUrl, // i.e. actualUrl
|
674
|
+
});
|
675
|
+
|
674
676
|
return;
|
675
677
|
}
|
676
678
|
|
@@ -679,12 +681,22 @@ const crawlDomain = async ({
|
|
679
681
|
numScanned: urlsCrawled.scanned.length,
|
680
682
|
urlScanned: request.url,
|
681
683
|
});
|
682
|
-
urlsCrawled.blacklisted.push(
|
684
|
+
urlsCrawled.blacklisted.push({
|
685
|
+
url: request.url,
|
686
|
+
pageTitle: request.url,
|
687
|
+
actualUrl: actualUrl, // i.e. actualUrl
|
688
|
+
});
|
689
|
+
|
683
690
|
return;
|
684
691
|
}
|
685
692
|
|
686
|
-
if (blacklistedPatterns && isSkippedUrl(actualUrl, blacklistedPatterns)) {
|
687
|
-
urlsCrawled.userExcluded.push(
|
693
|
+
if (!isFollowStrategy(url, actualUrl, strategy) && blacklistedPatterns && isSkippedUrl(actualUrl, blacklistedPatterns)) {
|
694
|
+
urlsCrawled.userExcluded.push({
|
695
|
+
url: request.url,
|
696
|
+
pageTitle: request.url,
|
697
|
+
actualUrl: actualUrl,
|
698
|
+
});
|
699
|
+
|
688
700
|
await enqueueProcess(page, enqueueLinks, browserContext);
|
689
701
|
return;
|
690
702
|
}
|
@@ -694,7 +706,12 @@ const crawlDomain = async ({
|
|
694
706
|
numScanned: urlsCrawled.scanned.length,
|
695
707
|
urlScanned: request.url,
|
696
708
|
});
|
697
|
-
urlsCrawled.forbidden.push(
|
709
|
+
urlsCrawled.forbidden.push({
|
710
|
+
url: request.url,
|
711
|
+
pageTitle: request.url,
|
712
|
+
actualUrl: actualUrl, // i.e. actualUrl
|
713
|
+
});
|
714
|
+
|
698
715
|
return;
|
699
716
|
}
|
700
717
|
|
@@ -703,24 +720,29 @@ const crawlDomain = async ({
|
|
703
720
|
numScanned: urlsCrawled.scanned.length,
|
704
721
|
urlScanned: request.url,
|
705
722
|
});
|
706
|
-
urlsCrawled.invalid.push(
|
723
|
+
urlsCrawled.invalid.push({
|
724
|
+
url: request.url,
|
725
|
+
pageTitle: request.url,
|
726
|
+
actualUrl: actualUrl, // i.e. actualUrl
|
727
|
+
});
|
728
|
+
|
707
729
|
return;
|
708
730
|
}
|
709
731
|
|
710
732
|
if (isScanHtml) {
|
711
733
|
// For deduplication, if the URL is redirected, we want to store the original URL and the redirected URL (actualUrl)
|
712
|
-
const isRedirected = !areLinksEqual(
|
734
|
+
const isRedirected = !areLinksEqual(actualUrl, request.url);
|
713
735
|
|
714
736
|
// check if redirected link is following strategy (same-domain/same-hostname)
|
715
737
|
const isLoadedUrlFollowStrategy = isFollowStrategy(
|
716
|
-
|
738
|
+
actualUrl,
|
717
739
|
request.url,
|
718
740
|
strategy,
|
719
741
|
);
|
720
742
|
if (isRedirected && !isLoadedUrlFollowStrategy) {
|
721
743
|
urlsCrawled.notScannedRedirects.push({
|
722
744
|
fromUrl: request.url,
|
723
|
-
toUrl:
|
745
|
+
toUrl: actualUrl, // i.e. actualUrl
|
724
746
|
});
|
725
747
|
return;
|
726
748
|
}
|
@@ -729,13 +751,13 @@ const crawlDomain = async ({
|
|
729
751
|
|
730
752
|
if (isRedirected) {
|
731
753
|
const isLoadedUrlInCrawledUrls = urlsCrawled.scanned.some(
|
732
|
-
item => (item.actualUrl || item.url) ===
|
754
|
+
item => (item.actualUrl || item.url) === actualUrl,
|
733
755
|
);
|
734
756
|
|
735
757
|
if (isLoadedUrlInCrawledUrls) {
|
736
758
|
urlsCrawled.notScannedRedirects.push({
|
737
759
|
fromUrl: request.url,
|
738
|
-
toUrl:
|
760
|
+
toUrl: actualUrl, // i.e. actualUrl
|
739
761
|
});
|
740
762
|
return;
|
741
763
|
}
|
@@ -750,16 +772,16 @@ const crawlDomain = async ({
|
|
750
772
|
urlsCrawled.scanned.push({
|
751
773
|
url: urlWithoutAuth(request.url),
|
752
774
|
pageTitle: results.pageTitle,
|
753
|
-
actualUrl:
|
775
|
+
actualUrl: actualUrl, // i.e. actualUrl
|
754
776
|
});
|
755
777
|
|
756
778
|
urlsCrawled.scannedRedirects.push({
|
757
779
|
fromUrl: urlWithoutAuth(request.url),
|
758
|
-
toUrl:
|
780
|
+
toUrl: actualUrl, // i.e. actualUrl
|
759
781
|
});
|
760
782
|
|
761
783
|
results.url = request.url;
|
762
|
-
results.actualUrl =
|
784
|
+
results.actualUrl = actualUrl;
|
763
785
|
await dataset.pushData(results);
|
764
786
|
}
|
765
787
|
} else {
|
@@ -782,7 +804,12 @@ const crawlDomain = async ({
|
|
782
804
|
numScanned: urlsCrawled.scanned.length,
|
783
805
|
urlScanned: request.url,
|
784
806
|
});
|
785
|
-
urlsCrawled.blacklisted.push(
|
807
|
+
urlsCrawled.blacklisted.push({
|
808
|
+
url: request.url,
|
809
|
+
pageTitle: request.url,
|
810
|
+
actualUrl: actualUrl, // i.e. actualUrl
|
811
|
+
});
|
812
|
+
|
786
813
|
}
|
787
814
|
|
788
815
|
if (followRobots) await getUrlsFromRobotsTxt(request.url, browser);
|
@@ -153,6 +153,8 @@ const crawlLocalFile = async (
|
|
153
153
|
await page.goto(request.url);
|
154
154
|
const results = await runAxeScript({ includeScreenshots, page, randomToken });
|
155
155
|
|
156
|
+
const actualUrl = page.url() || request.loadedUrl || request.url;
|
157
|
+
|
156
158
|
guiInfoLog(guiInfoStatusTypes.SCANNED, {
|
157
159
|
numScanned: urlsCrawled.scanned.length,
|
158
160
|
urlScanned: request.url,
|
@@ -161,16 +163,16 @@ const crawlLocalFile = async (
|
|
161
163
|
urlsCrawled.scanned.push({
|
162
164
|
url: request.url,
|
163
165
|
pageTitle: results.pageTitle,
|
164
|
-
actualUrl:
|
166
|
+
actualUrl: actualUrl, // i.e. actualUrl
|
165
167
|
});
|
166
168
|
|
167
169
|
urlsCrawled.scannedRedirects.push({
|
168
170
|
fromUrl: request.url,
|
169
|
-
toUrl:
|
171
|
+
toUrl: actualUrl, // i.e. actualUrl
|
170
172
|
});
|
171
173
|
|
172
174
|
results.url = request.url;
|
173
|
-
|
175
|
+
results.actualUrl = actualUrl;
|
174
176
|
|
175
177
|
await dataset.pushData(results);
|
176
178
|
} else {
|
@@ -18,7 +18,7 @@ import {
|
|
18
18
|
waitForPageLoaded,
|
19
19
|
isFilePath,
|
20
20
|
} from '../constants/common.js';
|
21
|
-
import { areLinksEqual, isWhitelistedContentType } from '../utils.js';
|
21
|
+
import { areLinksEqual, isWhitelistedContentType, isFollowStrategy } from '../utils.js';
|
22
22
|
import { handlePdfDownload, runPdfScan, mapPdfScanResults } from './pdfScanFunc.js';
|
23
23
|
import { guiInfoLog } from '../logs.js';
|
24
24
|
|
@@ -161,21 +161,67 @@ const crawlSitemap = async (
|
|
161
161
|
],
|
162
162
|
},
|
163
163
|
requestList,
|
164
|
+
postNavigationHooks: [
|
165
|
+
async ({ page, request }) => {
|
166
|
+
try {
|
167
|
+
// Wait for a quiet period in the DOM, but with safeguards
|
168
|
+
await page.evaluate(() => {
|
169
|
+
return new Promise((resolve) => {
|
170
|
+
let timeout;
|
171
|
+
let mutationCount = 0;
|
172
|
+
const MAX_MUTATIONS = 250; // Prevent infinite mutations
|
173
|
+
const OBSERVER_TIMEOUT = 5000; // Hard timeout to exit
|
174
|
+
|
175
|
+
const observer = new MutationObserver(() => {
|
176
|
+
clearTimeout(timeout);
|
177
|
+
|
178
|
+
mutationCount++;
|
179
|
+
if (mutationCount > MAX_MUTATIONS) {
|
180
|
+
observer.disconnect();
|
181
|
+
resolve('Too many mutations detected, exiting.');
|
182
|
+
return;
|
183
|
+
}
|
184
|
+
|
185
|
+
timeout = setTimeout(() => {
|
186
|
+
observer.disconnect();
|
187
|
+
resolve('DOM stabilized after mutations.');
|
188
|
+
}, 1000);
|
189
|
+
});
|
190
|
+
|
191
|
+
timeout = setTimeout(() => {
|
192
|
+
observer.disconnect();
|
193
|
+
resolve('Observer timeout reached, exiting.');
|
194
|
+
}, OBSERVER_TIMEOUT); // Ensure the observer stops after X seconds
|
195
|
+
|
196
|
+
observer.observe(document.documentElement, { childList: true, subtree: true });
|
197
|
+
|
198
|
+
});
|
199
|
+
});
|
200
|
+
} catch (err) {
|
201
|
+
// Handle page navigation errors gracefully
|
202
|
+
if (err.message.includes('was destroyed')) {
|
203
|
+
return; // Page navigated or closed, no need to handle
|
204
|
+
}
|
205
|
+
throw err; // Rethrow unknown errors
|
206
|
+
}
|
207
|
+
},
|
208
|
+
],
|
209
|
+
|
164
210
|
preNavigationHooks: isBasicAuth
|
165
211
|
? [
|
166
|
-
|
167
|
-
|
168
|
-
|
169
|
-
|
170
|
-
|
171
|
-
|
172
|
-
|
212
|
+
async ({ page }) => {
|
213
|
+
await page.setExtraHTTPHeaders({
|
214
|
+
Authorization: authHeader,
|
215
|
+
...extraHTTPHeaders,
|
216
|
+
});
|
217
|
+
},
|
218
|
+
]
|
173
219
|
: [
|
174
|
-
|
175
|
-
|
176
|
-
|
177
|
-
|
178
|
-
|
220
|
+
async () => {
|
221
|
+
preNavigationHooks(extraHTTPHeaders);
|
222
|
+
// insert other code here
|
223
|
+
},
|
224
|
+
],
|
179
225
|
requestHandlerTimeoutSecs: 90,
|
180
226
|
requestHandler: async ({ page, request, response, sendRequest }) => {
|
181
227
|
await waitForPageLoaded(page, 10000);
|
@@ -191,7 +237,7 @@ const crawlSitemap = async (
|
|
191
237
|
request.url = currentUrl.href;
|
192
238
|
}
|
193
239
|
|
194
|
-
const actualUrl = request.loadedUrl || request.url;
|
240
|
+
const actualUrl = page.url() || request.loadedUrl || request.url;
|
195
241
|
|
196
242
|
if (urlsCrawled.scanned.length >= maxRequestsPerCrawl) {
|
197
243
|
crawler.autoscaledPool.abort();
|
@@ -204,7 +250,12 @@ const crawlSitemap = async (
|
|
204
250
|
numScanned: urlsCrawled.scanned.length,
|
205
251
|
urlScanned: request.url,
|
206
252
|
});
|
207
|
-
urlsCrawled.blacklisted.push(
|
253
|
+
urlsCrawled.blacklisted.push({
|
254
|
+
url: request.url,
|
255
|
+
pageTitle: request.url,
|
256
|
+
actualUrl: actualUrl, // i.e. actualUrl
|
257
|
+
});
|
258
|
+
|
208
259
|
return;
|
209
260
|
}
|
210
261
|
// pushes download promise into pdfDownloads
|
@@ -223,8 +274,17 @@ const crawlSitemap = async (
|
|
223
274
|
const contentType = response.headers()['content-type'];
|
224
275
|
const status = response.status();
|
225
276
|
|
226
|
-
if (blacklistedPatterns && isSkippedUrl(actualUrl, blacklistedPatterns)) {
|
227
|
-
urlsCrawled.userExcluded.push(
|
277
|
+
if (blacklistedPatterns && !isFollowStrategy(actualUrl, request.url, "same-hostname") && isSkippedUrl(actualUrl, blacklistedPatterns)) {
|
278
|
+
urlsCrawled.userExcluded.push({
|
279
|
+
url: request.url,
|
280
|
+
pageTitle: request.url,
|
281
|
+
actualUrl: actualUrl,
|
282
|
+
});
|
283
|
+
|
284
|
+
guiInfoLog(guiInfoStatusTypes.SKIPPED, {
|
285
|
+
numScanned: urlsCrawled.scanned.length,
|
286
|
+
urlScanned: request.url,
|
287
|
+
});
|
228
288
|
return;
|
229
289
|
}
|
230
290
|
|
@@ -242,7 +302,12 @@ const crawlSitemap = async (
|
|
242
302
|
numScanned: urlsCrawled.scanned.length,
|
243
303
|
urlScanned: request.url,
|
244
304
|
});
|
245
|
-
urlsCrawled.invalid.push(
|
305
|
+
urlsCrawled.invalid.push({
|
306
|
+
url: request.url,
|
307
|
+
pageTitle: request.url,
|
308
|
+
actualUrl: actualUrl, // i.e. actualUrl
|
309
|
+
});
|
310
|
+
|
246
311
|
return;
|
247
312
|
}
|
248
313
|
|
@@ -255,16 +320,16 @@ const crawlSitemap = async (
|
|
255
320
|
urlScanned: request.url,
|
256
321
|
});
|
257
322
|
|
258
|
-
const isRedirected = !areLinksEqual(
|
323
|
+
const isRedirected = !areLinksEqual(page.url(), request.url);
|
259
324
|
if (isRedirected) {
|
260
325
|
const isLoadedUrlInCrawledUrls = urlsCrawled.scanned.some(
|
261
|
-
item => (item.actualUrl || item.url.href) ===
|
326
|
+
item => (item.actualUrl || item.url.href) === page,
|
262
327
|
);
|
263
328
|
|
264
329
|
if (isLoadedUrlInCrawledUrls) {
|
265
330
|
urlsCrawled.notScannedRedirects.push({
|
266
331
|
fromUrl: request.url,
|
267
|
-
toUrl:
|
332
|
+
toUrl: actualUrl, // i.e. actualUrl
|
268
333
|
});
|
269
334
|
return;
|
270
335
|
}
|
@@ -272,16 +337,16 @@ const crawlSitemap = async (
|
|
272
337
|
urlsCrawled.scanned.push({
|
273
338
|
url: urlWithoutAuth(request.url),
|
274
339
|
pageTitle: results.pageTitle,
|
275
|
-
actualUrl:
|
340
|
+
actualUrl: actualUrl, // i.e. actualUrl
|
276
341
|
});
|
277
342
|
|
278
343
|
urlsCrawled.scannedRedirects.push({
|
279
344
|
fromUrl: urlWithoutAuth(request.url),
|
280
|
-
toUrl:
|
345
|
+
toUrl: actualUrl,
|
281
346
|
});
|
282
347
|
|
283
348
|
results.url = request.url;
|
284
|
-
results.actualUrl =
|
349
|
+
results.actualUrl = actualUrl;
|
285
350
|
} else {
|
286
351
|
urlsCrawled.scanned.push({
|
287
352
|
url: urlWithoutAuth(request.url),
|
@@ -874,7 +874,6 @@ export const flagUnlabelledClickableElements = async (page: Page) => {
|
|
874
874
|
}
|
875
875
|
|
876
876
|
function flagElements() {
|
877
|
-
console.time('Accessibility Check Time');
|
878
877
|
|
879
878
|
const currentFlaggedElementsByDocument: Record<string, HTMLElement[]> = {}; // Temporary object to hold current flagged elements
|
880
879
|
|
@@ -1014,7 +1013,6 @@ export const flagUnlabelledClickableElements = async (page: Page) => {
|
|
1014
1013
|
previousFlaggedXPathsByDocument = { ...flaggedXPathsByDocument };
|
1015
1014
|
|
1016
1015
|
cleanupFlaggedElements();
|
1017
|
-
console.timeEnd('Accessibility Check Time');
|
1018
1016
|
return previousAllFlaggedElementsXPaths;
|
1019
1017
|
}
|
1020
1018
|
|
@@ -152,7 +152,12 @@ export const processPage = async (page, processPageParams) => {
|
|
152
152
|
window.confirm('Page has been excluded, would you still like to proceed with the scan?'),
|
153
153
|
);
|
154
154
|
if (!continueScan) {
|
155
|
-
urlsCrawled.userExcluded.push(
|
155
|
+
urlsCrawled.userExcluded.push({
|
156
|
+
url: pageUrl,
|
157
|
+
pageTitle: pageUrl,
|
158
|
+
actualUrl: pageUrl,
|
159
|
+
});
|
160
|
+
|
156
161
|
return;
|
157
162
|
}
|
158
163
|
}
|
@@ -396,7 +401,7 @@ export const initNewPage = async (page, pageClosePromises, processPageParams, pa
|
|
396
401
|
// eslint-disable-next-line no-underscore-dangle
|
397
402
|
const pageId = page._guid;
|
398
403
|
|
399
|
-
page.on('dialog', () => {});
|
404
|
+
page.on('dialog', () => { });
|
400
405
|
|
401
406
|
const pageClosePromise = new Promise(resolve => {
|
402
407
|
page.on('close', () => {
|
@@ -295,7 +295,11 @@ export const handlePdfDownload = (
|
|
295
295
|
numScanned: urlsCrawled.scanned.length,
|
296
296
|
urlScanned: request.url,
|
297
297
|
});
|
298
|
-
urlsCrawled.invalid.push(
|
298
|
+
urlsCrawled.invalid.push({
|
299
|
+
url: request.url,
|
300
|
+
pageTitle: url,
|
301
|
+
actualUrl: url, // i.e. actualUrl
|
302
|
+
});
|
299
303
|
}
|
300
304
|
resolve();
|
301
305
|
});
|
@@ -13,14 +13,15 @@
|
|
13
13
|
<div class="d-flex justify-content-between align-items-center">
|
14
14
|
<span class="fw-bold"> WCAG (A & AA) Passes </span>
|
15
15
|
<span aria-label="Pass percentage" class="ms-2">
|
16
|
-
<%= wcagPassPercentage %>% of automated checks
|
16
|
+
<%= wcagPassPercentage.passPercentageAA %>% of automated checks
|
17
17
|
</span>
|
18
18
|
</div>
|
19
19
|
<div class="wcag-compliance-passes-bar mb-5 d-flex">
|
20
20
|
<svg width="500" role="none" height="6" fill="none" xmlns="http://www.w3.org/2000/svg"
|
21
21
|
style="display: flex; width: 100%; position: absolute">
|
22
22
|
<rect width="100%" height="10" rx="3" fill="#E7ECEE" style="justify-content: left"></rect>
|
23
|
-
<rect width="<%= wcagPassPercentage %>%" height="6" rx="3" fill="#9021a6" style=""
|
23
|
+
<rect width="<%= wcagPassPercentage.passPercentageAA %>%" height="6" rx="3" fill="#9021a6" style="">
|
24
|
+
</rect>
|
24
25
|
</svg>
|
25
26
|
</div>
|
26
27
|
<ul class="unbulleted-list">
|
@@ -47,9 +47,9 @@
|
|
47
47
|
// Scan DATA FUNCTION TO REPLACE NA
|
48
48
|
const scanDataWCAGCompliance = () => {
|
49
49
|
const passPecentage = document.getElementById('passPercentage');
|
50
|
-
passPecentage.innerHTML = scanData.wcagPassPercentage + '% of automated checks';
|
50
|
+
passPecentage.innerHTML = scanData.wcagPassPercentage.passPercentageAA + '% of automated checks';
|
51
51
|
const wcagBarProgess = document.getElementById('wcag-compliance-passes-bar-progress');
|
52
|
-
wcagBarProgess.style.width = `${scanData.wcagPassPercentage}%`; // Set this to your desired width
|
52
|
+
wcagBarProgess.style.width = `${scanData.wcagPassPercentage.passPercentageAA}%`; // Set this to your desired width
|
53
53
|
|
54
54
|
const wcagLinksList = document.getElementById('wcagLinksList');
|
55
55
|
|
package/src/utils.ts
CHANGED
@@ -190,19 +190,23 @@ export const cleanUp = async pathToDelete => {
|
|
190
190
|
// timeZoneName: "longGeneric",
|
191
191
|
// });
|
192
192
|
|
193
|
-
export const getWcagPassPercentage = (wcagViolations: string[]): string => {
|
193
|
+
export const getWcagPassPercentage = (wcagViolations: string[]): { passPercentageAA: string; totalWcagChecksAA: number; totalWcagViolationsAA: number } => {
|
194
194
|
|
195
195
|
// These AAA rules should not be counted as WCAG Pass Percentage only contains A and AA
|
196
196
|
const wcagAAA = ['WCAG 1.4.6', 'WCAG 2.2.4', 'WCAG 2.4.9', 'WCAG 3.1.5', 'WCAG 3.2.5'];
|
197
197
|
|
198
198
|
const filteredWcagLinks = Object.keys(constants.wcagLinks).filter(key => !wcagAAA.includes(key));
|
199
199
|
const filteredWcagViolations = wcagViolations.filter(violation => !wcagAAA.includes(violation));
|
200
|
-
const
|
200
|
+
const totalChecksAA = filteredWcagLinks.length;
|
201
201
|
|
202
|
-
const passedChecks =
|
203
|
-
const
|
202
|
+
const passedChecks = totalChecksAA - filteredWcagViolations.length;
|
203
|
+
const passPercentageAA = (passedChecks / totalChecksAA) * 100;
|
204
204
|
|
205
|
-
return
|
205
|
+
return {
|
206
|
+
passPercentageAA: passPercentageAA.toFixed(2), // toFixed returns a string, which is correct here
|
207
|
+
totalWcagChecksAA: totalChecksAA,
|
208
|
+
totalWcagViolationsAA: filteredWcagViolations.length,
|
209
|
+
};
|
206
210
|
};
|
207
211
|
|
208
212
|
export const getFormattedTime = inputDate => {
|