@govtechsg/oobee 0.10.85 → 0.10.86
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.github/workflows/image.yml +3 -2
- package/.github/workflows/publish.yml +10 -0
- package/DETAILS.md +29 -0
- package/dist/combine.js +1 -1
- package/dist/constants/common.js +15 -4
- package/dist/constants/constants.js +604 -1
- package/dist/crawlers/commonCrawlerFunc.js +3 -2
- package/dist/crawlers/crawlSitemap.js +98 -80
- package/dist/crawlers/custom/utils.js +137 -31
- package/dist/crawlers/guards/urlGuard.js +8 -15
- package/dist/crawlers/runCustom.js +18 -11
- package/dist/generateOobeeClientScanner.js +570 -0
- package/dist/mergeAxeResults.js +5 -4
- package/dist/npmIndex.js +10 -2
- package/dist/proxyService.js +18 -3
- package/dist/services/s3Uploader.js +21 -10
- package/dist/static/ejs/partials/scripts/header/aboutScanModal/ScanConfiguration.ejs +2 -2
- package/dist/static/ejs/partials/scripts/ruleModal/constants.ejs +1 -761
- package/dist/static/ejs/summary.ejs +10 -5
- package/oobee-client-scanner.js +34992 -0
- package/package.json +2 -2
- package/src/combine.ts +3 -1
- package/src/constants/common.ts +22 -10
- package/src/constants/constants.ts +602 -1
- package/src/crawlers/commonCrawlerFunc.ts +4 -3
- package/src/crawlers/crawlSitemap.ts +116 -98
- package/src/crawlers/custom/utils.ts +143 -38
- package/src/crawlers/guards/urlGuard.ts +24 -31
- package/src/crawlers/runCustom.ts +29 -11
- package/src/generateOobeeClientScanner.ts +591 -0
- package/src/mergeAxeResults.ts +5 -3
- package/src/npmIndex.ts +12 -2
- package/src/proxyService.ts +25 -4
- package/src/services/s3Uploader.ts +23 -11
- package/src/static/ejs/partials/scripts/header/aboutScanModal/ScanConfiguration.ejs +2 -2
- package/src/static/ejs/partials/scripts/ruleModal/constants.ejs +1 -761
- package/src/static/ejs/summary.ejs +10 -5
- package/testStaticJSScanner.html +534 -0
package/dist/proxyService.js
CHANGED
|
@@ -57,7 +57,7 @@ function parseEnvProxyCommon() {
|
|
|
57
57
|
if (https)
|
|
58
58
|
info.https = stripScheme(https);
|
|
59
59
|
if (socks)
|
|
60
|
-
info.socks =
|
|
60
|
+
info.socks = socks; // keep original scheme so proxyInfoToResolution can use the right protocol
|
|
61
61
|
if (noProxy)
|
|
62
62
|
info.bypassList = semiJoin(noProxy.split(/[,;]/));
|
|
63
63
|
const { username, password } = readCredsFromEnv();
|
|
@@ -384,6 +384,14 @@ function buildIncludeOnlyPac(proxyServer, includeList) {
|
|
|
384
384
|
].join('\n');
|
|
385
385
|
return pac;
|
|
386
386
|
}
|
|
387
|
+
/**
|
|
388
|
+
* Convert an info.socks value to a full proxy server URL.
|
|
389
|
+
* When the value already carries a scheme (e.g. ALL_PROXY=http://..., socks4://...),
|
|
390
|
+
* it is used as-is. Bare host:port values (from scutil) default to socks5://.
|
|
391
|
+
*/
|
|
392
|
+
function toSocksServer(socks) {
|
|
393
|
+
return /^[a-zA-Z][a-zA-Z0-9+.-]*:\/\//.test(socks) ? socks : `socks5://${socks}`;
|
|
394
|
+
}
|
|
387
395
|
export function proxyInfoToResolution(info) {
|
|
388
396
|
if (!info)
|
|
389
397
|
return { kind: 'none' };
|
|
@@ -396,7 +404,7 @@ export function proxyInfoToResolution(info) {
|
|
|
396
404
|
else if (info.https)
|
|
397
405
|
proxyServer = `http://${info.https}`;
|
|
398
406
|
else if (info.socks)
|
|
399
|
-
proxyServer =
|
|
407
|
+
proxyServer = toSocksServer(info.socks);
|
|
400
408
|
if (proxyServer) {
|
|
401
409
|
// If credentials exist, embed them for the manual proxy auth
|
|
402
410
|
// PAC scripts themselves don't carry auth, but Playwright's proxy option can
|
|
@@ -408,6 +416,13 @@ export function proxyInfoToResolution(info) {
|
|
|
408
416
|
const pacDataUrl = `data:application/x-ns-proxy-autoconfig;base64,${Buffer.from(pac).toString('base64')}`;
|
|
409
417
|
return { kind: 'pac', pacUrl: pacDataUrl, bypass: info.bypassList };
|
|
410
418
|
}
|
|
419
|
+
// No direct proxy server was found — the configured proxy is PAC-based or auto-detect only.
|
|
420
|
+
// INCLUDE_PROXY needs a concrete server address to build a routing PAC script, so it cannot
|
|
421
|
+
// be applied here. Warn and fall through to use the existing PAC/autodetect as-is.
|
|
422
|
+
console.warn('INCLUDE_PROXY is set but no direct proxy server address was found. ' +
|
|
423
|
+
'INCLUDE_PROXY requires HTTP_PROXY, HTTPS_PROXY, or ALL_PROXY to be set with a direct ' +
|
|
424
|
+
'server address; it cannot be applied to a PAC URL or auto-detect proxy. ' +
|
|
425
|
+
'INCLUDE_PROXY will be ignored.');
|
|
411
426
|
}
|
|
412
427
|
// Prefer manual proxies first (these work with Playwright's proxy option)
|
|
413
428
|
if (info.http) {
|
|
@@ -428,7 +443,7 @@ export function proxyInfoToResolution(info) {
|
|
|
428
443
|
}
|
|
429
444
|
if (info.socks) {
|
|
430
445
|
return { kind: 'manual', settings: {
|
|
431
|
-
server:
|
|
446
|
+
server: toSocksServer(info.socks),
|
|
432
447
|
username: info.username,
|
|
433
448
|
password: info.password,
|
|
434
449
|
bypass: info.bypassList,
|
|
@@ -5,6 +5,17 @@ import mime from 'mime-types';
|
|
|
5
5
|
import { consoleLogger } from '../logs.js';
|
|
6
6
|
const REGION = process.env.AWS_REGION || 'ap-southeast-1';
|
|
7
7
|
const s3Client = new S3Client({ region: REGION });
|
|
8
|
+
// S3 user metadata is sent over REST as x-amz-meta-* HTTP headers.
|
|
9
|
+
// To avoid request-header validation failures in the Node/AWS SDK path,
|
|
10
|
+
// normalize to printable ASCII before attaching metadata values.
|
|
11
|
+
const sanitizeS3MetadataValue = (value) => {
|
|
12
|
+
return value
|
|
13
|
+
.normalize('NFKD') // e.g. "é" -> "e" + combining accent, "A" -> "A"
|
|
14
|
+
.replace(/[\u0300-\u036f]/g, '') // e.g. remove the combining accent from the decomposed "é"
|
|
15
|
+
.replace(/[^\x20-\x7E]+/g, ' ') // e.g. "公益金" or emoji -> " "
|
|
16
|
+
.replace(/\s+/g, ' ') // e.g. "Community Chest \n" -> "Community Chest "
|
|
17
|
+
.trim(); // e.g. " Homepage | Community Chest " -> "Homepage | Community Chest"
|
|
18
|
+
};
|
|
8
19
|
export const uploadFileToS3 = async (localFilePath, s3Key, metadata) => {
|
|
9
20
|
const fileStream = fs.readFileSync(localFilePath);
|
|
10
21
|
const contentType = mime.lookup(localFilePath) || 'application/octet-stream';
|
|
@@ -38,31 +49,31 @@ export const uploadFolderToS3 = async (localFolderPath, s3Prefix, scanMetadata)
|
|
|
38
49
|
const files = getAllFiles(localFolderPath, localFolderPath);
|
|
39
50
|
const allowedFileExtRegex = /\.(html|csv|pdf|zip)$/;
|
|
40
51
|
const metadata = {
|
|
41
|
-
scanid: scanMetadata.scanId,
|
|
42
|
-
userid: scanMetadata.userId,
|
|
43
|
-
useremail: scanMetadata.email,
|
|
52
|
+
scanid: sanitizeS3MetadataValue(scanMetadata.scanId),
|
|
53
|
+
userid: sanitizeS3MetadataValue(scanMetadata.userId),
|
|
54
|
+
useremail: sanitizeS3MetadataValue(scanMetadata.email),
|
|
44
55
|
};
|
|
45
56
|
// Add optional metadata fields if present
|
|
46
57
|
if (scanMetadata.messageId) {
|
|
47
|
-
metadata.messageid = scanMetadata.messageId;
|
|
58
|
+
metadata.messageid = sanitizeS3MetadataValue(scanMetadata.messageId);
|
|
48
59
|
}
|
|
49
60
|
if (scanMetadata.amplitudeUserId) {
|
|
50
|
-
metadata.amplitudeuserid = scanMetadata.amplitudeUserId;
|
|
61
|
+
metadata.amplitudeuserid = sanitizeS3MetadataValue(scanMetadata.amplitudeUserId);
|
|
51
62
|
}
|
|
52
63
|
if (scanMetadata.deviceId) {
|
|
53
|
-
metadata.deviceid = scanMetadata.deviceId;
|
|
64
|
+
metadata.deviceid = sanitizeS3MetadataValue(scanMetadata.deviceId);
|
|
54
65
|
}
|
|
55
66
|
if (scanMetadata.orgId) {
|
|
56
|
-
metadata.orgid = scanMetadata.orgId;
|
|
67
|
+
metadata.orgid = sanitizeS3MetadataValue(scanMetadata.orgId);
|
|
57
68
|
}
|
|
58
69
|
if (scanMetadata.userRole) {
|
|
59
|
-
metadata.userrole = scanMetadata.userRole;
|
|
70
|
+
metadata.userrole = sanitizeS3MetadataValue(scanMetadata.userRole);
|
|
60
71
|
}
|
|
61
72
|
if (scanMetadata.siteName) {
|
|
62
|
-
metadata.sitename = scanMetadata.siteName;
|
|
73
|
+
metadata.sitename = sanitizeS3MetadataValue(scanMetadata.siteName);
|
|
63
74
|
}
|
|
64
75
|
if (scanMetadata.durationExceeded !== undefined) {
|
|
65
|
-
metadata.durationexceeded = scanMetadata.durationExceeded;
|
|
76
|
+
metadata.durationexceeded = sanitizeS3MetadataValue(scanMetadata.durationExceeded);
|
|
66
77
|
}
|
|
67
78
|
consoleLogger.info(`Uploading ${files.length} files to S3...`);
|
|
68
79
|
const uploadPromises = files.map(async (relativePath) => {
|
|
@@ -21,7 +21,7 @@
|
|
|
21
21
|
>
|
|
22
22
|
</div>
|
|
23
23
|
<div class="display-url-container">
|
|
24
|
-
<a href="${page.url}" target="_blank">${page.pageTitle
|
|
24
|
+
<a href="${page.url}" target="_blank">${page.pageTitle?.length > 0 ? page.pageTitle : page.url}</a>
|
|
25
25
|
<p>${page.url}</p>
|
|
26
26
|
</div>
|
|
27
27
|
</div>
|
|
@@ -29,7 +29,7 @@
|
|
|
29
29
|
} else {
|
|
30
30
|
listItem.innerHTML = `
|
|
31
31
|
<a href="${page.url}" target="_blank">
|
|
32
|
-
${page.pageTitle
|
|
32
|
+
${page.pageTitle?.length > 0 ? page.pageTitle : page.url}
|
|
33
33
|
<svg class="link-external-icon" width="16" height="12" viewBox="0 0 8 8" aria-hidden="true" focusable="false">
|
|
34
34
|
<path d="M7.11111 7.11111H0.888889V0.888889H4V0H0.888889C0.395556 0 0 0.4 0 0.888889V7.11111C0 7.6 0.395556 8 0.888889 8H7.11111C7.6 8 8 7.6 8 7.11111V4H7.11111V7.11111ZM4.88889 0V0.888889H6.48444L2.11556 5.25778L2.74222 5.88444L7.11111 1.51556V3.11111H8V0H4.88889Z" fill="#5735DF"/>
|
|
35
35
|
</svg>
|