@govtechsg/oobee 0.10.85 → 0.10.86

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (38) hide show
  1. package/.github/workflows/image.yml +3 -2
  2. package/.github/workflows/publish.yml +10 -0
  3. package/DETAILS.md +29 -0
  4. package/dist/combine.js +1 -1
  5. package/dist/constants/common.js +15 -4
  6. package/dist/constants/constants.js +604 -1
  7. package/dist/crawlers/commonCrawlerFunc.js +3 -2
  8. package/dist/crawlers/crawlSitemap.js +98 -80
  9. package/dist/crawlers/custom/utils.js +137 -31
  10. package/dist/crawlers/guards/urlGuard.js +8 -15
  11. package/dist/crawlers/runCustom.js +18 -11
  12. package/dist/generateOobeeClientScanner.js +570 -0
  13. package/dist/mergeAxeResults.js +5 -4
  14. package/dist/npmIndex.js +10 -2
  15. package/dist/proxyService.js +18 -3
  16. package/dist/services/s3Uploader.js +21 -10
  17. package/dist/static/ejs/partials/scripts/header/aboutScanModal/ScanConfiguration.ejs +2 -2
  18. package/dist/static/ejs/partials/scripts/ruleModal/constants.ejs +1 -761
  19. package/dist/static/ejs/summary.ejs +10 -5
  20. package/oobee-client-scanner.js +34992 -0
  21. package/package.json +2 -2
  22. package/src/combine.ts +3 -1
  23. package/src/constants/common.ts +22 -10
  24. package/src/constants/constants.ts +602 -1
  25. package/src/crawlers/commonCrawlerFunc.ts +4 -3
  26. package/src/crawlers/crawlSitemap.ts +116 -98
  27. package/src/crawlers/custom/utils.ts +143 -38
  28. package/src/crawlers/guards/urlGuard.ts +24 -31
  29. package/src/crawlers/runCustom.ts +29 -11
  30. package/src/generateOobeeClientScanner.ts +591 -0
  31. package/src/mergeAxeResults.ts +5 -3
  32. package/src/npmIndex.ts +12 -2
  33. package/src/proxyService.ts +25 -4
  34. package/src/services/s3Uploader.ts +23 -11
  35. package/src/static/ejs/partials/scripts/header/aboutScanModal/ScanConfiguration.ejs +2 -2
  36. package/src/static/ejs/partials/scripts/ruleModal/constants.ejs +1 -761
  37. package/src/static/ejs/summary.ejs +10 -5
  38. package/testStaticJSScanner.html +534 -0
@@ -57,7 +57,7 @@ function parseEnvProxyCommon() {
57
57
  if (https)
58
58
  info.https = stripScheme(https);
59
59
  if (socks)
60
- info.socks = stripScheme(socks);
60
+ info.socks = socks; // keep original scheme so proxyInfoToResolution can use the right protocol
61
61
  if (noProxy)
62
62
  info.bypassList = semiJoin(noProxy.split(/[,;]/));
63
63
  const { username, password } = readCredsFromEnv();
@@ -384,6 +384,14 @@ function buildIncludeOnlyPac(proxyServer, includeList) {
384
384
  ].join('\n');
385
385
  return pac;
386
386
  }
387
+ /**
388
+ * Convert an info.socks value to a full proxy server URL.
389
+ * When the value already carries a scheme (e.g. ALL_PROXY=http://..., socks4://...),
390
+ * it is used as-is. Bare host:port values (from scutil) default to socks5://.
391
+ */
392
+ function toSocksServer(socks) {
393
+ return /^[a-zA-Z][a-zA-Z0-9+.-]*:\/\//.test(socks) ? socks : `socks5://${socks}`;
394
+ }
387
395
  export function proxyInfoToResolution(info) {
388
396
  if (!info)
389
397
  return { kind: 'none' };
@@ -396,7 +404,7 @@ export function proxyInfoToResolution(info) {
396
404
  else if (info.https)
397
405
  proxyServer = `http://${info.https}`;
398
406
  else if (info.socks)
399
- proxyServer = `socks5://${info.socks}`;
407
+ proxyServer = toSocksServer(info.socks);
400
408
  if (proxyServer) {
401
409
  // If credentials exist, embed them for the manual proxy auth
402
410
  // PAC scripts themselves don't carry auth, but Playwright's proxy option can
@@ -408,6 +416,13 @@ export function proxyInfoToResolution(info) {
408
416
  const pacDataUrl = `data:application/x-ns-proxy-autoconfig;base64,${Buffer.from(pac).toString('base64')}`;
409
417
  return { kind: 'pac', pacUrl: pacDataUrl, bypass: info.bypassList };
410
418
  }
419
+ // No direct proxy server was found — the configured proxy is PAC-based or auto-detect only.
420
+ // INCLUDE_PROXY needs a concrete server address to build a routing PAC script, so it cannot
421
+ // be applied here. Warn and fall through to use the existing PAC/autodetect as-is.
422
+ console.warn('INCLUDE_PROXY is set but no direct proxy server address was found. ' +
423
+ 'INCLUDE_PROXY requires HTTP_PROXY, HTTPS_PROXY, or ALL_PROXY to be set with a direct ' +
424
+ 'server address; it cannot be applied to a PAC URL or auto-detect proxy. ' +
425
+ 'INCLUDE_PROXY will be ignored.');
411
426
  }
412
427
  // Prefer manual proxies first (these work with Playwright's proxy option)
413
428
  if (info.http) {
@@ -428,7 +443,7 @@ export function proxyInfoToResolution(info) {
428
443
  }
429
444
  if (info.socks) {
430
445
  return { kind: 'manual', settings: {
431
- server: `socks5://${info.socks}`,
446
+ server: toSocksServer(info.socks),
432
447
  username: info.username,
433
448
  password: info.password,
434
449
  bypass: info.bypassList,
@@ -5,6 +5,17 @@ import mime from 'mime-types';
5
5
  import { consoleLogger } from '../logs.js';
6
6
  const REGION = process.env.AWS_REGION || 'ap-southeast-1';
7
7
  const s3Client = new S3Client({ region: REGION });
8
+ // S3 user metadata is sent over REST as x-amz-meta-* HTTP headers.
9
+ // To avoid request-header validation failures in the Node/AWS SDK path,
10
+ // normalize to printable ASCII before attaching metadata values.
11
+ const sanitizeS3MetadataValue = (value) => {
12
+ return value
13
+ .normalize('NFKD') // e.g. "é" -> "e" + combining accent, "A" -> "A"
14
+ .replace(/[\u0300-\u036f]/g, '') // e.g. remove the combining accent from the decomposed "é"
15
+ .replace(/[^\x20-\x7E]+/g, ' ') // e.g. "公益金" or emoji -> " "
16
+ .replace(/\s+/g, ' ') // e.g. "Community Chest \n" -> "Community Chest "
17
+ .trim(); // e.g. " Homepage | Community Chest " -> "Homepage | Community Chest"
18
+ };
8
19
  export const uploadFileToS3 = async (localFilePath, s3Key, metadata) => {
9
20
  const fileStream = fs.readFileSync(localFilePath);
10
21
  const contentType = mime.lookup(localFilePath) || 'application/octet-stream';
@@ -38,31 +49,31 @@ export const uploadFolderToS3 = async (localFolderPath, s3Prefix, scanMetadata)
38
49
  const files = getAllFiles(localFolderPath, localFolderPath);
39
50
  const allowedFileExtRegex = /\.(html|csv|pdf|zip)$/;
40
51
  const metadata = {
41
- scanid: scanMetadata.scanId,
42
- userid: scanMetadata.userId,
43
- useremail: scanMetadata.email,
52
+ scanid: sanitizeS3MetadataValue(scanMetadata.scanId),
53
+ userid: sanitizeS3MetadataValue(scanMetadata.userId),
54
+ useremail: sanitizeS3MetadataValue(scanMetadata.email),
44
55
  };
45
56
  // Add optional metadata fields if present
46
57
  if (scanMetadata.messageId) {
47
- metadata.messageid = scanMetadata.messageId;
58
+ metadata.messageid = sanitizeS3MetadataValue(scanMetadata.messageId);
48
59
  }
49
60
  if (scanMetadata.amplitudeUserId) {
50
- metadata.amplitudeuserid = scanMetadata.amplitudeUserId;
61
+ metadata.amplitudeuserid = sanitizeS3MetadataValue(scanMetadata.amplitudeUserId);
51
62
  }
52
63
  if (scanMetadata.deviceId) {
53
- metadata.deviceid = scanMetadata.deviceId;
64
+ metadata.deviceid = sanitizeS3MetadataValue(scanMetadata.deviceId);
54
65
  }
55
66
  if (scanMetadata.orgId) {
56
- metadata.orgid = scanMetadata.orgId;
67
+ metadata.orgid = sanitizeS3MetadataValue(scanMetadata.orgId);
57
68
  }
58
69
  if (scanMetadata.userRole) {
59
- metadata.userrole = scanMetadata.userRole;
70
+ metadata.userrole = sanitizeS3MetadataValue(scanMetadata.userRole);
60
71
  }
61
72
  if (scanMetadata.siteName) {
62
- metadata.sitename = scanMetadata.siteName;
73
+ metadata.sitename = sanitizeS3MetadataValue(scanMetadata.siteName);
63
74
  }
64
75
  if (scanMetadata.durationExceeded !== undefined) {
65
- metadata.durationexceeded = scanMetadata.durationExceeded;
76
+ metadata.durationexceeded = sanitizeS3MetadataValue(scanMetadata.durationExceeded);
66
77
  }
67
78
  consoleLogger.info(`Uploading ${files.length} files to S3...`);
68
79
  const uploadPromises = files.map(async (relativePath) => {
@@ -21,7 +21,7 @@
21
21
  >
22
22
  </div>
23
23
  <div class="display-url-container">
24
- <a href="${page.url}" target="_blank">${page.pageTitle.length > 0 ? page.pageTitle : page.url}</a>
24
+ <a href="${page.url}" target="_blank">${page.pageTitle?.length > 0 ? page.pageTitle : page.url}</a>
25
25
  <p>${page.url}</p>
26
26
  </div>
27
27
  </div>
@@ -29,7 +29,7 @@
29
29
  } else {
30
30
  listItem.innerHTML = `
31
31
  <a href="${page.url}" target="_blank">
32
- ${page.pageTitle.length > 0 ? page.pageTitle : page.url}
32
+ ${page.pageTitle?.length > 0 ? page.pageTitle : page.url}
33
33
  <svg class="link-external-icon" width="16" height="12" viewBox="0 0 8 8" aria-hidden="true" focusable="false">
34
34
  <path d="M7.11111 7.11111H0.888889V0.888889H4V0H0.888889C0.395556 0 0 0.4 0 0.888889V7.11111C0 7.6 0.395556 8 0.888889 8H7.11111C7.6 8 8 7.6 8 7.11111V4H7.11111V7.11111ZM4.88889 0V0.888889H6.48444L2.11556 5.25778L2.74222 5.88444L7.11111 1.51556V3.11111H8V0H4.88889Z" fill="#5735DF"/>
35
35
  </svg>