@govtechsg/oobee 0.10.84 → 0.10.86

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (40) hide show
  1. package/.github/workflows/image.yml +3 -2
  2. package/.github/workflows/publish.yml +10 -0
  3. package/DETAILS.md +29 -0
  4. package/dist/cli.js +7 -6
  5. package/dist/combine.js +1 -1
  6. package/dist/constants/common.js +15 -4
  7. package/dist/constants/constants.js +604 -1
  8. package/dist/crawlers/commonCrawlerFunc.js +3 -2
  9. package/dist/crawlers/crawlSitemap.js +98 -80
  10. package/dist/crawlers/custom/utils.js +218 -71
  11. package/dist/crawlers/guards/urlGuard.js +8 -15
  12. package/dist/crawlers/runCustom.js +24 -15
  13. package/dist/generateOobeeClientScanner.js +570 -0
  14. package/dist/mergeAxeResults.js +49 -29
  15. package/dist/npmIndex.js +10 -2
  16. package/dist/proxyService.js +18 -3
  17. package/dist/services/s3Uploader.js +21 -10
  18. package/dist/static/ejs/partials/scripts/header/aboutScanModal/ScanConfiguration.ejs +2 -2
  19. package/dist/static/ejs/partials/scripts/ruleModal/constants.ejs +1 -761
  20. package/dist/static/ejs/summary.ejs +10 -5
  21. package/oobee-client-scanner.js +34992 -0
  22. package/package.json +3 -3
  23. package/src/cli.ts +20 -15
  24. package/src/combine.ts +3 -1
  25. package/src/constants/common.ts +22 -10
  26. package/src/constants/constants.ts +602 -1
  27. package/src/crawlers/commonCrawlerFunc.ts +4 -3
  28. package/src/crawlers/crawlSitemap.ts +116 -98
  29. package/src/crawlers/custom/utils.ts +244 -84
  30. package/src/crawlers/guards/urlGuard.ts +24 -31
  31. package/src/crawlers/runCustom.ts +38 -15
  32. package/src/generateOobeeClientScanner.ts +591 -0
  33. package/src/mergeAxeResults.ts +48 -29
  34. package/src/npmIndex.ts +12 -2
  35. package/src/proxyService.ts +25 -4
  36. package/src/services/s3Uploader.ts +23 -11
  37. package/src/static/ejs/partials/scripts/header/aboutScanModal/ScanConfiguration.ejs +2 -2
  38. package/src/static/ejs/partials/scripts/ruleModal/constants.ejs +1 -761
  39. package/src/static/ejs/summary.ejs +10 -5
  40. package/testStaticJSScanner.html +534 -0
package/dist/npmIndex.js CHANGED
@@ -4,7 +4,7 @@ import axe from 'axe-core';
4
4
  import { JSDOM } from 'jsdom';
5
5
  import { fileURLToPath } from 'url';
6
6
  import { EnqueueStrategy } from 'crawlee';
7
- import constants, { BrowserTypes, RuleFlags, ScannerTypes } from './constants/constants.js';
7
+ import constants, { BrowserTypes, RuleFlags, ScannerTypes, a11yRuleShortDescriptionMap, a11yRuleLongDescriptionMap, a11yRuleStepByStepGuide } from './constants/constants.js';
8
8
  import { deleteClonedProfiles, getBrowserToRun, getPlaywrightLaunchOptions, submitForm, } from './constants/common.js';
9
9
  import { createCrawleeSubFolders, filterAxeResults } from './crawlers/commonCrawlerFunc.js';
10
10
  import { createAndUpdateResultsFolders, getVersion } from './utils.js';
@@ -489,6 +489,10 @@ const processAndSubmitResults = async (scanData, name, email, metadata) => {
489
489
  if (constants.a11yRuleShortDescriptionMap[ruleId]) {
490
490
  mergedResults[category].rules[ruleId].description = constants.a11yRuleShortDescriptionMap[ruleId];
491
491
  }
492
+ // Add short description, long description and step-by-step guide
493
+ mergedResults[category].rules[ruleId].shortDescription = a11yRuleShortDescriptionMap[ruleId];
494
+ mergedResults[category].rules[ruleId].longDescription = a11yRuleLongDescriptionMap[ruleId];
495
+ mergedResults[category].rules[ruleId].stepByStepGuide = a11yRuleStepByStepGuide[ruleId];
492
496
  // Add url to items
493
497
  mergedResults[category].rules[ruleId].items.forEach((item) => {
494
498
  item.url = result.url;
@@ -554,6 +558,10 @@ const processAndSubmitResults = async (scanData, name, email, metadata) => {
554
558
  if (constants.a11yRuleShortDescriptionMap[rule.rule]) {
555
559
  rule.description = constants.a11yRuleShortDescriptionMap[rule.rule];
556
560
  }
561
+ // Add short description, long description and step-by-step guide
562
+ rule.shortDescription = a11yRuleShortDescriptionMap[rule.rule];
563
+ rule.longDescription = a11yRuleLongDescriptionMap[rule.rule];
564
+ rule.stepByStepGuide = a11yRuleStepByStepGuide[rule.rule];
557
565
  if (rule.items) {
558
566
  rule.items.forEach((item) => {
559
567
  // Ensure item URL matches the result URL
@@ -637,4 +645,4 @@ export const scanPage = async (pages, config) => {
637
645
  }
638
646
  return processAndSubmitResults(scanData, name, email, metadata);
639
647
  };
640
- export { RuleFlags };
648
+ export { RuleFlags, a11yRuleLongDescriptionMap, a11yRuleStepByStepGuide, getOobeeFunctionsScript };
@@ -57,7 +57,7 @@ function parseEnvProxyCommon() {
57
57
  if (https)
58
58
  info.https = stripScheme(https);
59
59
  if (socks)
60
- info.socks = stripScheme(socks);
60
+ info.socks = socks; // keep original scheme so proxyInfoToResolution can use the right protocol
61
61
  if (noProxy)
62
62
  info.bypassList = semiJoin(noProxy.split(/[,;]/));
63
63
  const { username, password } = readCredsFromEnv();
@@ -384,6 +384,14 @@ function buildIncludeOnlyPac(proxyServer, includeList) {
384
384
  ].join('\n');
385
385
  return pac;
386
386
  }
387
+ /**
388
+ * Convert an info.socks value to a full proxy server URL.
389
+ * When the value already carries a scheme (e.g. ALL_PROXY=http://..., socks4://...),
390
+ * it is used as-is. Bare host:port values (from scutil) default to socks5://.
391
+ */
392
+ function toSocksServer(socks) {
393
+ return /^[a-zA-Z][a-zA-Z0-9+.-]*:\/\//.test(socks) ? socks : `socks5://${socks}`;
394
+ }
387
395
  export function proxyInfoToResolution(info) {
388
396
  if (!info)
389
397
  return { kind: 'none' };
@@ -396,7 +404,7 @@ export function proxyInfoToResolution(info) {
396
404
  else if (info.https)
397
405
  proxyServer = `http://${info.https}`;
398
406
  else if (info.socks)
399
- proxyServer = `socks5://${info.socks}`;
407
+ proxyServer = toSocksServer(info.socks);
400
408
  if (proxyServer) {
401
409
  // If credentials exist, embed them for the manual proxy auth
402
410
  // PAC scripts themselves don't carry auth, but Playwright's proxy option can
@@ -408,6 +416,13 @@ export function proxyInfoToResolution(info) {
408
416
  const pacDataUrl = `data:application/x-ns-proxy-autoconfig;base64,${Buffer.from(pac).toString('base64')}`;
409
417
  return { kind: 'pac', pacUrl: pacDataUrl, bypass: info.bypassList };
410
418
  }
419
+ // No direct proxy server was found — the configured proxy is PAC-based or auto-detect only.
420
+ // INCLUDE_PROXY needs a concrete server address to build a routing PAC script, so it cannot
421
+ // be applied here. Warn and fall through to use the existing PAC/autodetect as-is.
422
+ console.warn('INCLUDE_PROXY is set but no direct proxy server address was found. ' +
423
+ 'INCLUDE_PROXY requires HTTP_PROXY, HTTPS_PROXY, or ALL_PROXY to be set with a direct ' +
424
+ 'server address; it cannot be applied to a PAC URL or auto-detect proxy. ' +
425
+ 'INCLUDE_PROXY will be ignored.');
411
426
  }
412
427
  // Prefer manual proxies first (these work with Playwright's proxy option)
413
428
  if (info.http) {
@@ -428,7 +443,7 @@ export function proxyInfoToResolution(info) {
428
443
  }
429
444
  if (info.socks) {
430
445
  return { kind: 'manual', settings: {
431
- server: `socks5://${info.socks}`,
446
+ server: toSocksServer(info.socks),
432
447
  username: info.username,
433
448
  password: info.password,
434
449
  bypass: info.bypassList,
@@ -5,6 +5,17 @@ import mime from 'mime-types';
5
5
  import { consoleLogger } from '../logs.js';
6
6
  const REGION = process.env.AWS_REGION || 'ap-southeast-1';
7
7
  const s3Client = new S3Client({ region: REGION });
8
+ // S3 user metadata is sent over REST as x-amz-meta-* HTTP headers.
9
+ // To avoid request-header validation failures in the Node/AWS SDK path,
10
+ // normalize to printable ASCII before attaching metadata values.
11
+ const sanitizeS3MetadataValue = (value) => {
12
+ return value
13
+ .normalize('NFKD') // e.g. "é" -> "e" + combining accent, "A" -> "A"
14
+ .replace(/[\u0300-\u036f]/g, '') // e.g. remove the combining accent from the decomposed "é"
15
+ .replace(/[^\x20-\x7E]+/g, ' ') // e.g. "公益金" or emoji -> " "
16
+ .replace(/\s+/g, ' ') // e.g. "Community Chest \n" -> "Community Chest "
17
+ .trim(); // e.g. " Homepage | Community Chest " -> "Homepage | Community Chest"
18
+ };
8
19
  export const uploadFileToS3 = async (localFilePath, s3Key, metadata) => {
9
20
  const fileStream = fs.readFileSync(localFilePath);
10
21
  const contentType = mime.lookup(localFilePath) || 'application/octet-stream';
@@ -38,31 +49,31 @@ export const uploadFolderToS3 = async (localFolderPath, s3Prefix, scanMetadata)
38
49
  const files = getAllFiles(localFolderPath, localFolderPath);
39
50
  const allowedFileExtRegex = /\.(html|csv|pdf|zip)$/;
40
51
  const metadata = {
41
- scanid: scanMetadata.scanId,
42
- userid: scanMetadata.userId,
43
- useremail: scanMetadata.email,
52
+ scanid: sanitizeS3MetadataValue(scanMetadata.scanId),
53
+ userid: sanitizeS3MetadataValue(scanMetadata.userId),
54
+ useremail: sanitizeS3MetadataValue(scanMetadata.email),
44
55
  };
45
56
  // Add optional metadata fields if present
46
57
  if (scanMetadata.messageId) {
47
- metadata.messageid = scanMetadata.messageId;
58
+ metadata.messageid = sanitizeS3MetadataValue(scanMetadata.messageId);
48
59
  }
49
60
  if (scanMetadata.amplitudeUserId) {
50
- metadata.amplitudeuserid = scanMetadata.amplitudeUserId;
61
+ metadata.amplitudeuserid = sanitizeS3MetadataValue(scanMetadata.amplitudeUserId);
51
62
  }
52
63
  if (scanMetadata.deviceId) {
53
- metadata.deviceid = scanMetadata.deviceId;
64
+ metadata.deviceid = sanitizeS3MetadataValue(scanMetadata.deviceId);
54
65
  }
55
66
  if (scanMetadata.orgId) {
56
- metadata.orgid = scanMetadata.orgId;
67
+ metadata.orgid = sanitizeS3MetadataValue(scanMetadata.orgId);
57
68
  }
58
69
  if (scanMetadata.userRole) {
59
- metadata.userrole = scanMetadata.userRole;
70
+ metadata.userrole = sanitizeS3MetadataValue(scanMetadata.userRole);
60
71
  }
61
72
  if (scanMetadata.siteName) {
62
- metadata.sitename = scanMetadata.siteName;
73
+ metadata.sitename = sanitizeS3MetadataValue(scanMetadata.siteName);
63
74
  }
64
75
  if (scanMetadata.durationExceeded !== undefined) {
65
- metadata.durationexceeded = scanMetadata.durationExceeded;
76
+ metadata.durationexceeded = sanitizeS3MetadataValue(scanMetadata.durationExceeded);
66
77
  }
67
78
  consoleLogger.info(`Uploading ${files.length} files to S3...`);
68
79
  const uploadPromises = files.map(async (relativePath) => {
@@ -21,7 +21,7 @@
21
21
  >
22
22
  </div>
23
23
  <div class="display-url-container">
24
- <a href="${page.url}" target="_blank">${page.pageTitle.length > 0 ? page.pageTitle : page.url}</a>
24
+ <a href="${page.url}" target="_blank">${page.pageTitle?.length > 0 ? page.pageTitle : page.url}</a>
25
25
  <p>${page.url}</p>
26
26
  </div>
27
27
  </div>
@@ -29,7 +29,7 @@
29
29
  } else {
30
30
  listItem.innerHTML = `
31
31
  <a href="${page.url}" target="_blank">
32
- ${page.pageTitle.length > 0 ? page.pageTitle : page.url}
32
+ ${page.pageTitle?.length > 0 ? page.pageTitle : page.url}
33
33
  <svg class="link-external-icon" width="16" height="12" viewBox="0 0 8 8" aria-hidden="true" focusable="false">
34
34
  <path d="M7.11111 7.11111H0.888889V0.888889H4V0H0.888889C0.395556 0 0 0.4 0 0.888889V7.11111C0 7.6 0.395556 8 0.888889 8H7.11111C7.6 8 8 7.6 8 7.11111V4H7.11111V7.11111ZM4.88889 0V0.888889H6.48444L2.11556 5.25778L2.74222 5.88444L7.11111 1.51556V3.11111H8V0H4.88889Z" fill="#5735DF"/>
35
35
  </svg>