@govtechsg/oobee 0.10.85 → 0.10.87

package/src/mergeAxeResults/itemReferences.ts

@@ -1,5 +1,9 @@
 import type { AllIssues, ItemsInfo, RuleInfo } from './types.js';
 
+type ScanItems = AllIssues['items'];
+type ScanCategory = ScanItems[keyof ScanItems];
+type ScanItemsLight = Pick<ScanItems, 'mustFix' | 'goodToFix' | 'needsReview' | 'passed'>;
+
 /**
  * Builds pre-computed HTML groups to optimize Group by HTML Element functionality.
  * Keys are composite "html\x00xpath" strings to ensure unique matching per element instance.
@@ -31,32 +35,72 @@ export const buildHtmlGroups = (rule: RuleInfo, items: ItemsInfo[], pageUrl: str
   });
 };
 
+/*
+// Commenting this out for now as we are not including htmlGroups in the embedded report payload to keep it lean.
+// We can revisit this if we want to include htmlGroups in the future and need a reference builder for it.
+const toHtmlGroupReference = (item: any) => {
+  if (typeof item === 'string') {
+    return item;
+  }
+
+  return `${item?.html || 'No HTML element'}\x00${item?.xpath || ''}`;
+};
+
+const cloneCategoryWithReferenceItems = (category: ScanCategory): ScanCategory =>
+  ({
+    ...category,
+    rules: category.rules.map(
+      rule =>
+        ({
+          ...rule,
+          pagesAffected: rule.pagesAffected.map(
+            page => {
+              const { items, ...pageWithoutItems } = page;
+
+              return {
+                ...pageWithoutItems,
+                itemsCount: page.itemsCount ?? (Array.isArray(items) ? items.length : 0),
+                items: Array.isArray(items) ? items.map(toHtmlGroupReference) : items,
+              } as any;
+            },
+          ),
+        }) as any,
+    ),
+  }) as ScanCategory;
+*/
+
+const cloneCategoryLight = (category: ScanCategory, includeHtmlGroups: boolean): ScanCategory =>
+  ({
+    ...category,
+    rules: category.rules.map(
+      rule =>
+        ({
+          rule: rule.rule,
+          description: rule.description,
+          helpUrl: rule.helpUrl,
+          conformance: rule.conformance,
+          totalItems: rule.totalItems,
+          axeImpact: rule.axeImpact,
+          ...(includeHtmlGroups && rule.htmlGroups ? { htmlGroups: rule.htmlGroups } : {}),
+          pagesAffected: rule.pagesAffected.map(page => ({
+            url: page.url,
+            pageTitle: page.pageTitle,
+            itemsCount: page.itemsCount ?? (Array.isArray((page as any).items) ? (page as any).items.length : 0),
+          })),
+        }) as any,
+    ),
+  }) as ScanCategory;
+
 /**
- * Converts items in pagesAffected to references (html\x00xpath composite keys) for embedding in HTML report.
- * Additionally, it deep-clones allIssues, replaces page.items objects with composite reference keys.
- * Those refs are specifically for htmlGroups lookup (html + xpath).
+ * Builds the embedded HTML-report payload from the full scan items.
+ * Includes htmlGroups for non-passed categories (Group by HTML Element),
+ * excludes them from passed to keep payload within browser memory limits.
  */
-export const convertItemsToReferences = (allIssues: AllIssues): AllIssues => {
-  const cloned = JSON.parse(JSON.stringify(allIssues));
-
-  ['mustFix', 'goodToFix', 'needsReview', 'passed'].forEach(category => {
-    if (!cloned.items[category]?.rules) return;
-
-    cloned.items[category].rules.forEach((rule: any) => {
-      if (!rule.pagesAffected || !rule.htmlGroups) return;
-
-      rule.pagesAffected.forEach((page: any) => {
-        if (!page.items) return;
-
-        page.items = page.items.map((item: any) => {
-          if (typeof item === 'string') return item; // Already a reference
-          // Use composite key matching buildHtmlGroups
-          const htmlKey = `${item.html || 'No HTML element'}\x00${item.xpath || ''}`;
-          return htmlKey;
-        });
-      });
-    });
-  });
-
-  return cloned;
+export const convertItemsToReferences = (source: Pick<AllIssues, 'items'>): ScanItemsLight => {
+  return {
+    mustFix: cloneCategoryLight(source.items.mustFix, true),
+    goodToFix: cloneCategoryLight(source.items.goodToFix, true),
+    needsReview: cloneCategoryLight(source.items.needsReview, true),
+    passed: cloneCategoryLight(source.items.passed, false),
+  };
 };

package/src/mergeAxeResults/sentryTelemetry.ts

@@ -140,7 +140,10 @@ const sendWcagBreakdownToSentry = async (
         event_type: 'accessibility_scan',
         scanType: scanInfo.scanType,
         browser: scanInfo.browser,
-        entryUrl: scanInfo.entryUrl,
+        entryUrl: process.env.OOBEE_SCAN_METADATA ?? scanInfo.entryUrl,
+        ...(process.env.OOBEE_SCAN_PRODUCT && {
+          scanProduct: process.env.OOBEE_SCAN_PRODUCT,
+        }),
       },
       user: {
         ...(scanInfo.email && scanInfo.name

package/src/mergeAxeResults.ts

@@ -12,6 +12,7 @@ import constants, {
   a11yRuleShortDescriptionMap,
   disabilityBadgesMap,
   a11yRuleLongDescriptionMap,
+  a11yRuleStepByStepGuide,
 } from './constants/constants.js';
 import { getBrowserToRun, getPlaywrightLaunchOptions } from './constants/common.js';
 
@@ -184,15 +185,15 @@ const writeHTML = async (
     'utf-8',
   );
 
-  // Create lighter version with item references for embedding in HTML
-  const scanItemsWithHtmlGroupRefs = convertItemsToReferences(allIssues);
+  // Create the lighter scanItems payload for embedding in the HTML report.
+  const lightScanItemsPayload = convertItemsToReferences(allIssues);
 
   // Write the lighter items to a file and get the base64 path
   const {
-    jsonFilePath: scanItemsWithHtmlGroupRefsJsonFilePath,
-    base64FilePath: scanItemsWithHtmlGroupRefsBase64FilePath,
+    jsonFilePath: lightScanItemsPayloadJsonFilePath,
+    base64FilePath: lightScanItemsPayloadBase64FilePath,
   } = await writeJsonFileAndCompressedJsonFile(
-    scanItemsWithHtmlGroupRefs.items,
+    lightScanItemsPayload,
     storagePath,
     'scanItems-light',
   );
@@ -211,8 +212,8 @@ const writeHTML = async (
         await Promise.all([
           fs.promises.unlink(topFilePath),
           fs.promises.unlink(bottomFilePath),
-          fs.promises.unlink(scanItemsWithHtmlGroupRefsBase64FilePath),
-          fs.promises.unlink(scanItemsWithHtmlGroupRefsJsonFilePath),
+          fs.promises.unlink(lightScanItemsPayloadBase64FilePath),
+          fs.promises.unlink(lightScanItemsPayloadJsonFilePath),
         ]);
       } catch (err) {
         console.error('Error cleaning up temporary files:', err);
@@ -250,6 +251,9 @@ const writeHTML = async (
   } else {
     console.warn('Skipping fetch GenAI feature as it is local report');
   }
+
+  var scanData = null;
+  var scanItems = null;
   \n`);
 
     outputStream.write('</script>\n<script type="text/plain" id="scanDataRaw">');
@@ -258,22 +262,25 @@ const writeHTML = async (
     scanDetailsReadStream.on('end', async () => {
       outputStream.write('</script>\n<script>\n');
       outputStream.write(
-        "var scanDataPromise = (async () => { console.log('Loading scanData...'); scanData = await decodeUnzipParse(document.getElementById('scanDataRaw').textContent); })();\n",
+        "var scanDataPromise = (async () => { console.log('Loading scanData...'); scanData = await decodeUnzipParse(document.getElementById('scanDataRaw').textContent); console.log('[report] scanData loaded'); })();\n",
       );
       outputStream.write('</script>\n');
 
       // Write scanItems in 2MB chunks using a stream to avoid loading entire file into memory
       try {
         let chunkIndex = 1;
-        const scanItemsStream = fs.createReadStream(scanItemsWithHtmlGroupRefsBase64FilePath, {
+        const scanItemsStream = fs.createReadStream(lightScanItemsPayloadBase64FilePath, {
           encoding: 'utf8',
           highWaterMark: CHUNK_SIZE,
         });
 
         for await (const chunk of scanItemsStream) {
-          outputStream.write(
+          const ok = outputStream.write(
             `<script type="text/plain" id="scanItemsRaw${chunkIndex}">${chunk}</script>\n`,
           );
+          if (!ok) {
+            await new Promise<void>(resolve => outputStream.once('drain', resolve));
+          }
           chunkIndex++;
         }
 
@@ -290,6 +297,7 @@ var scanItemsPromise = (async () => {
     i++;
   }
   scanItems = await decodeUnzipParse(chunks);
+  console.log('[report] scanItems loaded');
 })();\n`);
         outputStream.write(suffixData);
         outputStream.end();
@@ -414,9 +422,9 @@ const pushResults = async (pageResults, allIssues, isCustomFlow) => {
   const { url, pageTitle, filePath } = pageResults;
 
   const totalIssuesInPage = new Set();
-  Object.keys(pageResults.mustFix.rules).forEach(k => totalIssuesInPage.add(k));
-  Object.keys(pageResults.goodToFix.rules).forEach(k => totalIssuesInPage.add(k));
-  Object.keys(pageResults.needsReview.rules).forEach(k => totalIssuesInPage.add(k));
+  Object.keys(pageResults.mustFix?.rules ?? {}).forEach(k => totalIssuesInPage.add(k));
+  Object.keys(pageResults.goodToFix?.rules ?? {}).forEach(k => totalIssuesInPage.add(k));
+  Object.keys(pageResults.needsReview?.rules ?? {}).forEach(k => totalIssuesInPage.add(k));
 
   allIssues.topFiveMostIssues.push({
     url,
@@ -784,6 +792,7 @@ const generateArtifacts = async (
     a11yRuleShortDescriptionMap,
     disabilityBadgesMap,
     a11yRuleLongDescriptionMap,
+    a11yRuleStepByStepGuide,
     wcagCriteriaLabels: constants.wcagCriteriaLabels,
     scanPagesDetail: {
       pagesAffected: [],
@@ -987,10 +996,13 @@ const generateArtifacts = async (
     1,
   );
 
+  // Brief delay to allow lingering async crawlee storage operations to flush
+  await new Promise(resolve => setTimeout(resolve, 3000));
+
   try {
     await fs.promises.rm(path.join(storagePath, 'crawlee'), { recursive: true, force: true });
   } catch (error) {
-    consoleLogger.warn(`Unable to force remove crawlee folder: ${error.message}`);
+    // Silently ignore — folder may already be gone or still locked
   }
 
   try {

package/src/npmIndex.ts

@@ -5,7 +5,7 @@ import axe, { AxeResults, ImpactValue } from 'axe-core';
 import { JSDOM } from 'jsdom';
 import { fileURLToPath } from 'url';
 import { EnqueueStrategy } from 'crawlee';
-import constants, { BrowserTypes, RuleFlags, ScannerTypes } from './constants/constants.js';
+import constants, { BrowserTypes, RuleFlags, ScannerTypes, a11yRuleShortDescriptionMap, a11yRuleLongDescriptionMap, a11yRuleStepByStepGuide } from './constants/constants.js';
 import {
   deleteClonedProfiles,
   getBrowserToRun,
@@ -638,6 +638,11 @@ const processAndSubmitResults = async (
             if (constants.a11yRuleShortDescriptionMap[ruleId]) {
               mergedResults[category].rules[ruleId].description = constants.a11yRuleShortDescriptionMap[ruleId];
             }
+
+            // Add short description, long description and step-by-step guide
+            mergedResults[category].rules[ruleId].shortDescription = a11yRuleShortDescriptionMap[ruleId];
+            mergedResults[category].rules[ruleId].longDescription = a11yRuleLongDescriptionMap[ruleId];
+            mergedResults[category].rules[ruleId].stepByStepGuide = a11yRuleStepByStepGuide[ruleId];
             
             // Add url to items
             mergedResults[category].rules[ruleId].items.forEach((item: any) => {
@@ -733,6 +738,11 @@ const processAndSubmitResults = async (
             rule.description = constants.a11yRuleShortDescriptionMap[rule.rule];
           }
 
+          // Add short description, long description and step-by-step guide
+          rule.shortDescription = a11yRuleShortDescriptionMap[rule.rule];
+          rule.longDescription = a11yRuleLongDescriptionMap[rule.rule];
+          rule.stepByStepGuide = a11yRuleStepByStepGuide[rule.rule];
+
           if (rule.items) {
            rule.items.forEach((item: any) => {
              // Ensure item URL matches the result URL
@@ -877,5 +887,5 @@ export const scanPage = async (
   );
 };
 
-export { RuleFlags };
+export { RuleFlags, a11yRuleLongDescriptionMap, a11yRuleStepByStepGuide, getOobeeFunctionsScript };
 

package/src/proxyService.ts

@@ -17,7 +17,9 @@ import path from 'path';
 import { spawnSync } from 'child_process';
 
 export interface ProxyInfo {
-  // host:port OR user:pass@host:port (no scheme)
+  // http/https: host:port OR user:pass@host:port (no scheme)
+  // socks: scheme://host:port OR scheme://user:pass@host:port (scheme preserved from ALL_PROXY)
+  //        OR bare host:port when sourced from scutil (defaults to socks5:// on use)
   http?: string;
   https?: string;
   socks?: string;
@@ -88,7 +90,7 @@ function parseEnvProxyCommon(): ProxyInfo | null {
   const info: ProxyInfo = {};
   if (http) info.http = stripScheme(http);
   if (https) info.https = stripScheme(https);
-  if (socks) info.socks = stripScheme(socks);
+  if (socks) info.socks = socks; // keep original scheme so proxyInfoToResolution can use the right protocol
   if (noProxy) info.bypassList = semiJoin(noProxy.split(/[,;]/));
 
   const { username, password } = readCredsFromEnv();
@@ -435,6 +437,15 @@ function buildIncludeOnlyPac(proxyServer: string, includeList: string[]): string
   return pac;
 }
 
+/**
+ * Convert an info.socks value to a full proxy server URL.
+ * When the value already carries a scheme (e.g. ALL_PROXY=http://..., socks4://...),
+ * it is used as-is. Bare host:port values (from scutil) default to socks5://.
+ */
+function toSocksServer(socks: string): string {
+  return /^[a-zA-Z][a-zA-Z0-9+.-]*:\/\//.test(socks) ? socks : `socks5://${socks}`;
+}
+
 export function proxyInfoToResolution(info: ProxyInfo | null): ProxyResolution {
   if (!info) return { kind: 'none' };
 
@@ -444,7 +455,7 @@ export function proxyInfoToResolution(info: ProxyInfo | null): ProxyResolution {
     let proxyServer: string | undefined;
     if (info.http) proxyServer = `http://${info.http}`;
     else if (info.https) proxyServer = `http://${info.https}`;
-    else if (info.socks) proxyServer = `socks5://${info.socks}`;
+    else if (info.socks) proxyServer = toSocksServer(info.socks);
 
     if (proxyServer) {
       // If credentials exist, embed them for the manual proxy auth
@@ -457,6 +468,16 @@ export function proxyInfoToResolution(info: ProxyInfo | null): ProxyResolution {
       const pacDataUrl = `data:application/x-ns-proxy-autoconfig;base64,${Buffer.from(pac).toString('base64')}`;
       return { kind: 'pac', pacUrl: pacDataUrl, bypass: info.bypassList };
     }
+
+    // No direct proxy server was found — the configured proxy is PAC-based or auto-detect only.
+    // INCLUDE_PROXY needs a concrete server address to build a routing PAC script, so it cannot
+    // be applied here. Warn and fall through to use the existing PAC/autodetect as-is.
+    console.warn(
+      'INCLUDE_PROXY is set but no direct proxy server address was found. ' +
+      'INCLUDE_PROXY requires HTTP_PROXY, HTTPS_PROXY, or ALL_PROXY to be set with a direct ' +
+      'server address; it cannot be applied to a PAC URL or auto-detect proxy. ' +
+      'INCLUDE_PROXY will be ignored.',
+    );
   }
 
   // Prefer manual proxies first (these work with Playwright's proxy option)
@@ -478,7 +499,7 @@ export function proxyInfoToResolution(info: ProxyInfo | null): ProxyResolution {
   }
   if (info.socks) {
     return { kind: 'manual', settings: {
-      server: `socks5://${info.socks}`,
+      server: toSocksServer(info.socks),
       username: info.username,
       password: info.password,
       bypass: info.bypassList,

package/src/services/s3Uploader.ts

@@ -7,6 +7,18 @@ import { consoleLogger } from '../logs.js';
 const REGION = process.env.AWS_REGION || 'ap-southeast-1';
 const s3Client = new S3Client({ region: REGION });
 
+// S3 user metadata is sent over REST as x-amz-meta-* HTTP headers.
+// To avoid request-header validation failures in the Node/AWS SDK path,
+// normalize to printable ASCII before attaching metadata values.
+const sanitizeS3MetadataValue = (value: string): string => {
+  return value
+    .normalize('NFKD') // e.g. "é" -> "e" + combining accent, "Ａ" -> "A"
+    .replace(/[\u0300-\u036f]/g, '') // e.g. remove the combining accent from the decomposed "é"
+    .replace(/[^\x20-\x7E]+/g, ' ') // e.g. "公益金" or emoji -> " "
+    .replace(/\s+/g, ' ') // e.g. "Community   Chest \n" -> "Community Chest "
+    .trim(); // e.g. " Homepage | Community Chest " -> "Homepage | Community Chest"
+};
+
 export interface UploadedFileInfo {
   filename: string;
   s3Path: string;
@@ -75,32 +87,32 @@ export const uploadFolderToS3 = async (
   const allowedFileExtRegex = /\.(html|csv|pdf|zip)$/;
 
   const metadata: Record<string, string> = {
-    scanid: scanMetadata.scanId,
-    userid: scanMetadata.userId,
-    useremail: scanMetadata.email,
+    scanid: sanitizeS3MetadataValue(scanMetadata.scanId),
+    userid: sanitizeS3MetadataValue(scanMetadata.userId),
+    useremail: sanitizeS3MetadataValue(scanMetadata.email),
   };
 
   // Add optional metadata fields if present
   if (scanMetadata.messageId) {
-    metadata.messageid = scanMetadata.messageId;
+    metadata.messageid = sanitizeS3MetadataValue(scanMetadata.messageId);
   }
   if (scanMetadata.amplitudeUserId) {
-    metadata.amplitudeuserid = scanMetadata.amplitudeUserId;
+    metadata.amplitudeuserid = sanitizeS3MetadataValue(scanMetadata.amplitudeUserId);
   }
   if (scanMetadata.deviceId) {
-    metadata.deviceid = scanMetadata.deviceId;
+    metadata.deviceid = sanitizeS3MetadataValue(scanMetadata.deviceId);
   }
   if (scanMetadata.orgId) {
-    metadata.orgid = scanMetadata.orgId;
+    metadata.orgid = sanitizeS3MetadataValue(scanMetadata.orgId);
   }
   if (scanMetadata.userRole) {
-    metadata.userrole = scanMetadata.userRole;
+    metadata.userrole = sanitizeS3MetadataValue(scanMetadata.userRole);
   }
   if (scanMetadata.siteName) {
-    metadata.sitename = scanMetadata.siteName;
+    metadata.sitename = sanitizeS3MetadataValue(scanMetadata.siteName);
   }
   if (scanMetadata.durationExceeded !== undefined) {
-    metadata.durationexceeded = scanMetadata.durationExceeded;
+    metadata.durationexceeded = sanitizeS3MetadataValue(scanMetadata.durationExceeded);
   }
 
   consoleLogger.info(`Uploading ${files.length} files to S3...`);
@@ -181,4 +193,4 @@ export const getS3UploadPrefix = (): string | null => {
   }
 
   return `users/${userId}/scans/${scanId}`;
-};
+};

package/src/static/ejs/partials/scripts/decodeUnzipParse.ejs

@@ -28,8 +28,11 @@ async function decodeUnzipParse(input) {
       offset += arr.length;
     }
 
-    // Step 2: Decompress with pako (GZIP)
-    const decompressed = pako.ungzip(merged, { to: 'string' });
+    // Step 2: Decompress with pako (GZIP) to bytes first to avoid large-string
+    // construction inside pako for very large payloads.
+    const decompressedBytes = pako.ungzip(merged);
+
+    const decompressed = new TextDecoder().decode(decompressedBytes);
 
     // Step 3: Parse JSON
     return JSON.parse(decompressed);
@@ -37,4 +40,4 @@ async function decodeUnzipParse(input) {
     throw new Error(`Failed to decode/unzip/parse: ${err.message}`);
   }
 }
-</script>
+</script>

package/src/static/ejs/partials/scripts/header/aboutScanModal/ScanConfiguration.ejs

@@ -21,7 +21,7 @@
             >
           </div>
           <div class="display-url-container">
-            <a href="${page.url}" target="_blank">${page.pageTitle.length > 0 ? page.pageTitle : page.url}</a>
+            <a href="${page.url}" target="_blank">${page.pageTitle?.length > 0 ? page.pageTitle : page.url}</a>
             <p>${page.url}</p>
           </div>
         </div>
@@ -29,7 +29,7 @@
     } else {
       listItem.innerHTML = `
         <a href="${page.url}" target="_blank">
-          ${page.pageTitle.length > 0 ? page.pageTitle : page.url}
+          ${page.pageTitle?.length > 0 ? page.pageTitle : page.url}
           <svg class="link-external-icon" width="16" height="12" viewBox="0 0 8 8" aria-hidden="true" focusable="false">
             <path d="M7.11111 7.11111H0.888889V0.888889H4V0H0.888889C0.395556 0 0 0.4 0 0.888889V7.11111C0 7.6 0.395556 8 0.888889 8H7.11111C7.6 8 8 7.6 8 7.11111V4H7.11111V7.11111ZM4.88889 0V0.888889H6.48444L2.11556 5.25778L2.74222 5.88444L7.11111 1.51556V3.11111H8V0H4.88889Z" fill="#5735DF"/>
           </svg>