@govtechsg/oobee 0.10.86 → 0.10.87

package/src/crawlers/crawlSitemap.ts

@@ -1,4 +1,4 @@
-import crawlee, { LaunchContext, Request, RequestList, Dataset } from 'crawlee';
+import crawlee, { EnqueueStrategy, LaunchContext, Request, RequestList, Dataset } from 'crawlee';
 import fs from 'fs';
 import * as path from 'path';
 import fsp from 'fs/promises';
@@ -23,7 +23,7 @@ import {
   waitForPageLoaded,
   isFilePath,
 } from '../constants/common.js';
-import { areLinksEqual, isWhitelistedContentType, register } from '../utils.js';
+import { areLinksEqual, isFollowStrategy, isWhitelistedContentType, normUrl, register } from '../utils.js';
 import {
   handlePdfDownload,
   runPdfScan,
@@ -46,6 +46,8 @@ const crawlSitemap = async ({
   blacklistedPatterns,
   includeScreenshots,
   extraHTTPHeaders,
+  strategy = EnqueueStrategy.All,
+  userUrl = '',
   scanDuration = 0,
   fromCrawlIntelligentSitemap = false,
   userUrlInputFromIntelligent = null,
@@ -65,6 +67,8 @@ const crawlSitemap = async ({
   blacklistedPatterns: string[];
   includeScreenshots: boolean;
   extraHTTPHeaders: Record<string, string>;
+  strategy?: EnqueueStrategy;
+  userUrl?: string;
   scanDuration?: number;
   fromCrawlIntelligentSitemap?: boolean;
   userUrlInputFromIntelligent?: string;
@@ -99,6 +103,8 @@ const crawlSitemap = async ({
     userUrlInputFromIntelligent,
     fromCrawlIntelligentSitemap,
     extraHTTPHeaders,
+    strategy,
+    userUrl || sitemapUrl,
   );
 
   sitemapUrl = encodeURI(sitemapUrl);
@@ -299,7 +305,7 @@ const crawlSitemap = async ({
           if (isScanHtml && status < 300 && isWhitelistedContentType(contentType)) {
             const isRedirected = !areLinksEqual(page.url(), request.url);
             const isLoadedUrlInCrawledUrls = urlsCrawled.scanned.some(
-              item => (item.actualUrl || item.url) === page.url(),
+              item => normUrl(item.actualUrl || item.url) === normUrl(page.url()),
             );
 
             if (isRedirected && isLoadedUrlInCrawledUrls) {
@@ -327,8 +333,49 @@ const crawlSitemap = async ({
               return;
             }
 
+            if (isRedirected && !isFollowStrategy(actualUrl, request.url, 'same-hostname')) {
+              urlsCrawled.notScannedRedirects.push({
+                fromUrl: request.url,
+                toUrl: actualUrl,
+              });
+              guiInfoLog(guiInfoStatusTypes.SKIPPED, {
+                numScanned: urlsCrawled.scanned.length,
+                urlScanned: request.url,
+              });
+              return;
+            }
+
             const results = await runAxeScript({ includeScreenshots, page, randomToken });
 
+            // Detect JS redirects that fire during/after axe scan.
+            // Listen for navigation, then give a brief window for pending redirects to complete.
+            try {
+              let navigatedToUrl: string | null = null;
+              const onFrameNavigated = (frame: any) => {
+                if (frame === page.mainFrame()) {
+                  navigatedToUrl = frame.url();
+                }
+              };
+              page.on('framenavigated', onFrameNavigated);
+              await page.waitForTimeout(1000);
+              page.off('framenavigated', onFrameNavigated);
+
+              const postScanUrl = navigatedToUrl || page.url();
+              if (postScanUrl && postScanUrl !== 'about:blank' && !isFollowStrategy(postScanUrl, request.url, 'same-hostname')) {
+                urlsCrawled.notScannedRedirects.push({
+                  fromUrl: request.url,
+                  toUrl: postScanUrl,
+                });
+                guiInfoLog(guiInfoStatusTypes.SKIPPED, {
+                  numScanned: urlsCrawled.scanned.length,
+                  urlScanned: request.url,
+                });
+                return;
+              }
+            } catch (_) {
+              // Page/context was destroyed during navigation — handled by outer catch
+            }
+
             guiInfoLog(guiInfoStatusTypes.SCANNED, {
               numScanned: urlsCrawled.scanned.length,
               urlScanned: request.url,

package/src/crawlers/custom/utils.ts

@@ -40,6 +40,7 @@ const RESTRICT_OVERLAY_TO_ENTRY_DOMAIN = parseBoolEnv(
   process.env.RESTRICT_OVERLAY_TO_ENTRY_DOMAIN,
   false,
 );
+const OVERLAY_OPERATION_TIMEOUT_MS = 5000;
 
 const isOverlayAllowed = (currentUrl: string, entryUrl: string) => {
   try {
@@ -306,7 +307,7 @@ export const addOverlayMenu = async (
     collapsed: false,
   },
 ) => {
-  await page.waitForLoadState('domcontentloaded');
+  await page.waitForLoadState('domcontentloaded', { timeout: OVERLAY_OPERATION_TIMEOUT_MS });
   consoleLogger.info(`Overlay menu: adding to ${menuPos}...`);
 
   // Add the overlay menu with initial styling
@@ -1143,6 +1144,7 @@ export const addOverlayMenu = async (
     })
     .catch(error => {
       consoleLogger.error('Overlay menu: failed to add', error);
+      throw error;
     });
 };
 
@@ -1165,6 +1167,8 @@ export const removeOverlayMenu = async page => {
 
 export const initNewPage = async (page, pageClosePromises, processPageParams, pagesDict) => {
   let menuPos = MENU_POSITION.right;
+  let overlayRefreshSeq = 0;
+  let overlayRefreshChain = Promise.resolve();
 
   // eslint-disable-next-line no-underscore-dangle
   const pageId = page._guid;
@@ -1194,6 +1198,83 @@ export const initNewPage = async (page, pageClosePromises, processPageParams, pa
     };
   }
 
+  const reconcileOverlayMenu = async (trigger: string) => {
+    // Mark this as the latest refresh so older ones can stop.
+    const refreshSeq = ++overlayRefreshSeq;
+
+    // Serialize overlay updates so multiple navigation events do not add/remove concurrently.
+    overlayRefreshChain = overlayRefreshChain
+      .catch(() => {})
+      .then(async () => {
+        if (refreshSeq !== overlayRefreshSeq || page.isClosed()) return;
+
+        try {
+          // `framenavigated` can fire before the new document is ready for DOM inspection/injection.
+          await page.waitForLoadState('domcontentloaded', { timeout: 5000 });
+        } catch {
+          // Best effort only. The page may still be mid-navigation.
+        }
+
+        try {
+          // Give fast redirect chains a brief chance to advance before we inject/remove the overlay.
+          await page.waitForTimeout(300);
+        } catch {
+          // Best effort only. The page may already be closing.
+        }
+
+        // Re-check staleness after waiting because a newer navigation may have happened meanwhile.
+        if (refreshSeq !== overlayRefreshSeq || page.isClosed()) return;
+
+        const allowed = isOverlayAllowed(page.url(), processPageParams.entryUrl);
+
+        if (!allowed) {
+          await Promise.race([
+            removeOverlayMenu(page),
+            new Promise((_, reject) => {
+              setTimeout(() => {
+                reject(
+                  new Error(
+                    `removeOverlayMenu timed out after ${OVERLAY_OPERATION_TIMEOUT_MS}ms`,
+                  ),
+                );
+              }, OVERLAY_OPERATION_TIMEOUT_MS);
+            }),
+          ]);
+          return;
+        }
+
+        const hasOverlay = await page.evaluate(() =>
+          Boolean(document.querySelector('#oobeeShadowHost')),
+        );
+
+        consoleLogger.info(`Overlay state (${trigger}): ${hasOverlay}`);
+
+        if (!hasOverlay) {
+          // Recreate the overlay after allowed redirects while preserving current UI state.
+          consoleLogger.info(`Adding overlay menu to page (${trigger}): ${page.url()}`);
+          await Promise.race([
+            addOverlayMenu(page, processPageParams.urlsCrawled, menuPos, {
+              inProgress: !!pagesDict[pageId]?.isScanning,
+              collapsed: !!pagesDict[pageId]?.collapsed,
+              hideStopInput: !!processPageParams.customFlowLabel,
+            }),
+            new Promise((_, reject) => {
+              setTimeout(() => {
+                reject(
+                  new Error(`addOverlayMenu timed out after ${OVERLAY_OPERATION_TIMEOUT_MS}ms`),
+                );
+              }, OVERLAY_OPERATION_TIMEOUT_MS);
+            }),
+          ]);
+        }
+      })
+      .catch(() => {
+        consoleLogger.info('Error in adding overlay menu to page');
+      });
+
+    await overlayRefreshChain;
+  };
+
   type handleOnScanClickFunction = () => void;
 
   // Window functions exposed in browser
@@ -1208,17 +1289,7 @@ export const initNewPage = async (page, pageClosePromises, processPageParams, pa
       pagesDict[pageId].isScanning = false;
 
       if (page.isClosed()) return;
-      const allowed = isOverlayAllowed(page.url(), processPageParams.entryUrl);
-
-      if (allowed) {
-        await addOverlayMenu(page, processPageParams.urlsCrawled, menuPos, {
-          inProgress: false,
-          collapsed: !!pagesDict[pageId]?.collapsed,
-          hideStopInput: !!processPageParams.customFlowLabel,
-        });
-      } else {
-        await removeOverlayMenu(page);
-      }
+      await reconcileOverlayMenu('scan-click');
     } catch (error) {
       log(`Scan failed ${error}`);
     }
@@ -1282,42 +1353,25 @@ export const initNewPage = async (page, pageClosePromises, processPageParams, pa
 
   page.on('domcontentloaded', async () => {
     if (page.isClosed()) return;
-    try {
-      const allowed = isOverlayAllowed(page.url(), processPageParams.entryUrl);
-
-      if (!allowed) {
-        await removeOverlayMenu(page);
-        return;
-      }
-
-      const existingOverlay = await page.evaluate(() => {
-        return document.querySelector('#oobeeShadowHost');
-      });
-
-      consoleLogger.info(`Overlay state: ${existingOverlay}`);
-
-      if (!existingOverlay) {
-        consoleLogger.info(`Adding overlay menu to page: ${page.url()}`);
-        await addOverlayMenu(page, processPageParams.urlsCrawled, menuPos, {
-          inProgress: !!pagesDict[pageId]?.isScanning,
-          collapsed: !!pagesDict[pageId]?.collapsed,
-          hideStopInput: !!processPageParams.customFlowLabel,
-        });
-      }
+    await reconcileOverlayMenu('domcontentloaded');
 
-      if (isCypressTest) {
-        try {
-          await handleOnScanClick();
-          page.close();
-        } catch {
-          consoleLogger.info(`Error in calling handleOnScanClick, isCypressTest: ${isCypressTest}`);
-        }
+    if (isCypressTest) {
+      try {
+        await handleOnScanClick();
+        page.close();
+      } catch {
+        consoleLogger.info(
+          `Error in calling handleOnScanClick, isCypressTest: ${isCypressTest}`,
+        );
       }
-    } catch {
-      consoleLogger.info('Error in adding overlay menu to page');
     }
   });
 
+  page.on('framenavigated', async (frame: any) => {
+    if (frame !== page.mainFrame() || page.isClosed()) return;
+    await reconcileOverlayMenu('framenavigated');
+  });
+
   try {
     if (page.isClosed()) return page;
     await page.exposeFunction('handleOnScanClick', handleOnScanClick);
@@ -1337,5 +1391,7 @@ export const initNewPage = async (page, pageClosePromises, processPageParams, pa
     log(`Error exposing functions on page: ${e}`);
   }
 
+  await reconcileOverlayMenu('init');
+
   return page;
 };

package/src/generateHtmlReport.ts

@@ -22,6 +22,7 @@ import constants, {
   a11yRuleShortDescriptionMap,
   disabilityBadgesMap,
   a11yRuleLongDescriptionMap,
+  a11yRuleStepByStepGuide,
 } from './constants/constants.js';
 
 import { consoleLogger } from './logs.js';
@@ -65,7 +66,12 @@ const ensureCategory = (
     if (typeof rule.totalItems !== 'number') {
       rule.totalItems = rule.pagesAffected.reduce(
         (accumulate: number, page: any) =>
-          accumulate + (Array.isArray(page.items) ? page.items.length : 0),
+          accumulate +
+          (Array.isArray(page.items)
+            ? page.items.length
+            : typeof page.itemsCount === 'number'
+              ? page.itemsCount
+              : 0),
         0,
       );
     }
@@ -87,7 +93,10 @@ const ensureCategory = (
   };
 };
 
-export const generateHtmlReport = async (resultDir: string): Promise<string> => {
+export const generateHtmlReport = async (
+  resultDir: string,
+  htmlFilename = 'report',
+): Promise<string> => {
   try {
     const storagePath = path.resolve(resultDir);
     const scanDataJsonPath = path.join(storagePath, 'scanData.json');
@@ -117,24 +126,22 @@ export const generateHtmlReport = async (resultDir: string): Promise<string> =>
     const scanData = JSON.parse(await fs.readFile(scanDataJsonPath, 'utf8'));
     const scanItemsAll = JSON.parse(await fs.readFile(scanItemsJsonPath, 'utf8'));
 
-    // Use convertItemsToReferences to normalize items structure to match scanItemsWithHtmlGroupRefs format
-    const scanItemsWithHtmlGroupRefs = convertItemsToReferences({
+    // Build the lighter scanItems payload used by the HTML report.
+    const lightScanItemsPayload = convertItemsToReferences({
       items: scanItemsAll,
-      ...scanData
     });
 
     const {
       mustFix = {},
       goodToFix = {},
       needsReview = {},
-      passed = {},
-    } = scanItemsWithHtmlGroupRefs;
+    } = lightScanItemsPayload;
 
     const items = {
       mustFix: ensureCategory(mustFix, 'mustFix'),
       goodToFix: ensureCategory(goodToFix, 'goodToFix'),
       needsReview: ensureCategory(needsReview, 'needsReview'),
-      passed: ensureCategory(passed, 'passed'),
+      passed: ensureCategory(scanItemsAll.passed || {}, 'passed'),
     };
 
     const pagesScanned = Array.isArray(scanData.pagesScanned) ? scanData.pagesScanned : [];
@@ -183,6 +190,8 @@ export const generateHtmlReport = async (resultDir: string): Promise<string> =>
       a11yRuleShortDescriptionMap,
       disabilityBadgesMap,
       a11yRuleLongDescriptionMap,
+      a11yRuleStepByStepGuide,
+      wcagCriteriaLabels: constants.wcagCriteriaLabels,
       advancedScanOptionsSummaryItems: {
         showIncludeScreenshots: !!scanData.advancedScanOptionsSummaryItems?.showIncludeScreenshots,
         showAllowSubdomains: !!scanData.advancedScanOptionsSummaryItems?.showAllowSubdomains,
@@ -217,10 +226,11 @@ export const generateHtmlReport = async (resultDir: string): Promise<string> =>
       (allIssues as any).advancedScanOptionsSummaryItems?.disableOobee,
     );
 
-    await writeHTML(allIssues, storagePath, 'report', scanDataB64Path, scanItemsB64Path);
+    await writeHTML(allIssues, storagePath, htmlFilename, scanDataB64Path, scanItemsB64Path);
 
-    consoleLogger.info(`Report generated at: ${path.join(storagePath, 'report.html')}`);
-    return path.join(storagePath, 'report.html');
+    const outputPath = path.join(storagePath, `${htmlFilename}.html`);
+    consoleLogger.info(`Report generated at: ${outputPath}`);
+    return outputPath;
   } catch (err: any) {
     consoleLogger.error(`generateHtmlReport failed: ${err?.message || err}`);
     throw err;

package/src/mergeAxeResults/itemReferences.ts

@@ -1,5 +1,9 @@
 import type { AllIssues, ItemsInfo, RuleInfo } from './types.js';
 
+type ScanItems = AllIssues['items'];
+type ScanCategory = ScanItems[keyof ScanItems];
+type ScanItemsLight = Pick<ScanItems, 'mustFix' | 'goodToFix' | 'needsReview' | 'passed'>;
+
 /**
  * Builds pre-computed HTML groups to optimize Group by HTML Element functionality.
  * Keys are composite "html\x00xpath" strings to ensure unique matching per element instance.
@@ -31,32 +35,72 @@ export const buildHtmlGroups = (rule: RuleInfo, items: ItemsInfo[], pageUrl: str
   });
 };
 
+/*
+// Commenting this out for now as we are not including htmlGroups in the embedded report payload to keep it lean.
+// We can revisit this if we want to include htmlGroups in the future and need a reference builder for it.
+const toHtmlGroupReference = (item: any) => {
+  if (typeof item === 'string') {
+    return item;
+  }
+
+  return `${item?.html || 'No HTML element'}\x00${item?.xpath || ''}`;
+};
+
+const cloneCategoryWithReferenceItems = (category: ScanCategory): ScanCategory =>
+  ({
+    ...category,
+    rules: category.rules.map(
+      rule =>
+        ({
+          ...rule,
+          pagesAffected: rule.pagesAffected.map(
+            page => {
+              const { items, ...pageWithoutItems } = page;
+
+              return {
+                ...pageWithoutItems,
+                itemsCount: page.itemsCount ?? (Array.isArray(items) ? items.length : 0),
+                items: Array.isArray(items) ? items.map(toHtmlGroupReference) : items,
+              } as any;
+            },
+          ),
+        }) as any,
+    ),
+  }) as ScanCategory;
+*/
+
+const cloneCategoryLight = (category: ScanCategory, includeHtmlGroups: boolean): ScanCategory =>
+  ({
+    ...category,
+    rules: category.rules.map(
+      rule =>
+        ({
+          rule: rule.rule,
+          description: rule.description,
+          helpUrl: rule.helpUrl,
+          conformance: rule.conformance,
+          totalItems: rule.totalItems,
+          axeImpact: rule.axeImpact,
+          ...(includeHtmlGroups && rule.htmlGroups ? { htmlGroups: rule.htmlGroups } : {}),
+          pagesAffected: rule.pagesAffected.map(page => ({
+            url: page.url,
+            pageTitle: page.pageTitle,
+            itemsCount: page.itemsCount ?? (Array.isArray((page as any).items) ? (page as any).items.length : 0),
+          })),
+        }) as any,
+    ),
+  }) as ScanCategory;
+
 /**
- * Converts items in pagesAffected to references (html\x00xpath composite keys) for embedding in HTML report.
- * Additionally, it deep-clones allIssues, replaces page.items objects with composite reference keys.
- * Those refs are specifically for htmlGroups lookup (html + xpath).
+ * Builds the embedded HTML-report payload from the full scan items.
+ * Includes htmlGroups for non-passed categories (Group by HTML Element),
+ * excludes them from passed to keep payload within browser memory limits.
  */
-export const convertItemsToReferences = (allIssues: AllIssues): AllIssues => {
-  const cloned = JSON.parse(JSON.stringify(allIssues));
-
-  ['mustFix', 'goodToFix', 'needsReview', 'passed'].forEach(category => {
-    if (!cloned.items[category]?.rules) return;
-
-    cloned.items[category].rules.forEach((rule: any) => {
-      if (!rule.pagesAffected || !rule.htmlGroups) return;
-
-      rule.pagesAffected.forEach((page: any) => {
-        if (!page.items) return;
-
-        page.items = page.items.map((item: any) => {
-          if (typeof item === 'string') return item; // Already a reference
-          // Use composite key matching buildHtmlGroups
-          const htmlKey = `${item.html || 'No HTML element'}\x00${item.xpath || ''}`;
-          return htmlKey;
-        });
-      });
-    });
-  });
-
-  return cloned;
+export const convertItemsToReferences = (source: Pick<AllIssues, 'items'>): ScanItemsLight => {
+  return {
+    mustFix: cloneCategoryLight(source.items.mustFix, true),
+    goodToFix: cloneCategoryLight(source.items.goodToFix, true),
+    needsReview: cloneCategoryLight(source.items.needsReview, true),
+    passed: cloneCategoryLight(source.items.passed, false),
+  };
 };

package/src/mergeAxeResults/sentryTelemetry.ts

@@ -140,7 +140,10 @@ const sendWcagBreakdownToSentry = async (
         event_type: 'accessibility_scan',
         scanType: scanInfo.scanType,
         browser: scanInfo.browser,
-        entryUrl: scanInfo.entryUrl,
+        entryUrl: process.env.OOBEE_SCAN_METADATA ?? scanInfo.entryUrl,
+        ...(process.env.OOBEE_SCAN_PRODUCT && {
+          scanProduct: process.env.OOBEE_SCAN_PRODUCT,
+        }),
       },
       user: {
         ...(scanInfo.email && scanInfo.name

package/src/mergeAxeResults.ts

@@ -185,15 +185,15 @@ const writeHTML = async (
     'utf-8',
   );
 
-  // Create lighter version with item references for embedding in HTML
-  const scanItemsWithHtmlGroupRefs = convertItemsToReferences(allIssues);
+  // Create the lighter scanItems payload for embedding in the HTML report.
+  const lightScanItemsPayload = convertItemsToReferences(allIssues);
 
   // Write the lighter items to a file and get the base64 path
   const {
-    jsonFilePath: scanItemsWithHtmlGroupRefsJsonFilePath,
-    base64FilePath: scanItemsWithHtmlGroupRefsBase64FilePath,
+    jsonFilePath: lightScanItemsPayloadJsonFilePath,
+    base64FilePath: lightScanItemsPayloadBase64FilePath,
   } = await writeJsonFileAndCompressedJsonFile(
-    scanItemsWithHtmlGroupRefs.items,
+    lightScanItemsPayload,
     storagePath,
     'scanItems-light',
   );
@@ -212,8 +212,8 @@ const writeHTML = async (
         await Promise.all([
           fs.promises.unlink(topFilePath),
           fs.promises.unlink(bottomFilePath),
-          fs.promises.unlink(scanItemsWithHtmlGroupRefsBase64FilePath),
-          fs.promises.unlink(scanItemsWithHtmlGroupRefsJsonFilePath),
+          fs.promises.unlink(lightScanItemsPayloadBase64FilePath),
+          fs.promises.unlink(lightScanItemsPayloadJsonFilePath),
         ]);
       } catch (err) {
         console.error('Error cleaning up temporary files:', err);
@@ -251,6 +251,9 @@ const writeHTML = async (
   } else {
     console.warn('Skipping fetch GenAI feature as it is local report');
   }
+
+  var scanData = null;
+  var scanItems = null;
   \n`);
 
     outputStream.write('</script>\n<script type="text/plain" id="scanDataRaw">');
@@ -259,22 +262,25 @@ const writeHTML = async (
     scanDetailsReadStream.on('end', async () => {
       outputStream.write('</script>\n<script>\n');
       outputStream.write(
-        "var scanDataPromise = (async () => { console.log('Loading scanData...'); scanData = await decodeUnzipParse(document.getElementById('scanDataRaw').textContent); })();\n",
+        "var scanDataPromise = (async () => { console.log('Loading scanData...'); scanData = await decodeUnzipParse(document.getElementById('scanDataRaw').textContent); console.log('[report] scanData loaded'); })();\n",
       );
       outputStream.write('</script>\n');
 
       // Write scanItems in 2MB chunks using a stream to avoid loading entire file into memory
       try {
         let chunkIndex = 1;
-        const scanItemsStream = fs.createReadStream(scanItemsWithHtmlGroupRefsBase64FilePath, {
+        const scanItemsStream = fs.createReadStream(lightScanItemsPayloadBase64FilePath, {
           encoding: 'utf8',
           highWaterMark: CHUNK_SIZE,
         });
 
         for await (const chunk of scanItemsStream) {
-          outputStream.write(
+          const ok = outputStream.write(
             `<script type="text/plain" id="scanItemsRaw${chunkIndex}">${chunk}</script>\n`,
           );
+          if (!ok) {
+            await new Promise<void>(resolve => outputStream.once('drain', resolve));
+          }
           chunkIndex++;
         }
 
@@ -291,6 +297,7 @@ var scanItemsPromise = (async () => {
     i++;
   }
   scanItems = await decodeUnzipParse(chunks);
+  console.log('[report] scanItems loaded');
 })();\n`);
         outputStream.write(suffixData);
         outputStream.end();
@@ -989,10 +996,13 @@ const generateArtifacts = async (
     1,
   );
 
+  // Brief delay to allow lingering async crawlee storage operations to flush
+  await new Promise(resolve => setTimeout(resolve, 3000));
+
   try {
     await fs.promises.rm(path.join(storagePath, 'crawlee'), { recursive: true, force: true });
   } catch (error) {
-    consoleLogger.warn(`Unable to force remove crawlee folder: ${error.message}`);
+    // Silently ignore — folder may already be gone or still locked
   }
 
   try {

package/src/static/ejs/partials/scripts/decodeUnzipParse.ejs

@@ -28,8 +28,11 @@ async function decodeUnzipParse(input) {
       offset += arr.length;
     }
 
-    // Step 2: Decompress with pako (GZIP)
-    const decompressed = pako.ungzip(merged, { to: 'string' });
+    // Step 2: Decompress with pako (GZIP) to bytes first to avoid large-string
+    // construction inside pako for very large payloads.
+    const decompressedBytes = pako.ungzip(merged);
+
+    const decompressed = new TextDecoder().decode(decompressedBytes);
 
     // Step 3: Parse JSON
     return JSON.parse(decompressed);
@@ -37,4 +40,4 @@ async function decodeUnzipParse(input) {
     throw new Error(`Failed to decode/unzip/parse: ${err.message}`);
   }
 }
-</script>
+</script>

package/src/static/ejs/partials/scripts/ruleModal/itemCardRenderer.ejs

@@ -1,10 +1,44 @@
 <script>
   /**
-   * Resolves item references (composite "html\x00xpath" strings) to full item data using htmlGroups.
+   * Rebuilds the item list for a page from pre-computed htmlGroups when the light report omits page.items.
+   */
+  function buildItemsFromHtmlGroupsForPage(page, ruleInCategory) {
+    const htmlGroups = ruleInCategory.htmlGroups || {};
+    const resolvedItems = [];
+
+    Object.values(htmlGroups).forEach(groupData => {
+      if (!Array.isArray(groupData.pageUrls) || !groupData.pageUrls.includes(page.url)) {
+        return;
+      }
+
+      resolvedItems.push({
+        html: groupData.html,
+        xpath: groupData.xpath,
+        message: groupData.message,
+        screenshotPath: groupData.screenshotPath,
+        displayNeedsReview: groupData.displayNeedsReview,
+        pageUrl: page.url,
+        pageTitle: page.pageTitle || page.metadata
+      });
+    });
+
+    return resolvedItems;
+  }
+
+  /**
+   * The embedded report payload now omits page.items and rebuilds occurrences from
+   * htmlGroups + page metadata. Keep the older page.items resolution logic below
+   * commented for an easy rollback if we need to restore mixed payload support.
    */
   function resolveItemReferencesForPage(page, ruleInCategory) {
+    return buildItemsFromHtmlGroupsForPage(page, ruleInCategory);
+
+    /*
     const items = page.items || [];
-    if (items.length === 0) return [];
+
+    if (items.length === 0) {
+      return buildItemsFromHtmlGroupsForPage(page, ruleInCategory);
+    }
 
     const isReference = typeof items[0] === 'string';
 
@@ -27,6 +61,7 @@
           pageTitle: page.pageTitle || page.metadata
         };
       }
+
       // Fallback: parse composite key
       const nullByteIndex = compositeKey.indexOf('\x00');
       const html = nullByteIndex !== -1 ? compositeKey.slice(0, nullByteIndex) : compositeKey;
@@ -40,6 +75,7 @@
         pageTitle: page.pageTitle || page.metadata
       };
     });
+    */
   }
 
   function buildItemCardsWithPagination(accordionId, category, ruleInCategory, page, index) {