@govtechsg/oobee 0.10.85 → 0.10.87

package/dist/static/ejs/partials/scripts/ruleModal/itemCardRenderer.ejs

@@ -1,10 +1,44 @@
 <script>
   /**
-   * Resolves item references (composite "html\x00xpath" strings) to full item data using htmlGroups.
+   * Rebuilds the item list for a page from pre-computed htmlGroups when the light report omits page.items.
+   */
+  function buildItemsFromHtmlGroupsForPage(page, ruleInCategory) {
+    const htmlGroups = ruleInCategory.htmlGroups || {};
+    const resolvedItems = [];
+
+    Object.values(htmlGroups).forEach(groupData => {
+      if (!Array.isArray(groupData.pageUrls) || !groupData.pageUrls.includes(page.url)) {
+        return;
+      }
+
+      resolvedItems.push({
+        html: groupData.html,
+        xpath: groupData.xpath,
+        message: groupData.message,
+        screenshotPath: groupData.screenshotPath,
+        displayNeedsReview: groupData.displayNeedsReview,
+        pageUrl: page.url,
+        pageTitle: page.pageTitle || page.metadata
+      });
+    });
+
+    return resolvedItems;
+  }
+
+  /**
+   * The embedded report payload now omits page.items and rebuilds occurrences from
+   * htmlGroups + page metadata. Keep the older page.items resolution logic below
+   * commented for an easy rollback if we need to restore mixed payload support.
    */
   function resolveItemReferencesForPage(page, ruleInCategory) {
+    return buildItemsFromHtmlGroupsForPage(page, ruleInCategory);
+
+    /*
     const items = page.items || [];
-    if (items.length === 0) return [];
+
+    if (items.length === 0) {
+      return buildItemsFromHtmlGroupsForPage(page, ruleInCategory);
+    }
 
     const isReference = typeof items[0] === 'string';
 
@@ -27,6 +61,7 @@
           pageTitle: page.pageTitle || page.metadata
         };
       }
+
       // Fallback: parse composite key
       const nullByteIndex = compositeKey.indexOf('\x00');
       const html = nullByteIndex !== -1 ? compositeKey.slice(0, nullByteIndex) : compositeKey;
@@ -40,6 +75,7 @@
         pageTitle: page.pageTitle || page.metadata
       };
     });
+    */
   }
 
   function buildItemCardsWithPagination(accordionId, category, ruleInCategory, page, index) {

package/dist/static/ejs/partials/scripts/ruleModal/pageAccordionBuilder.ejs

@@ -86,7 +86,7 @@
             // Use pre-computed htmlGroups for count if available, otherwise use pages
             const count = isHtmlGrouping && selectedCategory.htmlGroups
               ? Object.keys(selectedCategory.htmlGroups).length
-              : selectedCategory.pagesAffected.length;
+              : (selectedCategory.pagesAffectedCount || selectedCategory.pagesAffected.length);
             if (isHtmlGrouping) {
               dropdownTitle.innerText = `HTML elements affected by this issue (${count})`;
             } else {

package/dist/static/ejs/partials/scripts/ruleModal/ruleOffcanvas.ejs

@@ -270,8 +270,8 @@ include('./pageAccordionBuilder') %> <%- include('./constants') %>
           if (!Array.isArray(rule.pagesAffected)) return;
 
           rule.pagesAffected.sort((a, b) => {
-            const lenA = Array.isArray(a.items) ? a.items.length : 0;
-            const lenB = Array.isArray(b.items) ? b.items.length : 0;
+            const lenA = Array.isArray(a.items) ? a.items.length : a.itemsCount || 0;
+            const lenB = Array.isArray(b.items) ? b.items.length : b.itemsCount || 0;
             return lenB - lenA; // DESC
           });
         });
@@ -295,10 +295,10 @@ include('./pageAccordionBuilder') %> <%- include('./constants') %>
             dropdownToggle.innerText = `${ruleInCategory.totalItems} Total occ.`;
             dropdownToggle.setAttribute('aria-label', occurrencesText);
           document.getElementById('expandedRuleDropdownTitle').innerText =
-            `Pages affected by this issue (${ruleInCategory.pagesAffected.length})`;
+            `Pages affected by this issue (${(ruleInCategory.pagesAffectedCount || ruleInCategory.pagesAffected.length)})`;
           buildExpandedRuleCategoryContent(category, ruleInCategory);
           document.getElementById('expandedRulePageContent').innerText =
-            `Total ${ruleInCategory.pagesAffected.length} affected pages`;
+            `Total ${(ruleInCategory.pagesAffectedCount || ruleInCategory.pagesAffected.length)} affected pages`;
         }
       }
     });

package/dist/static/ejs/summary.ejs

@@ -21,14 +21,25 @@
     %>
     <script>
       const scanItems = <%- JSON.stringify(
-        {
-          ...items,
-          passed: {
-            rules: items.passed.rules.map(r => delete r.pagesAffected),
-            ...items.passed
-          },
-        }
-      ) %>
+        ['mustFix','goodToFix','needsReview','passed'].reduce((acc, cat) => {
+          if (items[cat]) {
+            acc[cat] = {
+              description: items[cat].description,
+              totalItems: items[cat].totalItems,
+              totalRuleIssues: items[cat].totalRuleIssues,
+              rules: (items[cat].rules || []).map(rule => ({
+                rule: rule.rule,
+                description: rule.description,
+                helpUrl: rule.helpUrl,
+                conformance: rule.conformance,
+                totalItems: rule.totalItems,
+                pagesAffected: { length: (rule.pagesAffected || []).length },
+              })),
+            };
+          }
+          return acc;
+        }, {})
+      ).replace(/<\//g, '<\\/') %>
     </script>
     <%- include('partials/scripts/summaryTable') %>
     <script>

package/dist/utils.js

@@ -4,6 +4,7 @@ import fs from 'fs-extra';
 import axe from 'axe-core';
 import { v4 as uuidv4 } from 'uuid';
 import { getDomain } from 'tldts';
+import { normalizeUrl } from '@apify/utilities';
 import constants, { destinationPath, getIntermediateScreenshotsPath, } from './constants/constants.js';
 import { consoleLogger, errorsTxtPath } from './logs.js';
 import { getAxeConfiguration } from './crawlers/custom/getAxeConfiguration.js';
@@ -852,13 +853,13 @@ export const randomThreeDigitNumberString = () => {
     const threeDigitNumber = Math.floor(scaledDecimal) + 100;
     return String(threeDigitNumber);
 };
+export const normUrl = (u) => (u ? normalizeUrl(u) || u : '');
 export const isFollowStrategy = (link1, link2, rule) => {
+    if (rule === 'all')
+        return true;
     try {
         const parsedLink1 = new URL(link1);
         const parsedLink2 = new URL(link2);
-        if (rule === 'all') {
-            return true;
-        }
         if (rule === 'same-origin') {
             return parsedLink1.origin === parsedLink2.origin;
         }

package/fix-summary-html-oom-pr.md

@@ -0,0 +1,62 @@
+# fix: prevent OOM and browser crash in report generation for large scans
+
+## Summary
+
+- Fix `summary.ejs` inlining the entire scan items payload (2 GB+ for 1000-page scans) via `JSON.stringify`, causing V8 OOM and killing the process
+- Fix `report.html` embedded scanItems exceeding browser memory limits (746 MB uncompressed JSON for 1000-page scans)
+- Fix write stream backpressure handling when embedding chunked base64 data
+- `writeSummaryHTML` crash also blocked `report.html` generation since it runs first
+
+## Problem 1: OOM in summary.html generation (server-side)
+
+For large scans (e.g. 1000 pages, 2.5M+ passed occurrences), `summary.ejs` serialized the full `items` object — including every rule's `pagesAffected` array with all individual issue items — into an inline `<script>` tag. This produced a string exceeding V8's limits, crashing the process silently.
+
+The result: neither `summary.html` nor `report.html` were generated, even though all JSON artifacts (`scanData.json`, `scanItems.json`, etc.) were written successfully.
+
+## Problem 2: Browser cannot parse embedded scanItems (client-side)
+
+Even with report generation fixed, the browser failed to load the All Issues view:
+```
+Failed to decode/unzip/parse: Unexpected end of JSON input
+```
+
+Root cause: `convertItemsToReferences` stripped per-page `items` arrays but still embedded the full `pagesAffected` array (url, pageTitle, actualUrl, metadata, etc. for every page × every rule). For 1000-page scans this produced **746 MB of uncompressed JSON** after base64-decode and gunzip — exceeding browser string/memory limits during `JSON.parse()`.
+
+## Problem 3: Write stream backpressure (server-side)
+
+The `writeHTML` function writes scan items as 2 MB base64 chunks via a `for await` loop over a read stream. `outputStream.write()` was not being checked for backpressure — when the write buffer filled up, subsequent writes could be silently dropped, producing truncated base64.
+
+## Fix
+
+### summary.ejs (OOM fix)
+Strip the inline JSON to only what `summaryTable.ejs` actually needs:
+- Rule-level metadata: `description`, `helpUrl`, `conformance`, `totalItems`
+- `pagesAffected: { length: N }` (just the count object, not the full array)
+
+This reduces the serialized payload from potentially gigabytes to a few kilobytes regardless of scan size.
+
+### itemReferences.ts (browser payload fix)
+`convertItemsToReferences` now strips each `pagesAffected` entry down to only `url`, `pageTitle`, and `itemsCount` — removing all per-item details (html snippets, screenshots, xpath, metadata, etc.) that constituted the bulk of the data. The All Issues list renders rule totals, and the "Group By Page" view in the rule modal still shows page URLs with occurrence counts.
+
+This reduces the embedded payload from 746 MB (uncompressed) to ~11 MB for a 1000-page scan — well within browser memory limits.
+
+### mergeAxeResults.ts (backpressure fix)
+Await the `drain` event on the output stream when `write()` returns `false` before writing the next chunk. This ensures all base64 data is fully written to the report regardless of payload size.
+
+## Files changed
+
+| File | Change |
+|------|--------|
+| `src/static/ejs/summary.ejs` | Strip inline JSON to rule counts only |
+| `src/mergeAxeResults/itemReferences.ts` | Strip `pagesAffected` to lightweight entries (url, pageTitle, itemsCount only) |
+| `src/mergeAxeResults.ts` | Await drain on backpressure during chunked write |
+| `src/static/ejs/partials/scripts/ruleModal/ruleOffcanvas.ejs` | Fall back to `pagesAffectedCount` |
+| `src/static/ejs/partials/scripts/ruleModal/pageAccordionBuilder.ejs` | Fall back to `pagesAffectedCount` |
+
+## Test plan
+
+- [ ] Run a large scan (500+ pages) and verify both `summary.html` and `report.html` are generated
+- [ ] Open `summary.html` in a browser and verify the summary table renders correctly (issue counts, page counts, help links)
+- [ ] Open `report.html` and verify the All Issues list loads and displays rule counts correctly
+- [ ] Verify the rule modal shows correct "Pages affected" count
+- [ ] Verify small scans still produce correct reports (no regression)