@govtechsg/oobee 0.10.86 → 0.10.88

package/dist/crawlers/crawlIntelligentSitemap.js

@@ -3,7 +3,7 @@ import constants, { guiInfoStatusTypes, sitemapPaths } from '../constants/consta
 import { consoleLogger, guiInfoLog } from '../logs.js';
 import crawlDomain from './crawlDomain.js';
 import crawlSitemap from './crawlSitemap.js';
-import { getPlaywrightLaunchOptions } from '../constants/common.js';
+import { getPlaywrightLaunchOptions, getSitemapsFromRobotsTxt } from '../constants/common.js';
 import { register } from '../utils.js';
 const crawlIntelligentSitemap = async (url, randomToken, host, viewportSettings, maxRequestsPerCrawl, browser, userDataDirectory, strategy, specifiedMaxConcurrency, fileTypes, blacklistedPatterns, includeScreenshots, followRobots, extraHTTPHeaders, safeMode, scanDuration) => {
     const startTime = Date.now(); // Track start time
@@ -66,12 +66,30 @@ const crawlIntelligentSitemap = async (url, randomToken, host, viewportSettings,
             return false;
         }
     };
+    // Discover sitemaps from robots.txt first (supports multiple Sitemap: directives)
+    let sitemapUrls = [];
     try {
-        sitemapUrl = await findSitemap(url, userDataDirectory, extraHTTPHeaders);
+        sitemapUrls = await getSitemapsFromRobotsTxt(url, browser, userDataDirectory, extraHTTPHeaders);
+        if (sitemapUrls.length > 0) {
+            console.log(`Found ${sitemapUrls.length} sitemap(s) in robots.txt: ${sitemapUrls.join(', ')}`);
+            sitemapExist = true;
+        }
     }
     catch (error) {
         consoleLogger.error(error);
     }
+    // Fall back to hardcoded path probing if robots.txt had no sitemaps
+    if (!sitemapExist) {
+        try {
+            sitemapUrl = await findSitemap(url, userDataDirectory, extraHTTPHeaders);
+            if (sitemapExist) {
+                sitemapUrls = [sitemapUrl];
+            }
+        }
+        catch (error) {
+            consoleLogger.error(error);
+        }
+    }
     if (!sitemapExist) {
         console.log('Unable to find sitemap. Commencing website crawl instead.');
         return await crawlDomain({
@@ -90,34 +108,48 @@ const crawlIntelligentSitemap = async (url, randomToken, host, viewportSettings,
             followRobots,
             extraHTTPHeaders,
             safeMode,
-            scanDuration, // Use full duration since no sitemap
+            scanDuration,
+        });
+    }
+    // Process all discovered sitemaps sequentially, sharing dataset and urlsCrawled
+    for (const currentSitemapUrl of sitemapUrls) {
+        if (urlsCrawled.scanned.length >= maxRequestsPerCrawl)
+            break;
+        const elapsed = Date.now() - startTime;
+        const remainingDuration = scanDuration > 0 ? Math.max(scanDuration - elapsed / 1000, 0) : scanDuration;
+        if (scanDuration > 0 && remainingDuration <= 0) {
+            durationExceeded = true;
+            break;
+        }
+        console.log(`Processing sitemap: ${currentSitemapUrl}`);
+        urlsCrawledFinal = await crawlSitemap({
+            sitemapUrl: currentSitemapUrl,
+            randomToken,
+            host,
+            viewportSettings,
+            maxRequestsPerCrawl,
+            browser,
+            userDataDirectory,
+            specifiedMaxConcurrency,
+            fileTypes,
+            blacklistedPatterns,
+            includeScreenshots,
+            extraHTTPHeaders,
+            strategy,
+            userUrl: url,
+            fromCrawlIntelligentSitemap,
+            userUrlInputFromIntelligent: url,
+            datasetFromIntelligent: dataset,
+            urlsCrawledFromIntelligent: urlsCrawled,
+            crawledFromLocalFile: false,
+            scanDuration: scanDuration > 0 ? remainingDuration : 0,
         });
     }
-    console.log(`Sitemap found at ${sitemapUrl}`);
-    urlsCrawledFinal = await crawlSitemap({
-        sitemapUrl,
-        randomToken,
-        host,
-        viewportSettings,
-        maxRequestsPerCrawl,
-        browser,
-        userDataDirectory,
-        specifiedMaxConcurrency,
-        fileTypes,
-        blacklistedPatterns,
-        includeScreenshots,
-        extraHTTPHeaders,
-        fromCrawlIntelligentSitemap,
-        userUrlInputFromIntelligent: url,
-        datasetFromIntelligent: dataset,
-        urlsCrawledFromIntelligent: urlsCrawled,
-        crawledFromLocalFile: false,
-        scanDuration,
-    });
     const elapsed = Date.now() - startTime;
-    const remainingScanDuration = Math.max(scanDuration - elapsed / 1000, 0); // in seconds
-    if (urlsCrawledFinal.scanned.length < maxRequestsPerCrawl && remainingScanDuration > 0) {
-        console.log(`Continuing crawl from root website. Remaining scan time: ${remainingScanDuration.toFixed(1)}s`);
+    const remainingScanDuration = scanDuration > 0 ? Math.max(scanDuration - elapsed / 1000, 0) : 0;
+    const hasDurationRemaining = scanDuration === 0 || remainingScanDuration > 0;
+    if (urlsCrawled.scanned.length < maxRequestsPerCrawl && hasDurationRemaining) {
+        console.log(`Continuing crawl from root website.${scanDuration > 0 ? ` Remaining scan time: ${remainingScanDuration.toFixed(1)}s` : ''}`);
         urlsCrawledFinal = await crawlDomain({
             url,
             randomToken,
@@ -136,15 +168,15 @@ const crawlIntelligentSitemap = async (url, randomToken, host, viewportSettings,
             safeMode,
             fromCrawlIntelligentSitemap,
             datasetFromIntelligent: dataset,
-            urlsCrawledFromIntelligent: urlsCrawledFinal,
+            urlsCrawledFromIntelligent: urlsCrawled,
             scanDuration: remainingScanDuration,
         });
     }
-    else if (remainingScanDuration <= 0) {
+    else if (!hasDurationRemaining) {
         console.log(`Crawl duration exceeded before more pages could be found (limit: ${scanDuration}s).`);
         durationExceeded = true;
     }
     guiInfoLog(guiInfoStatusTypes.COMPLETED, {});
-    return { urlsCrawled: urlsCrawledFinal, durationExceeded };
+    return { urlsCrawled, durationExceeded };
 };
 export default crawlIntelligentSitemap;

package/dist/crawlers/crawlLocalFile.js

@@ -8,7 +8,7 @@ import { runPdfScan, mapPdfScanResults, doPdfScreenshots } from './pdfScanFunc.j
 import { guiInfoLog } from '../logs.js';
 import crawlSitemap from './crawlSitemap.js';
 import { getPdfStoragePath, register } from '../utils.js';
-export const crawlLocalFile = async ({ url, randomToken, host, viewportSettings, maxRequestsPerCrawl, browser, userDataDirectory, specifiedMaxConcurrency, fileTypes, blacklistedPatterns, includeScreenshots, extraHTTPHeaders, scanDuration = 0, fromCrawlIntelligentSitemap = false, userUrlInputFromIntelligent = null, datasetFromIntelligent = null, urlsCrawledFromIntelligent = null, }) => {
+export const crawlLocalFile = async ({ url, randomToken, host, viewportSettings, maxRequestsPerCrawl, browser, userDataDirectory, specifiedMaxConcurrency, fileTypes, blacklistedPatterns, includeScreenshots, extraHTTPHeaders, scanDuration = 0, ruleset = [], fromCrawlIntelligentSitemap = false, userUrlInputFromIntelligent = null, datasetFromIntelligent = null, urlsCrawledFromIntelligent = null, }) => {
     let dataset;
     let urlsCrawled;
     let linksFromSitemap = [];
@@ -105,7 +105,7 @@ export const crawlLocalFile = async ({ url, randomToken, host, viewportSettings,
             await browserContext.close().catch(() => { });
             return urlsCrawled;
         }
-        const results = await runAxeScript({ includeScreenshots, page, randomToken });
+        const results = await runAxeScript({ includeScreenshots, page, randomToken, ruleset });
         const actualUrl = page.url() || request.loadedUrl || url;
         guiInfoLog(guiInfoStatusTypes.SCANNED, {
             numScanned: urlsCrawled.scanned.length,

package/dist/crawlers/crawlSitemap.js

@@ -1,13 +1,13 @@
-import crawlee, { RequestList } from 'crawlee';
+import crawlee, { EnqueueStrategy, RequestList } from 'crawlee';
 import * as path from 'path';
 import fsp from 'fs/promises';
 import { createCrawleeSubFolders, preNavigationHooks, runAxeScript, } from './commonCrawlerFunc.js';
 import constants, { STATUS_CODE_METADATA, guiInfoStatusTypes, disallowedListOfPatterns, FileTypes, } from '../constants/constants.js';
 import { getLinksFromSitemap, getPlaywrightLaunchOptions, isSkippedUrl, waitForPageLoaded, isFilePath, } from '../constants/common.js';
-import { areLinksEqual, isWhitelistedContentType, register } from '../utils.js';
+import { areLinksEqual, isFollowStrategy, isWhitelistedContentType, normUrl, register } from '../utils.js';
 import { handlePdfDownload, runPdfScan, mapPdfScanResults, doPdfScreenshots, } from './pdfScanFunc.js';
 import { guiInfoLog } from '../logs.js';
-const crawlSitemap = async ({ sitemapUrl, randomToken, host, viewportSettings, maxRequestsPerCrawl, browser, userDataDirectory, specifiedMaxConcurrency, fileTypes, blacklistedPatterns, includeScreenshots, extraHTTPHeaders, scanDuration = 0, fromCrawlIntelligentSitemap = false, userUrlInputFromIntelligent = null, datasetFromIntelligent = null, urlsCrawledFromIntelligent = null, crawledFromLocalFile = false, }) => {
+const crawlSitemap = async ({ sitemapUrl, randomToken, host, viewportSettings, maxRequestsPerCrawl, browser, userDataDirectory, specifiedMaxConcurrency, fileTypes, blacklistedPatterns, includeScreenshots, extraHTTPHeaders, strategy = EnqueueStrategy.All, userUrl = '', scanDuration = 0, fromCrawlIntelligentSitemap = false, userUrlInputFromIntelligent = null, datasetFromIntelligent = null, urlsCrawledFromIntelligent = null, crawledFromLocalFile = false, }) => {
     const crawlStartTime = Date.now();
     let dataset;
     let urlsCrawled;
@@ -25,7 +25,7 @@ const crawlSitemap = async ({ sitemapUrl, randomToken, host, viewportSettings, m
         console.log('Local file crawling not supported for sitemap. Please provide a valid URL.');
         return;
     }
-    const linksFromSitemap = await getLinksFromSitemap(sitemapUrl, maxRequestsPerCrawl, browser, userDataDirectory, userUrlInputFromIntelligent, fromCrawlIntelligentSitemap, extraHTTPHeaders);
+    const linksFromSitemap = await getLinksFromSitemap(sitemapUrl, maxRequestsPerCrawl, browser, userDataDirectory, userUrlInputFromIntelligent, fromCrawlIntelligentSitemap, extraHTTPHeaders, strategy, userUrl || sitemapUrl);
     sitemapUrl = encodeURI(sitemapUrl);
     const pdfDownloads = [];
     const uuidToPdfMapping = {};
@@ -182,7 +182,7 @@ const crawlSitemap = async ({ sitemapUrl, randomToken, host, viewportSettings, m
                 const status = response ? response.status() : 0;
                 if (isScanHtml && status < 300 && isWhitelistedContentType(contentType)) {
                     const isRedirected = !areLinksEqual(page.url(), request.url);
-                    const isLoadedUrlInCrawledUrls = urlsCrawled.scanned.some(item => (item.actualUrl || item.url) === page.url());
+                    const isLoadedUrlInCrawledUrls = urlsCrawled.scanned.some(item => normUrl(item.actualUrl || item.url) === normUrl(page.url()));
                     if (isRedirected && isLoadedUrlInCrawledUrls) {
                         urlsCrawled.notScannedRedirects.push({
                             fromUrl: request.url,
@@ -205,7 +205,46 @@ const crawlSitemap = async ({ sitemapUrl, randomToken, host, viewportSettings, m
                         });
                         return;
                     }
+                    if (isRedirected && !isFollowStrategy(actualUrl, request.url, 'same-hostname')) {
+                        urlsCrawled.notScannedRedirects.push({
+                            fromUrl: request.url,
+                            toUrl: actualUrl,
+                        });
+                        guiInfoLog(guiInfoStatusTypes.SKIPPED, {
+                            numScanned: urlsCrawled.scanned.length,
+                            urlScanned: request.url,
+                        });
+                        return;
+                    }
                     const results = await runAxeScript({ includeScreenshots, page, randomToken });
+                    // Detect JS redirects that fire during/after axe scan.
+                    // Listen for navigation, then give a brief window for pending redirects to complete.
+                    try {
+                        let navigatedToUrl = null;
+                        const onFrameNavigated = (frame) => {
+                            if (frame === page.mainFrame()) {
+                                navigatedToUrl = frame.url();
+                            }
+                        };
+                        page.on('framenavigated', onFrameNavigated);
+                        await page.waitForTimeout(1000);
+                        page.off('framenavigated', onFrameNavigated);
+                        const postScanUrl = navigatedToUrl || page.url();
+                        if (postScanUrl && postScanUrl !== 'about:blank' && !isFollowStrategy(postScanUrl, request.url, 'same-hostname')) {
+                            urlsCrawled.notScannedRedirects.push({
+                                fromUrl: request.url,
+                                toUrl: postScanUrl,
+                            });
+                            guiInfoLog(guiInfoStatusTypes.SKIPPED, {
+                                numScanned: urlsCrawled.scanned.length,
+                                urlScanned: request.url,
+                            });
+                            return;
+                        }
+                    }
+                    catch (_) {
+                        // Page/context was destroyed during navigation — handled by outer catch
+                    }
                     guiInfoLog(guiInfoStatusTypes.SCANNED, {
                         numScanned: urlsCrawled.scanned.length,
                         urlScanned: request.url,

package/dist/crawlers/custom/extractAndGradeText.js

@@ -34,7 +34,7 @@ export async function extractAndGradeText(page) {
         const readabilityScore = wordCount >= 20 ? textReadability.fleschReadingEase(filteredText) : 0;
         // Log details for debugging
         // Determine the return value
-        const result = readabilityScore === 0 || readabilityScore > 50 ? '' : readabilityScore.toString(); // Convert readabilityScore to string
+        const result = readabilityScore <= 0 || readabilityScore > 50 ? '' : readabilityScore.toString();
         return result;
     }
     catch (error) {

package/dist/crawlers/custom/getAxeConfiguration.js

@@ -1,5 +1,13 @@
 import { evaluateAltText } from "./evaluateAltText.js";
 export function getAxeConfiguration({ enableWcagAaa = false, gradingReadabilityFlag = '', disableOobee = false, }) {
+    function getReadabilityInterpretation(score) {
+        const num = parseFloat(score);
+        if (Number.isNaN(num))
+            return '';
+        if (num > 30)
+            return 'It is targeted for junior college (JC) level comprehension and above.';
+        return 'It is targeted for university graduate level comprehension and above.';
+    }
     return {
         branding: {
             application: 'oobee',
@@ -29,7 +37,7 @@ export function getAxeConfiguration({ enableWcagAaa = false, gradingReadabilityF
                     return !node.dataset.flagged; // fail any element with a data-flagged attribute set to true
                 },
             },
-            ...(enableWcagAaa
+            ...((enableWcagAaa && gradingReadabilityFlag !== '')
                 ? [
                     {
                         id: 'oobee-grading-text-contents',
@@ -37,16 +45,11 @@ export function getAxeConfiguration({ enableWcagAaa = false, gradingReadabilityF
                             impact: 'moderate',
                             messages: {
                                 pass: 'The text content is easy to understand.',
-                                fail: 'The text content is potentially difficult to understand.',
-                                incomplete: `The text content is potentially difficult to read, with a Flesch-Kincaid Reading Ease score of ${gradingReadabilityFlag}.\nThe target passing score is above 50, indicating content readable by university students and lower grade levels.\nA higher score reflects better readability.`,
+                                fail: `Text content is potentially difficult to read.\n  It scored ${gradingReadabilityFlag} out of 50 on the Flesch-Kincaid Readability Test.\n  ${getReadabilityInterpretation(gradingReadabilityFlag)}`,
+                                incomplete: `Text content is potentially difficult to read.\n  It scored ${gradingReadabilityFlag} out of 50 on the Flesch-Kincaid Readability Test.\n  ${getReadabilityInterpretation(gradingReadabilityFlag)}`,
                             },
                         },
-                        evaluate: (_node) => {
-                            if (gradingReadabilityFlag === '') {
-                                return true; // Pass if no readability issues
-                            }
-                            // Fail if readability issues are detected
-                        },
+                        evaluate: (_node) => false,
                     },
                 ]
                 : []),
@@ -77,18 +80,20 @@ export function getAxeConfiguration({ enableWcagAaa = false, gradingReadabilityF
                     helpUrl: 'https://www.deque.com/blog/accessible-aria-buttons',
                 },
             },
-            {
-                id: 'oobee-grading-text-contents',
-                selector: 'html',
-                enabled: true,
-                any: ['oobee-grading-text-contents'],
-                tags: ['wcag2aaa', 'wcag315'],
-                metadata: {
-                    description: 'Text content should be easy to understand for individuals with education levels up to university graduates. If the text content is difficult to understand, provide supplemental content or a version that is easy to understand.',
-                    help: 'Text content should be clear and plain to ensure that it is easily understood.',
-                    helpUrl: 'https://www.wcag.com/uncategorized/3-1-5-reading-level/',
-                },
-            },
+            ...((enableWcagAaa && gradingReadabilityFlag !== '')
+                ? [{
+                        id: 'oobee-grading-text-contents',
+                        selector: 'html',
+                        enabled: true,
+                        any: ['oobee-grading-text-contents'],
+                        tags: ['wcag2aaa', 'wcag315'],
+                        metadata: {
+                            description: 'Text content should be easy to understand for individuals with education levels up to university graduates. If the text content is difficult to understand, provide supplemental content or a version that is easy to understand.',
+                            help: 'Text content should be clear and plain to ensure that it is easily understood.',
+                            helpUrl: 'https://www.wcag.com/uncategorized/3-1-5-reading-level/',
+                        },
+                    }]
+                : []),
         ]
             .filter(rule => (disableOobee ? !rule.id.startsWith('oobee') : true))
             .concat(enableWcagAaa

package/dist/crawlers/custom/gradeReadability.js

@@ -13,7 +13,7 @@ export function gradeReadability(sentences) {
         const readabilityScore = wordCount >= 20 ? textReadability.fleschReadingEase(filteredText) : 0;
         // Log details for debugging
         // Determine the return value
-        const result = readabilityScore === 0 || readabilityScore > 50 ? '' : readabilityScore.toString(); // Convert readabilityScore to string
+        const result = readabilityScore <= 0 || readabilityScore > 50 ? '' : readabilityScore.toString();
         return result;
     }
     catch (error) {

package/dist/crawlers/custom/utils.js

@@ -25,6 +25,7 @@ const parseBoolEnv = (val, defaultVal) => {
     return defaultVal;
 };
 const RESTRICT_OVERLAY_TO_ENTRY_DOMAIN = parseBoolEnv(process.env.RESTRICT_OVERLAY_TO_ENTRY_DOMAIN, false);
+const OVERLAY_OPERATION_TIMEOUT_MS = 5000;
 const isOverlayAllowed = (currentUrl, entryUrl) => {
     try {
         const cur = new URL(currentUrl);
@@ -206,7 +207,7 @@ export const addOverlayMenu = async (page, urlsCrawled, menuPos, opts = {
     inProgress: false,
     collapsed: false,
 }) => {
-    await page.waitForLoadState('domcontentloaded');
+    await page.waitForLoadState('domcontentloaded', { timeout: OVERLAY_OPERATION_TIMEOUT_MS });
     consoleLogger.info(`Overlay menu: adding to ${menuPos}...`);
     // Add the overlay menu with initial styling
     return page
@@ -987,6 +988,7 @@ export const addOverlayMenu = async (page, urlsCrawled, menuPos, opts = {
     })
         .catch(error => {
         consoleLogger.error('Overlay menu: failed to add', error);
+        throw error;
     });
 };
 export const removeOverlayMenu = async (page) => {
@@ -1007,6 +1009,8 @@ export const removeOverlayMenu = async (page) => {
 };
 export const initNewPage = async (page, pageClosePromises, processPageParams, pagesDict) => {
     let menuPos = MENU_POSITION.right;
+    let overlayRefreshSeq = 0;
+    let overlayRefreshChain = Promise.resolve();
     // eslint-disable-next-line no-underscore-dangle
     const pageId = page._guid;
     page.on('dialog', async (dialog) => {
@@ -1032,6 +1036,68 @@ export const initNewPage = async (page, pageClosePromises, processPageParams, pa
             collapsed: false,
         };
     }
+    const reconcileOverlayMenu = async (trigger) => {
+        // Mark this as the latest refresh so older ones can stop.
+        const refreshSeq = ++overlayRefreshSeq;
+        // Serialize overlay updates so multiple navigation events do not add/remove concurrently.
+        overlayRefreshChain = overlayRefreshChain
+            .catch(() => { })
+            .then(async () => {
+            if (refreshSeq !== overlayRefreshSeq || page.isClosed())
+                return;
+            try {
+                // `framenavigated` can fire before the new document is ready for DOM inspection/injection.
+                await page.waitForLoadState('domcontentloaded', { timeout: 5000 });
+            }
+            catch {
+                // Best effort only. The page may still be mid-navigation.
+            }
+            try {
+                // Give fast redirect chains a brief chance to advance before we inject/remove the overlay.
+                await page.waitForTimeout(300);
+            }
+            catch {
+                // Best effort only. The page may already be closing.
+            }
+            // Re-check staleness after waiting because a newer navigation may have happened meanwhile.
+            if (refreshSeq !== overlayRefreshSeq || page.isClosed())
+                return;
+            const allowed = isOverlayAllowed(page.url(), processPageParams.entryUrl);
+            if (!allowed) {
+                await Promise.race([
+                    removeOverlayMenu(page),
+                    new Promise((_, reject) => {
+                        setTimeout(() => {
+                            reject(new Error(`removeOverlayMenu timed out after ${OVERLAY_OPERATION_TIMEOUT_MS}ms`));
+                        }, OVERLAY_OPERATION_TIMEOUT_MS);
+                    }),
+                ]);
+                return;
+            }
+            const hasOverlay = await page.evaluate(() => Boolean(document.querySelector('#oobeeShadowHost')));
+            consoleLogger.info(`Overlay state (${trigger}): ${hasOverlay}`);
+            if (!hasOverlay) {
+                // Recreate the overlay after allowed redirects while preserving current UI state.
+                consoleLogger.info(`Adding overlay menu to page (${trigger}): ${page.url()}`);
+                await Promise.race([
+                    addOverlayMenu(page, processPageParams.urlsCrawled, menuPos, {
+                        inProgress: !!pagesDict[pageId]?.isScanning,
+                        collapsed: !!pagesDict[pageId]?.collapsed,
+                        hideStopInput: !!processPageParams.customFlowLabel,
+                    }),
+                    new Promise((_, reject) => {
+                        setTimeout(() => {
+                            reject(new Error(`addOverlayMenu timed out after ${OVERLAY_OPERATION_TIMEOUT_MS}ms`));
+                        }, OVERLAY_OPERATION_TIMEOUT_MS);
+                    }),
+                ]);
+            }
+        })
+            .catch(() => {
+            consoleLogger.info('Error in adding overlay menu to page');
+        });
+        await overlayRefreshChain;
+    };
     // Window functions exposed in browser
     const handleOnScanClick = async () => {
         consoleLogger.info('Scan: click detected');
@@ -1044,17 +1110,7 @@ export const initNewPage = async (page, pageClosePromises, processPageParams, pa
             pagesDict[pageId].isScanning = false;
             if (page.isClosed())
                 return;
-            const allowed = isOverlayAllowed(page.url(), processPageParams.entryUrl);
-            if (allowed) {
-                await addOverlayMenu(page, processPageParams.urlsCrawled, menuPos, {
-                    inProgress: false,
-                    collapsed: !!pagesDict[pageId]?.collapsed,
-                    hideStopInput: !!processPageParams.customFlowLabel,
-                });
-            }
-            else {
-                await removeOverlayMenu(page);
-            }
+            await reconcileOverlayMenu('scan-click');
         }
         catch (error) {
             log(`Scan failed ${error}`);
@@ -1118,37 +1174,21 @@ export const initNewPage = async (page, pageClosePromises, processPageParams, pa
     page.on('domcontentloaded', async () => {
         if (page.isClosed())
             return;
-        try {
-            const allowed = isOverlayAllowed(page.url(), processPageParams.entryUrl);
-            if (!allowed) {
-                await removeOverlayMenu(page);
-                return;
-            }
-            const existingOverlay = await page.evaluate(() => {
-                return document.querySelector('#oobeeShadowHost');
-            });
-            consoleLogger.info(`Overlay state: ${existingOverlay}`);
-            if (!existingOverlay) {
-                consoleLogger.info(`Adding overlay menu to page: ${page.url()}`);
-                await addOverlayMenu(page, processPageParams.urlsCrawled, menuPos, {
-                    inProgress: !!pagesDict[pageId]?.isScanning,
-                    collapsed: !!pagesDict[pageId]?.collapsed,
-                    hideStopInput: !!processPageParams.customFlowLabel,
-                });
+        await reconcileOverlayMenu('domcontentloaded');
+        if (isCypressTest) {
+            try {
+                await handleOnScanClick();
+                page.close();
             }
-            if (isCypressTest) {
-                try {
-                    await handleOnScanClick();
-                    page.close();
-                }
-                catch {
-                    consoleLogger.info(`Error in calling handleOnScanClick, isCypressTest: ${isCypressTest}`);
-                }
+            catch {
+                consoleLogger.info(`Error in calling handleOnScanClick, isCypressTest: ${isCypressTest}`);
             }
         }
-        catch {
-            consoleLogger.info('Error in adding overlay menu to page');
-        }
+    });
+    page.on('framenavigated', async (frame) => {
+        if (frame !== page.mainFrame() || page.isClosed())
+            return;
+        await reconcileOverlayMenu('framenavigated');
     });
     try {
         if (page.isClosed())
@@ -1167,5 +1207,6 @@ export const initNewPage = async (page, pageClosePromises, processPageParams, pa
     catch (e) {
         log(`Error exposing functions on page: ${e}`);
     }
+    await reconcileOverlayMenu('init');
     return page;
 };

package/dist/generateHtmlReport.js

@@ -1,7 +1,7 @@
 import fs from 'fs-extra';
 import path from 'path';
 import { compressJsonFileStreaming, writeHTML, flattenAndSortResults, populateScanPagesDetail, getWcagPassPercentage, getProgressPercentage, getIssuesPercentage, itemTypeDescription, oobeeAiHtmlETL, oobeeAiRules, formatAboutStartTime, convertItemsToReferences, } from './mergeAxeResults.js';
-import constants, { ScannerTypes, WCAGclauses, a11yRuleShortDescriptionMap, disabilityBadgesMap, a11yRuleLongDescriptionMap, } from './constants/constants.js';
+import constants, { ScannerTypes, WCAGclauses, a11yRuleShortDescriptionMap, disabilityBadgesMap, a11yRuleLongDescriptionMap, a11yRuleStepByStepGuide, } from './constants/constants.js';
 import { consoleLogger } from './logs.js';
 const ensureCategory = (categoryObj, categoryName) => {
     const rulesRaw = categoryObj?.rules ?? [];
@@ -23,7 +23,12 @@ const ensureCategory = (categoryObj, categoryName) => {
             rule.pagesAffected = [];
         }
         if (typeof rule.totalItems !== 'number') {
-            rule.totalItems = rule.pagesAffected.reduce((accumulate, page) => accumulate + (Array.isArray(page.items) ? page.items.length : 0), 0);
+            rule.totalItems = rule.pagesAffected.reduce((accumulate, page) => accumulate +
+                (Array.isArray(page.items)
+                    ? page.items.length
+                    : typeof page.itemsCount === 'number'
+                        ? page.itemsCount
+                        : 0), 0);
         }
     });
     const totals = {
@@ -38,7 +43,7 @@ const ensureCategory = (categoryObj, categoryName) => {
         rules,
     };
 };
-export const generateHtmlReport = async (resultDir) => {
+export const generateHtmlReport = async (resultDir, htmlFilename = 'report') => {
     try {
         const storagePath = path.resolve(resultDir);
         const scanDataJsonPath = path.join(storagePath, 'scanData.json');
@@ -61,17 +66,16 @@ export const generateHtmlReport = async (resultDir) => {
         }
         const scanData = JSON.parse(await fs.readFile(scanDataJsonPath, 'utf8'));
         const scanItemsAll = JSON.parse(await fs.readFile(scanItemsJsonPath, 'utf8'));
-        // Use convertItemsToReferences to normalize items structure to match scanItemsWithHtmlGroupRefs format
-        const scanItemsWithHtmlGroupRefs = convertItemsToReferences({
+        // Build the lighter scanItems payload used by the HTML report.
+        const lightScanItemsPayload = convertItemsToReferences({
             items: scanItemsAll,
-            ...scanData
         });
-        const { mustFix = {}, goodToFix = {}, needsReview = {}, passed = {}, } = scanItemsWithHtmlGroupRefs;
+        const { mustFix = {}, goodToFix = {}, needsReview = {}, } = lightScanItemsPayload;
         const items = {
             mustFix: ensureCategory(mustFix, 'mustFix'),
             goodToFix: ensureCategory(goodToFix, 'goodToFix'),
             needsReview: ensureCategory(needsReview, 'needsReview'),
-            passed: ensureCategory(passed, 'passed'),
+            passed: ensureCategory(scanItemsAll.passed || {}, 'passed'),
         };
         const pagesScanned = Array.isArray(scanData.pagesScanned) ? scanData.pagesScanned : [];
         const pagesNotScanned = Array.isArray(scanData.pagesNotScanned) ? scanData.pagesNotScanned : [];
@@ -116,6 +120,8 @@ export const generateHtmlReport = async (resultDir) => {
             a11yRuleShortDescriptionMap,
             disabilityBadgesMap,
             a11yRuleLongDescriptionMap,
+            a11yRuleStepByStepGuide,
+            wcagCriteriaLabels: constants.wcagCriteriaLabels,
             advancedScanOptionsSummaryItems: {
                 showIncludeScreenshots: !!scanData.advancedScanOptionsSummaryItems?.showIncludeScreenshots,
                 showAllowSubdomains: !!scanData.advancedScanOptionsSummaryItems?.showAllowSubdomains,
@@ -137,9 +143,10 @@ export const generateHtmlReport = async (resultDir) => {
         allIssues.wcagPassPercentage = getWcagPassPercentage(allIssues.wcagViolations, allIssues.advancedScanOptionsSummaryItems.showEnableWcagAaa);
         allIssues.progressPercentage = getProgressPercentage(allIssues.scanPagesDetail, allIssues.advancedScanOptionsSummaryItems.showEnableWcagAaa);
         allIssues.issuesPercentage = await getIssuesPercentage(allIssues.scanPagesDetail, allIssues.advancedScanOptionsSummaryItems.showEnableWcagAaa, allIssues.advancedScanOptionsSummaryItems?.disableOobee);
-        await writeHTML(allIssues, storagePath, 'report', scanDataB64Path, scanItemsB64Path);
-        consoleLogger.info(`Report generated at: ${path.join(storagePath, 'report.html')}`);
-        return path.join(storagePath, 'report.html');
+        await writeHTML(allIssues, storagePath, htmlFilename, scanDataB64Path, scanItemsB64Path);
+        const outputPath = path.join(storagePath, `${htmlFilename}.html`);
+        consoleLogger.info(`Report generated at: ${outputPath}`);
+        return outputPath;
     }
     catch (err) {
         consoleLogger.error(`generateHtmlReport failed: ${err?.message || err}`);