@govtechsg/oobee 0.10.83 → 0.10.85

package/dist/crawlers/crawlIntelligentSitemap.js

@@ -22,13 +22,25 @@ const crawlIntelligentSitemap = async (url, randomToken, host, viewportSettings,
     async function findSitemap(link, userDataDirectory, extraHTTPHeaders) {
         const homeUrl = getHomeUrl(link);
         let sitemapLink = '';
-        const effectiveUserDataDirectory = process.env.CRAWLEE_HEADLESS === '1' ? userDataDirectory : '';
-        const context = await constants.launcher.launchPersistentContext(effectiveUserDataDirectory, {
-            headless: process.env.CRAWLEE_HEADLESS === '1',
-            ...getPlaywrightLaunchOptions(browser),
-            ...(extraHTTPHeaders && { extraHTTPHeaders }),
-        });
-        register(context);
+        const launchOptions = getPlaywrightLaunchOptions(browser);
+        let context;
+        let browserInstance;
+        if (process.env.CRAWLEE_HEADLESS === '1') {
+            const effectiveUserDataDirectory = userDataDirectory || '';
+            context = await constants.launcher.launchPersistentContext(effectiveUserDataDirectory, {
+                ...launchOptions,
+                ...(extraHTTPHeaders && { extraHTTPHeaders }),
+            });
+            register(context);
+        }
+        else {
+            // In headful mode, avoid launchPersistentContext to prevent "Browser window not found"
+            browserInstance = await constants.launcher.launch(launchOptions);
+            register(browserInstance);
+            context = await browserInstance.newContext({
+                ...(extraHTTPHeaders && { extraHTTPHeaders }),
+            });
+        }
         const page = await context.newPage();
         for (const path of sitemapPaths) {
             sitemapLink = homeUrl + path;
@@ -39,6 +51,9 @@ const crawlIntelligentSitemap = async (url, randomToken, host, viewportSettings,
         }
         await page.close();
         await context.close().catch(() => { });
+        if (browserInstance) {
+            await browserInstance.close().catch(() => { });
+        }
         return sitemapExist ? sitemapLink : '';
     }
     const checkUrlExists = async (page, parsedUrl) => {

package/dist/crawlers/custom/utils.js

@@ -1,12 +1,44 @@
-/* eslint-disable no-shadow */
 /* eslint-disable no-alert */
 /* eslint-disable no-param-reassign */
 /* eslint-env browser */
 import path from 'path';
+import { getDomain } from 'tldts';
 import { runAxeScript } from '../commonCrawlerFunc.js';
 import { consoleLogger, guiInfoLog } from '../../logs.js';
 import { guiInfoStatusTypes } from '../../constants/constants.js';
 import { isSkippedUrl, validateCustomFlowLabel } from '../../constants/common.js';
+const sameRegistrableDomain = (hostA, hostB) => {
+    const domainA = getDomain(hostA);
+    const domainB = getDomain(hostB);
+    if (!domainA || !domainB)
+        return hostA === hostB;
+    return domainA === domainB;
+};
+const parseBoolEnv = (val, defaultVal) => {
+    if (val == null)
+        return defaultVal;
+    const v = String(val).trim().toLowerCase();
+    if (['1', 'true', 'yes', 'y', 'on'].includes(v))
+        return true;
+    if (['0', 'false', 'no', 'n', 'off'].includes(v))
+        return false;
+    return defaultVal;
+};
+const RESTRICT_OVERLAY_TO_ENTRY_DOMAIN = parseBoolEnv(process.env.RESTRICT_OVERLAY_TO_ENTRY_DOMAIN, false);
+const isOverlayAllowed = (currentUrl, entryUrl) => {
+    try {
+        const cur = new URL(currentUrl);
+        if (cur.protocol !== 'http:' && cur.protocol !== 'https:')
+            return false;
+        if (!RESTRICT_OVERLAY_TO_ENTRY_DOMAIN)
+            return true;
+        const base = new URL(entryUrl);
+        return sameRegistrableDomain(cur.hostname, base.hostname);
+    }
+    catch {
+        return false;
+    }
+};
 //! For Cypress Test
 // env to check if Cypress test is running
 const isCypressTest = process.env.IS_CYPRESS_TEST === 'true';
@@ -33,9 +65,7 @@ export const screenshotFullPage = async (page, screenshotsDir, screenshotIdx) =>
             const isLoadMoreContent = async () => new Promise(resolve => {
                 setTimeout(async () => {
                     await page.waitForLoadState('domcontentloaded');
-                    const newHeight = await page.evaluate(
-                    // eslint-disable-next-line no-shadow
-                    () => document.body.scrollHeight);
+                    const newHeight = await page.evaluate(() => document.body.scrollHeight);
                     const result = newHeight > prevHeight;
                     resolve(result);
                 }, 2500);
@@ -157,7 +187,7 @@ export const MENU_POSITION = {
 export const updateMenu = async (page, urlsCrawled) => {
     log(`Overlay menu: updating: ${page.url()}`);
     await page.evaluate(vars => {
-        const shadowHost = document.querySelector('#oobee-shadow-host');
+        const shadowHost = document.querySelector('#oobeeShadowHost');
         if (shadowHost) {
             const p = shadowHost.shadowRoot.querySelector('#oobee-p-pages-scanned');
             if (p) {
@@ -200,7 +230,7 @@ export const addOverlayMenu = async (page, urlsCrawled, menuPos, opts = {
           </svg>
         `;
         minBtn.innerHTML = MINBTN_SVG;
-        let currentPos = (vars.menuPos || 'RIGHT');
+        let currentPos = vars.menuPos || 'RIGHT';
         const isCollapsed = () => panel.classList.contains('collapsed');
         const setPosClass = (pos) => {
             panel.classList.remove('pos-left', 'pos-right');
@@ -217,7 +247,7 @@ export const addOverlayMenu = async (page, urlsCrawled, menuPos, opts = {
             setDraggableSidebarMenu();
         };
         const toggleCollapsed = (force) => {
-            const willCollapse = (typeof force === 'boolean') ? force : !isCollapsed();
+            const willCollapse = typeof force === 'boolean' ? force : !isCollapsed();
             if (willCollapse) {
                 panel.classList.add('collapsed');
                 localStorage.setItem('oobee:overlay-collapsed', '1');
@@ -292,12 +322,12 @@ export const addOverlayMenu = async (page, urlsCrawled, menuPos, opts = {
             }
             const ol = document.createElement('ol');
             ol.className = 'oobee-ol';
-            scanned.forEach((item) => {
+            scanned.forEach(item => {
                 const li = document.createElement('li');
                 li.className = 'oobee-li';
                 const title = document.createElement('div');
                 title.className = 'oobee-item-title';
-                title.textContent = (item.pageTitle && item.pageTitle.trim()) ? item.pageTitle : item.url;
+                title.textContent = item.pageTitle && item.pageTitle.trim() ? item.pageTitle : item.url;
                 const url = document.createElement('div');
                 url.className = 'oobee-item-url';
                 url.textContent = item.url;
@@ -596,8 +626,7 @@ export const addOverlayMenu = async (page, urlsCrawled, menuPos, opts = {
             if (!icon)
                 return;
             const closed = isCollapsed();
-            const arrowPointsRight = (currentPos === 'RIGHT' && !closed) ||
-                (currentPos === 'LEFT' && closed);
+            const arrowPointsRight = (currentPos === 'RIGHT' && !closed) || (currentPos === 'LEFT' && closed);
             icon.classList.toggle('is-left', !arrowPointsRight);
             minBtn.setAttribute('aria-label', closed ? 'Expand panel' : 'Collapse panel');
         }
@@ -652,7 +681,7 @@ export const addOverlayMenu = async (page, urlsCrawled, menuPos, opts = {
             borderRadius: '16px',
             overflow: 'hidden',
             boxShadow: '0 10px 40px rgba(0,0,0,.35)',
-            fontFamily: 'system-ui, -apple-system, Segoe UI, Roboto, Arial, sans-serif'
+            fontFamily: 'system-ui, -apple-system, Segoe UI, Roboto, Arial, sans-serif',
         });
         const dialogSheet = new CSSStyleSheet();
         dialogSheet.replaceSync(`
@@ -689,12 +718,17 @@ export const addOverlayMenu = async (page, urlsCrawled, menuPos, opts = {
             display: 'flex',
             alignItems: 'center',
             justifyContent: 'space-between',
-            gap: '8px'
+            gap: '8px',
         });
         const title = document.createElement('h2');
         title.id = 'oobee-stop-title';
         title.textContent = 'Are you sure you want to stop this scan?';
-        Object.assign(title.style, { margin: '0', fontSize: '22px', fontWeight: '700', lineHeight: '1.25' });
+        Object.assign(title.style, {
+            margin: '0',
+            fontSize: '22px',
+            fontWeight: '700',
+            lineHeight: '1.25',
+        });
         const closeX = document.createElement('button');
         closeX.type = 'button';
         closeX.setAttribute('aria-label', 'Close');
@@ -711,13 +745,13 @@ export const addOverlayMenu = async (page, urlsCrawled, menuPos, opts = {
             height: '36px',
             borderRadius: '12px',
             display: 'grid',
-            placeItems: 'center'
+            placeItems: 'center',
         });
         head.appendChild(title);
         head.appendChild(closeX);
         const bodyWrap = document.createElement('div');
         Object.assign(bodyWrap.style, {
-            padding: '12px 20px 20px 20px'
+            padding: '12px 20px 20px 20px',
         });
         const form = document.createElement('form');
         form.noValidate = true;
@@ -725,7 +759,7 @@ export const addOverlayMenu = async (page, urlsCrawled, menuPos, opts = {
         Object.assign(form.style, {
             display: 'grid',
             gridTemplateColumns: '1fr',
-            rowGap: '12px'
+            rowGap: '12px',
         });
         const label = document.createElement('label');
         label.setAttribute('for', 'oobee-stop-input');
@@ -741,7 +775,7 @@ export const addOverlayMenu = async (page, urlsCrawled, menuPos, opts = {
             padding: '12px 14px',
             fontSize: '14px',
             outline: 'none',
-            boxSizing: 'border-box'
+            boxSizing: 'border-box',
         });
         input.addEventListener('focus', () => {
             input.style.borderColor = '#7b4dff';
@@ -765,7 +799,7 @@ export const addOverlayMenu = async (page, urlsCrawled, menuPos, opts = {
             fontWeight: '600',
             color: '#fff',
             background: '#9021A6',
-            cursor: 'pointer'
+            cursor: 'pointer',
         });
         const cancel = document.createElement('button');
         cancel.type = 'button';
@@ -777,7 +811,7 @@ export const addOverlayMenu = async (page, urlsCrawled, menuPos, opts = {
             fontSize: '14px',
             justifySelf: 'center',
             cursor: 'pointer',
-            padding: '6px'
+            padding: '6px',
         });
         actions.appendChild(primary);
         actions.appendChild(cancel);
@@ -792,10 +826,13 @@ export const addOverlayMenu = async (page, urlsCrawled, menuPos, opts = {
         stopDialog.appendChild(bodyWrap);
         shadowRoot.appendChild(stopDialog);
         let stopResolver = null;
-        const hideStop = () => { try {
-            stopDialog.close();
-        }
-        catch { } stopResolver = null; };
+        const hideStop = () => {
+            try {
+                stopDialog.close();
+            }
+            catch { }
+            stopResolver = null;
+        };
         const showStop = () => {
             if (!shouldHideInput)
                 input.value = '';
@@ -813,7 +850,7 @@ export const addOverlayMenu = async (page, urlsCrawled, menuPos, opts = {
                 });
             }
         };
-        form.addEventListener('submit', (e) => {
+        form.addEventListener('submit', e => {
             e.preventDefault();
             const v = (input.value || '').trim();
             if (stopResolver)
@@ -830,13 +867,13 @@ export const addOverlayMenu = async (page, urlsCrawled, menuPos, opts = {
                 stopResolver({ confirmed: false, label: '' });
             hideStop();
         });
-        stopDialog.addEventListener('cancel', (e) => {
+        stopDialog.addEventListener('cancel', e => {
             e.preventDefault();
             if (stopResolver)
                 stopResolver({ confirmed: false, label: '' });
             hideStop();
         });
-        customWindow.oobeeShowStopModal = () => new Promise((resolve) => {
+        customWindow.oobeeShowStopModal = () => new Promise(resolve => {
             stopResolver = resolve;
             showStop();
         });
@@ -861,7 +898,7 @@ export const addOverlayMenu = async (page, urlsCrawled, menuPos, opts = {
         log('Overlay menu: successfully added');
     })
         .catch(error => {
-        error('Overlay menu: failed to add', error);
+        consoleLogger.error('Overlay menu: failed to add', error);
     });
 };
 export const removeOverlayMenu = async (page) => {
@@ -910,11 +947,17 @@ export const initNewPage = async (page, pageClosePromises, processPageParams, pa
             await processPage(page, processPageParams);
             log('Scan: success');
             pagesDict[pageId].isScanning = false;
-            await addOverlayMenu(page, processPageParams.urlsCrawled, menuPos, {
-                inProgress: false,
-                collapsed: !!pagesDict[pageId]?.collapsed,
-                hideStopInput: !!processPageParams.customFlowLabel,
-            });
+            const allowed = isOverlayAllowed(page.url(), processPageParams.entryUrl);
+            if (allowed) {
+                await addOverlayMenu(page, processPageParams.urlsCrawled, menuPos, {
+                    inProgress: false,
+                    collapsed: !!pagesDict[pageId]?.collapsed,
+                    hideStopInput: !!processPageParams.customFlowLabel,
+                });
+            }
+            else {
+                await removeOverlayMenu(page);
+            }
         }
         catch (error) {
             log(`Scan failed ${error}`);
@@ -977,6 +1020,11 @@ export const initNewPage = async (page, pageClosePromises, processPageParams, pa
     };
     page.on('domcontentloaded', async () => {
         try {
+            const allowed = isOverlayAllowed(page.url(), processPageParams.entryUrl);
+            if (!allowed) {
+                await removeOverlayMenu(page);
+                return;
+            }
             const existingOverlay = await page.evaluate(() => {
                 return document.querySelector('#oobeeShadowHost');
             });
@@ -989,11 +1037,6 @@ export const initNewPage = async (page, pageClosePromises, processPageParams, pa
                     hideStopInput: !!processPageParams.customFlowLabel,
                 });
             }
-            setTimeout(() => {
-                // Timeout here to slow things down a little
-            }, 1000);
-            //! For Cypress Test
-            // Auto-clicks 'Scan this page' button only once
             if (isCypressTest) {
                 try {
                     await handleOnScanClick();
@@ -1003,11 +1046,9 @@ export const initNewPage = async (page, pageClosePromises, processPageParams, pa
                     consoleLogger.info(`Error in calling handleOnScanClick, isCypressTest: ${isCypressTest}`);
                 }
             }
-            consoleLogger.info(`Overlay state: ${existingOverlay}`);
         }
         catch {
             consoleLogger.info('Error in adding overlay menu to page');
-            consoleLogger.info('Error in adding overlay menu to page');
         }
     });
     await page.exposeFunction('handleOnScanClick', handleOnScanClick);

package/dist/crawlers/runCustom.js

@@ -6,6 +6,7 @@ import constants, { getIntermediateScreenshotsPath, guiInfoStatusTypes, } from '
 import { initNewPage, log } from './custom/utils.js';
 import { guiInfoLog } from '../logs.js';
 import { addUrlGuardScript } from './guards/urlGuard.js';
+import { getPlaywrightLaunchOptions } from '../constants/common.js';
 // Export of classes
 export class ProcessPageParams {
     constructor(scannedIdx, blacklistedPatterns, includeScreenshots, dataset, intermediateScreenshotsPath, urlsCrawled, randomToken) {
@@ -26,6 +27,7 @@ const runCustom = async (url, randomToken, viewportSettings, blacklistedPatterns
     const intermediateScreenshotsPath = getIntermediateScreenshotsPath(randomToken);
     const processPageParams = new ProcessPageParams(0, // scannedIdx
     blacklistedPatterns, includeScreenshots, dataset, intermediateScreenshotsPath, urlsCrawled, randomToken);
+    processPageParams.entryUrl = url;
     if (initialCustomFlowLabel && initialCustomFlowLabel.trim()) {
         processPageParams.customFlowLabel = initialCustomFlowLabel.trim();
     }
@@ -34,11 +36,18 @@ const runCustom = async (url, randomToken, viewportSettings, blacklistedPatterns
     try {
         const deviceConfig = viewportSettings.playwrightDeviceDetailsObject;
         const hasCustomViewport = !!deviceConfig;
+        const baseLaunchOptions = getPlaywrightLaunchOptions();
+        // Merge base args with custom flow specific args
+        const baseArgs = baseLaunchOptions.args || [];
+        const customArgs = hasCustomViewport ? ['--window-size=1920,1040'] : ['--start-maximized'];
+        const mergedArgs = [
+            ...baseArgs.filter(a => !a.startsWith('--window-size') && a !== '--start-maximized'),
+            ...customArgs,
+        ];
         const browser = await chromium.launch({
-            args: hasCustomViewport ? ['--window-size=1920,1040'] : ['--start-maximized'],
+            ...baseLaunchOptions,
+            args: mergedArgs,
             headless: false,
-            channel: 'chrome',
-            // bypassCSP: true,
         });
         const context = await browser.newContext({
             ignoreHTTPSErrors: true,
@@ -52,8 +61,7 @@ const runCustom = async (url, randomToken, viewportSettings, blacklistedPatterns
                 await context.close().catch(() => { });
                 await browser.close().catch(() => { });
             }
-            catch {
-            }
+            catch { }
         };
         // For handling closing playwright browser and continue generate artifacts etc
         registerSoftClose(processPageParams.stopAll);

package/dist/mergeAxeResults/itemReferences.js

@@ -0,0 +1,55 @@
+/**
+ * Builds pre-computed HTML groups to optimize Group by HTML Element functionality.
+ * Keys are composite "html\x00xpath" strings to ensure unique matching per element instance.
+ */
+export const buildHtmlGroups = (rule, items, pageUrl) => {
+    if (!rule.htmlGroups) {
+        rule.htmlGroups = {};
+    }
+    items.forEach(item => {
+        // Use composite key of html + xpath for precise matching
+        const htmlKey = `${item.html || 'No HTML element'}\x00${item.xpath || ''}`;
+        if (!rule.htmlGroups[htmlKey]) {
+            // Create new group with the first occurrence
+            rule.htmlGroups[htmlKey] = {
+                html: item.html || '',
+                xpath: item.xpath || '',
+                message: item.message || '',
+                screenshotPath: item.screenshotPath || '',
+                displayNeedsReview: item.displayNeedsReview,
+                pageUrls: [],
+            };
+        }
+        if (!rule.htmlGroups[htmlKey].pageUrls.includes(pageUrl)) {
+            rule.htmlGroups[htmlKey].pageUrls.push(pageUrl);
+        }
+    });
+};
+/**
+ * Converts items in pagesAffected to references (html\x00xpath composite keys) for embedding in HTML report.
+ * Additionally, it deep-clones allIssues, replaces page.items objects with composite reference keys.
+ * Those refs are specifically for htmlGroups lookup (html + xpath).
+ */
+export const convertItemsToReferences = (allIssues) => {
+    const cloned = JSON.parse(JSON.stringify(allIssues));
+    ['mustFix', 'goodToFix', 'needsReview', 'passed'].forEach(category => {
+        if (!cloned.items[category]?.rules)
+            return;
+        cloned.items[category].rules.forEach((rule) => {
+            if (!rule.pagesAffected || !rule.htmlGroups)
+                return;
+            rule.pagesAffected.forEach((page) => {
+                if (!page.items)
+                    return;
+                page.items = page.items.map((item) => {
+                    if (typeof item === 'string')
+                        return item; // Already a reference
+                    // Use composite key matching buildHtmlGroups
+                    const htmlKey = `${item.html || 'No HTML element'}\x00${item.xpath || ''}`;
+                    return htmlKey;
+                });
+            });
+        });
+    });
+    return cloned;
+};