@govtechsg/oobee 0.10.84 → 0.10.86

package/src/crawlers/custom/utils.ts

@@ -1,8 +1,8 @@
-/* eslint-disable no-shadow */
 /* eslint-disable no-alert */
 /* eslint-disable no-param-reassign */
 /* eslint-env browser */
 import path from 'path';
+import { getDomain } from 'tldts';
 import { runAxeScript } from '../commonCrawlerFunc.js';
 import { consoleLogger, guiInfoLog, silentLogger } from '../../logs.js';
 import { guiInfoStatusTypes } from '../../constants/constants.js';
@@ -19,6 +19,44 @@ declare global {
   }
 }
 
+const sameRegistrableDomain = (hostA: string, hostB: string) => {
+  const domainA = getDomain(hostA);
+  const domainB = getDomain(hostB);
+
+  if (!domainA || !domainB) return hostA === hostB;
+
+  return domainA === domainB;
+};
+
+const parseBoolEnv = (val: string | undefined, defaultVal: boolean) => {
+  if (val == null) return defaultVal;
+  const v = String(val).trim().toLowerCase();
+  if (['1', 'true', 'yes', 'y', 'on'].includes(v)) return true;
+  if (['0', 'false', 'no', 'n', 'off'].includes(v)) return false;
+  return defaultVal;
+};
+
+const RESTRICT_OVERLAY_TO_ENTRY_DOMAIN = parseBoolEnv(
+  process.env.RESTRICT_OVERLAY_TO_ENTRY_DOMAIN,
+  false,
+);
+
+const isOverlayAllowed = (currentUrl: string, entryUrl: string) => {
+  try {
+    const cur = new URL(currentUrl);
+
+    if (cur.protocol !== 'http:' && cur.protocol !== 'https:') return false;
+
+    if (!RESTRICT_OVERLAY_TO_ENTRY_DOMAIN) return true;
+
+    const base = new URL(entryUrl);
+
+    return sameRegistrableDomain(cur.hostname, base.hostname);
+  } catch {
+    return false;
+  }
+};
+
 //! For Cypress Test
 // env to check if Cypress test is running
 const isCypressTest = process.env.IS_CYPRESS_TEST === 'true';
@@ -62,20 +100,17 @@ export const screenshotFullPage = async (page, screenshotsDir: string, screensho
         window.scrollTo(0, document.body.scrollHeight);
       });
 
-      const isLoadMoreContent = async () =>
-        new Promise(resolve => {
-          setTimeout(async () => {
-            await page.waitForLoadState('domcontentloaded');
-
-            const newHeight = await page.evaluate(
-              // eslint-disable-next-line no-shadow
-              () => document.body.scrollHeight,
-            );
-            const result = newHeight > prevHeight;
-
-            resolve(result);
-          }, 2500);
-        });
+      const isLoadMoreContent = async () => {
+        await new Promise(resolve => setTimeout(resolve, 2500));
+        if (page.isClosed()) return false;
+        try {
+          await page.waitForLoadState('domcontentloaded');
+          const newHeight = await page.evaluate(() => document.body.scrollHeight);
+          return newHeight > prevHeight;
+        } catch {
+          return false;
+        }
+      };
 
       const result = await isLoadMoreContent();
       return result;
@@ -248,7 +283,7 @@ export const updateMenu = async (page, urlsCrawled) => {
   log(`Overlay menu: updating: ${page.url()}`);
   await page.evaluate(
     vars => {
-      const shadowHost = document.querySelector('#oobee-shadow-host');
+      const shadowHost = document.querySelector('#oobeeShadowHost');
       if (shadowHost) {
         const p = shadowHost.shadowRoot.querySelector('#oobee-p-pages-scanned');
         if (p) {
@@ -262,7 +297,6 @@ export const updateMenu = async (page, urlsCrawled) => {
   consoleLogger.info(`Overlay menu updated`);
 };
 
-
 export const addOverlayMenu = async (
   page,
   urlsCrawled,
@@ -307,7 +341,7 @@ export const addOverlayMenu = async (
         `;
         minBtn.innerHTML = MINBTN_SVG;
 
-        let currentPos: 'LEFT' | 'RIGHT' = (vars.menuPos || 'RIGHT');
+        let currentPos: 'LEFT' | 'RIGHT' = vars.menuPos || 'RIGHT';
         const isCollapsed = () => panel.classList.contains('collapsed');
 
         const setPosClass = (pos: 'LEFT' | 'RIGHT') => {
@@ -325,7 +359,7 @@ export const addOverlayMenu = async (
         };
 
         const toggleCollapsed = (force?: boolean) => {
-          const willCollapse = (typeof force === 'boolean') ? force : !isCollapsed();
+          const willCollapse = typeof force === 'boolean' ? force : !isCollapsed();
           if (willCollapse) {
             panel.classList.add('collapsed');
             localStorage.setItem('oobee:overlay-collapsed', '1');
@@ -375,25 +409,72 @@ export const addOverlayMenu = async (
         const h2 = document.createElement('h2');
         h2.id = 'oobeeHPagesScanned';
         h2.className = 'oobee-section-title';
-        h2.textContent = 'Pages Scanned';
-
+        h2.textContent = `Pages Scanned (${vars.urlsCrawled.scanned.length || 0})`;
+
+        const scanIcon = document.createElement('span');
+        scanIcon.className = 'oobee-btn-icon';
+
+        const SCAN_SVG = `
+          <svg xmlns="http://www.w3.org/2000/svg" width="20" height="20" viewBox="0 0 20 20" fill="none">
+            <g clip-path="url(#clip0_1421_431)">
+              <path d="M12.5763 11.5472L12.2958 11.2857L12.1037 11.1005C12.776 10.3183 12.9194 9.56432 12.9194 8.45969C12.9194 5.99657 10.9228 4 8.45969 4C5.99657 4 4 5.99657 4 8.45969C4 10.9228 5.99657 12.9194 8.45969 12.9194C9.56432 12.9194 10.3183 12.776 11.1005 12.1037L11.2857 12.2958L11.5472 12.5763L14.9777 16L16 14.9777L12.5763 11.5472ZM8.45969 11.5472C6.75129 11.5472 5.37221 10.1681 5.37221 8.45969C5.37221 6.75129 6.75129 5.37221 8.45969 5.37221C10.1681 5.37221 11.5472 6.75129 11.5472 8.45969C11.5472 10.1681 10.1681 11.5472 8.45969 11.5472Z" fill="white"/>
+              <path d="M18.5 0H19.5C19.7761 0 20 0.223858 20 0.5V5H18.5V0Z" fill="white"/>
+              <path d="M19.5 2.18552e-08L19.5 1.5L15 1.5L15 -2.18556e-07L19.5 2.18552e-08Z" fill="white"/>
+              <path d="M1.5 0H0.5C0.223858 0 0 0.223858 0 0.5V5H1.5V0Z" fill="white"/>
+              <path d="M0.5 2.18552e-08L0.5 1.5L5 1.5L5 -2.18556e-07L0.5 2.18552e-08Z" fill="white"/>
+              <path d="M1.5 20H0.5C0.223858 20 0 19.7761 0 19.5V15H1.5V20Z" fill="white"/>
+              <path d="M0.5 20L0.5 18.5L5 18.5L5 20L0.5 20Z" fill="white"/>
+              <path d="M18.5 20H19.5C19.7761 20 20 19.7761 20 19.5V15H18.5V20Z" fill="white"/>
+              <path d="M19.5 20L19.5 18.5L15 18.5L15 20L19.5 20Z" fill="white"/>
+            </g>
+            <defs>
+              <clipPath id="clip0_1421_431">
+                <rect width="20" height="20" fill="white"/>
+              </clipPath>
+            </defs>
+          </svg>
+        `;
+        
+        scanIcon.innerHTML = SCAN_SVG; 
         const scanBtn = document.createElement('button');
         scanBtn.id = 'oobeeBtnScan';
         scanBtn.className = 'oobee-btn oobee-btn-primary';
-        scanBtn.innerText = 'Scan this page';
         scanBtn.disabled = inProgress;
+        scanBtn.appendChild(scanIcon);
+
+        const scanText = document.createElement('span');
+        scanText.className = 'oobee-btn-text';
+        scanText.innerText = 'Scan page';
+        scanBtn.appendChild(scanText);
+
         scanBtn.addEventListener('click', async () => customWindow.handleOnScanClick?.());
 
-        const stopBtn = document.createElement('button');
-        stopBtn.id = 'oobeeBtnStop';
-        stopBtn.className = 'oobee-btn oobee-btn-secondary';
-        stopBtn.innerText = 'Stop scan';
-        stopBtn.addEventListener('click', async () => customWindow.handleOnStopClick?.());
+        const endScanIcon = document.createElement('span');
+        endScanIcon.className = 'oobee-btn-icon';
+
+        const ENDSCAN_SVG = 
+          `<svg xmlns="http://www.w3.org/2000/svg" width="20" height="20" viewBox="0 0 20 20" fill="none">
+            <path d="M10 0C4.47 0 0 4.47 0 10C0 15.53 4.47 20 10 20C15.53 20 20 15.53 20 10C20 4.47 15.53 0 10 0ZM10 18C5.59 18 2 14.41 2 10C2 5.59 5.59 2 10 2C14.41 2 18 5.59 18 10C18 14.41 14.41 18 10 18ZM13.59 5L10 8.59L6.41 5L5 6.41L8.59 10L5 13.59L6.41 15L10 11.41L13.59 15L15 13.59L11.41 10L15 6.41L13.59 5Z" fill="#9021A6"/>
+          </svg>
+        `;
+
+        endScanIcon.innerHTML = ENDSCAN_SVG;
+        const endScanBtn = document.createElement('button');
+        endScanBtn.id = 'oobeeBtnEndScan';
+        endScanBtn.className = 'oobee-btn oobee-btn-secondary';
+        endScanBtn.appendChild(endScanIcon);
+        
+        const endScanText = document.createElement('span');
+        endScanText.className = 'oobee-btn-text';
+        endScanText.innerText = 'End scan';
+        endScanBtn.appendChild(endScanText);
+
+        endScanBtn.addEventListener('click', async () => customWindow.handleOnStopClick?.());
 
         const btnGroup = document.createElement('div');
         btnGroup.className = 'oobee-actions';
         btnGroup.appendChild(scanBtn);
-        btnGroup.appendChild(stopBtn);
+        btnGroup.appendChild(endScanBtn);
 
         const listWrap = document.createElement('div');
         listWrap.id = 'oobeeList';
@@ -414,13 +495,13 @@ export const addOverlayMenu = async (
           const ol = document.createElement('ol');
           ol.className = 'oobee-ol';
 
-          scanned.forEach((item) => {
+          scanned.forEach(item => {
             const li = document.createElement('li');
             li.className = 'oobee-li';
 
             const title = document.createElement('div');
             title.className = 'oobee-item-title';
-            title.textContent = (item.pageTitle && item.pageTitle.trim()) ? item.pageTitle : item.url;
+            title.textContent = item.pageTitle && item.pageTitle.trim() ? item.pageTitle : item.url;
 
             const url = document.createElement('div');
             url.className = 'oobee-item-url';
@@ -469,7 +550,7 @@ export const addOverlayMenu = async (
             border-right: 1px solid rgba(0,0,0,.08)
           }
           .oobee-panel.collapsed {
-            width: 56px;
+            width: 58px;
             overflow: hidden
           }
 
@@ -546,6 +627,12 @@ export const addOverlayMenu = async (
             padding: 1rem;
           }
 
+          .oobee-panel.collapsed .oobee-actions {
+            display: flex;
+            justify-content: center;
+            padding: 1rem 0.7rem;
+          }
+
           /* Base button */
           .oobee-btn {
             width: 100%;
@@ -556,6 +643,10 @@ export const addOverlayMenu = async (
             line-height: 1.2;
             font-weight: 400;
             cursor: pointer;
+            display: flex;
+            align-items: center;
+            justify-content: center;
+            gap: 10px;
             transition: {
               box-shadow .12s ease,
               transform .02s ease,
@@ -569,6 +660,19 @@ export const addOverlayMenu = async (
             cursor:not-allowed
           }
 
+          .oobee-panel.collapsed .oobee-btn {
+            width: 44px !important;
+            height: 44px !important;
+            min-width: 44px !important;
+            min-height: 44px !important;
+            max-width: 44px !important;
+            max-height: 44px !important;
+            border-radius: 50% !important;
+            padding: 0 !important;
+            justify-content: center;
+            gap: 0;
+          }
+
           /* Primary (filled) */
           .oobee-btn-primary {
             background: #9021a6;
@@ -624,6 +728,25 @@ export const addOverlayMenu = async (
             display: none;
           }
 
+          .oobee-btn-icon {
+            display: inline-flex;
+            align-items: center;
+            justify-content: center;
+            width: 20px;
+            height: 20px;
+            vertical-align: middle;
+          }
+
+          .oobee-btn-text {
+            display: inline;
+            white-space: nowrap;
+            vertical-align: middle;
+          }
+          
+          .oobee-panel.collapsed .oobee-btn-text {
+            display: none;
+          }
+
           #oobeeStopOverlay[hidden] {
             display:none !important;
           }
@@ -641,7 +764,10 @@ export const addOverlayMenu = async (
           }
 
           .oobee-panel.collapsed .oobee-section-title {
-            display: none;
+            font-size: 14px;
+            display: flex;
+            justify-content: center;
+            text-align: center;
           }
 
           .oobee-ol {
@@ -730,8 +856,7 @@ export const addOverlayMenu = async (
 
           const closed = isCollapsed();
           const arrowPointsRight =
-            (currentPos === 'RIGHT' && !closed) ||
-            (currentPos === 'LEFT'  &&  closed);
+            (currentPos === 'RIGHT' && !closed) || (currentPos === 'LEFT' && closed);
 
           icon.classList.toggle('is-left', !arrowPointsRight);
           minBtn.setAttribute('aria-label', closed ? 'Expand panel' : 'Collapse panel');
@@ -761,11 +886,11 @@ export const addOverlayMenu = async (
 
         grip.addEventListener('pointerdown', (e: PointerEvent) => {
           startX = e.clientX;
-          grip.setPointerCapture(e.pointerId);       // <-- use the button
+          grip.setPointerCapture(e.pointerId); // <-- use the button
         });
 
         grip.addEventListener('pointermove', (e: PointerEvent) => {
-          if (!grip.hasPointerCapture?.(e.pointerId)) return;  // <-- check the button
+          if (!grip.hasPointerCapture?.(e.pointerId)) return; // <-- check the button
           const dx = e.clientX - startX;
           if (Math.abs(dx) >= THRESH) {
             const nextPos: 'LEFT' | 'RIGHT' = dx < 0 ? 'LEFT' : 'RIGHT';
@@ -779,7 +904,9 @@ export const addOverlayMenu = async (
         });
 
         grip.addEventListener('pointerup', (e: PointerEvent) => {
-          try { grip.releasePointerCapture(e.pointerId); } catch {}
+          try {
+            grip.releasePointerCapture(e.pointerId);
+          } catch {}
         });
 
         const stopDialog = document.createElement('dialog');
@@ -791,7 +918,7 @@ export const addOverlayMenu = async (
           borderRadius: '16px',
           overflow: 'hidden',
           boxShadow: '0 10px 40px rgba(0,0,0,.35)',
-          fontFamily: 'system-ui, -apple-system, Segoe UI, Roboto, Arial, sans-serif'
+          fontFamily: 'system-ui, -apple-system, Segoe UI, Roboto, Arial, sans-serif',
         });
         const dialogSheet = new CSSStyleSheet();
         dialogSheet.replaceSync(`
@@ -829,13 +956,18 @@ export const addOverlayMenu = async (
           display: 'flex',
           alignItems: 'center',
           justifyContent: 'space-between',
-          gap: '8px'
+          gap: '8px',
         });
 
         const title = document.createElement('h2');
         title.id = 'oobee-stop-title';
         title.textContent = 'Are you sure you want to stop this scan?';
-        Object.assign(title.style, { margin: '0', fontSize: '22px', fontWeight: '700', lineHeight: '1.25' });
+        Object.assign(title.style, {
+          margin: '0',
+          fontSize: '22px',
+          fontWeight: '700',
+          lineHeight: '1.25',
+        });
 
         const closeX = document.createElement('button');
         closeX.type = 'button';
@@ -853,14 +985,14 @@ export const addOverlayMenu = async (
           height: '36px',
           borderRadius: '12px',
           display: 'grid',
-          placeItems: 'center'
+          placeItems: 'center',
         });
         head.appendChild(title);
         head.appendChild(closeX);
 
         const bodyWrap = document.createElement('div');
         Object.assign(bodyWrap.style, {
-          padding: '12px 20px 20px 20px'
+          padding: '12px 20px 20px 20px',
         });
 
         const form = document.createElement('form');
@@ -869,7 +1001,7 @@ export const addOverlayMenu = async (
         Object.assign(form.style, {
           display: 'grid',
           gridTemplateColumns: '1fr',
-          rowGap: '12px'
+          rowGap: '12px',
         });
 
         const label = document.createElement('label');
@@ -887,7 +1019,7 @@ export const addOverlayMenu = async (
           padding: '12px 14px',
           fontSize: '14px',
           outline: 'none',
-          boxSizing: 'border-box'
+          boxSizing: 'border-box',
         });
         input.addEventListener('focus', () => {
           input.style.borderColor = '#7b4dff';
@@ -913,7 +1045,7 @@ export const addOverlayMenu = async (
           fontWeight: '600',
           color: '#fff',
           background: '#9021A6',
-          cursor: 'pointer'
+          cursor: 'pointer',
         });
 
         const cancel = document.createElement('button');
@@ -926,7 +1058,7 @@ export const addOverlayMenu = async (
           fontSize: '14px',
           justifySelf: 'center',
           cursor: 'pointer',
-          padding: '6px'
+          padding: '6px',
         });
 
         actions.appendChild(primary);
@@ -936,7 +1068,7 @@ export const addOverlayMenu = async (
           form.appendChild(label);
           form.appendChild(input);
         }
-         form.appendChild(actions);
+        form.appendChild(actions);
         bodyWrap.appendChild(form);
 
         stopDialog.appendChild(head);
@@ -944,17 +1076,27 @@ export const addOverlayMenu = async (
         shadowRoot.appendChild(stopDialog);
 
         let stopResolver: null | ((v: { confirmed: boolean; label: string }) => void) = null;
-        const hideStop = () => { try { stopDialog.close(); } catch {} stopResolver = null; };
+        const hideStop = () => {
+          try {
+            stopDialog.close();
+          } catch {}
+          stopResolver = null;
+        };
         const showStop = () => {
           if (!shouldHideInput) input.value = '';
-          try { stopDialog.showModal(); } catch {}
+          try {
+            stopDialog.showModal();
+          } catch {}
           if (!shouldHideInput) {
             requestAnimationFrame(() => {
-              try { input.focus({ preventScroll: true }); input.select(); } catch {}
+              try {
+                input.focus({ preventScroll: true });
+                input.select();
+              } catch {}
             });
           }
         };
-        form.addEventListener('submit', (e) => {
+        form.addEventListener('submit', e => {
           e.preventDefault();
           const v = (input.value || '').trim();
           if (stopResolver) stopResolver({ confirmed: true, label: v });
@@ -968,13 +1110,13 @@ export const addOverlayMenu = async (
           if (stopResolver) stopResolver({ confirmed: false, label: '' });
           hideStop();
         });
-        stopDialog.addEventListener('cancel', (e) => {
+        stopDialog.addEventListener('cancel', e => {
           e.preventDefault();
           if (stopResolver) stopResolver({ confirmed: false, label: '' });
           hideStop();
         });
         (customWindow as Window).oobeeShowStopModal = () =>
-          new Promise<{ confirmed: boolean; label: string }>((resolve) => {
+          new Promise<{ confirmed: boolean; label: string }>(resolve => {
             stopResolver = resolve;
             showStop();
           });
@@ -1000,7 +1142,7 @@ export const addOverlayMenu = async (
       log('Overlay menu: successfully added');
     })
     .catch(error => {
-      error('Overlay menu: failed to add', error);
+      consoleLogger.error('Overlay menu: failed to add', error);
     });
 };
 
@@ -1027,7 +1169,13 @@ export const initNewPage = async (page, pageClosePromises, processPageParams, pa
   // eslint-disable-next-line no-underscore-dangle
   const pageId = page._guid;
 
-  page.on('dialog', () => { });
+  page.on('dialog', async dialog => {
+    try {
+      await dialog.dismiss();
+    } catch {
+      // dialog may already be closed
+    }
+  });
 
   const pageClosePromise = new Promise(resolve => {
     page.on('close', () => {
@@ -1058,11 +1206,19 @@ export const initNewPage = async (page, pageClosePromises, processPageParams, pa
       await processPage(page, processPageParams);
       log('Scan: success');
       pagesDict[pageId].isScanning = false;
-       await addOverlayMenu(page, processPageParams.urlsCrawled, menuPos, {
-         inProgress: false,
-         collapsed: !!pagesDict[pageId]?.collapsed,
-         hideStopInput: !!processPageParams.customFlowLabel,
-       });
+
+      if (page.isClosed()) return;
+      const allowed = isOverlayAllowed(page.url(), processPageParams.entryUrl);
+
+      if (allowed) {
+        await addOverlayMenu(page, processPageParams.urlsCrawled, menuPos, {
+          inProgress: false,
+          collapsed: !!pagesDict[pageId]?.collapsed,
+          hideStopInput: !!processPageParams.customFlowLabel,
+        });
+      } else {
+        await removeOverlayMenu(page);
+      }
     } catch (error) {
       log(`Scan failed ${error}`);
     }
@@ -1093,10 +1249,10 @@ export const initNewPage = async (page, pageClosePromises, processPageParams, pa
 
       if (!inputValue?.confirmed) {
         await page.evaluate(() => {
-          const stopBtn = document.getElementById('oobeeBtnStop') as HTMLButtonElement | null;
-          if (stopBtn) {
-            stopBtn.disabled = false;
-            stopBtn.textContent = 'Stop';
+          const endScanBtn = document.getElementById('oobeeBtnEndScan') as HTMLButtonElement | null;
+          if (endScanBtn) {
+            endScanBtn.disabled = false;
+            endScanBtn.textContent = 'Stop';
           }
         });
         return;
@@ -1125,7 +1281,15 @@ export const initNewPage = async (page, pageClosePromises, processPageParams, pa
   };
 
   page.on('domcontentloaded', async () => {
+    if (page.isClosed()) return;
     try {
+      const allowed = isOverlayAllowed(page.url(), processPageParams.entryUrl);
+
+      if (!allowed) {
+        await removeOverlayMenu(page);
+        return;
+      }
+
       const existingOverlay = await page.evaluate(() => {
         return document.querySelector('#oobeeShadowHost');
       });
@@ -1141,12 +1305,6 @@ export const initNewPage = async (page, pageClosePromises, processPageParams, pa
         });
       }
 
-      setTimeout(() => {
-        // Timeout here to slow things down a little
-      }, 1000);
-
-      //! For Cypress Test
-      // Auto-clicks 'Scan this page' button only once
       if (isCypressTest) {
         try {
           await handleOnScanClick();
@@ -1155,27 +1313,29 @@ export const initNewPage = async (page, pageClosePromises, processPageParams, pa
           consoleLogger.info(`Error in calling handleOnScanClick, isCypressTest: ${isCypressTest}`);
         }
       }
-
-      consoleLogger.info(`Overlay state: ${existingOverlay}`);
     } catch {
       consoleLogger.info('Error in adding overlay menu to page');
-      consoleLogger.info('Error in adding overlay menu to page');
     }
   });
 
-  await page.exposeFunction('handleOnScanClick', handleOnScanClick);
-  await page.exposeFunction('handleOnStopClick', handleOnStopClick);
+  try {
+    if (page.isClosed()) return page;
+    await page.exposeFunction('handleOnScanClick', handleOnScanClick);
+    await page.exposeFunction('handleOnStopClick', handleOnStopClick);
 
-  type UpdateMenuPosFunction = (newPos: any) => void;
+    type UpdateMenuPosFunction = (newPos: any) => void;
 
-  // Define the updateMenuPos function
-  const updateMenuPos: UpdateMenuPosFunction = newPos => {
-    const prevPos = menuPos;
-    if (prevPos !== newPos) {
-      menuPos = newPos;
-    }
-  };
-  await page.exposeFunction('updateMenuPos', updateMenuPos);
+    // Define the updateMenuPos function
+    const updateMenuPos: UpdateMenuPosFunction = newPos => {
+      const prevPos = menuPos;
+      if (prevPos !== newPos) {
+        menuPos = newPos;
+      }
+    };
+    await page.exposeFunction('updateMenuPos', updateMenuPos);
+  } catch (e) {
+    log(`Error exposing functions on page: ${e}`);
+  }
 
   return page;
 };