@govtechsg/oobee 0.10.84 → 0.10.86

package/dist/crawlers/custom/utils.js

@@ -1,12 +1,44 @@
-/* eslint-disable no-shadow */
 /* eslint-disable no-alert */
 /* eslint-disable no-param-reassign */
 /* eslint-env browser */
 import path from 'path';
+import { getDomain } from 'tldts';
 import { runAxeScript } from '../commonCrawlerFunc.js';
 import { consoleLogger, guiInfoLog } from '../../logs.js';
 import { guiInfoStatusTypes } from '../../constants/constants.js';
 import { isSkippedUrl, validateCustomFlowLabel } from '../../constants/common.js';
+const sameRegistrableDomain = (hostA, hostB) => {
+    const domainA = getDomain(hostA);
+    const domainB = getDomain(hostB);
+    if (!domainA || !domainB)
+        return hostA === hostB;
+    return domainA === domainB;
+};
+const parseBoolEnv = (val, defaultVal) => {
+    if (val == null)
+        return defaultVal;
+    const v = String(val).trim().toLowerCase();
+    if (['1', 'true', 'yes', 'y', 'on'].includes(v))
+        return true;
+    if (['0', 'false', 'no', 'n', 'off'].includes(v))
+        return false;
+    return defaultVal;
+};
+const RESTRICT_OVERLAY_TO_ENTRY_DOMAIN = parseBoolEnv(process.env.RESTRICT_OVERLAY_TO_ENTRY_DOMAIN, false);
+const isOverlayAllowed = (currentUrl, entryUrl) => {
+    try {
+        const cur = new URL(currentUrl);
+        if (cur.protocol !== 'http:' && cur.protocol !== 'https:')
+            return false;
+        if (!RESTRICT_OVERLAY_TO_ENTRY_DOMAIN)
+            return true;
+        const base = new URL(entryUrl);
+        return sameRegistrableDomain(cur.hostname, base.hostname);
+    }
+    catch {
+        return false;
+    }
+};
 //! For Cypress Test
 // env to check if Cypress test is running
 const isCypressTest = process.env.IS_CYPRESS_TEST === 'true';
@@ -30,16 +62,19 @@ export const screenshotFullPage = async (page, screenshotsDir, screenshotIdx) =>
             await page.evaluate(() => {
                 window.scrollTo(0, document.body.scrollHeight);
             });
-            const isLoadMoreContent = async () => new Promise(resolve => {
-                setTimeout(async () => {
+            const isLoadMoreContent = async () => {
+                await new Promise(resolve => setTimeout(resolve, 2500));
+                if (page.isClosed())
+                    return false;
+                try {
                     await page.waitForLoadState('domcontentloaded');
-                    const newHeight = await page.evaluate(
-                    // eslint-disable-next-line no-shadow
-                    () => document.body.scrollHeight);
-                    const result = newHeight > prevHeight;
-                    resolve(result);
-                }, 2500);
-            });
+                    const newHeight = await page.evaluate(() => document.body.scrollHeight);
+                    return newHeight > prevHeight;
+                }
+                catch {
+                    return false;
+                }
+            };
             const result = await isLoadMoreContent();
             return result;
         };
@@ -157,7 +192,7 @@ export const MENU_POSITION = {
 export const updateMenu = async (page, urlsCrawled) => {
     log(`Overlay menu: updating: ${page.url()}`);
     await page.evaluate(vars => {
-        const shadowHost = document.querySelector('#oobee-shadow-host');
+        const shadowHost = document.querySelector('#oobeeShadowHost');
         if (shadowHost) {
             const p = shadowHost.shadowRoot.querySelector('#oobee-p-pages-scanned');
             if (p) {
@@ -200,7 +235,7 @@ export const addOverlayMenu = async (page, urlsCrawled, menuPos, opts = {
           </svg>
         `;
         minBtn.innerHTML = MINBTN_SVG;
-        let currentPos = (vars.menuPos || 'RIGHT');
+        let currentPos = vars.menuPos || 'RIGHT';
         const isCollapsed = () => panel.classList.contains('collapsed');
         const setPosClass = (pos) => {
             panel.classList.remove('pos-left', 'pos-right');
@@ -217,7 +252,7 @@ export const addOverlayMenu = async (page, urlsCrawled, menuPos, opts = {
             setDraggableSidebarMenu();
         };
         const toggleCollapsed = (force) => {
-            const willCollapse = (typeof force === 'boolean') ? force : !isCollapsed();
+            const willCollapse = typeof force === 'boolean' ? force : !isCollapsed();
             if (willCollapse) {
                 panel.classList.add('collapsed');
                 localStorage.setItem('oobee:overlay-collapsed', '1');
@@ -261,22 +296,60 @@ export const addOverlayMenu = async (page, urlsCrawled, menuPos, opts = {
         const h2 = document.createElement('h2');
         h2.id = 'oobeeHPagesScanned';
         h2.className = 'oobee-section-title';
-        h2.textContent = 'Pages Scanned';
+        h2.textContent = `Pages Scanned (${vars.urlsCrawled.scanned.length || 0})`;
+        const scanIcon = document.createElement('span');
+        scanIcon.className = 'oobee-btn-icon';
+        const SCAN_SVG = `
+          <svg xmlns="http://www.w3.org/2000/svg" width="20" height="20" viewBox="0 0 20 20" fill="none">
+            <g clip-path="url(#clip0_1421_431)">
+              <path d="M12.5763 11.5472L12.2958 11.2857L12.1037 11.1005C12.776 10.3183 12.9194 9.56432 12.9194 8.45969C12.9194 5.99657 10.9228 4 8.45969 4C5.99657 4 4 5.99657 4 8.45969C4 10.9228 5.99657 12.9194 8.45969 12.9194C9.56432 12.9194 10.3183 12.776 11.1005 12.1037L11.2857 12.2958L11.5472 12.5763L14.9777 16L16 14.9777L12.5763 11.5472ZM8.45969 11.5472C6.75129 11.5472 5.37221 10.1681 5.37221 8.45969C5.37221 6.75129 6.75129 5.37221 8.45969 5.37221C10.1681 5.37221 11.5472 6.75129 11.5472 8.45969C11.5472 10.1681 10.1681 11.5472 8.45969 11.5472Z" fill="white"/>
+              <path d="M18.5 0H19.5C19.7761 0 20 0.223858 20 0.5V5H18.5V0Z" fill="white"/>
+              <path d="M19.5 2.18552e-08L19.5 1.5L15 1.5L15 -2.18556e-07L19.5 2.18552e-08Z" fill="white"/>
+              <path d="M1.5 0H0.5C0.223858 0 0 0.223858 0 0.5V5H1.5V0Z" fill="white"/>
+              <path d="M0.5 2.18552e-08L0.5 1.5L5 1.5L5 -2.18556e-07L0.5 2.18552e-08Z" fill="white"/>
+              <path d="M1.5 20H0.5C0.223858 20 0 19.7761 0 19.5V15H1.5V20Z" fill="white"/>
+              <path d="M0.5 20L0.5 18.5L5 18.5L5 20L0.5 20Z" fill="white"/>
+              <path d="M18.5 20H19.5C19.7761 20 20 19.7761 20 19.5V15H18.5V20Z" fill="white"/>
+              <path d="M19.5 20L19.5 18.5L15 18.5L15 20L19.5 20Z" fill="white"/>
+            </g>
+            <defs>
+              <clipPath id="clip0_1421_431">
+                <rect width="20" height="20" fill="white"/>
+              </clipPath>
+            </defs>
+          </svg>
+        `;
+        scanIcon.innerHTML = SCAN_SVG;
         const scanBtn = document.createElement('button');
         scanBtn.id = 'oobeeBtnScan';
         scanBtn.className = 'oobee-btn oobee-btn-primary';
-        scanBtn.innerText = 'Scan this page';
         scanBtn.disabled = inProgress;
+        scanBtn.appendChild(scanIcon);
+        const scanText = document.createElement('span');
+        scanText.className = 'oobee-btn-text';
+        scanText.innerText = 'Scan page';
+        scanBtn.appendChild(scanText);
         scanBtn.addEventListener('click', async () => customWindow.handleOnScanClick?.());
-        const stopBtn = document.createElement('button');
-        stopBtn.id = 'oobeeBtnStop';
-        stopBtn.className = 'oobee-btn oobee-btn-secondary';
-        stopBtn.innerText = 'Stop scan';
-        stopBtn.addEventListener('click', async () => customWindow.handleOnStopClick?.());
+        const endScanIcon = document.createElement('span');
+        endScanIcon.className = 'oobee-btn-icon';
+        const ENDSCAN_SVG = `<svg xmlns="http://www.w3.org/2000/svg" width="20" height="20" viewBox="0 0 20 20" fill="none">
+            <path d="M10 0C4.47 0 0 4.47 0 10C0 15.53 4.47 20 10 20C15.53 20 20 15.53 20 10C20 4.47 15.53 0 10 0ZM10 18C5.59 18 2 14.41 2 10C2 5.59 5.59 2 10 2C14.41 2 18 5.59 18 10C18 14.41 14.41 18 10 18ZM13.59 5L10 8.59L6.41 5L5 6.41L8.59 10L5 13.59L6.41 15L10 11.41L13.59 15L15 13.59L11.41 10L15 6.41L13.59 5Z" fill="#9021A6"/>
+          </svg>
+        `;
+        endScanIcon.innerHTML = ENDSCAN_SVG;
+        const endScanBtn = document.createElement('button');
+        endScanBtn.id = 'oobeeBtnEndScan';
+        endScanBtn.className = 'oobee-btn oobee-btn-secondary';
+        endScanBtn.appendChild(endScanIcon);
+        const endScanText = document.createElement('span');
+        endScanText.className = 'oobee-btn-text';
+        endScanText.innerText = 'End scan';
+        endScanBtn.appendChild(endScanText);
+        endScanBtn.addEventListener('click', async () => customWindow.handleOnStopClick?.());
         const btnGroup = document.createElement('div');
         btnGroup.className = 'oobee-actions';
         btnGroup.appendChild(scanBtn);
-        btnGroup.appendChild(stopBtn);
+        btnGroup.appendChild(endScanBtn);
         const listWrap = document.createElement('div');
         listWrap.id = 'oobeeList';
         listWrap.className = 'oobee-list';
@@ -292,12 +365,12 @@ export const addOverlayMenu = async (page, urlsCrawled, menuPos, opts = {
             }
             const ol = document.createElement('ol');
             ol.className = 'oobee-ol';
-            scanned.forEach((item) => {
+            scanned.forEach(item => {
                 const li = document.createElement('li');
                 li.className = 'oobee-li';
                 const title = document.createElement('div');
                 title.className = 'oobee-item-title';
-                title.textContent = (item.pageTitle && item.pageTitle.trim()) ? item.pageTitle : item.url;
+                title.textContent = item.pageTitle && item.pageTitle.trim() ? item.pageTitle : item.url;
                 const url = document.createElement('div');
                 url.className = 'oobee-item-url';
                 url.textContent = item.url;
@@ -340,7 +413,7 @@ export const addOverlayMenu = async (page, urlsCrawled, menuPos, opts = {
             border-right: 1px solid rgba(0,0,0,.08)
           }
           .oobee-panel.collapsed {
-            width: 56px;
+            width: 58px;
             overflow: hidden
           }
 
@@ -417,6 +490,12 @@ export const addOverlayMenu = async (page, urlsCrawled, menuPos, opts = {
             padding: 1rem;
           }
 
+          .oobee-panel.collapsed .oobee-actions {
+            display: flex;
+            justify-content: center;
+            padding: 1rem 0.7rem;
+          }
+
           /* Base button */
           .oobee-btn {
             width: 100%;
@@ -427,6 +506,10 @@ export const addOverlayMenu = async (page, urlsCrawled, menuPos, opts = {
             line-height: 1.2;
             font-weight: 400;
             cursor: pointer;
+            display: flex;
+            align-items: center;
+            justify-content: center;
+            gap: 10px;
             transition: {
               box-shadow .12s ease,
               transform .02s ease,
@@ -440,6 +523,19 @@ export const addOverlayMenu = async (page, urlsCrawled, menuPos, opts = {
             cursor:not-allowed
           }
 
+          .oobee-panel.collapsed .oobee-btn {
+            width: 44px !important;
+            height: 44px !important;
+            min-width: 44px !important;
+            min-height: 44px !important;
+            max-width: 44px !important;
+            max-height: 44px !important;
+            border-radius: 50% !important;
+            padding: 0 !important;
+            justify-content: center;
+            gap: 0;
+          }
+
           /* Primary (filled) */
           .oobee-btn-primary {
             background: #9021a6;
@@ -495,6 +591,25 @@ export const addOverlayMenu = async (page, urlsCrawled, menuPos, opts = {
             display: none;
           }
 
+          .oobee-btn-icon {
+            display: inline-flex;
+            align-items: center;
+            justify-content: center;
+            width: 20px;
+            height: 20px;
+            vertical-align: middle;
+          }
+
+          .oobee-btn-text {
+            display: inline;
+            white-space: nowrap;
+            vertical-align: middle;
+          }
+          
+          .oobee-panel.collapsed .oobee-btn-text {
+            display: none;
+          }
+
           #oobeeStopOverlay[hidden] {
             display:none !important;
           }
@@ -512,7 +627,10 @@ export const addOverlayMenu = async (page, urlsCrawled, menuPos, opts = {
           }
 
           .oobee-panel.collapsed .oobee-section-title {
-            display: none;
+            font-size: 14px;
+            display: flex;
+            justify-content: center;
+            text-align: center;
           }
 
           .oobee-ol {
@@ -596,8 +714,7 @@ export const addOverlayMenu = async (page, urlsCrawled, menuPos, opts = {
             if (!icon)
                 return;
             const closed = isCollapsed();
-            const arrowPointsRight = (currentPos === 'RIGHT' && !closed) ||
-                (currentPos === 'LEFT' && closed);
+            const arrowPointsRight = (currentPos === 'RIGHT' && !closed) || (currentPos === 'LEFT' && closed);
             icon.classList.toggle('is-left', !arrowPointsRight);
             minBtn.setAttribute('aria-label', closed ? 'Expand panel' : 'Collapse panel');
         }
@@ -652,7 +769,7 @@ export const addOverlayMenu = async (page, urlsCrawled, menuPos, opts = {
             borderRadius: '16px',
             overflow: 'hidden',
             boxShadow: '0 10px 40px rgba(0,0,0,.35)',
-            fontFamily: 'system-ui, -apple-system, Segoe UI, Roboto, Arial, sans-serif'
+            fontFamily: 'system-ui, -apple-system, Segoe UI, Roboto, Arial, sans-serif',
         });
         const dialogSheet = new CSSStyleSheet();
         dialogSheet.replaceSync(`
@@ -689,12 +806,17 @@ export const addOverlayMenu = async (page, urlsCrawled, menuPos, opts = {
             display: 'flex',
             alignItems: 'center',
             justifyContent: 'space-between',
-            gap: '8px'
+            gap: '8px',
         });
         const title = document.createElement('h2');
         title.id = 'oobee-stop-title';
         title.textContent = 'Are you sure you want to stop this scan?';
-        Object.assign(title.style, { margin: '0', fontSize: '22px', fontWeight: '700', lineHeight: '1.25' });
+        Object.assign(title.style, {
+            margin: '0',
+            fontSize: '22px',
+            fontWeight: '700',
+            lineHeight: '1.25',
+        });
         const closeX = document.createElement('button');
         closeX.type = 'button';
         closeX.setAttribute('aria-label', 'Close');
@@ -711,13 +833,13 @@ export const addOverlayMenu = async (page, urlsCrawled, menuPos, opts = {
             height: '36px',
             borderRadius: '12px',
             display: 'grid',
-            placeItems: 'center'
+            placeItems: 'center',
         });
         head.appendChild(title);
         head.appendChild(closeX);
         const bodyWrap = document.createElement('div');
         Object.assign(bodyWrap.style, {
-            padding: '12px 20px 20px 20px'
+            padding: '12px 20px 20px 20px',
         });
         const form = document.createElement('form');
         form.noValidate = true;
@@ -725,7 +847,7 @@ export const addOverlayMenu = async (page, urlsCrawled, menuPos, opts = {
         Object.assign(form.style, {
             display: 'grid',
             gridTemplateColumns: '1fr',
-            rowGap: '12px'
+            rowGap: '12px',
         });
         const label = document.createElement('label');
         label.setAttribute('for', 'oobee-stop-input');
@@ -741,7 +863,7 @@ export const addOverlayMenu = async (page, urlsCrawled, menuPos, opts = {
             padding: '12px 14px',
             fontSize: '14px',
             outline: 'none',
-            boxSizing: 'border-box'
+            boxSizing: 'border-box',
         });
         input.addEventListener('focus', () => {
             input.style.borderColor = '#7b4dff';
@@ -765,7 +887,7 @@ export const addOverlayMenu = async (page, urlsCrawled, menuPos, opts = {
             fontWeight: '600',
             color: '#fff',
             background: '#9021A6',
-            cursor: 'pointer'
+            cursor: 'pointer',
         });
         const cancel = document.createElement('button');
         cancel.type = 'button';
@@ -777,7 +899,7 @@ export const addOverlayMenu = async (page, urlsCrawled, menuPos, opts = {
             fontSize: '14px',
             justifySelf: 'center',
             cursor: 'pointer',
-            padding: '6px'
+            padding: '6px',
         });
         actions.appendChild(primary);
         actions.appendChild(cancel);
@@ -792,10 +914,13 @@ export const addOverlayMenu = async (page, urlsCrawled, menuPos, opts = {
         stopDialog.appendChild(bodyWrap);
         shadowRoot.appendChild(stopDialog);
         let stopResolver = null;
-        const hideStop = () => { try {
-            stopDialog.close();
-        }
-        catch { } stopResolver = null; };
+        const hideStop = () => {
+            try {
+                stopDialog.close();
+            }
+            catch { }
+            stopResolver = null;
+        };
         const showStop = () => {
             if (!shouldHideInput)
                 input.value = '';
@@ -813,7 +938,7 @@ export const addOverlayMenu = async (page, urlsCrawled, menuPos, opts = {
                 });
             }
         };
-        form.addEventListener('submit', (e) => {
+        form.addEventListener('submit', e => {
             e.preventDefault();
             const v = (input.value || '').trim();
             if (stopResolver)
@@ -830,13 +955,13 @@ export const addOverlayMenu = async (page, urlsCrawled, menuPos, opts = {
                 stopResolver({ confirmed: false, label: '' });
             hideStop();
         });
-        stopDialog.addEventListener('cancel', (e) => {
+        stopDialog.addEventListener('cancel', e => {
             e.preventDefault();
             if (stopResolver)
                 stopResolver({ confirmed: false, label: '' });
             hideStop();
         });
-        customWindow.oobeeShowStopModal = () => new Promise((resolve) => {
+        customWindow.oobeeShowStopModal = () => new Promise(resolve => {
             stopResolver = resolve;
             showStop();
         });
@@ -861,7 +986,7 @@ export const addOverlayMenu = async (page, urlsCrawled, menuPos, opts = {
         log('Overlay menu: successfully added');
     })
         .catch(error => {
-        error('Overlay menu: failed to add', error);
+        consoleLogger.error('Overlay menu: failed to add', error);
     });
 };
 export const removeOverlayMenu = async (page) => {
@@ -884,7 +1009,14 @@ export const initNewPage = async (page, pageClosePromises, processPageParams, pa
     let menuPos = MENU_POSITION.right;
     // eslint-disable-next-line no-underscore-dangle
     const pageId = page._guid;
-    page.on('dialog', () => { });
+    page.on('dialog', async (dialog) => {
+        try {
+            await dialog.dismiss();
+        }
+        catch {
+            // dialog may already be closed
+        }
+    });
     const pageClosePromise = new Promise(resolve => {
         page.on('close', () => {
             log(`Page: close detected: ${page.url()}`);
@@ -910,11 +1042,19 @@ export const initNewPage = async (page, pageClosePromises, processPageParams, pa
             await processPage(page, processPageParams);
             log('Scan: success');
             pagesDict[pageId].isScanning = false;
-            await addOverlayMenu(page, processPageParams.urlsCrawled, menuPos, {
-                inProgress: false,
-                collapsed: !!pagesDict[pageId]?.collapsed,
-                hideStopInput: !!processPageParams.customFlowLabel,
-            });
+            if (page.isClosed())
+                return;
+            const allowed = isOverlayAllowed(page.url(), processPageParams.entryUrl);
+            if (allowed) {
+                await addOverlayMenu(page, processPageParams.urlsCrawled, menuPos, {
+                    inProgress: false,
+                    collapsed: !!pagesDict[pageId]?.collapsed,
+                    hideStopInput: !!processPageParams.customFlowLabel,
+                });
+            }
+            else {
+                await removeOverlayMenu(page);
+            }
         }
         catch (error) {
             log(`Scan failed ${error}`);
@@ -944,10 +1084,10 @@ export const initNewPage = async (page, pageClosePromises, processPageParams, pa
             });
             if (!inputValue?.confirmed) {
                 await page.evaluate(() => {
-                    const stopBtn = document.getElementById('oobeeBtnStop');
-                    if (stopBtn) {
-                        stopBtn.disabled = false;
-                        stopBtn.textContent = 'Stop';
+                    const endScanBtn = document.getElementById('oobeeBtnEndScan');
+                    if (endScanBtn) {
+                        endScanBtn.disabled = false;
+                        endScanBtn.textContent = 'Stop';
                     }
                 });
                 return;
@@ -976,7 +1116,14 @@ export const initNewPage = async (page, pageClosePromises, processPageParams, pa
         }
     };
     page.on('domcontentloaded', async () => {
+        if (page.isClosed())
+            return;
         try {
+            const allowed = isOverlayAllowed(page.url(), processPageParams.entryUrl);
+            if (!allowed) {
+                await removeOverlayMenu(page);
+                return;
+            }
             const existingOverlay = await page.evaluate(() => {
                 return document.querySelector('#oobeeShadowHost');
             });
@@ -989,11 +1136,6 @@ export const initNewPage = async (page, pageClosePromises, processPageParams, pa
                     hideStopInput: !!processPageParams.customFlowLabel,
                 });
             }
-            setTimeout(() => {
-                // Timeout here to slow things down a little
-            }, 1000);
-            //! For Cypress Test
-            // Auto-clicks 'Scan this page' button only once
             if (isCypressTest) {
                 try {
                     await handleOnScanClick();
@@ -1003,22 +1145,27 @@ export const initNewPage = async (page, pageClosePromises, processPageParams, pa
                     consoleLogger.info(`Error in calling handleOnScanClick, isCypressTest: ${isCypressTest}`);
                 }
             }
-            consoleLogger.info(`Overlay state: ${existingOverlay}`);
         }
         catch {
             consoleLogger.info('Error in adding overlay menu to page');
-            consoleLogger.info('Error in adding overlay menu to page');
         }
     });
-    await page.exposeFunction('handleOnScanClick', handleOnScanClick);
-    await page.exposeFunction('handleOnStopClick', handleOnStopClick);
-    // Define the updateMenuPos function
-    const updateMenuPos = newPos => {
-        const prevPos = menuPos;
-        if (prevPos !== newPos) {
-            menuPos = newPos;
-        }
-    };
-    await page.exposeFunction('updateMenuPos', updateMenuPos);
+    try {
+        if (page.isClosed())
+            return page;
+        await page.exposeFunction('handleOnScanClick', handleOnScanClick);
+        await page.exposeFunction('handleOnStopClick', handleOnStopClick);
+        // Define the updateMenuPos function
+        const updateMenuPos = newPos => {
+            const prevPos = menuPos;
+            if (prevPos !== newPos) {
+                menuPos = newPos;
+            }
+        };
+        await page.exposeFunction('updateMenuPos', updateMenuPos);
+    }
+    catch (e) {
+        log(`Error exposing functions on page: ${e}`);
+    }
     return page;
 };

package/dist/crawlers/guards/urlGuard.js

@@ -2,15 +2,16 @@ const ALLOWED_PROTOCOLS = new Set(['http:', 'https:']);
 export function addUrlGuardScript(context, opts = {}) {
     const { fallbackUrl } = opts;
     const lastAllowedUrlByPage = new WeakMap();
-    const attachGuardsToPage = (page) => {
+    const attachGuardsToPage = page => {
         if (!lastAllowedUrlByPage.has(page) && fallbackUrl) {
             lastAllowedUrlByPage.set(page, String(fallbackUrl));
         }
-        page.addInitScript(() => {
-            const isAllowedProtocol = (value) => {
+        page
+            .addInitScript(() => {
+            const isAllowedProtocol = value => {
                 try {
                     const s = value instanceof URL ? value.toString() : String(value);
-                    const protocol = new URL(s, window.location.href).protocol;
+                    const { protocol } = new URL(s, window.location.href);
                     return protocol === 'http:' || protocol === 'https:';
                 }
                 catch {
@@ -24,17 +25,9 @@ export function addUrlGuardScript(context, opts = {}) {
                     return null;
                 return openOriginal.call(this, targetUrl, ...args);
             };
-            const assignOriginal = win.location.assign.bind(win.location);
-            const replaceOriginal = win.location.replace.bind(win.location);
-            win.location.assign = (nextUrl) => { if (isAllowedProtocol(nextUrl))
-                assignOriginal(nextUrl); };
-            win.location.replace = (nextUrl) => { if (isAllowedProtocol(nextUrl))
-                replaceOriginal(nextUrl); };
-            Object.defineProperty(win.location, 'href', {
-                get() { return String(win.location.toString()); },
-                set(nextUrl) { if (isAllowedProtocol(nextUrl))
-                    assignOriginal(nextUrl); },
-            });
+        })
+            .catch(() => {
+            // page may have closed before addInitScript completed; safe to ignore
         });
         const restoreToSafeUrl = async (page, attemptedUrl) => {
             try {

package/dist/crawlers/runCustom.js

@@ -1,12 +1,11 @@
 /* eslint-env browser */
-import { chromium } from 'playwright';
 import { createCrawleeSubFolders } from './commonCrawlerFunc.js';
 import { cleanUpAndExit, register, registerSoftClose } from '../utils.js';
 import constants, { getIntermediateScreenshotsPath, guiInfoStatusTypes, } from '../constants/constants.js';
 import { initNewPage, log } from './custom/utils.js';
 import { guiInfoLog } from '../logs.js';
 import { addUrlGuardScript } from './guards/urlGuard.js';
-import { getPlaywrightLaunchOptions } from '../constants/common.js';
+import { getBrowserToRun, getPlaywrightLaunchOptions, initModifiedUserAgent, } from '../constants/common.js';
 // Export of classes
 export class ProcessPageParams {
     constructor(scannedIdx, blacklistedPatterns, includeScreenshots, dataset, intermediateScreenshotsPath, urlsCrawled, randomToken) {
@@ -19,7 +18,7 @@ export class ProcessPageParams {
         this.randomToken = randomToken;
     }
 }
-const runCustom = async (url, randomToken, viewportSettings, blacklistedPatterns, includeScreenshots, initialCustomFlowLabel) => {
+const runCustom = async (url, randomToken, browserToRun, userDataDirectory, viewportSettings, blacklistedPatterns, includeScreenshots, initialCustomFlowLabel) => {
     // checks and delete datasets path if it already exists
     process.env.CRAWLEE_STORAGE_DIR = randomToken;
     const urlsCrawled = { ...constants.urlsCrawledObj };
@@ -27,48 +26,58 @@ const runCustom = async (url, randomToken, viewportSettings, blacklistedPatterns
     const intermediateScreenshotsPath = getIntermediateScreenshotsPath(randomToken);
     const processPageParams = new ProcessPageParams(0, // scannedIdx
     blacklistedPatterns, includeScreenshots, dataset, intermediateScreenshotsPath, urlsCrawled, randomToken);
+    processPageParams.entryUrl = url;
     if (initialCustomFlowLabel && initialCustomFlowLabel.trim()) {
         processPageParams.customFlowLabel = initialCustomFlowLabel.trim();
     }
     const pagesDict = {};
     const pageClosePromises = [];
     try {
+        const { browserToRun: resolvedBrowserToRun } = getBrowserToRun(randomToken, browserToRun, false);
         const deviceConfig = viewportSettings.playwrightDeviceDetailsObject;
         const hasCustomViewport = !!deviceConfig;
-        const baseLaunchOptions = getPlaywrightLaunchOptions('chrome');
+        const rawDevice = (deviceConfig || {});
+        const { userAgent: deviceUserAgent, ...contextDeviceOptions } = rawDevice;
+        await initModifiedUserAgent(resolvedBrowserToRun, viewportSettings.playwrightDeviceDetailsObject);
+        const baseLaunchOptions = getPlaywrightLaunchOptions(resolvedBrowserToRun);
         // Merge base args with custom flow specific args
         const baseArgs = baseLaunchOptions.args || [];
         const customArgs = hasCustomViewport ? ['--window-size=1920,1040'] : ['--start-maximized'];
-        const mergedArgs = [...baseArgs.filter(a => !a.startsWith('--window-size') && a !== '--start-maximized'), ...customArgs];
-        const browser = await chromium.launch({
+        const mergedArgs = [
+            ...baseArgs.filter(a => !a.startsWith('--window-size') && a !== '--start-maximized'),
+            ...customArgs,
+        ];
+        const context = await constants.launcher.launchPersistentContext(userDataDirectory, {
             ...baseLaunchOptions,
             args: mergedArgs,
             headless: false,
-            channel: 'chrome',
-        });
-        const context = await browser.newContext({
             ignoreHTTPSErrors: true,
             serviceWorkers: 'block',
             viewport: null,
-            ...(hasCustomViewport ? deviceConfig : {}),
+            ...(hasCustomViewport ? contextDeviceOptions : {}),
+            userAgent: process.env.OOBEE_USER_AGENT || deviceUserAgent,
         });
         register(context);
         processPageParams.stopAll = async () => {
             try {
                 await context.close().catch(() => { });
-                await browser.close().catch(() => { });
-            }
-            catch {
             }
+            catch { }
         };
         // For handling closing playwright browser and continue generate artifacts etc
         registerSoftClose(processPageParams.stopAll);
         addUrlGuardScript(context, { fallbackUrl: url });
+        const page = context.pages().find(existingPage => !existingPage.isClosed()) || (await context.newPage());
+        await initNewPage(page, pageClosePromises, processPageParams, pagesDict);
         // Detection of new page
         context.on('page', async (newPage) => {
-            await initNewPage(newPage, pageClosePromises, processPageParams, pagesDict);
+            try {
+                await initNewPage(newPage, pageClosePromises, processPageParams, pagesDict);
+            }
+            catch (e) {
+                log(`Error initializing new page: ${e}`);
+            }
         });
-        const page = await context.newPage();
         await page.goto(url, { timeout: 0 });
         // to execute and wait for all pages to close
         // idea is for promise to be pending until page.on('close') detected