@govtechsg/oobee 0.10.85 → 0.10.86

package/dist/crawlers/commonCrawlerFunc.js

@@ -116,7 +116,7 @@ export const filterAxeResults = (results, pageTitle, customFlowDetails) => {
             return;
         const conformance = tags.filter(tag => tag.startsWith('wcag') || tag === 'best-practice');
         nodes.forEach(node => {
-            const { html } = node;
+            const { html, target } = node;
             if (!(rule in passed.rules)) {
                 passed.rules[rule] = {
                     description,
@@ -128,7 +128,8 @@ export const filterAxeResults = (results, pageTitle, customFlowDetails) => {
                 };
             }
             const finalHtml = truncateHtml(html);
-            passed.rules[rule].items.push({ html: finalHtml, screenshotPath: '', message: '', xpath: '' });
+            const xpath = target.length === 1 && typeof target[0] === 'string' ? target[0] : undefined;
+            passed.rules[rule].items.push({ html: finalHtml, screenshotPath: '', message: '', xpath: xpath || '' });
             passed.totalItems += 1;
             passed.rules[rule].totalItems += 1;
             totalItems += 1;

package/dist/crawlers/crawlSitemap.js

@@ -12,6 +12,7 @@ const crawlSitemap = async ({ sitemapUrl, randomToken, host, viewportSettings, m
     let dataset;
     let urlsCrawled;
     let durationExceeded = false;
+    let isAbortingScan = false;
     if (fromCrawlIntelligentSitemap) {
         dataset = datasetFromIntelligent;
         urlsCrawled = urlsCrawledFromIntelligent;
@@ -144,106 +145,123 @@ const crawlSitemap = async ({ sitemapUrl, randomToken, host, viewportSettings, m
                 });
                 return;
             }
-            await waitForPageLoaded(page, 10000);
-            const actualUrl = page.url() || request.loadedUrl || request.url;
-            const hasExceededDuration = scanDuration > 0 && Date.now() - crawlStartTime > scanDuration * 1000;
-            if (urlsCrawled.scanned.length >= maxRequestsPerCrawl || hasExceededDuration) {
-                if (hasExceededDuration) {
-                    console.log(`Crawl duration of ${scanDuration}s exceeded. Aborting sitemap crawl.`);
-                    durationExceeded = true;
-                }
-                crawler.autoscaledPool.abort(); // stops new requests
-                return;
-            }
-            if (request.skipNavigation && actualUrl === 'about:blank') {
-                if (isScanPdfs) {
-                    // pushes download promise into pdfDownloads
-                    const { pdfFileName, url } = handlePdfDownload(randomToken, pdfDownloads, request, sendRequest, urlsCrawled);
-                    uuidToPdfMapping[pdfFileName] = url;
+            try {
+                await waitForPageLoaded(page, 10000);
+                const actualUrl = page.url() || request.loadedUrl || request.url;
+                const hasExceededDuration = scanDuration > 0 && Date.now() - crawlStartTime > scanDuration * 1000;
+                if (urlsCrawled.scanned.length >= maxRequestsPerCrawl || hasExceededDuration) {
+                    isAbortingScan = true;
+                    if (hasExceededDuration) {
+                        console.log(`Crawl duration of ${scanDuration}s exceeded. Aborting sitemap crawl.`);
+                        durationExceeded = true;
+                    }
+                    crawler.autoscaledPool.abort(); // stops new requests
                     return;
                 }
-                guiInfoLog(guiInfoStatusTypes.SKIPPED, {
-                    numScanned: urlsCrawled.scanned.length,
-                    urlScanned: request.url,
-                });
-                urlsCrawled.userExcluded.push({
-                    url: request.url,
-                    pageTitle: request.url,
-                    actualUrl: request.url, // because about:blank is not useful
-                    metadata: STATUS_CODE_METADATA[1],
-                    httpStatusCode: 1,
-                });
-                return;
-            }
-            const contentType = response?.headers?.()['content-type'] || '';
-            const status = response ? response.status() : 0;
-            if (isScanHtml && status < 300 && isWhitelistedContentType(contentType)) {
-                const isRedirected = !areLinksEqual(page.url(), request.url);
-                const isLoadedUrlInCrawledUrls = urlsCrawled.scanned.some(item => (item.actualUrl || item.url) === page.url());
-                if (isRedirected && isLoadedUrlInCrawledUrls) {
-                    urlsCrawled.notScannedRedirects.push({
-                        fromUrl: request.url,
-                        toUrl: actualUrl, // i.e. actualUrl
+                if (request.skipNavigation && actualUrl === 'about:blank') {
+                    if (isScanPdfs) {
+                        // pushes download promise into pdfDownloads
+                        const { pdfFileName, url } = handlePdfDownload(randomToken, pdfDownloads, request, sendRequest, urlsCrawled);
+                        uuidToPdfMapping[pdfFileName] = url;
+                        return;
+                    }
+                    guiInfoLog(guiInfoStatusTypes.SKIPPED, {
+                        numScanned: urlsCrawled.scanned.length,
+                        urlScanned: request.url,
                     });
-                    return;
-                }
-                // This logic is different from crawlDomain, as it also checks if the pae is redirected before checking if it is excluded using exclusions.txt
-                if (isRedirected && blacklistedPatterns && isSkippedUrl(actualUrl, blacklistedPatterns)) {
                     urlsCrawled.userExcluded.push({
                         url: request.url,
                         pageTitle: request.url,
-                        actualUrl,
-                        metadata: STATUS_CODE_METADATA[0],
-                        httpStatusCode: 0,
+                        actualUrl: request.url, // because about:blank is not useful
+                        metadata: STATUS_CODE_METADATA[1],
+                        httpStatusCode: 1,
                     });
+                    return;
+                }
+                const contentType = response?.headers?.()['content-type'] || '';
+                const status = response ? response.status() : 0;
+                if (isScanHtml && status < 300 && isWhitelistedContentType(contentType)) {
+                    const isRedirected = !areLinksEqual(page.url(), request.url);
+                    const isLoadedUrlInCrawledUrls = urlsCrawled.scanned.some(item => (item.actualUrl || item.url) === page.url());
+                    if (isRedirected && isLoadedUrlInCrawledUrls) {
+                        urlsCrawled.notScannedRedirects.push({
+                            fromUrl: request.url,
+                            toUrl: actualUrl, // i.e. actualUrl
+                        });
+                        return;
+                    }
+                    // This logic is different from crawlDomain, as it also checks if the pae is redirected before checking if it is excluded using exclusions.txt
+                    if (isRedirected && blacklistedPatterns && isSkippedUrl(actualUrl, blacklistedPatterns)) {
+                        urlsCrawled.userExcluded.push({
+                            url: request.url,
+                            pageTitle: request.url,
+                            actualUrl,
+                            metadata: STATUS_CODE_METADATA[0],
+                            httpStatusCode: 0,
+                        });
+                        guiInfoLog(guiInfoStatusTypes.SKIPPED, {
+                            numScanned: urlsCrawled.scanned.length,
+                            urlScanned: request.url,
+                        });
+                        return;
+                    }
+                    const results = await runAxeScript({ includeScreenshots, page, randomToken });
+                    guiInfoLog(guiInfoStatusTypes.SCANNED, {
+                        numScanned: urlsCrawled.scanned.length,
+                        urlScanned: request.url,
+                    });
+                    urlsCrawled.scanned.push({
+                        url: request.url,
+                        pageTitle: results.pageTitle,
+                        actualUrl, // i.e. actualUrl
+                    });
+                    urlsCrawled.scannedRedirects.push({
+                        fromUrl: request.url,
+                        toUrl: actualUrl,
+                    });
+                    results.url = request.url;
+                    results.actualUrl = actualUrl;
+                    await dataset.pushData(results);
+                }
+                else {
                     guiInfoLog(guiInfoStatusTypes.SKIPPED, {
                         numScanned: urlsCrawled.scanned.length,
                         urlScanned: request.url,
                     });
-                    return;
+                    if (isScanHtml) {
+                        // carry through the HTTP status metadata
+                        const status = response?.status();
+                        const metadata = typeof status === 'number'
+                            ? STATUS_CODE_METADATA[status] || STATUS_CODE_METADATA[599]
+                            : STATUS_CODE_METADATA[2];
+                        urlsCrawled.invalid.push({
+                            actualUrl,
+                            url: request.url,
+                            pageTitle: request.url,
+                            metadata,
+                            httpStatusCode: typeof status === 'number' ? status : 0,
+                        });
+                    }
                 }
-                const results = await runAxeScript({ includeScreenshots, page, randomToken });
-                guiInfoLog(guiInfoStatusTypes.SCANNED, {
-                    numScanned: urlsCrawled.scanned.length,
-                    urlScanned: request.url,
-                });
-                urlsCrawled.scanned.push({
-                    url: request.url,
-                    pageTitle: results.pageTitle,
-                    actualUrl, // i.e. actualUrl
-                });
-                urlsCrawled.scannedRedirects.push({
-                    fromUrl: request.url,
-                    toUrl: actualUrl,
-                });
-                results.url = request.url;
-                results.actualUrl = actualUrl;
-                await dataset.pushData(results);
             }
-            else {
-                guiInfoLog(guiInfoStatusTypes.SKIPPED, {
-                    numScanned: urlsCrawled.scanned.length,
-                    urlScanned: request.url,
-                });
-                if (isScanHtml) {
-                    // carry through the HTTP status metadata
-                    const status = response?.status();
-                    const metadata = typeof status === 'number'
-                        ? STATUS_CODE_METADATA[status] || STATUS_CODE_METADATA[599]
-                        : STATUS_CODE_METADATA[2];
-                    urlsCrawled.invalid.push({
-                        actualUrl,
+            catch (e) {
+                if (!isAbortingScan) {
+                    guiInfoLog(guiInfoStatusTypes.ERROR, {
+                        numScanned: urlsCrawled.scanned.length,
+                        urlScanned: request.url,
+                    });
+                    urlsCrawled.error.push({
                         url: request.url,
                         pageTitle: request.url,
-                        metadata,
-                        httpStatusCode: typeof status === 'number' ? status : 0,
+                        actualUrl: request.url,
+                        metadata: STATUS_CODE_METADATA[2],
+                        httpStatusCode: 0,
                     });
                 }
             }
         },
         failedRequestHandler: async ({ request, response, error }) => {
-            // check if scanned pages have reached limit due to multi-instances of handler running
-            if (urlsCrawled.scanned.length >= maxRequestsPerCrawl) {
+            if (isAbortingScan) {
                 return;
             }
             guiInfoLog(guiInfoStatusTypes.ERROR, {

package/dist/crawlers/custom/utils.js

@@ -62,14 +62,19 @@ export const screenshotFullPage = async (page, screenshotsDir, screenshotIdx) =>
             await page.evaluate(() => {
                 window.scrollTo(0, document.body.scrollHeight);
             });
-            const isLoadMoreContent = async () => new Promise(resolve => {
-                setTimeout(async () => {
+            const isLoadMoreContent = async () => {
+                await new Promise(resolve => setTimeout(resolve, 2500));
+                if (page.isClosed())
+                    return false;
+                try {
                     await page.waitForLoadState('domcontentloaded');
                     const newHeight = await page.evaluate(() => document.body.scrollHeight);
-                    const result = newHeight > prevHeight;
-                    resolve(result);
-                }, 2500);
-            });
+                    return newHeight > prevHeight;
+                }
+                catch {
+                    return false;
+                }
+            };
             const result = await isLoadMoreContent();
             return result;
         };
@@ -291,22 +296,60 @@ export const addOverlayMenu = async (page, urlsCrawled, menuPos, opts = {
         const h2 = document.createElement('h2');
         h2.id = 'oobeeHPagesScanned';
         h2.className = 'oobee-section-title';
-        h2.textContent = 'Pages Scanned';
+        h2.textContent = `Pages Scanned (${vars.urlsCrawled.scanned.length || 0})`;
+        const scanIcon = document.createElement('span');
+        scanIcon.className = 'oobee-btn-icon';
+        const SCAN_SVG = `
+          <svg xmlns="http://www.w3.org/2000/svg" width="20" height="20" viewBox="0 0 20 20" fill="none">
+            <g clip-path="url(#clip0_1421_431)">
+              <path d="M12.5763 11.5472L12.2958 11.2857L12.1037 11.1005C12.776 10.3183 12.9194 9.56432 12.9194 8.45969C12.9194 5.99657 10.9228 4 8.45969 4C5.99657 4 4 5.99657 4 8.45969C4 10.9228 5.99657 12.9194 8.45969 12.9194C9.56432 12.9194 10.3183 12.776 11.1005 12.1037L11.2857 12.2958L11.5472 12.5763L14.9777 16L16 14.9777L12.5763 11.5472ZM8.45969 11.5472C6.75129 11.5472 5.37221 10.1681 5.37221 8.45969C5.37221 6.75129 6.75129 5.37221 8.45969 5.37221C10.1681 5.37221 11.5472 6.75129 11.5472 8.45969C11.5472 10.1681 10.1681 11.5472 8.45969 11.5472Z" fill="white"/>
+              <path d="M18.5 0H19.5C19.7761 0 20 0.223858 20 0.5V5H18.5V0Z" fill="white"/>
+              <path d="M19.5 2.18552e-08L19.5 1.5L15 1.5L15 -2.18556e-07L19.5 2.18552e-08Z" fill="white"/>
+              <path d="M1.5 0H0.5C0.223858 0 0 0.223858 0 0.5V5H1.5V0Z" fill="white"/>
+              <path d="M0.5 2.18552e-08L0.5 1.5L5 1.5L5 -2.18556e-07L0.5 2.18552e-08Z" fill="white"/>
+              <path d="M1.5 20H0.5C0.223858 20 0 19.7761 0 19.5V15H1.5V20Z" fill="white"/>
+              <path d="M0.5 20L0.5 18.5L5 18.5L5 20L0.5 20Z" fill="white"/>
+              <path d="M18.5 20H19.5C19.7761 20 20 19.7761 20 19.5V15H18.5V20Z" fill="white"/>
+              <path d="M19.5 20L19.5 18.5L15 18.5L15 20L19.5 20Z" fill="white"/>
+            </g>
+            <defs>
+              <clipPath id="clip0_1421_431">
+                <rect width="20" height="20" fill="white"/>
+              </clipPath>
+            </defs>
+          </svg>
+        `;
+        scanIcon.innerHTML = SCAN_SVG;
         const scanBtn = document.createElement('button');
         scanBtn.id = 'oobeeBtnScan';
         scanBtn.className = 'oobee-btn oobee-btn-primary';
-        scanBtn.innerText = 'Scan this page';
         scanBtn.disabled = inProgress;
+        scanBtn.appendChild(scanIcon);
+        const scanText = document.createElement('span');
+        scanText.className = 'oobee-btn-text';
+        scanText.innerText = 'Scan page';
+        scanBtn.appendChild(scanText);
         scanBtn.addEventListener('click', async () => customWindow.handleOnScanClick?.());
-        const stopBtn = document.createElement('button');
-        stopBtn.id = 'oobeeBtnStop';
-        stopBtn.className = 'oobee-btn oobee-btn-secondary';
-        stopBtn.innerText = 'Stop scan';
-        stopBtn.addEventListener('click', async () => customWindow.handleOnStopClick?.());
+        const endScanIcon = document.createElement('span');
+        endScanIcon.className = 'oobee-btn-icon';
+        const ENDSCAN_SVG = `<svg xmlns="http://www.w3.org/2000/svg" width="20" height="20" viewBox="0 0 20 20" fill="none">
+            <path d="M10 0C4.47 0 0 4.47 0 10C0 15.53 4.47 20 10 20C15.53 20 20 15.53 20 10C20 4.47 15.53 0 10 0ZM10 18C5.59 18 2 14.41 2 10C2 5.59 5.59 2 10 2C14.41 2 18 5.59 18 10C18 14.41 14.41 18 10 18ZM13.59 5L10 8.59L6.41 5L5 6.41L8.59 10L5 13.59L6.41 15L10 11.41L13.59 15L15 13.59L11.41 10L15 6.41L13.59 5Z" fill="#9021A6"/>
+          </svg>
+        `;
+        endScanIcon.innerHTML = ENDSCAN_SVG;
+        const endScanBtn = document.createElement('button');
+        endScanBtn.id = 'oobeeBtnEndScan';
+        endScanBtn.className = 'oobee-btn oobee-btn-secondary';
+        endScanBtn.appendChild(endScanIcon);
+        const endScanText = document.createElement('span');
+        endScanText.className = 'oobee-btn-text';
+        endScanText.innerText = 'End scan';
+        endScanBtn.appendChild(endScanText);
+        endScanBtn.addEventListener('click', async () => customWindow.handleOnStopClick?.());
         const btnGroup = document.createElement('div');
         btnGroup.className = 'oobee-actions';
         btnGroup.appendChild(scanBtn);
-        btnGroup.appendChild(stopBtn);
+        btnGroup.appendChild(endScanBtn);
         const listWrap = document.createElement('div');
         listWrap.id = 'oobeeList';
         listWrap.className = 'oobee-list';
@@ -370,7 +413,7 @@ export const addOverlayMenu = async (page, urlsCrawled, menuPos, opts = {
             border-right: 1px solid rgba(0,0,0,.08)
           }
           .oobee-panel.collapsed {
-            width: 56px;
+            width: 58px;
             overflow: hidden
           }
 
@@ -447,6 +490,12 @@ export const addOverlayMenu = async (page, urlsCrawled, menuPos, opts = {
             padding: 1rem;
           }
 
+          .oobee-panel.collapsed .oobee-actions {
+            display: flex;
+            justify-content: center;
+            padding: 1rem 0.7rem;
+          }
+
           /* Base button */
           .oobee-btn {
             width: 100%;
@@ -457,6 +506,10 @@ export const addOverlayMenu = async (page, urlsCrawled, menuPos, opts = {
             line-height: 1.2;
             font-weight: 400;
             cursor: pointer;
+            display: flex;
+            align-items: center;
+            justify-content: center;
+            gap: 10px;
             transition: {
               box-shadow .12s ease,
               transform .02s ease,
@@ -470,6 +523,19 @@ export const addOverlayMenu = async (page, urlsCrawled, menuPos, opts = {
             cursor:not-allowed
           }
 
+          .oobee-panel.collapsed .oobee-btn {
+            width: 44px !important;
+            height: 44px !important;
+            min-width: 44px !important;
+            min-height: 44px !important;
+            max-width: 44px !important;
+            max-height: 44px !important;
+            border-radius: 50% !important;
+            padding: 0 !important;
+            justify-content: center;
+            gap: 0;
+          }
+
           /* Primary (filled) */
           .oobee-btn-primary {
             background: #9021a6;
@@ -525,6 +591,25 @@ export const addOverlayMenu = async (page, urlsCrawled, menuPos, opts = {
             display: none;
           }
 
+          .oobee-btn-icon {
+            display: inline-flex;
+            align-items: center;
+            justify-content: center;
+            width: 20px;
+            height: 20px;
+            vertical-align: middle;
+          }
+
+          .oobee-btn-text {
+            display: inline;
+            white-space: nowrap;
+            vertical-align: middle;
+          }
+          
+          .oobee-panel.collapsed .oobee-btn-text {
+            display: none;
+          }
+
           #oobeeStopOverlay[hidden] {
             display:none !important;
           }
@@ -542,7 +627,10 @@ export const addOverlayMenu = async (page, urlsCrawled, menuPos, opts = {
           }
 
           .oobee-panel.collapsed .oobee-section-title {
-            display: none;
+            font-size: 14px;
+            display: flex;
+            justify-content: center;
+            text-align: center;
           }
 
           .oobee-ol {
@@ -921,7 +1009,14 @@ export const initNewPage = async (page, pageClosePromises, processPageParams, pa
     let menuPos = MENU_POSITION.right;
     // eslint-disable-next-line no-underscore-dangle
     const pageId = page._guid;
-    page.on('dialog', () => { });
+    page.on('dialog', async (dialog) => {
+        try {
+            await dialog.dismiss();
+        }
+        catch {
+            // dialog may already be closed
+        }
+    });
     const pageClosePromise = new Promise(resolve => {
         page.on('close', () => {
             log(`Page: close detected: ${page.url()}`);
@@ -947,6 +1042,8 @@ export const initNewPage = async (page, pageClosePromises, processPageParams, pa
             await processPage(page, processPageParams);
             log('Scan: success');
             pagesDict[pageId].isScanning = false;
+            if (page.isClosed())
+                return;
             const allowed = isOverlayAllowed(page.url(), processPageParams.entryUrl);
             if (allowed) {
                 await addOverlayMenu(page, processPageParams.urlsCrawled, menuPos, {
@@ -987,10 +1084,10 @@ export const initNewPage = async (page, pageClosePromises, processPageParams, pa
             });
             if (!inputValue?.confirmed) {
                 await page.evaluate(() => {
-                    const stopBtn = document.getElementById('oobeeBtnStop');
-                    if (stopBtn) {
-                        stopBtn.disabled = false;
-                        stopBtn.textContent = 'Stop';
+                    const endScanBtn = document.getElementById('oobeeBtnEndScan');
+                    if (endScanBtn) {
+                        endScanBtn.disabled = false;
+                        endScanBtn.textContent = 'Stop';
                     }
                 });
                 return;
@@ -1019,6 +1116,8 @@ export const initNewPage = async (page, pageClosePromises, processPageParams, pa
         }
     };
     page.on('domcontentloaded', async () => {
+        if (page.isClosed())
+            return;
         try {
             const allowed = isOverlayAllowed(page.url(), processPageParams.entryUrl);
             if (!allowed) {
@@ -1051,15 +1150,22 @@ export const initNewPage = async (page, pageClosePromises, processPageParams, pa
             consoleLogger.info('Error in adding overlay menu to page');
         }
     });
-    await page.exposeFunction('handleOnScanClick', handleOnScanClick);
-    await page.exposeFunction('handleOnStopClick', handleOnStopClick);
-    // Define the updateMenuPos function
-    const updateMenuPos = newPos => {
-        const prevPos = menuPos;
-        if (prevPos !== newPos) {
-            menuPos = newPos;
-        }
-    };
-    await page.exposeFunction('updateMenuPos', updateMenuPos);
+    try {
+        if (page.isClosed())
+            return page;
+        await page.exposeFunction('handleOnScanClick', handleOnScanClick);
+        await page.exposeFunction('handleOnStopClick', handleOnStopClick);
+        // Define the updateMenuPos function
+        const updateMenuPos = newPos => {
+            const prevPos = menuPos;
+            if (prevPos !== newPos) {
+                menuPos = newPos;
+            }
+        };
+        await page.exposeFunction('updateMenuPos', updateMenuPos);
+    }
+    catch (e) {
+        log(`Error exposing functions on page: ${e}`);
+    }
     return page;
 };

package/dist/crawlers/guards/urlGuard.js

@@ -2,15 +2,16 @@ const ALLOWED_PROTOCOLS = new Set(['http:', 'https:']);
 export function addUrlGuardScript(context, opts = {}) {
     const { fallbackUrl } = opts;
     const lastAllowedUrlByPage = new WeakMap();
-    const attachGuardsToPage = (page) => {
+    const attachGuardsToPage = page => {
         if (!lastAllowedUrlByPage.has(page) && fallbackUrl) {
             lastAllowedUrlByPage.set(page, String(fallbackUrl));
         }
-        page.addInitScript(() => {
-            const isAllowedProtocol = (value) => {
+        page
+            .addInitScript(() => {
+            const isAllowedProtocol = value => {
                 try {
                     const s = value instanceof URL ? value.toString() : String(value);
-                    const protocol = new URL(s, window.location.href).protocol;
+                    const { protocol } = new URL(s, window.location.href);
                     return protocol === 'http:' || protocol === 'https:';
                 }
                 catch {
@@ -24,17 +25,9 @@ export function addUrlGuardScript(context, opts = {}) {
                     return null;
                 return openOriginal.call(this, targetUrl, ...args);
             };
-            const assignOriginal = win.location.assign.bind(win.location);
-            const replaceOriginal = win.location.replace.bind(win.location);
-            win.location.assign = (nextUrl) => { if (isAllowedProtocol(nextUrl))
-                assignOriginal(nextUrl); };
-            win.location.replace = (nextUrl) => { if (isAllowedProtocol(nextUrl))
-                replaceOriginal(nextUrl); };
-            Object.defineProperty(win.location, 'href', {
-                get() { return String(win.location.toString()); },
-                set(nextUrl) { if (isAllowedProtocol(nextUrl))
-                    assignOriginal(nextUrl); },
-            });
+        })
+            .catch(() => {
+            // page may have closed before addInitScript completed; safe to ignore
         });
         const restoreToSafeUrl = async (page, attemptedUrl) => {
             try {

package/dist/crawlers/runCustom.js

@@ -1,12 +1,11 @@
 /* eslint-env browser */
-import { chromium } from 'playwright';
 import { createCrawleeSubFolders } from './commonCrawlerFunc.js';
 import { cleanUpAndExit, register, registerSoftClose } from '../utils.js';
 import constants, { getIntermediateScreenshotsPath, guiInfoStatusTypes, } from '../constants/constants.js';
 import { initNewPage, log } from './custom/utils.js';
 import { guiInfoLog } from '../logs.js';
 import { addUrlGuardScript } from './guards/urlGuard.js';
-import { getPlaywrightLaunchOptions } from '../constants/common.js';
+import { getBrowserToRun, getPlaywrightLaunchOptions, initModifiedUserAgent, } from '../constants/common.js';
 // Export of classes
 export class ProcessPageParams {
     constructor(scannedIdx, blacklistedPatterns, includeScreenshots, dataset, intermediateScreenshotsPath, urlsCrawled, randomToken) {
@@ -19,7 +18,7 @@ export class ProcessPageParams {
         this.randomToken = randomToken;
     }
 }
-const runCustom = async (url, randomToken, viewportSettings, blacklistedPatterns, includeScreenshots, initialCustomFlowLabel) => {
+const runCustom = async (url, randomToken, browserToRun, userDataDirectory, viewportSettings, blacklistedPatterns, includeScreenshots, initialCustomFlowLabel) => {
     // checks and delete datasets path if it already exists
     process.env.CRAWLEE_STORAGE_DIR = randomToken;
     const urlsCrawled = { ...constants.urlsCrawledObj };
@@ -34,9 +33,13 @@ const runCustom = async (url, randomToken, viewportSettings, blacklistedPatterns
     const pagesDict = {};
     const pageClosePromises = [];
     try {
+        const { browserToRun: resolvedBrowserToRun } = getBrowserToRun(randomToken, browserToRun, false);
         const deviceConfig = viewportSettings.playwrightDeviceDetailsObject;
         const hasCustomViewport = !!deviceConfig;
-        const baseLaunchOptions = getPlaywrightLaunchOptions();
+        const rawDevice = (deviceConfig || {});
+        const { userAgent: deviceUserAgent, ...contextDeviceOptions } = rawDevice;
+        await initModifiedUserAgent(resolvedBrowserToRun, viewportSettings.playwrightDeviceDetailsObject);
+        const baseLaunchOptions = getPlaywrightLaunchOptions(resolvedBrowserToRun);
         // Merge base args with custom flow specific args
         const baseArgs = baseLaunchOptions.args || [];
         const customArgs = hasCustomViewport ? ['--window-size=1920,1040'] : ['--start-maximized'];
@@ -44,33 +47,37 @@ const runCustom = async (url, randomToken, viewportSettings, blacklistedPatterns
             ...baseArgs.filter(a => !a.startsWith('--window-size') && a !== '--start-maximized'),
             ...customArgs,
         ];
-        const browser = await chromium.launch({
+        const context = await constants.launcher.launchPersistentContext(userDataDirectory, {
             ...baseLaunchOptions,
             args: mergedArgs,
             headless: false,
-        });
-        const context = await browser.newContext({
             ignoreHTTPSErrors: true,
             serviceWorkers: 'block',
             viewport: null,
-            ...(hasCustomViewport ? deviceConfig : {}),
+            ...(hasCustomViewport ? contextDeviceOptions : {}),
+            userAgent: process.env.OOBEE_USER_AGENT || deviceUserAgent,
         });
         register(context);
         processPageParams.stopAll = async () => {
             try {
                 await context.close().catch(() => { });
-                await browser.close().catch(() => { });
             }
             catch { }
         };
         // For handling closing playwright browser and continue generate artifacts etc
         registerSoftClose(processPageParams.stopAll);
         addUrlGuardScript(context, { fallbackUrl: url });
+        const page = context.pages().find(existingPage => !existingPage.isClosed()) || (await context.newPage());
+        await initNewPage(page, pageClosePromises, processPageParams, pagesDict);
         // Detection of new page
         context.on('page', async (newPage) => {
-            await initNewPage(newPage, pageClosePromises, processPageParams, pagesDict);
+            try {
+                await initNewPage(newPage, pageClosePromises, processPageParams, pagesDict);
+            }
+            catch (e) {
+                log(`Error initializing new page: ${e}`);
+            }
         });
-        const page = await context.newPage();
         await page.goto(url, { timeout: 0 });
         // to execute and wait for all pages to close
         // idea is for promise to be pending until page.on('close') detected