@govtechsg/oobee 0.10.83 → 0.10.84

package/README.md

@@ -90,6 +90,11 @@ verapdf --version
 | WARN_LEVEL | Only used in tests. |  |
 | OOBEE_DISABLE_BROWSER_DOWNLOAD | Experimental flag to disable file downloads on Chrome/Chromium/Edge.  Does not affect Local File scan | |
 | OOBEE_SLOWMO | Experimental flag to slow down web browser behaviour by specified duration (in miliseconds) | |
+| HTTP_PROXY | URL of the proxy server to be used for HTTP requests (e.g. `http://proxy.example.com:8080`). | |
+| HTTPS_PROXY | URL of the proxy server to be used for HTTPS requests (e.g. `https://proxy.example.com:8080`). | |
+| ALL_PROXY | URL of the proxy server to be used for all requests, typically used for SOCKS5 proxies (e.g. `socks5://proxy.example.com:1080`. Note: IPv6 direct connections may still continue even though socks5 proxy is specified due to a known issue with Chrome/Chromium. (Recommended workaround is to turn off IPv6 at host-level). | |
+| NO_PROXY | Comma-separated list of domains that should bypass the proxy (e.g. `localhost,127.0.0.1,.example.com`). | |
+| INCLUDE_PROXY | Comma-separated list of domains that should specifically be routed through the proxy. | |
 
 #### Environment variables used internally (Do not set)
 Do not set these environment variables or behaviour might change unexpectedly.
@@ -677,4 +682,4 @@ It uses the existing report *.json files for the embedded HTML dataset.
 
 ```
 npx tsx dev/runGenerateJustHtmlReport.ts results/<report directory>
-```
+```

package/dist/constants/common.js

@@ -364,7 +364,19 @@ const checkUrlConnectivityWithBrowser = async (url, browserToRun, clonedDataDir,
         });
         if (!response)
             throw new Error('No response from navigation');
-        // We use the response headers from the navigation we just performed.
+        // Wait briefly for JS/meta-refresh redirects to settle before reading the final URL.
+        // Server-side redirects are already reflected after goto(), but client-side redirects
+        // (e.g. domain.tld -> www.domain.tld via JS or meta-refresh) need extra time.
+        try {
+            await Promise.race([
+                page.waitForURL(currentUrl => currentUrl !== url, { timeout: 5000 }),
+                new Promise(resolve => setTimeout(resolve, 1000)), // minimum settle time
+            ]);
+        }
+        catch {
+            // No redirect happened within the window — that's fine, continue with current URL
+        }
+        // Re-read page.url() AFTER potential client-side redirects have resolved
         const finalUrl = page.url();
         const finalStatus = response.status();
         const headers = response.headers();

package/dist/crawlers/crawlDomain.js

@@ -24,7 +24,7 @@ const crawlDomain = async ({ url, randomToken, host: _host, viewportSettings, ma
     const crawlStartTime = Date.now();
     let dataset;
     let urlsCrawled;
-    let requestQueue;
+    const { requestQueue } = await createCrawleeSubFolders(randomToken);
     let durationExceeded = false;
     if (fromCrawlIntelligentSitemap) {
         dataset = datasetFromIntelligent;
@@ -34,65 +34,41 @@ const crawlDomain = async ({ url, randomToken, host: _host, viewportSettings, ma
         ({ dataset } = await createCrawleeSubFolders(randomToken));
         urlsCrawled = { ...constants.urlsCrawledObj };
     }
-    ({ requestQueue } = await createCrawleeSubFolders(randomToken));
     const pdfDownloads = [];
     const uuidToPdfMapping = {};
+    const queuedUrlSet = new Set();
+    const scannedUrlSet = new Set(urlsCrawled.scanned.map(item => item.url));
+    const scannedResolvedUrlSet = new Set(urlsCrawled.scanned.map(item => item.actualUrl || item.url));
     const isScanHtml = [FileTypes.All, FileTypes.HtmlOnly].includes(fileTypes);
     const isScanPdfs = [FileTypes.All, FileTypes.PdfOnly].includes(fileTypes);
     const { maxConcurrency } = constants;
     const { playwrightDeviceDetailsObject } = viewportSettings;
-    await requestQueue.addRequest({
-        url,
-        skipNavigation: isUrlPdf(url),
-        label: url,
-    });
-    const enqueueProcess = async (page, enqueueLinks, browserContext) => {
+    const enqueueUniqueRequest = async ({ url, skipNavigation, label, }) => {
+        if (queuedUrlSet.has(url)) {
+            return;
+        }
+        queuedUrlSet.add(url);
         try {
-            await enqueueLinks({
-                // set selector matches anchor elements with href but not contains # or starting with mailto:
-                selector: `a:not(${disallowedSelectorPatterns})`,
-                strategy,
-                requestQueue,
-                transformRequestFunction: (req) => {
-                    try {
-                        req.url = req.url.replace(/(?<=&|\?)utm_.*?(&|$)/gim, '');
-                    }
-                    catch (e) {
-                        consoleLogger.error(e);
-                    }
-                    if (urlsCrawled.scanned.some(item => item.url === req.url)) {
-                        req.skipNavigation = true;
-                    }
-                    if (isDisallowedInRobotsTxt(req.url))
-                        return null;
-                    if (isBlacklisted(req.url, blacklistedPatterns))
-                        return null;
-                    if (isUrlPdf(req.url)) {
-                        // playwright headless mode does not support navigation to pdf document
-                        req.skipNavigation = true;
-                    }
-                    req.label = req.url;
-                    return req;
-                },
+            await requestQueue.addRequest({
+                url,
+                skipNavigation,
+                label,
             });
-            // If safeMode flag is enabled, skip enqueueLinksByClickingElements
-            if (!safeMode) {
-                // Try catch is necessary as clicking links is best effort, it may result in new pages that cause browser load or navigation errors that PlaywrightCrawler does not handle
-                try {
-                    await customEnqueueLinksByClickingElements(page, browserContext);
-                }
-                catch (e) {
-                    // do nothing;
-                }
-            }
         }
-        catch {
-            // No logging for this case as it is best effort to handle dynamic client-side JavaScript redirects and clicks.
-            // Handles browser page object been closed.
+        catch (error) {
+            queuedUrlSet.delete(url);
+            throw error;
         }
     };
-    const customEnqueueLinksByClickingElements = async (page, browserContext) => {
-        const initialPageUrl = page.url().toString();
+    await enqueueUniqueRequest({
+        url,
+        skipNavigation: isUrlPdf(url),
+        label: url,
+    });
+    const customEnqueueLinksByClickingElements = async (currentPage, browserContext) => {
+        let workingPage = currentPage;
+        const initialPageUrl = workingPage.url().toString();
+        const selectedElementsString = cssQuerySelectors.join(', ');
         const isExcluded = (newPageUrl) => {
             const isAlreadyScanned = urlsCrawled.scanned.some(item => item.url === newPageUrl);
             const isBlacklistedUrl = isBlacklisted(newPageUrl, blacklistedPatterns);
@@ -100,13 +76,13 @@ const crawlDomain = async ({ url, randomToken, host: _host, viewportSettings, ma
             const isNotSupportedDocument = disallowedListOfPatterns.some(pattern => newPageUrl.toLowerCase().startsWith(pattern));
             return isNotSupportedDocument || isAlreadyScanned || isBlacklistedUrl || isNotFollowStrategy;
         };
-        const setPageListeners = (page) => {
+        const setPageListeners = (pageListener) => {
             // event listener to handle new page popups upon button click
-            page.on('popup', async (newPage) => {
+            pageListener.on('popup', async (newPage) => {
                 try {
-                    if (newPage.url() != initialPageUrl && !isExcluded(newPage.url())) {
+                    if (newPage.url() !== initialPageUrl && !isExcluded(newPage.url())) {
                         const newPageUrl = newPage.url().replace(/(?<=&|\?)utm_.*?(&|$)/gim, '');
-                        await requestQueue.addRequest({
+                        await enqueueUniqueRequest({
                             url: newPageUrl,
                             skipNavigation: isUrlPdf(newPage.url()),
                             label: newPageUrl,
@@ -128,13 +104,13 @@ const crawlDomain = async ({ url, randomToken, host: _host, viewportSettings, ma
                 }
             });
             // event listener to handle navigation to new url within same page upon element click
-            page.on('framenavigated', async (newFrame) => {
+            pageListener.on('framenavigated', async (newFrame) => {
                 try {
                     if (newFrame.url() !== initialPageUrl &&
                         !isExcluded(newFrame.url()) &&
-                        !(newFrame.url() == 'about:blank')) {
+                        !(newFrame.url() === 'about:blank')) {
                         const newFrameUrl = newFrame.url().replace(/(?<=&|\?)utm_.*?(&|$)/gim, '');
-                        await requestQueue.addRequest({
+                        await enqueueUniqueRequest({
                             url: newFrameUrl,
                             skipNavigation: isUrlPdf(newFrame.url()),
                             label: newFrameUrl,
@@ -147,28 +123,32 @@ const crawlDomain = async ({ url, randomToken, host: _host, viewportSettings, ma
                 }
             });
         };
-        setPageListeners(page);
+        setPageListeners(workingPage);
         let currentElementIndex = 0;
         let isAllElementsHandled = false;
+        // This loop is intentionally sequential because each step depends on the latest page state
+        // (navigation, popup/frame events, and potential page recreation).
+        // Running iterations in parallel (for example with Promise.all) would race on shared `page`
+        // state, causing stale element handles and nondeterministic enqueue/navigation behavior.
+        /* eslint-disable no-await-in-loop */
         while (!isAllElementsHandled) {
             try {
                 // navigate back to initial page if clicking on a element previously caused it to navigate to a new url
-                if (page.url() != initialPageUrl) {
+                if (workingPage.url() !== initialPageUrl) {
                     try {
-                        await page.close();
+                        await workingPage.close();
                     }
                     catch {
                         // No logging for this case as it is best effort to handle dynamic client-side JavaScript redirects and clicks.
                         // Handles browser page object been closed.
                     }
-                    page = await browserContext.newPage();
-                    await page.goto(initialPageUrl, {
+                    workingPage = await browserContext.newPage();
+                    await workingPage.goto(initialPageUrl, {
                         waitUntil: 'domcontentloaded',
                     });
-                    setPageListeners(page);
+                    setPageListeners(workingPage);
                 }
-                const selectedElementsString = cssQuerySelectors.join(', ');
-                const selectedElements = await page.$$(selectedElementsString);
+                const selectedElements = await workingPage.$$(selectedElementsString);
                 // edge case where there might be elements on page that appears intermittently
                 if (currentElementIndex + 1 > selectedElements.length || !selectedElements) {
                     break;
@@ -181,36 +161,34 @@ const crawlDomain = async ({ url, randomToken, host: _host, viewportSettings, ma
                 currentElementIndex += 1;
                 let newUrlFoundInElement = null;
                 if (await element.isVisible()) {
+                    const currentPageUrl = workingPage.url();
                     // Find url in html elements without clicking them
-                    await page
-                        .evaluate(element => {
+                    const result = await workingPage.evaluate(pageElement => {
                         // find href attribute
-                        const hrefUrl = element.getAttribute('href');
+                        const hrefUrl = pageElement.getAttribute('href');
                         // find url in datapath
-                        const dataPathUrl = element.getAttribute('data-path');
+                        const dataPathUrl = pageElement.getAttribute('data-path');
                         return hrefUrl || dataPathUrl;
-                    }, element)
-                        .then(result => {
-                        if (result) {
-                            newUrlFoundInElement = result;
-                            const pageUrl = new URL(page.url());
-                            const baseUrl = `${pageUrl.protocol}//${pageUrl.host}`;
-                            let absoluteUrl;
-                            // Construct absolute URL using base URL
-                            try {
-                                // Check if newUrlFoundInElement is a valid absolute URL
-                                absoluteUrl = new URL(newUrlFoundInElement);
-                            }
-                            catch (e) {
-                                // If it's not a valid URL, treat it as a relative URL
-                                absoluteUrl = new URL(newUrlFoundInElement, baseUrl);
-                            }
-                            newUrlFoundInElement = absoluteUrl.href;
+                    }, element);
+                    if (result) {
+                        newUrlFoundInElement = result;
+                        const pageUrl = new URL(currentPageUrl);
+                        const baseUrl = `${pageUrl.protocol}//${pageUrl.host}`;
+                        let absoluteUrl;
+                        // Construct absolute URL using base URL
+                        try {
+                            // Check if newUrlFoundInElement is a valid absolute URL
+                            absoluteUrl = new URL(newUrlFoundInElement);
                         }
-                    });
+                        catch {
+                            // If it's not a valid URL, treat it as a relative URL
+                            absoluteUrl = new URL(newUrlFoundInElement, baseUrl);
+                        }
+                        newUrlFoundInElement = absoluteUrl.href;
+                    }
                     if (newUrlFoundInElement && !isExcluded(newUrlFoundInElement)) {
                         const newUrlFoundInElementUrl = newUrlFoundInElement.replace(/(?<=&|\?)utm_.*?(&|$)/gim, '');
-                        await requestQueue.addRequest({
+                        await enqueueUniqueRequest({
                             url: newUrlFoundInElementUrl,
                             skipNavigation: isUrlPdf(newUrlFoundInElement),
                             label: newUrlFoundInElementUrl,
@@ -218,15 +196,16 @@ const crawlDomain = async ({ url, randomToken, host: _host, viewportSettings, ma
                     }
                     else if (!newUrlFoundInElement) {
                         try {
-                            const shouldSkip = await shouldSkipClickDueToDisallowedHref(page, element);
+                            const shouldSkip = await shouldSkipClickDueToDisallowedHref(workingPage, element);
                             if (shouldSkip) {
-                                const elementHtml = await page.evaluate(el => el.outerHTML, element);
+                                const elementHtml = await workingPage.evaluate(el => el.outerHTML, element);
                                 consoleLogger.info('Skipping a click due to disallowed href nearby. Element HTML:', elementHtml);
-                                continue;
                             }
-                            // Find url in html elements by manually clicking them. New page navigation/popups will be handled by event listeners above
-                            await element.click({ force: true });
-                            await page.waitForTimeout(1000); // Add a delay of 1 second between each Element click
+                            else {
+                                // Find url in html elements by manually clicking them. New page navigation/popups will be handled by event listeners above
+                                await element.click({ force: true });
+                                await workingPage.waitForTimeout(1000); // Add a delay of 1 second between each Element click
+                            }
                         }
                         catch {
                             // No logging for this case as it is best effort to handle dynamic client-side JavaScript redirects and clicks.
@@ -240,6 +219,61 @@ const crawlDomain = async ({ url, randomToken, host: _host, viewportSettings, ma
                 // Handles browser page object been closed.
             }
         }
+        /* eslint-enable no-await-in-loop */
+    };
+    const enqueueProcess = async (page, enqueueLinks, browserContext) => {
+        try {
+            await enqueueLinks({
+                // set selector matches anchor elements with href but not contains # or starting with mailto:
+                selector: `a:not(${disallowedSelectorPatterns})`,
+                strategy,
+                requestQueue,
+                transformRequestFunction: (req) => {
+                    try {
+                        req.url = req.url.replace(/(?<=&|\?)utm_.*?(&|$)/gim, '');
+                    }
+                    catch (e) {
+                        consoleLogger.error(e);
+                    }
+                    if (scannedUrlSet.has(req.url)) {
+                        req.skipNavigation = true;
+                    }
+                    if (isDisallowedInRobotsTxt(req.url))
+                        return null;
+                    if (isBlacklisted(req.url, blacklistedPatterns))
+                        return null;
+                    if (isUrlPdf(req.url)) {
+                        // playwright headless mode does not support navigation to pdf document
+                        req.skipNavigation = true;
+                    }
+                    req.label = req.url;
+                    return req;
+                },
+            });
+            // If safeMode flag is enabled, skip enqueueLinksByClickingElements
+            if (!safeMode) {
+                // Only run the expensive element-clicking discovery on pages sharing the
+                // same hostname as the seed URL.  Cross-subdomain pages (reachable via
+                // same-domain strategy) still contribute their <a> links above, but
+                // clicking every interactive element on them is too slow and starves
+                // the crawler of time to discover pages on the primary hostname.
+                const currentHostname = new URL(page.url()).hostname;
+                const seedHostname = new URL(url).hostname;
+                if (currentHostname === seedHostname) {
+                    // Try catch is necessary as clicking links is best effort, it may result in new pages that cause browser load or navigation errors that PlaywrightCrawler does not handle
+                    try {
+                        await customEnqueueLinksByClickingElements(page, browserContext);
+                    }
+                    catch {
+                        // do nothing;
+                    }
+                }
+            }
+        }
+        catch {
+            // No logging for this case as it is best effort to handle dynamic client-side JavaScript redirects and clicks.
+            // Handles browser page object been closed.
+        }
     };
     let isAbortingScanNow = false;
     const crawler = register(new crawlee.PlaywrightCrawler({
@@ -261,8 +295,11 @@ const crawlDomain = async ({ url, randomToken, host: _host, viewportSettings, ma
                     const subProfileDir = path.join(baseDir, `profile-${Date.now()}-${Math.random().toString(36).slice(2, 8)}`);
                     await fsp.mkdir(subProfileDir, { recursive: true });
                     // Assign to Crawlee's launcher
+                    // Crawlee preLaunchHooks expects launchContext to be mutated in-place.
+                    // eslint-disable-next-line no-param-reassign
                     launchContext.userDataDir = subProfileDir;
                     // Safely extend launchOptions
+                    // eslint-disable-next-line no-param-reassign
                     launchContext.launchOptions = {
                         ...launchContext.launchOptions,
                         ignoreHTTPSErrors: true,
@@ -287,7 +324,7 @@ const crawlDomain = async ({ url, randomToken, host: _host, viewportSettings, ma
                         const OBSERVER_TIMEOUT = 5000; // hard cap on total wait
                         const observer = new MutationObserver(() => {
                             clearTimeout(timeout);
-                            mutationCount++;
+                            mutationCount += 1;
                             if (mutationCount > MAX_MUTATIONS) {
                                 observer.disconnect();
                                 resolve('Too many mutations, exiting.');
@@ -308,6 +345,9 @@ const crawlDomain = async ({ url, randomToken, host: _host, viewportSettings, ma
                         if (!root || typeof observer.observe !== 'function') {
                             resolve('No root node to observe.');
                         }
+                        else {
+                            observer.observe(root, { childList: true, subtree: true });
+                        }
                     });
                 });
                 let finalUrl = page.url();
@@ -319,7 +359,7 @@ const crawlDomain = async ({ url, randomToken, host: _host, viewportSettings, ma
                 }
                 const isRedirected = !areLinksEqual(finalUrl, requestLabelUrl);
                 if (isRedirected) {
-                    await requestQueue.addRequest({ url: finalUrl, label: finalUrl });
+                    await enqueueUniqueRequest({ url: finalUrl, label: finalUrl });
                 }
                 else {
                     request.skipNavigation = false;
@@ -327,7 +367,7 @@ const crawlDomain = async ({ url, randomToken, host: _host, viewportSettings, ma
             },
         ],
         requestHandlerTimeoutSecs: 90, // Allow each page to be processed by up from default 60 seconds
-        requestHandler: async ({ page, request, response, crawler, sendRequest, enqueueLinks }) => {
+        requestHandler: async ({ page, request, response, crawler: activeCrawler, sendRequest, enqueueLinks, }) => {
             const browserContext = page.context();
             try {
                 await waitForPageLoaded(page, 10000);
@@ -335,6 +375,11 @@ const crawlDomain = async ({ url, randomToken, host: _host, viewportSettings, ma
                 if (page.url() !== 'about:blank') {
                     actualUrl = page.url();
                 }
+                // Second-pass requests: only do click-discovery, skip scanning
+                if (request.label?.startsWith('__clickpass__')) {
+                    await enqueueProcess(page, enqueueLinks, browserContext);
+                    return;
+                }
                 if (!isFollowStrategy(url, actualUrl, strategy) &&
                     (isBlacklisted(actualUrl, blacklistedPatterns) || (isUrlPdf(actualUrl) && !isScanPdfs))) {
                     guiInfoLog(guiInfoStatusTypes.SKIPPED, {
@@ -350,12 +395,12 @@ const crawlDomain = async ({ url, randomToken, host: _host, viewportSettings, ma
                         durationExceeded = true;
                     }
                     isAbortingScanNow = true;
-                    crawler.autoscaledPool.abort();
+                    activeCrawler.autoscaledPool.abort();
                     return;
                 }
                 // if URL has already been scanned
-                if (urlsCrawled.scanned.some(item => item.url === request.url)) {
-                    // await enqueueProcess(page, enqueueLinks, browserContext);
+                if (scannedUrlSet.has(request.url)) {
+                    await enqueueProcess(page, enqueueLinks, browserContext);
                     return;
                 }
                 if (isDisallowedInRobotsTxt(request.url)) {
@@ -382,8 +427,8 @@ const crawlDomain = async ({ url, randomToken, host: _host, viewportSettings, ma
                       */
                         return;
                     }
-                    const { pdfFileName, url } = handlePdfDownload(randomToken, pdfDownloads, request, sendRequest, urlsCrawled);
-                    uuidToPdfMapping[pdfFileName] = url;
+                    const { pdfFileName, url: downloadedPdfUrl } = handlePdfDownload(randomToken, pdfDownloads, request, sendRequest, urlsCrawled);
+                    uuidToPdfMapping[pdfFileName] = downloadedPdfUrl;
                     return;
                 }
                 if (isBlacklistedFileExtensions(actualUrl, blackListedFileExtensions)) {
@@ -449,7 +494,7 @@ const crawlDomain = async ({ url, randomToken, host: _host, viewportSettings, ma
                     }
                     const results = await runAxeScript({ includeScreenshots, page, randomToken, ruleset });
                     if (isRedirected) {
-                        const isLoadedUrlInCrawledUrls = urlsCrawled.scanned.some(item => (item.actualUrl || item.url) === actualUrl);
+                        const isLoadedUrlInCrawledUrls = scannedResolvedUrlSet.has(actualUrl);
                         if (isLoadedUrlInCrawledUrls) {
                             urlsCrawled.notScannedRedirects.push({
                                 fromUrl: request.url,
@@ -468,6 +513,8 @@ const crawlDomain = async ({ url, randomToken, host: _host, viewportSettings, ma
                                 pageTitle: results.pageTitle,
                                 actualUrl, // i.e. actualUrl
                             });
+                            scannedUrlSet.add(request.url);
+                            scannedResolvedUrlSet.add(actualUrl);
                             urlsCrawled.scannedRedirects.push({
                                 fromUrl: request.url,
                                 toUrl: actualUrl, // i.e. actualUrl
@@ -477,20 +524,20 @@ const crawlDomain = async ({ url, randomToken, host: _host, viewportSettings, ma
                             await dataset.pushData(results);
                         }
                     }
-                    else {
+                    else if (urlsCrawled.scanned.length < maxRequestsPerCrawl) {
                         // One more check if scanned pages have reached limit due to multi-instances of handler running
-                        if (urlsCrawled.scanned.length < maxRequestsPerCrawl) {
-                            guiInfoLog(guiInfoStatusTypes.SCANNED, {
-                                numScanned: urlsCrawled.scanned.length,
-                                urlScanned: request.url,
-                            });
-                            urlsCrawled.scanned.push({
-                                url: request.url,
-                                actualUrl: request.url,
-                                pageTitle: results.pageTitle,
-                            });
-                            await dataset.pushData(results);
-                        }
+                        guiInfoLog(guiInfoStatusTypes.SCANNED, {
+                            numScanned: urlsCrawled.scanned.length,
+                            urlScanned: request.url,
+                        });
+                        urlsCrawled.scanned.push({
+                            url: request.url,
+                            actualUrl: request.url,
+                            pageTitle: results.pageTitle,
+                        });
+                        scannedUrlSet.add(request.url);
+                        scannedResolvedUrlSet.add(request.url);
+                        await dataset.pushData(results);
                     }
                 }
                 else {
@@ -521,15 +568,15 @@ const crawlDomain = async ({ url, randomToken, host: _host, viewportSettings, ma
                             numScanned: urlsCrawled.scanned.length,
                             urlScanned: request.url,
                         });
-                        page = await browserContext.newPage();
-                        await page.goto(request.url);
-                        await page.route('**/*', async (route) => {
+                        const recoveryPage = await browserContext.newPage();
+                        await recoveryPage.goto(request.url);
+                        await recoveryPage.route('**/*', async (route) => {
                             const interceptedRequest = route.request();
                             if (interceptedRequest.resourceType() === 'document') {
                                 const interceptedRequestUrl = interceptedRequest
                                     .url()
                                     .replace(/(?<=&|\?)utm_.*?(&|$)/gim, '');
-                                await requestQueue.addRequest({
+                                await enqueueUniqueRequest({
                                     url: interceptedRequestUrl,
                                     skipNavigation: isUrlPdf(interceptedRequest.url()),
                                     label: interceptedRequestUrl,
@@ -587,6 +634,59 @@ const crawlDomain = async ({ url, randomToken, host: _host, viewportSettings, ma
         }),
     }));
     await crawler.run();
+    // Additional passes: keep re-visiting scanned seed-hostname pages for
+    // click-discovery until no new pages are found or limits are reached.
+    if (!safeMode && !isAbortingScanNow && !durationExceeded) {
+        const seedHostname = new URL(url).hostname;
+        const clickPassVisited = new Set();
+        let prevScannedCount;
+        do {
+            prevScannedCount = urlsCrawled.scanned.length;
+            if (prevScannedCount >= maxRequestsPerCrawl)
+                break;
+            if (scanDuration > 0 && Date.now() - crawlStartTime > scanDuration * 1000)
+                break;
+            const seedHostnamePages = urlsCrawled.scanned
+                .map(item => item.actualUrl || item.url)
+                .filter(pageUrl => {
+                try {
+                    return new URL(pageUrl).hostname === seedHostname && !clickPassVisited.has(pageUrl);
+                }
+                catch {
+                    return false;
+                }
+            });
+            if (seedHostnamePages.length === 0)
+                break;
+            let enqueued = 0;
+            for (const pageUrl of seedHostnamePages) {
+                if (urlsCrawled.scanned.length >= maxRequestsPerCrawl)
+                    break;
+                if (scanDuration > 0 && Date.now() - crawlStartTime > scanDuration * 1000)
+                    break;
+                clickPassVisited.add(pageUrl);
+                try {
+                    const clickPassLabel = `__clickpass__${pageUrl}`;
+                    if (!queuedUrlSet.has(clickPassLabel)) {
+                        queuedUrlSet.add(clickPassLabel);
+                        await requestQueue.addRequest({
+                            url: pageUrl,
+                            label: clickPassLabel,
+                            skipNavigation: false,
+                        });
+                        enqueued += 1;
+                    }
+                }
+                catch {
+                    // ignore enqueue errors
+                }
+            }
+            if (enqueued === 0)
+                break;
+            await crawler.run();
+            // Stop looping if no new pages were discovered in this pass
+        } while (urlsCrawled.scanned.length > prevScannedCount);
+    }
     if (pdfDownloads.length > 0) {
         // wait for pdf downloads to complete
         await Promise.all(pdfDownloads);
@@ -596,7 +696,7 @@ const crawlDomain = async ({ url, randomToken, host: _host, viewportSettings, ma
         const pdfResults = await mapPdfScanResults(randomToken, uuidToPdfMapping);
         // get screenshots from pdf docs
         if (includeScreenshots) {
-            await Promise.all(pdfResults.map(async (result) => await doPdfScreenshots(randomToken, result)));
+            await Promise.all(pdfResults.map(result => doPdfScreenshots(randomToken, result)));
         }
         // push results for each pdf document to key value store
         await Promise.all(pdfResults.map(result => dataset.pushData(result)));

package/dist/crawlers/crawlIntelligentSitemap.js

@@ -22,13 +22,25 @@ const crawlIntelligentSitemap = async (url, randomToken, host, viewportSettings,
     async function findSitemap(link, userDataDirectory, extraHTTPHeaders) {
         const homeUrl = getHomeUrl(link);
         let sitemapLink = '';
-        const effectiveUserDataDirectory = process.env.CRAWLEE_HEADLESS === '1' ? userDataDirectory : '';
-        const context = await constants.launcher.launchPersistentContext(effectiveUserDataDirectory, {
-            headless: process.env.CRAWLEE_HEADLESS === '1',
-            ...getPlaywrightLaunchOptions(browser),
-            ...(extraHTTPHeaders && { extraHTTPHeaders }),
-        });
-        register(context);
+        const launchOptions = getPlaywrightLaunchOptions(browser);
+        let context;
+        let browserInstance;
+        if (process.env.CRAWLEE_HEADLESS === '1') {
+            const effectiveUserDataDirectory = userDataDirectory || '';
+            context = await constants.launcher.launchPersistentContext(effectiveUserDataDirectory, {
+                ...launchOptions,
+                ...(extraHTTPHeaders && { extraHTTPHeaders }),
+            });
+            register(context);
+        }
+        else {
+            // In headful mode, avoid launchPersistentContext to prevent "Browser window not found"
+            browserInstance = await constants.launcher.launch(launchOptions);
+            register(browserInstance);
+            context = await browserInstance.newContext({
+                ...(extraHTTPHeaders && { extraHTTPHeaders }),
+            });
+        }
         const page = await context.newPage();
         for (const path of sitemapPaths) {
             sitemapLink = homeUrl + path;
@@ -39,6 +51,9 @@ const crawlIntelligentSitemap = async (url, randomToken, host, viewportSettings,
         }
         await page.close();
         await context.close().catch(() => { });
+        if (browserInstance) {
+            await browserInstance.close().catch(() => { });
+        }
         return sitemapExist ? sitemapLink : '';
     }
     const checkUrlExists = async (page, parsedUrl) => {

package/dist/crawlers/runCustom.js

@@ -6,6 +6,7 @@ import constants, { getIntermediateScreenshotsPath, guiInfoStatusTypes, } from '
 import { initNewPage, log } from './custom/utils.js';
 import { guiInfoLog } from '../logs.js';
 import { addUrlGuardScript } from './guards/urlGuard.js';
+import { getPlaywrightLaunchOptions } from '../constants/common.js';
 // Export of classes
 export class ProcessPageParams {
     constructor(scannedIdx, blacklistedPatterns, includeScreenshots, dataset, intermediateScreenshotsPath, urlsCrawled, randomToken) {
@@ -34,11 +35,16 @@ const runCustom = async (url, randomToken, viewportSettings, blacklistedPatterns
     try {
         const deviceConfig = viewportSettings.playwrightDeviceDetailsObject;
         const hasCustomViewport = !!deviceConfig;
+        const baseLaunchOptions = getPlaywrightLaunchOptions('chrome');
+        // Merge base args with custom flow specific args
+        const baseArgs = baseLaunchOptions.args || [];
+        const customArgs = hasCustomViewport ? ['--window-size=1920,1040'] : ['--start-maximized'];
+        const mergedArgs = [...baseArgs.filter(a => !a.startsWith('--window-size') && a !== '--start-maximized'), ...customArgs];
         const browser = await chromium.launch({
-            args: hasCustomViewport ? ['--window-size=1920,1040'] : ['--start-maximized'],
+            ...baseLaunchOptions,
+            args: mergedArgs,
             headless: false,
             channel: 'chrome',
-            // bypassCSP: true,
         });
         const context = await browser.newContext({
             ignoreHTTPSErrors: true,