@govtechsg/oobee 0.10.83 → 0.10.84

package/src/crawlers/crawlDomain.ts

@@ -1,9 +1,6 @@
 import crawlee, { EnqueueStrategy } from 'crawlee';
-import fs from 'fs';
 import type { BrowserContext, ElementHandle, Frame, Page } from 'playwright';
-import type { EnqueueLinksOptions, RequestOptions } from 'crawlee';
-import https from 'https';
-import type { BatchAddRequestsResult } from '@crawlee/types';
+import type { PlaywrightCrawlingContext, RequestOptions } from 'crawlee';
 import * as path from 'path';
 import fsp from 'fs/promises';
 import {
@@ -39,7 +36,7 @@ import {
   mapPdfScanResults,
   doPdfScreenshots,
 } from './pdfScanFunc.js';
-import { consoleLogger, guiInfoLog, silentLogger } from '../logs.js';
+import { consoleLogger, guiInfoLog } from '../logs.js';
 import { ViewportSettingsClass } from '../combine.js';
 
 const isBlacklisted = (url: string, blacklistedPatterns: string[]) => {
@@ -104,7 +101,8 @@ const crawlDomain = async ({
   const crawlStartTime = Date.now();
   let dataset: crawlee.Dataset;
   let urlsCrawled: UrlsCrawled;
-  let requestQueue: crawlee.RequestQueue;
+  const { requestQueue }: { requestQueue: crawlee.RequestQueue } =
+    await createCrawleeSubFolders(randomToken);
   let durationExceeded = false;
 
   if (fromCrawlIntelligentSitemap) {
@@ -115,73 +113,57 @@ const crawlDomain = async ({
     urlsCrawled = { ...constants.urlsCrawledObj };
   }
 
-  ({ requestQueue } = await createCrawleeSubFolders(randomToken));
-
   const pdfDownloads: Promise<void>[] = [];
   const uuidToPdfMapping: Record<string, string> = {};
+  const queuedUrlSet = new Set<string>();
+  const scannedUrlSet = new Set<string>(urlsCrawled.scanned.map(item => item.url));
+  const scannedResolvedUrlSet = new Set<string>(
+    urlsCrawled.scanned.map(item => item.actualUrl || item.url),
+  );
   const isScanHtml = [FileTypes.All, FileTypes.HtmlOnly].includes(fileTypes as FileTypes);
   const isScanPdfs = [FileTypes.All, FileTypes.PdfOnly].includes(fileTypes as FileTypes);
   const { maxConcurrency } = constants;
   const { playwrightDeviceDetailsObject } = viewportSettings;
 
-  await requestQueue.addRequest({
+  const enqueueUniqueRequest = async ({
     url,
-    skipNavigation: isUrlPdf(url),
-    label: url,
-  });
+    skipNavigation,
+    label,
+  }: {
+    url: string;
+    skipNavigation?: boolean;
+    label?: string;
+  }) => {
+    if (queuedUrlSet.has(url)) {
+      return;
+    }
+    queuedUrlSet.add(url);
 
-  const enqueueProcess = async (
-    page: Page,
-    enqueueLinks: (options: EnqueueLinksOptions) => Promise<BatchAddRequestsResult>,
-    browserContext: BrowserContext,
-  ) => {
     try {
-      await enqueueLinks({
-        // set selector matches anchor elements with href but not contains # or starting with mailto:
-        selector: `a:not(${disallowedSelectorPatterns})`,
-        strategy,
-        requestQueue,
-        transformRequestFunction: (req: RequestOptions): RequestOptions | null => {
-          try {
-            req.url = req.url.replace(/(?<=&|\?)utm_.*?(&|$)/gim, '');
-          } catch (e) {
-            consoleLogger.error(e);
-          }
-          if (urlsCrawled.scanned.some(item => item.url === req.url)) {
-            req.skipNavigation = true;
-          }
-          if (isDisallowedInRobotsTxt(req.url)) return null;
-          if (isBlacklisted(req.url, blacklistedPatterns)) return null;
-          if (isUrlPdf(req.url)) {
-            // playwright headless mode does not support navigation to pdf document
-            req.skipNavigation = true;
-          }
-          req.label = req.url;
-
-          return req;
-        },
+      await requestQueue.addRequest({
+        url,
+        skipNavigation,
+        label,
       });
-
-      // If safeMode flag is enabled, skip enqueueLinksByClickingElements
-      if (!safeMode) {
-        // Try catch is necessary as clicking links is best effort, it may result in new pages that cause browser load or navigation errors that PlaywrightCrawler does not handle
-        try {
-          await customEnqueueLinksByClickingElements(page, browserContext);
-        } catch (e) {
-          // do nothing;
-        }
-      }
-    } catch {
-      // No logging for this case as it is best effort to handle dynamic client-side JavaScript redirects and clicks.
-      // Handles browser page object been closed.
+    } catch (error) {
+      queuedUrlSet.delete(url);
+      throw error;
     }
   };
 
+  await enqueueUniqueRequest({
+    url,
+    skipNavigation: isUrlPdf(url),
+    label: url,
+  });
+
   const customEnqueueLinksByClickingElements = async (
-    page: Page,
+    currentPage: Page,
     browserContext: BrowserContext,
   ): Promise<void> => {
-    const initialPageUrl: string = page.url().toString();
+    let workingPage = currentPage;
+    const initialPageUrl: string = workingPage.url().toString();
+    const selectedElementsString = cssQuerySelectors.join(', ');
 
     const isExcluded = (newPageUrl: string): boolean => {
       const isAlreadyScanned: boolean = urlsCrawled.scanned.some(item => item.url === newPageUrl);
@@ -192,13 +174,13 @@ const crawlDomain = async ({
       );
       return isNotSupportedDocument || isAlreadyScanned || isBlacklistedUrl || isNotFollowStrategy;
     };
-    const setPageListeners = (page: Page): void => {
+    const setPageListeners = (pageListener: Page): void => {
       // event listener to handle new page popups upon button click
-      page.on('popup', async (newPage: Page) => {
+      pageListener.on('popup', async (newPage: Page) => {
         try {
-          if (newPage.url() != initialPageUrl && !isExcluded(newPage.url())) {
+          if (newPage.url() !== initialPageUrl && !isExcluded(newPage.url())) {
             const newPageUrl: string = newPage.url().replace(/(?<=&|\?)utm_.*?(&|$)/gim, '');
-            await requestQueue.addRequest({
+            await enqueueUniqueRequest({
               url: newPageUrl,
               skipNavigation: isUrlPdf(newPage.url()),
               label: newPageUrl,
@@ -218,15 +200,15 @@ const crawlDomain = async ({
       });
 
       // event listener to handle navigation to new url within same page upon element click
-      page.on('framenavigated', async (newFrame: Frame) => {
+      pageListener.on('framenavigated', async (newFrame: Frame) => {
         try {
           if (
             newFrame.url() !== initialPageUrl &&
             !isExcluded(newFrame.url()) &&
-            !(newFrame.url() == 'about:blank')
+            !(newFrame.url() === 'about:blank')
           ) {
             const newFrameUrl: string = newFrame.url().replace(/(?<=&|\?)utm_.*?(&|$)/gim, '');
-            await requestQueue.addRequest({
+            await enqueueUniqueRequest({
               url: newFrameUrl,
               skipNavigation: isUrlPdf(newFrame.url()),
               label: newFrameUrl,
@@ -238,28 +220,32 @@ const crawlDomain = async ({
         }
       });
     };
-    setPageListeners(page);
+    setPageListeners(workingPage);
     let currentElementIndex: number = 0;
     let isAllElementsHandled: boolean = false;
+    // This loop is intentionally sequential because each step depends on the latest page state
+    // (navigation, popup/frame events, and potential page recreation).
+    // Running iterations in parallel (for example with Promise.all) would race on shared `page`
+    // state, causing stale element handles and nondeterministic enqueue/navigation behavior.
+    /* eslint-disable no-await-in-loop */
     while (!isAllElementsHandled) {
       try {
         // navigate back to initial page if clicking on a element previously caused it to navigate to a new url
-        if (page.url() != initialPageUrl) {
+        if (workingPage.url() !== initialPageUrl) {
           try {
-            await page.close();
+            await workingPage.close();
           } catch {
             // No logging for this case as it is best effort to handle dynamic client-side JavaScript redirects and clicks.
             // Handles browser page object been closed.
           }
-          page = await browserContext.newPage();
-          await page.goto(initialPageUrl, {
+          workingPage = await browserContext.newPage();
+          await workingPage.goto(initialPageUrl, {
             waitUntil: 'domcontentloaded',
           });
-          setPageListeners(page);
+          setPageListeners(workingPage);
         }
-        const selectedElementsString = cssQuerySelectors.join(', ');
         const selectedElements: ElementHandle<SVGElement | HTMLElement>[] =
-          await page.$$(selectedElementsString);
+          await workingPage.$$(selectedElementsString);
         // edge case where there might be elements on page that appears intermittently
         if (currentElementIndex + 1 > selectedElements.length || !selectedElements) {
           break;
@@ -273,60 +259,57 @@ const crawlDomain = async ({
         currentElementIndex += 1;
         let newUrlFoundInElement: string = null;
         if (await element.isVisible()) {
+          const currentPageUrl = workingPage.url();
           // Find url in html elements without clicking them
-          await page
-            .evaluate(element => {
-              // find href attribute
-              const hrefUrl: string = element.getAttribute('href');
-
-              // find url in datapath
-              const dataPathUrl: string = element.getAttribute('data-path');
-
-              return hrefUrl || dataPathUrl;
-            }, element)
-            .then(result => {
-              if (result) {
-                newUrlFoundInElement = result;
-                const pageUrl: URL = new URL(page.url());
-                const baseUrl: string = `${pageUrl.protocol}//${pageUrl.host}`;
-                let absoluteUrl: URL;
-                // Construct absolute URL using base URL
-                try {
-                  // Check if newUrlFoundInElement is a valid absolute URL
-                  absoluteUrl = new URL(newUrlFoundInElement);
-                } catch (e) {
-                  // If it's not a valid URL, treat it as a relative URL
-                  absoluteUrl = new URL(newUrlFoundInElement, baseUrl);
-                }
-                newUrlFoundInElement = absoluteUrl.href;
-              }
-            });
+          const result = await workingPage.evaluate(pageElement => {
+            // find href attribute
+            const hrefUrl: string = pageElement.getAttribute('href');
+
+            // find url in datapath
+            const dataPathUrl: string = pageElement.getAttribute('data-path');
+
+            return hrefUrl || dataPathUrl;
+          }, element);
+          if (result) {
+            newUrlFoundInElement = result;
+            const pageUrl: URL = new URL(currentPageUrl);
+            const baseUrl: string = `${pageUrl.protocol}//${pageUrl.host}`;
+            let absoluteUrl: URL;
+            // Construct absolute URL using base URL
+            try {
+              // Check if newUrlFoundInElement is a valid absolute URL
+              absoluteUrl = new URL(newUrlFoundInElement);
+            } catch {
+              // If it's not a valid URL, treat it as a relative URL
+              absoluteUrl = new URL(newUrlFoundInElement, baseUrl);
+            }
+            newUrlFoundInElement = absoluteUrl.href;
+          }
           if (newUrlFoundInElement && !isExcluded(newUrlFoundInElement)) {
             const newUrlFoundInElementUrl: string = newUrlFoundInElement.replace(
               /(?<=&|\?)utm_.*?(&|$)/gim,
               '',
             );
 
-            await requestQueue.addRequest({
+            await enqueueUniqueRequest({
               url: newUrlFoundInElementUrl,
               skipNavigation: isUrlPdf(newUrlFoundInElement),
               label: newUrlFoundInElementUrl,
             });
           } else if (!newUrlFoundInElement) {
             try {
-              const shouldSkip = await shouldSkipClickDueToDisallowedHref(page, element);
+              const shouldSkip = await shouldSkipClickDueToDisallowedHref(workingPage, element);
               if (shouldSkip) {
-                const elementHtml = await page.evaluate(el => el.outerHTML, element);
+                const elementHtml = await workingPage.evaluate(el => el.outerHTML, element);
                 consoleLogger.info(
                   'Skipping a click due to disallowed href nearby. Element HTML:',
                   elementHtml,
                 );
-                continue;
+              } else {
+                // Find url in html elements by manually clicking them. New page navigation/popups will be handled by event listeners above
+                await element.click({ force: true });
+                await workingPage.waitForTimeout(1000); // Add a delay of 1 second between each Element click
               }
-
-              // Find url in html elements by manually clicking them. New page navigation/popups will be handled by event listeners above
-              await element.click({ force: true });
-              await page.waitForTimeout(1000); // Add a delay of 1 second between each Element click
             } catch {
               // No logging for this case as it is best effort to handle dynamic client-side JavaScript redirects and clicks.
               // Handles browser page object been closed.
@@ -338,6 +321,63 @@ const crawlDomain = async ({
         // Handles browser page object been closed.
       }
     }
+    /* eslint-enable no-await-in-loop */
+  };
+
+  const enqueueProcess = async (
+    page: Page,
+    enqueueLinks: PlaywrightCrawlingContext['enqueueLinks'],
+    browserContext: BrowserContext,
+  ) => {
+    try {
+      await enqueueLinks({
+        // set selector matches anchor elements with href but not contains # or starting with mailto:
+        selector: `a:not(${disallowedSelectorPatterns})`,
+        strategy,
+        requestQueue,
+        transformRequestFunction: (req: RequestOptions): RequestOptions | null => {
+          try {
+            req.url = req.url.replace(/(?<=&|\?)utm_.*?(&|$)/gim, '');
+          } catch (e) {
+            consoleLogger.error(e);
+          }
+          if (scannedUrlSet.has(req.url)) {
+            req.skipNavigation = true;
+          }
+          if (isDisallowedInRobotsTxt(req.url)) return null;
+          if (isBlacklisted(req.url, blacklistedPatterns)) return null;
+          if (isUrlPdf(req.url)) {
+            // playwright headless mode does not support navigation to pdf document
+            req.skipNavigation = true;
+          }
+          req.label = req.url;
+
+          return req;
+        },
+      });
+
+      // If safeMode flag is enabled, skip enqueueLinksByClickingElements
+      if (!safeMode) {
+        // Only run the expensive element-clicking discovery on pages sharing the
+        // same hostname as the seed URL.  Cross-subdomain pages (reachable via
+        // same-domain strategy) still contribute their <a> links above, but
+        // clicking every interactive element on them is too slow and starves
+        // the crawler of time to discover pages on the primary hostname.
+        const currentHostname = new URL(page.url()).hostname;
+        const seedHostname = new URL(url).hostname;
+        if (currentHostname === seedHostname) {
+          // Try catch is necessary as clicking links is best effort, it may result in new pages that cause browser load or navigation errors that PlaywrightCrawler does not handle
+          try {
+            await customEnqueueLinksByClickingElements(page, browserContext);
+          } catch {
+            // do nothing;
+          }
+        }
+      }
+    } catch {
+      // No logging for this case as it is best effort to handle dynamic client-side JavaScript redirects and clicks.
+      // Handles browser page object been closed.
+    }
   };
 
   let isAbortingScanNow = false;
@@ -368,9 +408,12 @@ const crawlDomain = async ({
             await fsp.mkdir(subProfileDir, { recursive: true });
 
             // Assign to Crawlee's launcher
+            // Crawlee preLaunchHooks expects launchContext to be mutated in-place.
+            // eslint-disable-next-line no-param-reassign
             launchContext.userDataDir = subProfileDir;
 
             // Safely extend launchOptions
+            // eslint-disable-next-line no-param-reassign
             launchContext.launchOptions = {
               ...launchContext.launchOptions,
               ignoreHTTPSErrors: true,
@@ -399,7 +442,7 @@ const crawlDomain = async ({
               const observer = new MutationObserver(() => {
                 clearTimeout(timeout);
 
-                mutationCount++;
+                mutationCount += 1;
                 if (mutationCount > MAX_MUTATIONS) {
                   observer.disconnect();
                   resolve('Too many mutations, exiting.');
@@ -422,6 +465,8 @@ const crawlDomain = async ({
               const root = document.documentElement || document.body || document;
               if (!root || typeof observer.observe !== 'function') {
                 resolve('No root node to observe.');
+              } else {
+                observer.observe(root, { childList: true, subtree: true });
               }
             });
           });
@@ -437,14 +482,21 @@ const crawlDomain = async ({
 
           const isRedirected = !areLinksEqual(finalUrl, requestLabelUrl);
           if (isRedirected) {
-            await requestQueue.addRequest({ url: finalUrl, label: finalUrl });
+            await enqueueUniqueRequest({ url: finalUrl, label: finalUrl });
           } else {
             request.skipNavigation = false;
           }
         },
       ],
       requestHandlerTimeoutSecs: 90, // Allow each page to be processed by up from default 60 seconds
-      requestHandler: async ({ page, request, response, crawler, sendRequest, enqueueLinks }) => {
+      requestHandler: async ({
+        page,
+        request,
+        response,
+        crawler: activeCrawler,
+        sendRequest,
+        enqueueLinks,
+      }) => {
         const browserContext: BrowserContext = page.context();
         try {
           await waitForPageLoaded(page, 10000);
@@ -454,6 +506,12 @@ const crawlDomain = async ({
             actualUrl = page.url();
           }
 
+          // Second-pass requests: only do click-discovery, skip scanning
+          if (request.label?.startsWith('__clickpass__')) {
+            await enqueueProcess(page, enqueueLinks, browserContext);
+            return;
+          }
+
           if (
             !isFollowStrategy(url, actualUrl, strategy) &&
             (isBlacklisted(actualUrl, blacklistedPatterns) || (isUrlPdf(actualUrl) && !isScanPdfs))
@@ -474,13 +532,13 @@ const crawlDomain = async ({
               durationExceeded = true;
             }
             isAbortingScanNow = true;
-            crawler.autoscaledPool.abort();
+            activeCrawler.autoscaledPool.abort();
             return;
           }
 
           // if URL has already been scanned
-          if (urlsCrawled.scanned.some(item => item.url === request.url)) {
-            // await enqueueProcess(page, enqueueLinks, browserContext);
+          if (scannedUrlSet.has(request.url)) {
+            await enqueueProcess(page, enqueueLinks, browserContext);
             return;
           }
 
@@ -512,7 +570,7 @@ const crawlDomain = async ({
 
               return;
             }
-            const { pdfFileName, url } = handlePdfDownload(
+            const { pdfFileName, url: downloadedPdfUrl } = handlePdfDownload(
               randomToken,
               pdfDownloads,
               request,
@@ -520,7 +578,7 @@ const crawlDomain = async ({
               urlsCrawled,
             );
 
-            uuidToPdfMapping[pdfFileName] = url;
+            uuidToPdfMapping[pdfFileName] = downloadedPdfUrl;
             return;
           }
 
@@ -597,9 +655,7 @@ const crawlDomain = async ({
             const results = await runAxeScript({ includeScreenshots, page, randomToken, ruleset });
 
             if (isRedirected) {
-              const isLoadedUrlInCrawledUrls = urlsCrawled.scanned.some(
-                item => (item.actualUrl || item.url) === actualUrl,
-              );
+              const isLoadedUrlInCrawledUrls = scannedResolvedUrlSet.has(actualUrl);
 
               if (isLoadedUrlInCrawledUrls) {
                 urlsCrawled.notScannedRedirects.push({
@@ -621,6 +677,8 @@ const crawlDomain = async ({
                   pageTitle: results.pageTitle,
                   actualUrl, // i.e. actualUrl
                 });
+                scannedUrlSet.add(request.url);
+                scannedResolvedUrlSet.add(actualUrl);
 
                 urlsCrawled.scannedRedirects.push({
                   fromUrl: request.url,
@@ -631,20 +689,20 @@ const crawlDomain = async ({
                 results.actualUrl = actualUrl;
                 await dataset.pushData(results);
               }
-            } else {
+            } else if (urlsCrawled.scanned.length < maxRequestsPerCrawl) {
               // One more check if scanned pages have reached limit due to multi-instances of handler running
-              if (urlsCrawled.scanned.length < maxRequestsPerCrawl) {
-                guiInfoLog(guiInfoStatusTypes.SCANNED, {
-                  numScanned: urlsCrawled.scanned.length,
-                  urlScanned: request.url,
-                });
-                urlsCrawled.scanned.push({
-                  url: request.url,
-                  actualUrl: request.url,
-                  pageTitle: results.pageTitle,
-                });
-                await dataset.pushData(results);
-              }
+              guiInfoLog(guiInfoStatusTypes.SCANNED, {
+                numScanned: urlsCrawled.scanned.length,
+                urlScanned: request.url,
+              });
+              urlsCrawled.scanned.push({
+                url: request.url,
+                actualUrl: request.url,
+                pageTitle: results.pageTitle,
+              });
+              scannedUrlSet.add(request.url);
+              scannedResolvedUrlSet.add(request.url);
+              await dataset.pushData(results);
             }
           } else {
             // Don't inform the user it is skipped since web crawler is best-effort.
@@ -675,16 +733,16 @@ const crawlDomain = async ({
                 urlScanned: request.url,
               });
 
-              page = await browserContext.newPage();
-              await page.goto(request.url);
+              const recoveryPage = await browserContext.newPage();
+              await recoveryPage.goto(request.url);
 
-              await page.route('**/*', async route => {
+              await recoveryPage.route('**/*', async route => {
                 const interceptedRequest = route.request();
                 if (interceptedRequest.resourceType() === 'document') {
                   const interceptedRequestUrl = interceptedRequest
                     .url()
                     .replace(/(?<=&|\?)utm_.*?(&|$)/gim, '');
-                  await requestQueue.addRequest({
+                  await enqueueUniqueRequest({
                     url: interceptedRequestUrl,
                     skipNavigation: isUrlPdf(interceptedRequest.url()),
                     label: interceptedRequestUrl,
@@ -749,6 +807,61 @@ const crawlDomain = async ({
 
   await crawler.run();
 
+  // Additional passes: keep re-visiting scanned seed-hostname pages for
+  // click-discovery until no new pages are found or limits are reached.
+  if (!safeMode && !isAbortingScanNow && !durationExceeded) {
+    const seedHostname = new URL(url).hostname;
+    const clickPassVisited = new Set<string>();
+    let prevScannedCount: number;
+
+    do {
+      prevScannedCount = urlsCrawled.scanned.length;
+
+      if (prevScannedCount >= maxRequestsPerCrawl) break;
+      if (scanDuration > 0 && Date.now() - crawlStartTime > scanDuration * 1000) break;
+
+      const seedHostnamePages = urlsCrawled.scanned
+        .map(item => item.actualUrl || item.url)
+        .filter(pageUrl => {
+          try {
+            return new URL(pageUrl).hostname === seedHostname && !clickPassVisited.has(pageUrl);
+          } catch {
+            return false;
+          }
+        });
+
+      if (seedHostnamePages.length === 0) break;
+
+      let enqueued = 0;
+      for (const pageUrl of seedHostnamePages) {
+        if (urlsCrawled.scanned.length >= maxRequestsPerCrawl) break;
+        if (scanDuration > 0 && Date.now() - crawlStartTime > scanDuration * 1000) break;
+
+        clickPassVisited.add(pageUrl);
+        try {
+          const clickPassLabel = `__clickpass__${pageUrl}`;
+          if (!queuedUrlSet.has(clickPassLabel)) {
+            queuedUrlSet.add(clickPassLabel);
+            await requestQueue.addRequest({
+              url: pageUrl,
+              label: clickPassLabel,
+              skipNavigation: false,
+            });
+            enqueued += 1;
+          }
+        } catch {
+          // ignore enqueue errors
+        }
+      }
+
+      if (enqueued === 0) break;
+
+      await crawler.run();
+
+      // Stop looping if no new pages were discovered in this pass
+    } while (urlsCrawled.scanned.length > prevScannedCount);
+  }
+
   if (pdfDownloads.length > 0) {
     // wait for pdf downloads to complete
     await Promise.all(pdfDownloads);
@@ -761,9 +874,7 @@ const crawlDomain = async ({
 
     // get screenshots from pdf docs
     if (includeScreenshots) {
-      await Promise.all(
-        pdfResults.map(async result => await doPdfScreenshots(randomToken, result)),
-      );
+      await Promise.all(pdfResults.map(result => doPdfScreenshots(randomToken, result)));
     }
 
     // push results for each pdf document to key value store

package/src/crawlers/crawlIntelligentSitemap.ts

@@ -53,14 +53,25 @@ const crawlIntelligentSitemap = async (
     const homeUrl = getHomeUrl(link);
     let sitemapLink = '';
 
-    const effectiveUserDataDirectory =
-      process.env.CRAWLEE_HEADLESS === '1' ? userDataDirectory : '';
-    const context = await constants.launcher.launchPersistentContext(effectiveUserDataDirectory, {
-      headless: process.env.CRAWLEE_HEADLESS === '1',
-      ...getPlaywrightLaunchOptions(browser),
-      ...(extraHTTPHeaders && { extraHTTPHeaders }),
-    });
-    register(context);
+    const launchOptions = getPlaywrightLaunchOptions(browser);
+    let context;
+    let browserInstance;
+
+    if (process.env.CRAWLEE_HEADLESS === '1') {
+      const effectiveUserDataDirectory = userDataDirectory || '';
+      context = await constants.launcher.launchPersistentContext(effectiveUserDataDirectory, {
+        ...launchOptions,
+        ...(extraHTTPHeaders && { extraHTTPHeaders }),
+      });
+      register(context);
+    } else {
+      // In headful mode, avoid launchPersistentContext to prevent "Browser window not found"
+      browserInstance = await constants.launcher.launch(launchOptions);
+      register(browserInstance as unknown as { close: () => Promise<void> });
+      context = await browserInstance.newContext({
+        ...(extraHTTPHeaders && { extraHTTPHeaders }),
+      });
+    }
 
     const page = await context.newPage();
 
@@ -73,6 +84,9 @@ const crawlIntelligentSitemap = async (
     }
     await page.close();
     await context.close().catch(() => {});
+    if (browserInstance) {
+      await browserInstance.close().catch(() => {});
+    }
     return sitemapExist ? sitemapLink : '';
   }