@govtechsg/oobee 0.10.86 → 0.10.87

package/fix-summary-html-oom-pr.md

@@ -0,0 +1,62 @@
+# fix: prevent OOM and browser crash in report generation for large scans
+
+## Summary
+
+- Fix `summary.ejs` inlining the entire scan items payload (2 GB+ for 1000-page scans) via `JSON.stringify`, causing V8 OOM and killing the process
+- Fix `report.html` embedded scanItems exceeding browser memory limits (746 MB uncompressed JSON for 1000-page scans)
+- Fix write stream backpressure handling when embedding chunked base64 data
+- `writeSummaryHTML` crash also blocked `report.html` generation since it runs first
+
+## Problem 1: OOM in summary.html generation (server-side)
+
+For large scans (e.g. 1000 pages, 2.5M+ passed occurrences), `summary.ejs` serialized the full `items` object — including every rule's `pagesAffected` array with all individual issue items — into an inline `<script>` tag. This produced a string exceeding V8's limits, crashing the process silently.
+
+The result: neither `summary.html` nor `report.html` were generated, even though all JSON artifacts (`scanData.json`, `scanItems.json`, etc.) were written successfully.
+
+## Problem 2: Browser cannot parse embedded scanItems (client-side)
+
+Even with report generation fixed, the browser failed to load the All Issues view:
+```
+Failed to decode/unzip/parse: Unexpected end of JSON input
+```
+
+Root cause: `convertItemsToReferences` stripped per-page `items` arrays but still embedded the full `pagesAffected` array (url, pageTitle, actualUrl, metadata, etc. for every page × every rule). For 1000-page scans this produced **746 MB of uncompressed JSON** after base64-decode and gunzip — exceeding browser string/memory limits during `JSON.parse()`.
+
+## Problem 3: Write stream backpressure (server-side)
+
+The `writeHTML` function writes scan items as 2 MB base64 chunks via a `for await` loop over a read stream. `outputStream.write()` was not being checked for backpressure — when the write buffer filled up, subsequent writes could be silently dropped, producing truncated base64.
+
+## Fix
+
+### summary.ejs (OOM fix)
+Strip the inline JSON to only what `summaryTable.ejs` actually needs:
+- Rule-level metadata: `description`, `helpUrl`, `conformance`, `totalItems`
+- `pagesAffected: { length: N }` (just the count object, not the full array)
+
+This reduces the serialized payload from potentially gigabytes to a few kilobytes regardless of scan size.
+
+### itemReferences.ts (browser payload fix)
+`convertItemsToReferences` now strips each `pagesAffected` entry down to only `url`, `pageTitle`, and `itemsCount` — removing all per-item details (html snippets, screenshots, xpath, metadata, etc.) that constituted the bulk of the data. The All Issues list renders rule totals, and the "Group By Page" view in the rule modal still shows page URLs with occurrence counts.
+
+This reduces the embedded payload from 746 MB (uncompressed) to ~11 MB for a 1000-page scan — well within browser memory limits.
+
+### mergeAxeResults.ts (backpressure fix)
+Await the `drain` event on the output stream when `write()` returns `false` before writing the next chunk. This ensures all base64 data is fully written to the report regardless of payload size.
+
+## Files changed
+
+| File | Change |
+|------|--------|
+| `src/static/ejs/summary.ejs` | Strip inline JSON to rule counts only |
+| `src/mergeAxeResults/itemReferences.ts` | Strip `pagesAffected` to lightweight entries (url, pageTitle, itemsCount only) |
+| `src/mergeAxeResults.ts` | Await drain on backpressure during chunked write |
+| `src/static/ejs/partials/scripts/ruleModal/ruleOffcanvas.ejs` | Fall back to `pagesAffectedCount` |
+| `src/static/ejs/partials/scripts/ruleModal/pageAccordionBuilder.ejs` | Fall back to `pagesAffectedCount` |
+
+## Test plan
+
+- [ ] Run a large scan (500+ pages) and verify both `summary.html` and `report.html` are generated
+- [ ] Open `summary.html` in a browser and verify the summary table renders correctly (issue counts, page counts, help links)
+- [ ] Open `report.html` and verify the All Issues list loads and displays rule counts correctly
+- [ ] Verify the rule modal shows correct "Pages affected" count
+- [ ] Verify small scans still produce correct reports (no regression)

package/package.json

@@ -1,19 +1,19 @@
 {
   "name": "@govtechsg/oobee",
   "main": "dist/npmIndex.js",
-  "version": "0.10.86",
+  "version": "0.10.87",
   "type": "module",
   "author": "Government Technology Agency <info@tech.gov.sg>",
   "bin": {
     "oobee": "./dist/cli.js"
   },
   "dependencies": {
-    "@aws-sdk/client-s3": "^3.893.0",
+    "@aws-sdk/client-s3": "^3.1049.0",
     "@json2csv/node": "^7.0.3",
     "@napi-rs/canvas": "^0.1.53",
     "@sentry/node": "^9.13.0",
     "@types/aws-sdk": "^0.0.42",
-    "axe-core": "^4.11.1",
+    "axe-core": "^4.11.4",
     "axios": "^1.8.2",
     "base64-stream": "^1.0.0",
     "cheerio": "^1.0.0-rc.12",
@@ -39,7 +39,7 @@
     "tldts": "^7.0.27",
     "typescript": "^5.4.5",
     "url": "^0.11.3",
-    "uuid": "^11.0.3",
+    "uuid": "^14.0.0",
     "validator": "^13.11.0",
     "which": "^4.0.0",
     "winston": "^3.11.0",
@@ -86,7 +86,7 @@
     "fast-xml-parser": ">=5.3.8",
     "js-yaml": "^4.1.1",
     "minimatch": "^10.2.4",
-    "brace-expansion": "^5.0.5",
+    "brace-expansion": "^5.0.6",
     "glob": "^13.0.6",
     "flatted": "^3.4.1",
     "file-type": "^21.3.3"

package/src/cli.ts

@@ -193,8 +193,11 @@ Usage: npm run cli -- -c <crawler> -d <device> -w <viewport> -u <url> OPTIONS`,
   .check(argvs => {
     const scanner = String(argvs.scanner ?? '');
 
-    if (argvs.strategy && scanner !== ScannerTypes.WEBSITE && scanner !== ScannerTypes.CUSTOM) {
-      throw new Error('-s or --strategy is only available in website and custom flow scans.');
+    if (argvs.strategy && scanner !== ScannerTypes.WEBSITE && scanner !== ScannerTypes.CUSTOM && scanner !== ScannerTypes.INTELLIGENT && scanner !== ScannerTypes.SITEMAP) {
+      throw new Error('-s or --strategy is only available in website, custom flow, intelligent, and sitemap scans.');
+    }
+    if (argvs.strategy === 'ignore' && scanner !== ScannerTypes.SITEMAP) {
+      throw new Error('-s ignore is only available for sitemap scans.');
     }
     return true;
   })
@@ -210,14 +213,21 @@ Usage: npm run cli -- -c <crawler> -d <device> -w <viewport> -u <url> OPTIONS`,
     return duration;
   })
   .check(argvs => {
-    if (argvs.scanner !== ScannerTypes.WEBSITE && argvs.strategy) {
-      throw new Error('-s or --strategy is only available in website scans.');
+    if (argvs.scanner !== ScannerTypes.WEBSITE && argvs.scanner !== ScannerTypes.CUSTOM && argvs.scanner !== ScannerTypes.INTELLIGENT && argvs.scanner !== ScannerTypes.SITEMAP && argvs.strategy) {
+      throw new Error('-s or --strategy is only available in website, custom flow, intelligent, and sitemap scans.');
+    }
+    if (argvs.strategy === 'ignore' && argvs.scanner !== ScannerTypes.SITEMAP) {
+      throw new Error('-s ignore is only available for sitemap scans.');
     }
     return true;
   })
   .conflicts('d', 'w')
   .parse() as unknown as Answers;
 
+if (!options.strategy) {
+  options.strategy = options.scanner === ScannerTypes.SITEMAP ? 'ignore' : 'same-domain';
+}
+
 const scanInit = async (argvs: Answers): Promise<string> => {
   const updatedArgvs = { ...argvs };
 
@@ -250,7 +260,11 @@ const scanInit = async (argvs: Answers): Promise<string> => {
     consoleLogger.info(`Connectivity Check HTTP Response Code: ${res.httpStatus}`);
 
   if (res.status === statuses.success.code) {
-    data.url = res.url;
+    // Custom flow should continue from the user-provided entry URL so auth redirects
+    // do not replace the original domain used for overlay gating and navigation.
+    if (data.type !== ScannerTypes.CUSTOM) {
+      data.url = res.url;
+    }
     if (process.env.OOBEE_VALIDATE_URL) {
       consoleLogger.info('Url is valid');
       cleanUpAndExit(0, data.randomToken);

package/src/combine.ts

@@ -161,6 +161,8 @@ const combineRun = async (details: Data, deviceToScan: string) => {
         blacklistedPatterns,
         includeScreenshots,
         extraHTTPHeaders,
+        strategy,
+        userUrl: url,
         scanDuration,
       });
       urlsCrawledObj = sitemapResult.urlsCrawled;

package/src/constants/cliFunctions.ts

@@ -168,8 +168,8 @@ export const cliOptions: { [key: string]: Options } = {
   s: {
     alias: 'strategy',
     describe:
-      'Crawls up to general (same parent) domains, or only specific hostname. Defaults to "same-domain".',
-    choices: ['same-domain', 'same-hostname'],
+      'Crawls up to general (same parent) domains, or only specific hostname. Use "ignore" to disable URL filtering (default for sitemap scans). Defaults to "same-domain".',
+    choices: ['same-domain', 'same-hostname', 'ignore'],
     requiresArg: true,
     demandOption: false,
   },

package/src/constants/common.ts

@@ -33,7 +33,7 @@ import constants, {
 } from './constants.js';
 import { consoleLogger } from '../logs.js';
 import { isUrlPdf } from '../crawlers/commonCrawlerFunc.js';
-import { cleanUpAndExit, randomThreeDigitNumberString, register } from '../utils.js';
+import { cleanUpAndExit, isFollowStrategy, randomThreeDigitNumberString, register } from '../utils.js';
 import { Answers, Data } from '../index.js';
 import { DeviceDescriptor } from '../types/types.js';
 import { getProxyInfo, proxyInfoToResolution, ProxySettings } from '../proxyService.js';
@@ -746,7 +746,9 @@ export const prepareData = async (argv: Answers): Promise<Data> => {
     playwrightDeviceDetailsObject,
     maxRequestsPerCrawl: maxpages || constants.maxRequestsPerCrawl,
     strategy:
-      strategy === 'same-hostname' ? EnqueueStrategy.SameHostname : EnqueueStrategy.SameDomain,
+      strategy === 'same-hostname' ? EnqueueStrategy.SameHostname
+      : strategy === 'ignore' ? EnqueueStrategy.All
+      : EnqueueStrategy.SameDomain,
     isLocalFileScan,
     browser: browserToRun,
     nameEmail,
@@ -804,7 +806,11 @@ export const getUrlsFromRobotsTxt = async (
   const disallowedUrls = [];
   const allowedUrls = [];
 
-  const sanitisePattern = (pattern: string): string => {
+  // Returns 1–2 minimatch glob patterns for a single robots.txt path pattern.
+  // Two patterns are returned for bare paths (no trailing wildcard) so that
+  // both the exact URL and all child paths are blocked, matching robots.txt
+  // prefix semantics.
+  const sanitisePattern = (pattern: string): string[] => {
     const directoryRegex = /^\/(?:[^?#/]+\/)*[^?#]*$/;
     const subdirWildcardRegex = /\/\*\//g;
     const filePathRegex = /^\/(?:[^\/]+\/)*[^\/]+\.[a-zA-Z0-9]{1,6}$/;
@@ -812,16 +818,30 @@ export const getUrlsFromRobotsTxt = async (
     if (subdirWildcardRegex.test(pattern)) {
       pattern = pattern.replace(subdirWildcardRegex, '/**/');
     }
+
+    // Query-string patterns (e.g. /faq?faqItem= or /faq/?faq&faqItem=):
+    // '?' is the query separator in robots.txt but a single-char wildcard in
+    // minimatch. Escape it to a literal match and append '*' so any query
+    // value after the stated prefix is also blocked.
+    if (pattern.includes('?')) {
+      return [domain + pattern.replace('?', '\\?') + '*'];
+    }
+
     if (pattern.match(directoryRegex) && !pattern.match(filePathRegex)) {
       if (pattern.endsWith('*')) {
-        pattern = pattern.concat('*');
+        // e.g. /ebook/* → /ebook/** (already covers all children)
+        return [domain + pattern.concat('*')];
       } else {
-        if (!pattern.endsWith('/')) pattern = pattern.concat('/');
-        pattern = pattern.concat('**');
+        // Bare path (e.g. /subscription/unsubscribe): robots.txt blocks the
+        // exact URL *and* every descendant. minimatch's '/**' glob does not
+        // match the bare path itself (no trailing slash), so we emit both the
+        // exact-path pattern and a children glob.
+        const base = domain + pattern;
+        const children = domain + (pattern.endsWith('/') ? pattern : pattern + '/') + '**';
+        return [base, children];
       }
     }
-    const final = domain.concat(pattern);
-    return final;
+    return [domain + pattern];
   };
 
   for (const line of lines) {
@@ -832,14 +852,12 @@ export const getUrlsFromRobotsTxt = async (
     } else if (shouldCapture && line.toLowerCase().startsWith('disallow:')) {
       let disallowed = line.substring('disallow: '.length).trim();
       if (disallowed) {
-        disallowed = sanitisePattern(disallowed);
-        disallowedUrls.push(disallowed);
+        disallowedUrls.push(...sanitisePattern(disallowed));
       }
     } else if (shouldCapture && line.toLowerCase().startsWith('allow:')) {
       let allowed = line.substring('allow: '.length).trim();
       if (allowed) {
-        allowed = sanitisePattern(allowed);
-        allowedUrls.push(allowed);
+        allowedUrls.push(...sanitisePattern(allowed));
       }
     }
   }
@@ -899,6 +917,38 @@ const getRobotsTxtViaPlaywright = async (
   }
 };
 
+export const getSitemapsFromRobotsTxt = async (
+  url: string,
+  browser: string,
+  userDataDirectory: string,
+  extraHTTPHeaders: Record<string, string>,
+): Promise<string[]> => {
+  const domain = new URL(url).origin;
+  const robotsUrl = domain.concat('/robots.txt');
+
+  let robotsTxt: string;
+  try {
+    robotsTxt = await getRobotsTxtViaPlaywright(robotsUrl, browser, userDataDirectory, extraHTTPHeaders);
+  } catch (e) {
+    consoleLogger.info(`Unable to fetch robots.txt from ${robotsUrl} for sitemap discovery`);
+    return [];
+  }
+
+  if (!robotsTxt) return [];
+
+  const sitemaps: string[] = [];
+  const lines = robotsTxt.split(/\r?\n/);
+  for (const line of lines) {
+    if (line.toLowerCase().startsWith('sitemap:')) {
+      const sitemapUrl = line.substring('sitemap:'.length).trim();
+      if (sitemapUrl) {
+        sitemaps.push(sitemapUrl);
+      }
+    }
+  }
+  return sitemaps;
+};
+
 export const isDisallowedInRobotsTxt = (url: string): boolean => {
   if (!constants.robotsTxtUrls) return;
 
@@ -931,6 +981,8 @@ export const getLinksFromSitemap = async (
   userUrlInput: string,
   isIntelligent: boolean,
   extraHTTPHeaders: Record<string, string>,
+  strategy: EnqueueStrategy = EnqueueStrategy.All,
+  userUrl: string = userUrlInput,
 ) => {
   const scannedSitemaps = new Set<string>();
   const urls: Record<string, Request> = {}; // dictionary of requests to urls to be scanned
@@ -940,6 +992,7 @@ export const getLinksFromSitemap = async (
   const addToUrlList = (url: string) => {
     if (!url) return;
     if (isDisallowedInRobotsTxt(url)) return;
+    if (!isFilePath(userUrl) && !isFollowStrategy(url, userUrl, strategy)) return;
 
     url = convertPathToLocalFile(url);
 

package/src/crawlers/crawlDomain.ts

@@ -29,7 +29,7 @@ import {
   getUrlsFromRobotsTxt,
   waitForPageLoaded,
 } from '../constants/common.js';
-import { areLinksEqual, isFollowStrategy, register } from '../utils.js';
+import { areLinksEqual, isFollowStrategy, normUrl, register } from '../utils.js';
 import {
   handlePdfDownload,
   runPdfScan,
@@ -116,9 +116,9 @@ const crawlDomain = async ({
   const pdfDownloads: Promise<void>[] = [];
   const uuidToPdfMapping: Record<string, string> = {};
   const queuedUrlSet = new Set<string>();
-  const scannedUrlSet = new Set<string>(urlsCrawled.scanned.map(item => item.url));
+  const scannedUrlSet = new Set<string>(urlsCrawled.scanned.map(item => normUrl(item.url)));
   const scannedResolvedUrlSet = new Set<string>(
-    urlsCrawled.scanned.map(item => item.actualUrl || item.url),
+    urlsCrawled.scanned.map(item => normUrl(item.actualUrl || item.url)),
   );
   const isScanHtml = [FileTypes.All, FileTypes.HtmlOnly].includes(fileTypes as FileTypes);
   const isScanPdfs = [FileTypes.All, FileTypes.PdfOnly].includes(fileTypes as FileTypes);
@@ -166,13 +166,14 @@ const crawlDomain = async ({
     const selectedElementsString = cssQuerySelectors.join(', ');
 
     const isExcluded = (newPageUrl: string): boolean => {
-      const isAlreadyScanned: boolean = urlsCrawled.scanned.some(item => item.url === newPageUrl);
+      const isAlreadyScanned: boolean = scannedUrlSet.has(normUrl(newPageUrl));
       const isBlacklistedUrl: boolean = isBlacklisted(newPageUrl, blacklistedPatterns);
       const isNotFollowStrategy: boolean = !isFollowStrategy(newPageUrl, initialPageUrl, strategy);
       const isNotSupportedDocument: boolean = disallowedListOfPatterns.some(pattern =>
         newPageUrl.toLowerCase().startsWith(pattern),
       );
-      return isNotSupportedDocument || isAlreadyScanned || isBlacklistedUrl || isNotFollowStrategy;
+      const isRobotsDisallowed: boolean = isDisallowedInRobotsTxt(newPageUrl);
+      return isNotSupportedDocument || isAlreadyScanned || isBlacklistedUrl || isNotFollowStrategy || isRobotsDisallowed;
     };
     const setPageListeners = (pageListener: Page): void => {
       // event listener to handle new page popups upon button click
@@ -341,7 +342,7 @@ const crawlDomain = async ({
           } catch (e) {
             consoleLogger.error(e);
           }
-          if (scannedUrlSet.has(req.url)) {
+          if (scannedUrlSet.has(normUrl(req.url))) {
             req.skipNavigation = true;
           }
           if (isDisallowedInRobotsTxt(req.url)) return null;
@@ -481,7 +482,7 @@ const crawlDomain = async ({
           }
 
           const isRedirected = !areLinksEqual(finalUrl, requestLabelUrl);
-          if (isRedirected) {
+          if (isRedirected && !isDisallowedInRobotsTxt(finalUrl)) {
             await enqueueUniqueRequest({ url: finalUrl, label: finalUrl });
           } else {
             request.skipNavigation = false;
@@ -537,7 +538,7 @@ const crawlDomain = async ({
           }
 
           // if URL has already been scanned
-          if (scannedUrlSet.has(request.url)) {
+          if (scannedUrlSet.has(normUrl(request.url))) {
             await enqueueProcess(page, enqueueLinks, browserContext);
             return;
           }
@@ -654,8 +655,33 @@ const crawlDomain = async ({
 
             const results = await runAxeScript({ includeScreenshots, page, randomToken, ruleset });
 
+            // Detect JS redirects that fire during/after axe scan.
+            // Listen for navigation, then give a brief window for pending redirects to complete.
+            try {
+              let navigatedToUrl: string | null = null;
+              const onFrameNavigated = (frame: Frame) => {
+                if (frame === page.mainFrame()) {
+                  navigatedToUrl = frame.url();
+                }
+              };
+              page.on('framenavigated', onFrameNavigated);
+              await page.waitForTimeout(1000);
+              page.off('framenavigated', onFrameNavigated);
+
+              const postScanUrl = navigatedToUrl || page.url();
+              if (postScanUrl && postScanUrl !== 'about:blank' && !isFollowStrategy(postScanUrl, request.url, 'same-hostname')) {
+                urlsCrawled.notScannedRedirects.push({
+                  fromUrl: request.url,
+                  toUrl: postScanUrl,
+                });
+                return;
+              }
+            } catch (_) {
+              // Page/context was destroyed during navigation — handled by outer catch
+            }
+
             if (isRedirected) {
-              const isLoadedUrlInCrawledUrls = scannedResolvedUrlSet.has(actualUrl);
+              const isLoadedUrlInCrawledUrls = scannedResolvedUrlSet.has(normUrl(actualUrl));
 
               if (isLoadedUrlInCrawledUrls) {
                 urlsCrawled.notScannedRedirects.push({
@@ -677,8 +703,8 @@ const crawlDomain = async ({
                   pageTitle: results.pageTitle,
                   actualUrl, // i.e. actualUrl
                 });
-                scannedUrlSet.add(request.url);
-                scannedResolvedUrlSet.add(actualUrl);
+                scannedUrlSet.add(normUrl(request.url));
+                scannedResolvedUrlSet.add(normUrl(actualUrl));
 
                 urlsCrawled.scannedRedirects.push({
                   fromUrl: request.url,
@@ -700,8 +726,8 @@ const crawlDomain = async ({
                 actualUrl: request.url,
                 pageTitle: results.pageTitle,
               });
-              scannedUrlSet.add(request.url);
-              scannedResolvedUrlSet.add(request.url);
+              scannedUrlSet.add(normUrl(request.url));
+              scannedResolvedUrlSet.add(normUrl(request.url));
               await dataset.pushData(results);
             }
           } else {

package/src/crawlers/crawlIntelligentSitemap.ts

@@ -7,7 +7,7 @@ import { consoleLogger, guiInfoLog } from '../logs.js';
 import crawlDomain from './crawlDomain.js';
 import crawlSitemap from './crawlSitemap.js';
 import { ViewportSettingsClass } from '../combine.js';
-import { getPlaywrightLaunchOptions } from '../constants/common.js';
+import { getPlaywrightLaunchOptions, getSitemapsFromRobotsTxt } from '../constants/common.js';
 import { register } from '../utils.js';
 
 const crawlIntelligentSitemap = async (
@@ -100,12 +100,30 @@ const crawlIntelligentSitemap = async (
     }
   };
 
+  // Discover sitemaps from robots.txt first (supports multiple Sitemap: directives)
+  let sitemapUrls: string[] = [];
   try {
-    sitemapUrl = await findSitemap(url, userDataDirectory, extraHTTPHeaders);
+    sitemapUrls = await getSitemapsFromRobotsTxt(url, browser, userDataDirectory, extraHTTPHeaders);
+    if (sitemapUrls.length > 0) {
+      console.log(`Found ${sitemapUrls.length} sitemap(s) in robots.txt: ${sitemapUrls.join(', ')}`);
+      sitemapExist = true;
+    }
   } catch (error) {
     consoleLogger.error(error);
   }
 
+  // Fall back to hardcoded path probing if robots.txt had no sitemaps
+  if (!sitemapExist) {
+    try {
+      sitemapUrl = await findSitemap(url, userDataDirectory, extraHTTPHeaders);
+      if (sitemapExist) {
+        sitemapUrls = [sitemapUrl];
+      }
+    } catch (error) {
+      consoleLogger.error(error);
+    }
+  }
+
   if (!sitemapExist) {
     console.log('Unable to find sitemap. Commencing website crawl instead.');
     return await crawlDomain({
@@ -124,38 +142,53 @@ const crawlIntelligentSitemap = async (
       followRobots,
       extraHTTPHeaders,
       safeMode,
-      scanDuration, // Use full duration since no sitemap
+      scanDuration,
     });
   }
 
-  console.log(`Sitemap found at ${sitemapUrl}`);
-  urlsCrawledFinal = await crawlSitemap({
-    sitemapUrl,
-    randomToken,
-    host,
-    viewportSettings,
-    maxRequestsPerCrawl,
-    browser,
-    userDataDirectory,
-    specifiedMaxConcurrency,
-    fileTypes,
-    blacklistedPatterns,
-    includeScreenshots,
-    extraHTTPHeaders,
-    fromCrawlIntelligentSitemap,
-    userUrlInputFromIntelligent: url,
-    datasetFromIntelligent: dataset,
-    urlsCrawledFromIntelligent: urlsCrawled,
-    crawledFromLocalFile: false,
-    scanDuration,
-  });
+  // Process all discovered sitemaps sequentially, sharing dataset and urlsCrawled
+  for (const currentSitemapUrl of sitemapUrls) {
+    if (urlsCrawled.scanned.length >= maxRequestsPerCrawl) break;
+
+    const elapsed = Date.now() - startTime;
+    const remainingDuration = scanDuration > 0 ? Math.max(scanDuration - elapsed / 1000, 0) : scanDuration;
+    if (scanDuration > 0 && remainingDuration <= 0) {
+      durationExceeded = true;
+      break;
+    }
+
+    console.log(`Processing sitemap: ${currentSitemapUrl}`);
+    urlsCrawledFinal = await crawlSitemap({
+      sitemapUrl: currentSitemapUrl,
+      randomToken,
+      host,
+      viewportSettings,
+      maxRequestsPerCrawl,
+      browser,
+      userDataDirectory,
+      specifiedMaxConcurrency,
+      fileTypes,
+      blacklistedPatterns,
+      includeScreenshots,
+      extraHTTPHeaders,
+      strategy,
+      userUrl: url,
+      fromCrawlIntelligentSitemap,
+      userUrlInputFromIntelligent: url,
+      datasetFromIntelligent: dataset,
+      urlsCrawledFromIntelligent: urlsCrawled,
+      crawledFromLocalFile: false,
+      scanDuration: scanDuration > 0 ? remainingDuration : 0,
+    });
+  }
 
   const elapsed = Date.now() - startTime;
-  const remainingScanDuration = Math.max(scanDuration - elapsed / 1000, 0); // in seconds
+  const remainingScanDuration = scanDuration > 0 ? Math.max(scanDuration - elapsed / 1000, 0) : 0;
+  const hasDurationRemaining = scanDuration === 0 || remainingScanDuration > 0;
 
-  if (urlsCrawledFinal.scanned.length < maxRequestsPerCrawl && remainingScanDuration > 0) {
+  if (urlsCrawled.scanned.length < maxRequestsPerCrawl && hasDurationRemaining) {
     console.log(
-      `Continuing crawl from root website. Remaining scan time: ${remainingScanDuration.toFixed(1)}s`,
+      `Continuing crawl from root website.${scanDuration > 0 ? ` Remaining scan time: ${remainingScanDuration.toFixed(1)}s` : ''}`,
     );
     urlsCrawledFinal = await crawlDomain({
       url,
@@ -175,10 +208,10 @@ const crawlIntelligentSitemap = async (
       safeMode,
       fromCrawlIntelligentSitemap,
       datasetFromIntelligent: dataset,
-      urlsCrawledFromIntelligent: urlsCrawledFinal,
+      urlsCrawledFromIntelligent: urlsCrawled,
       scanDuration: remainingScanDuration,
     });
-  } else if (remainingScanDuration <= 0) {
+  } else if (!hasDurationRemaining) {
     console.log(
       `Crawl duration exceeded before more pages could be found (limit: ${scanDuration}s).`,
     );
@@ -186,7 +219,7 @@ const crawlIntelligentSitemap = async (
   }
 
   guiInfoLog(guiInfoStatusTypes.COMPLETED, {});
-  return { urlsCrawled: urlsCrawledFinal, durationExceeded };
+  return { urlsCrawled, durationExceeded };
 };
 
 export default crawlIntelligentSitemap;