@govtechsg/oobee 0.10.86 → 0.10.87

package/.github/workflows/image.yml

@@ -146,18 +146,17 @@ jobs:
           chmod -R u+w "$GITHUB_WORKSPACE/oobee"
 
           # Sign all Mach-O (exec bits OR dylib OR node native addons)
-          # Search $GITHUB_WORKSPACE (not just oobee/) to cover scripts copied to the parent dir
           while IFS= read -r f; do
             echo "Signing $f"
             codesign --force --options runtime --timestamp --sign "${CERTIFICATE_NAME}" "$f"
           done < <(
-            find "$GITHUB_WORKSPACE" -type f \
+            find "$GITHUB_WORKSPACE/oobee" -type f \
               \( -perm -111 -o -name "*.dylib" -o -name "*.node" \) \
               ! -path "*/.git/*"
           )
 
           echo "Verifying signatures of Mach-O files..."
-          find "$GITHUB_WORKSPACE" -type f \( -perm -111 -o -name "*.dylib" -o -name "*.node" \) \
+          find "$GITHUB_WORKSPACE/oobee" -type f \( -perm -111 -o -name "*.dylib" -o -name "*.node" \) \
             -exec codesign --verify --strict --verbose=2 {} \; || true
           
       - name: Cleanup keychain

package/dist/cli.js

@@ -147,8 +147,11 @@ Usage: npm run cli -- -c <crawler> -d <device> -w <viewport> -u <url> OPTIONS`)
 })
     .check(argvs => {
     const scanner = String(argvs.scanner ?? '');
-    if (argvs.strategy && scanner !== ScannerTypes.WEBSITE && scanner !== ScannerTypes.CUSTOM) {
-        throw new Error('-s or --strategy is only available in website and custom flow scans.');
+    if (argvs.strategy && scanner !== ScannerTypes.WEBSITE && scanner !== ScannerTypes.CUSTOM && scanner !== ScannerTypes.INTELLIGENT && scanner !== ScannerTypes.SITEMAP) {
+        throw new Error('-s or --strategy is only available in website, custom flow, intelligent, and sitemap scans.');
+    }
+    if (argvs.strategy === 'ignore' && scanner !== ScannerTypes.SITEMAP) {
+        throw new Error('-s ignore is only available for sitemap scans.');
     }
     return true;
 })
@@ -161,13 +164,19 @@ Usage: npm run cli -- -c <crawler> -d <device> -w <viewport> -u <url> OPTIONS`)
     return duration;
 })
     .check(argvs => {
-    if (argvs.scanner !== ScannerTypes.WEBSITE && argvs.strategy) {
-        throw new Error('-s or --strategy is only available in website scans.');
+    if (argvs.scanner !== ScannerTypes.WEBSITE && argvs.scanner !== ScannerTypes.CUSTOM && argvs.scanner !== ScannerTypes.INTELLIGENT && argvs.scanner !== ScannerTypes.SITEMAP && argvs.strategy) {
+        throw new Error('-s or --strategy is only available in website, custom flow, intelligent, and sitemap scans.');
+    }
+    if (argvs.strategy === 'ignore' && argvs.scanner !== ScannerTypes.SITEMAP) {
+        throw new Error('-s ignore is only available for sitemap scans.');
     }
     return true;
 })
     .conflicts('d', 'w')
     .parse();
+if (!options.strategy) {
+    options.strategy = options.scanner === ScannerTypes.SITEMAP ? 'ignore' : 'same-domain';
+}
 const scanInit = async (argvs) => {
     const updatedArgvs = { ...argvs };
     // Cannot use data.browser and data.isHeadless as the connectivity check comes first before prepareData
@@ -187,7 +196,11 @@ const scanInit = async (argvs) => {
     if (res.httpStatus)
         consoleLogger.info(`Connectivity Check HTTP Response Code: ${res.httpStatus}`);
     if (res.status === statuses.success.code) {
-        data.url = res.url;
+        // Custom flow should continue from the user-provided entry URL so auth redirects
+        // do not replace the original domain used for overlay gating and navigation.
+        if (data.type !== ScannerTypes.CUSTOM) {
+            data.url = res.url;
+        }
         if (process.env.OOBEE_VALIDATE_URL) {
             consoleLogger.info('Url is valid');
             cleanUpAndExit(0, data.randomToken);

package/dist/combine.js

@@ -95,6 +95,8 @@ const combineRun = async (details, deviceToScan) => {
                 blacklistedPatterns,
                 includeScreenshots,
                 extraHTTPHeaders,
+                strategy,
+                userUrl: url,
                 scanDuration,
             });
             urlsCrawledObj = sitemapResult.urlsCrawled;

package/dist/constants/cliFunctions.js

@@ -147,8 +147,8 @@ export const cliOptions = {
     },
     s: {
         alias: 'strategy',
-        describe: 'Crawls up to general (same parent) domains, or only specific hostname. Defaults to "same-domain".',
-        choices: ['same-domain', 'same-hostname'],
+        describe: 'Crawls up to general (same parent) domains, or only specific hostname. Use "ignore" to disable URL filtering (default for sitemap scans). Defaults to "same-domain".',
+        choices: ['same-domain', 'same-hostname', 'ignore'],
         requiresArg: true,
         demandOption: false,
     },

package/dist/constants/common.js

@@ -26,7 +26,7 @@ formDataFields,
 ScannerTypes, BrowserTypes, FileTypes, getEnumKey, } from './constants.js';
 import { consoleLogger } from '../logs.js';
 import { isUrlPdf } from '../crawlers/commonCrawlerFunc.js';
-import { cleanUpAndExit, randomThreeDigitNumberString, register } from '../utils.js';
+import { cleanUpAndExit, isFollowStrategy, randomThreeDigitNumberString, register } from '../utils.js';
 import { getProxyInfo, proxyInfoToResolution } from '../proxyService.js';
 // validateDirPath validates a provided directory path
 // returns null if no error
@@ -592,7 +592,9 @@ export const prepareData = async (argv) => {
         viewportWidth,
         playwrightDeviceDetailsObject,
         maxRequestsPerCrawl: maxpages || constants.maxRequestsPerCrawl,
-        strategy: strategy === 'same-hostname' ? EnqueueStrategy.SameHostname : EnqueueStrategy.SameDomain,
+        strategy: strategy === 'same-hostname' ? EnqueueStrategy.SameHostname
+            : strategy === 'ignore' ? EnqueueStrategy.All
+                : EnqueueStrategy.SameDomain,
         isLocalFileScan,
         browser: browserToRun,
         nameEmail,
@@ -637,6 +639,10 @@ export const getUrlsFromRobotsTxt = async (url, browserToRun, userDataDirectory,
     let shouldCapture = false;
     const disallowedUrls = [];
     const allowedUrls = [];
+    // Returns 1–2 minimatch glob patterns for a single robots.txt path pattern.
+    // Two patterns are returned for bare paths (no trailing wildcard) so that
+    // both the exact URL and all child paths are blocked, matching robots.txt
+    // prefix semantics.
     const sanitisePattern = (pattern) => {
         const directoryRegex = /^\/(?:[^?#/]+\/)*[^?#]*$/;
         const subdirWildcardRegex = /\/\*\//g;
@@ -644,18 +650,29 @@ export const getUrlsFromRobotsTxt = async (url, browserToRun, userDataDirectory,
         if (subdirWildcardRegex.test(pattern)) {
             pattern = pattern.replace(subdirWildcardRegex, '/**/');
         }
+        // Query-string patterns (e.g. /faq?faqItem= or /faq/?faq&faqItem=):
+        // '?' is the query separator in robots.txt but a single-char wildcard in
+        // minimatch. Escape it to a literal match and append '*' so any query
+        // value after the stated prefix is also blocked.
+        if (pattern.includes('?')) {
+            return [domain + pattern.replace('?', '\\?') + '*'];
+        }
         if (pattern.match(directoryRegex) && !pattern.match(filePathRegex)) {
             if (pattern.endsWith('*')) {
-                pattern = pattern.concat('*');
+                // e.g. /ebook/* → /ebook/** (already covers all children)
+                return [domain + pattern.concat('*')];
             }
             else {
-                if (!pattern.endsWith('/'))
-                    pattern = pattern.concat('/');
-                pattern = pattern.concat('**');
+                // Bare path (e.g. /subscription/unsubscribe): robots.txt blocks the
+                // exact URL *and* every descendant. minimatch's '/**' glob does not
+                // match the bare path itself (no trailing slash), so we emit both the
+                // exact-path pattern and a children glob.
+                const base = domain + pattern;
+                const children = domain + (pattern.endsWith('/') ? pattern : pattern + '/') + '**';
+                return [base, children];
             }
         }
-        const final = domain.concat(pattern);
-        return final;
+        return [domain + pattern];
     };
     for (const line of lines) {
         if (line.toLowerCase().startsWith('user-agent: *')) {
@@ -667,15 +684,13 @@ export const getUrlsFromRobotsTxt = async (url, browserToRun, userDataDirectory,
         else if (shouldCapture && line.toLowerCase().startsWith('disallow:')) {
             let disallowed = line.substring('disallow: '.length).trim();
             if (disallowed) {
-                disallowed = sanitisePattern(disallowed);
-                disallowedUrls.push(disallowed);
+                disallowedUrls.push(...sanitisePattern(disallowed));
             }
         }
         else if (shouldCapture && line.toLowerCase().startsWith('allow:')) {
             let allowed = line.substring('allow: '.length).trim();
             if (allowed) {
-                allowed = sanitisePattern(allowed);
-                allowedUrls.push(allowed);
+                allowedUrls.push(...sanitisePattern(allowed));
             }
         }
     }
@@ -726,6 +741,31 @@ const getRobotsTxtViaPlaywright = async (robotsUrl, browser, userDataDirectory,
         }
     }
 };
+export const getSitemapsFromRobotsTxt = async (url, browser, userDataDirectory, extraHTTPHeaders) => {
+    const domain = new URL(url).origin;
+    const robotsUrl = domain.concat('/robots.txt');
+    let robotsTxt;
+    try {
+        robotsTxt = await getRobotsTxtViaPlaywright(robotsUrl, browser, userDataDirectory, extraHTTPHeaders);
+    }
+    catch (e) {
+        consoleLogger.info(`Unable to fetch robots.txt from ${robotsUrl} for sitemap discovery`);
+        return [];
+    }
+    if (!robotsTxt)
+        return [];
+    const sitemaps = [];
+    const lines = robotsTxt.split(/\r?\n/);
+    for (const line of lines) {
+        if (line.toLowerCase().startsWith('sitemap:')) {
+            const sitemapUrl = line.substring('sitemap:'.length).trim();
+            if (sitemapUrl) {
+                sitemaps.push(sitemapUrl);
+            }
+        }
+    }
+    return sitemaps;
+};
 export const isDisallowedInRobotsTxt = (url) => {
     if (!constants.robotsTxtUrls)
         return;
@@ -744,7 +784,7 @@ export const isDisallowedInRobotsTxt = (url) => {
     }
     return false;
 };
-export const getLinksFromSitemap = async (sitemapUrl, maxLinksCount, browser, userDataDirectory, userUrlInput, isIntelligent, extraHTTPHeaders) => {
+export const getLinksFromSitemap = async (sitemapUrl, maxLinksCount, browser, userDataDirectory, userUrlInput, isIntelligent, extraHTTPHeaders, strategy = EnqueueStrategy.All, userUrl = userUrlInput) => {
     const scannedSitemaps = new Set();
     const urls = {}; // dictionary of requests to urls to be scanned
     const isLimitReached = () => Object.keys(urls).length >= maxLinksCount;
@@ -753,6 +793,8 @@ export const getLinksFromSitemap = async (sitemapUrl, maxLinksCount, browser, us
             return;
         if (isDisallowedInRobotsTxt(url))
             return;
+        if (!isFilePath(userUrl) && !isFollowStrategy(url, userUrl, strategy))
+            return;
         url = convertPathToLocalFile(url);
         let request;
         try {

package/dist/crawlers/crawlDomain.js

@@ -4,7 +4,7 @@ import fsp from 'fs/promises';
 import { createCrawleeSubFolders, runAxeScript, isUrlPdf, shouldSkipClickDueToDisallowedHref, shouldSkipDueToUnsupportedContent, } from './commonCrawlerFunc.js';
 import constants, { blackListedFileExtensions, guiInfoStatusTypes, cssQuerySelectors, STATUS_CODE_METADATA, disallowedListOfPatterns, disallowedSelectorPatterns, FileTypes, } from '../constants/constants.js';
 import { getPlaywrightLaunchOptions, isBlacklistedFileExtensions, isSkippedUrl, isDisallowedInRobotsTxt, getUrlsFromRobotsTxt, waitForPageLoaded, } from '../constants/common.js';
-import { areLinksEqual, isFollowStrategy, register } from '../utils.js';
+import { areLinksEqual, isFollowStrategy, normUrl, register } from '../utils.js';
 import { handlePdfDownload, runPdfScan, mapPdfScanResults, doPdfScreenshots, } from './pdfScanFunc.js';
 import { consoleLogger, guiInfoLog } from '../logs.js';
 const isBlacklisted = (url, blacklistedPatterns) => {
@@ -37,8 +37,8 @@ const crawlDomain = async ({ url, randomToken, host: _host, viewportSettings, ma
     const pdfDownloads = [];
     const uuidToPdfMapping = {};
     const queuedUrlSet = new Set();
-    const scannedUrlSet = new Set(urlsCrawled.scanned.map(item => item.url));
-    const scannedResolvedUrlSet = new Set(urlsCrawled.scanned.map(item => item.actualUrl || item.url));
+    const scannedUrlSet = new Set(urlsCrawled.scanned.map(item => normUrl(item.url)));
+    const scannedResolvedUrlSet = new Set(urlsCrawled.scanned.map(item => normUrl(item.actualUrl || item.url)));
     const isScanHtml = [FileTypes.All, FileTypes.HtmlOnly].includes(fileTypes);
     const isScanPdfs = [FileTypes.All, FileTypes.PdfOnly].includes(fileTypes);
     const { maxConcurrency } = constants;
@@ -70,11 +70,12 @@ const crawlDomain = async ({ url, randomToken, host: _host, viewportSettings, ma
         const initialPageUrl = workingPage.url().toString();
         const selectedElementsString = cssQuerySelectors.join(', ');
         const isExcluded = (newPageUrl) => {
-            const isAlreadyScanned = urlsCrawled.scanned.some(item => item.url === newPageUrl);
+            const isAlreadyScanned = scannedUrlSet.has(normUrl(newPageUrl));
             const isBlacklistedUrl = isBlacklisted(newPageUrl, blacklistedPatterns);
             const isNotFollowStrategy = !isFollowStrategy(newPageUrl, initialPageUrl, strategy);
             const isNotSupportedDocument = disallowedListOfPatterns.some(pattern => newPageUrl.toLowerCase().startsWith(pattern));
-            return isNotSupportedDocument || isAlreadyScanned || isBlacklistedUrl || isNotFollowStrategy;
+            const isRobotsDisallowed = isDisallowedInRobotsTxt(newPageUrl);
+            return isNotSupportedDocument || isAlreadyScanned || isBlacklistedUrl || isNotFollowStrategy || isRobotsDisallowed;
         };
         const setPageListeners = (pageListener) => {
             // event listener to handle new page popups upon button click
@@ -235,7 +236,7 @@ const crawlDomain = async ({ url, randomToken, host: _host, viewportSettings, ma
                     catch (e) {
                         consoleLogger.error(e);
                     }
-                    if (scannedUrlSet.has(req.url)) {
+                    if (scannedUrlSet.has(normUrl(req.url))) {
                         req.skipNavigation = true;
                     }
                     if (isDisallowedInRobotsTxt(req.url))
@@ -358,7 +359,7 @@ const crawlDomain = async ({ url, randomToken, host: _host, viewportSettings, ma
                     finalUrl = requestLabelUrl;
                 }
                 const isRedirected = !areLinksEqual(finalUrl, requestLabelUrl);
-                if (isRedirected) {
+                if (isRedirected && !isDisallowedInRobotsTxt(finalUrl)) {
                     await enqueueUniqueRequest({ url: finalUrl, label: finalUrl });
                 }
                 else {
@@ -399,7 +400,7 @@ const crawlDomain = async ({ url, randomToken, host: _host, viewportSettings, ma
                     return;
                 }
                 // if URL has already been scanned
-                if (scannedUrlSet.has(request.url)) {
+                if (scannedUrlSet.has(normUrl(request.url))) {
                     await enqueueProcess(page, enqueueLinks, browserContext);
                     return;
                 }
@@ -493,8 +494,32 @@ const crawlDomain = async ({ url, randomToken, host: _host, viewportSettings, ma
                         return;
                     }
                     const results = await runAxeScript({ includeScreenshots, page, randomToken, ruleset });
+                    // Detect JS redirects that fire during/after axe scan.
+                    // Listen for navigation, then give a brief window for pending redirects to complete.
+                    try {
+                        let navigatedToUrl = null;
+                        const onFrameNavigated = (frame) => {
+                            if (frame === page.mainFrame()) {
+                                navigatedToUrl = frame.url();
+                            }
+                        };
+                        page.on('framenavigated', onFrameNavigated);
+                        await page.waitForTimeout(1000);
+                        page.off('framenavigated', onFrameNavigated);
+                        const postScanUrl = navigatedToUrl || page.url();
+                        if (postScanUrl && postScanUrl !== 'about:blank' && !isFollowStrategy(postScanUrl, request.url, 'same-hostname')) {
+                            urlsCrawled.notScannedRedirects.push({
+                                fromUrl: request.url,
+                                toUrl: postScanUrl,
+                            });
+                            return;
+                        }
+                    }
+                    catch (_) {
+                        // Page/context was destroyed during navigation — handled by outer catch
+                    }
                     if (isRedirected) {
-                        const isLoadedUrlInCrawledUrls = scannedResolvedUrlSet.has(actualUrl);
+                        const isLoadedUrlInCrawledUrls = scannedResolvedUrlSet.has(normUrl(actualUrl));
                         if (isLoadedUrlInCrawledUrls) {
                             urlsCrawled.notScannedRedirects.push({
                                 fromUrl: request.url,
@@ -513,8 +538,8 @@ const crawlDomain = async ({ url, randomToken, host: _host, viewportSettings, ma
                                 pageTitle: results.pageTitle,
                                 actualUrl, // i.e. actualUrl
                             });
-                            scannedUrlSet.add(request.url);
-                            scannedResolvedUrlSet.add(actualUrl);
+                            scannedUrlSet.add(normUrl(request.url));
+                            scannedResolvedUrlSet.add(normUrl(actualUrl));
                             urlsCrawled.scannedRedirects.push({
                                 fromUrl: request.url,
                                 toUrl: actualUrl, // i.e. actualUrl
@@ -535,8 +560,8 @@ const crawlDomain = async ({ url, randomToken, host: _host, viewportSettings, ma
                             actualUrl: request.url,
                             pageTitle: results.pageTitle,
                         });
-                        scannedUrlSet.add(request.url);
-                        scannedResolvedUrlSet.add(request.url);
+                        scannedUrlSet.add(normUrl(request.url));
+                        scannedResolvedUrlSet.add(normUrl(request.url));
                         await dataset.pushData(results);
                     }
                 }

package/dist/crawlers/crawlIntelligentSitemap.js

@@ -3,7 +3,7 @@ import constants, { guiInfoStatusTypes, sitemapPaths } from '../constants/consta
 import { consoleLogger, guiInfoLog } from '../logs.js';
 import crawlDomain from './crawlDomain.js';
 import crawlSitemap from './crawlSitemap.js';
-import { getPlaywrightLaunchOptions } from '../constants/common.js';
+import { getPlaywrightLaunchOptions, getSitemapsFromRobotsTxt } from '../constants/common.js';
 import { register } from '../utils.js';
 const crawlIntelligentSitemap = async (url, randomToken, host, viewportSettings, maxRequestsPerCrawl, browser, userDataDirectory, strategy, specifiedMaxConcurrency, fileTypes, blacklistedPatterns, includeScreenshots, followRobots, extraHTTPHeaders, safeMode, scanDuration) => {
     const startTime = Date.now(); // Track start time
@@ -66,12 +66,30 @@ const crawlIntelligentSitemap = async (url, randomToken, host, viewportSettings,
             return false;
         }
     };
+    // Discover sitemaps from robots.txt first (supports multiple Sitemap: directives)
+    let sitemapUrls = [];
     try {
-        sitemapUrl = await findSitemap(url, userDataDirectory, extraHTTPHeaders);
+        sitemapUrls = await getSitemapsFromRobotsTxt(url, browser, userDataDirectory, extraHTTPHeaders);
+        if (sitemapUrls.length > 0) {
+            console.log(`Found ${sitemapUrls.length} sitemap(s) in robots.txt: ${sitemapUrls.join(', ')}`);
+            sitemapExist = true;
+        }
     }
     catch (error) {
         consoleLogger.error(error);
     }
+    // Fall back to hardcoded path probing if robots.txt had no sitemaps
+    if (!sitemapExist) {
+        try {
+            sitemapUrl = await findSitemap(url, userDataDirectory, extraHTTPHeaders);
+            if (sitemapExist) {
+                sitemapUrls = [sitemapUrl];
+            }
+        }
+        catch (error) {
+            consoleLogger.error(error);
+        }
+    }
     if (!sitemapExist) {
         console.log('Unable to find sitemap. Commencing website crawl instead.');
         return await crawlDomain({
@@ -90,34 +108,48 @@ const crawlIntelligentSitemap = async (url, randomToken, host, viewportSettings,
             followRobots,
             extraHTTPHeaders,
             safeMode,
-            scanDuration, // Use full duration since no sitemap
+            scanDuration,
+        });
+    }
+    // Process all discovered sitemaps sequentially, sharing dataset and urlsCrawled
+    for (const currentSitemapUrl of sitemapUrls) {
+        if (urlsCrawled.scanned.length >= maxRequestsPerCrawl)
+            break;
+        const elapsed = Date.now() - startTime;
+        const remainingDuration = scanDuration > 0 ? Math.max(scanDuration - elapsed / 1000, 0) : scanDuration;
+        if (scanDuration > 0 && remainingDuration <= 0) {
+            durationExceeded = true;
+            break;
+        }
+        console.log(`Processing sitemap: ${currentSitemapUrl}`);
+        urlsCrawledFinal = await crawlSitemap({
+            sitemapUrl: currentSitemapUrl,
+            randomToken,
+            host,
+            viewportSettings,
+            maxRequestsPerCrawl,
+            browser,
+            userDataDirectory,
+            specifiedMaxConcurrency,
+            fileTypes,
+            blacklistedPatterns,
+            includeScreenshots,
+            extraHTTPHeaders,
+            strategy,
+            userUrl: url,
+            fromCrawlIntelligentSitemap,
+            userUrlInputFromIntelligent: url,
+            datasetFromIntelligent: dataset,
+            urlsCrawledFromIntelligent: urlsCrawled,
+            crawledFromLocalFile: false,
+            scanDuration: scanDuration > 0 ? remainingDuration : 0,
         });
     }
-    console.log(`Sitemap found at ${sitemapUrl}`);
-    urlsCrawledFinal = await crawlSitemap({
-        sitemapUrl,
-        randomToken,
-        host,
-        viewportSettings,
-        maxRequestsPerCrawl,
-        browser,
-        userDataDirectory,
-        specifiedMaxConcurrency,
-        fileTypes,
-        blacklistedPatterns,
-        includeScreenshots,
-        extraHTTPHeaders,
-        fromCrawlIntelligentSitemap,
-        userUrlInputFromIntelligent: url,
-        datasetFromIntelligent: dataset,
-        urlsCrawledFromIntelligent: urlsCrawled,
-        crawledFromLocalFile: false,
-        scanDuration,
-    });
     const elapsed = Date.now() - startTime;
-    const remainingScanDuration = Math.max(scanDuration - elapsed / 1000, 0); // in seconds
-    if (urlsCrawledFinal.scanned.length < maxRequestsPerCrawl && remainingScanDuration > 0) {
-        console.log(`Continuing crawl from root website. Remaining scan time: ${remainingScanDuration.toFixed(1)}s`);
+    const remainingScanDuration = scanDuration > 0 ? Math.max(scanDuration - elapsed / 1000, 0) : 0;
+    const hasDurationRemaining = scanDuration === 0 || remainingScanDuration > 0;
+    if (urlsCrawled.scanned.length < maxRequestsPerCrawl && hasDurationRemaining) {
+        console.log(`Continuing crawl from root website.${scanDuration > 0 ? ` Remaining scan time: ${remainingScanDuration.toFixed(1)}s` : ''}`);
         urlsCrawledFinal = await crawlDomain({
             url,
             randomToken,
@@ -136,15 +168,15 @@ const crawlIntelligentSitemap = async (url, randomToken, host, viewportSettings,
             safeMode,
             fromCrawlIntelligentSitemap,
             datasetFromIntelligent: dataset,
-            urlsCrawledFromIntelligent: urlsCrawledFinal,
+            urlsCrawledFromIntelligent: urlsCrawled,
             scanDuration: remainingScanDuration,
         });
     }
-    else if (remainingScanDuration <= 0) {
+    else if (!hasDurationRemaining) {
         console.log(`Crawl duration exceeded before more pages could be found (limit: ${scanDuration}s).`);
         durationExceeded = true;
     }
     guiInfoLog(guiInfoStatusTypes.COMPLETED, {});
-    return { urlsCrawled: urlsCrawledFinal, durationExceeded };
+    return { urlsCrawled, durationExceeded };
 };
 export default crawlIntelligentSitemap;

package/dist/crawlers/crawlSitemap.js

@@ -1,13 +1,13 @@
-import crawlee, { RequestList } from 'crawlee';
+import crawlee, { EnqueueStrategy, RequestList } from 'crawlee';
 import * as path from 'path';
 import fsp from 'fs/promises';
 import { createCrawleeSubFolders, preNavigationHooks, runAxeScript, } from './commonCrawlerFunc.js';
 import constants, { STATUS_CODE_METADATA, guiInfoStatusTypes, disallowedListOfPatterns, FileTypes, } from '../constants/constants.js';
 import { getLinksFromSitemap, getPlaywrightLaunchOptions, isSkippedUrl, waitForPageLoaded, isFilePath, } from '../constants/common.js';
-import { areLinksEqual, isWhitelistedContentType, register } from '../utils.js';
+import { areLinksEqual, isFollowStrategy, isWhitelistedContentType, normUrl, register } from '../utils.js';
 import { handlePdfDownload, runPdfScan, mapPdfScanResults, doPdfScreenshots, } from './pdfScanFunc.js';
 import { guiInfoLog } from '../logs.js';
-const crawlSitemap = async ({ sitemapUrl, randomToken, host, viewportSettings, maxRequestsPerCrawl, browser, userDataDirectory, specifiedMaxConcurrency, fileTypes, blacklistedPatterns, includeScreenshots, extraHTTPHeaders, scanDuration = 0, fromCrawlIntelligentSitemap = false, userUrlInputFromIntelligent = null, datasetFromIntelligent = null, urlsCrawledFromIntelligent = null, crawledFromLocalFile = false, }) => {
+const crawlSitemap = async ({ sitemapUrl, randomToken, host, viewportSettings, maxRequestsPerCrawl, browser, userDataDirectory, specifiedMaxConcurrency, fileTypes, blacklistedPatterns, includeScreenshots, extraHTTPHeaders, strategy = EnqueueStrategy.All, userUrl = '', scanDuration = 0, fromCrawlIntelligentSitemap = false, userUrlInputFromIntelligent = null, datasetFromIntelligent = null, urlsCrawledFromIntelligent = null, crawledFromLocalFile = false, }) => {
     const crawlStartTime = Date.now();
     let dataset;
     let urlsCrawled;
@@ -25,7 +25,7 @@ const crawlSitemap = async ({ sitemapUrl, randomToken, host, viewportSettings, m
         console.log('Local file crawling not supported for sitemap. Please provide a valid URL.');
         return;
     }
-    const linksFromSitemap = await getLinksFromSitemap(sitemapUrl, maxRequestsPerCrawl, browser, userDataDirectory, userUrlInputFromIntelligent, fromCrawlIntelligentSitemap, extraHTTPHeaders);
+    const linksFromSitemap = await getLinksFromSitemap(sitemapUrl, maxRequestsPerCrawl, browser, userDataDirectory, userUrlInputFromIntelligent, fromCrawlIntelligentSitemap, extraHTTPHeaders, strategy, userUrl || sitemapUrl);
     sitemapUrl = encodeURI(sitemapUrl);
     const pdfDownloads = [];
     const uuidToPdfMapping = {};
@@ -182,7 +182,7 @@ const crawlSitemap = async ({ sitemapUrl, randomToken, host, viewportSettings, m
                 const status = response ? response.status() : 0;
                 if (isScanHtml && status < 300 && isWhitelistedContentType(contentType)) {
                     const isRedirected = !areLinksEqual(page.url(), request.url);
-                    const isLoadedUrlInCrawledUrls = urlsCrawled.scanned.some(item => (item.actualUrl || item.url) === page.url());
+                    const isLoadedUrlInCrawledUrls = urlsCrawled.scanned.some(item => normUrl(item.actualUrl || item.url) === normUrl(page.url()));
                     if (isRedirected && isLoadedUrlInCrawledUrls) {
                         urlsCrawled.notScannedRedirects.push({
                             fromUrl: request.url,
@@ -205,7 +205,46 @@ const crawlSitemap = async ({ sitemapUrl, randomToken, host, viewportSettings, m
                         });
                         return;
                     }
+                    if (isRedirected && !isFollowStrategy(actualUrl, request.url, 'same-hostname')) {
+                        urlsCrawled.notScannedRedirects.push({
+                            fromUrl: request.url,
+                            toUrl: actualUrl,
+                        });
+                        guiInfoLog(guiInfoStatusTypes.SKIPPED, {
+                            numScanned: urlsCrawled.scanned.length,
+                            urlScanned: request.url,
+                        });
+                        return;
+                    }
                     const results = await runAxeScript({ includeScreenshots, page, randomToken });
+                    // Detect JS redirects that fire during/after axe scan.
+                    // Listen for navigation, then give a brief window for pending redirects to complete.
+                    try {
+                        let navigatedToUrl = null;
+                        const onFrameNavigated = (frame) => {
+                            if (frame === page.mainFrame()) {
+                                navigatedToUrl = frame.url();
+                            }
+                        };
+                        page.on('framenavigated', onFrameNavigated);
+                        await page.waitForTimeout(1000);
+                        page.off('framenavigated', onFrameNavigated);
+                        const postScanUrl = navigatedToUrl || page.url();
+                        if (postScanUrl && postScanUrl !== 'about:blank' && !isFollowStrategy(postScanUrl, request.url, 'same-hostname')) {
+                            urlsCrawled.notScannedRedirects.push({
+                                fromUrl: request.url,
+                                toUrl: postScanUrl,
+                            });
+                            guiInfoLog(guiInfoStatusTypes.SKIPPED, {
+                                numScanned: urlsCrawled.scanned.length,
+                                urlScanned: request.url,
+                            });
+                            return;
+                        }
+                    }
+                    catch (_) {
+                        // Page/context was destroyed during navigation — handled by outer catch
+                    }
                     guiInfoLog(guiInfoStatusTypes.SCANNED, {
                         numScanned: urlsCrawled.scanned.length,
                         urlScanned: request.url,