@govtechsg/oobee 0.10.86 → 0.10.88

package/src/crawlers/crawlDomain.ts

@@ -29,7 +29,7 @@ import {
   getUrlsFromRobotsTxt,
   waitForPageLoaded,
 } from '../constants/common.js';
-import { areLinksEqual, isFollowStrategy, register } from '../utils.js';
+import { areLinksEqual, isFollowStrategy, normUrl, register } from '../utils.js';
 import {
   handlePdfDownload,
   runPdfScan,
@@ -116,9 +116,9 @@ const crawlDomain = async ({
   const pdfDownloads: Promise<void>[] = [];
   const uuidToPdfMapping: Record<string, string> = {};
   const queuedUrlSet = new Set<string>();
-  const scannedUrlSet = new Set<string>(urlsCrawled.scanned.map(item => item.url));
+  const scannedUrlSet = new Set<string>(urlsCrawled.scanned.map(item => normUrl(item.url)));
   const scannedResolvedUrlSet = new Set<string>(
-    urlsCrawled.scanned.map(item => item.actualUrl || item.url),
+    urlsCrawled.scanned.map(item => normUrl(item.actualUrl || item.url)),
   );
   const isScanHtml = [FileTypes.All, FileTypes.HtmlOnly].includes(fileTypes as FileTypes);
   const isScanPdfs = [FileTypes.All, FileTypes.PdfOnly].includes(fileTypes as FileTypes);
@@ -166,13 +166,14 @@ const crawlDomain = async ({
     const selectedElementsString = cssQuerySelectors.join(', ');
 
     const isExcluded = (newPageUrl: string): boolean => {
-      const isAlreadyScanned: boolean = urlsCrawled.scanned.some(item => item.url === newPageUrl);
+      const isAlreadyScanned: boolean = scannedUrlSet.has(normUrl(newPageUrl));
       const isBlacklistedUrl: boolean = isBlacklisted(newPageUrl, blacklistedPatterns);
       const isNotFollowStrategy: boolean = !isFollowStrategy(newPageUrl, initialPageUrl, strategy);
       const isNotSupportedDocument: boolean = disallowedListOfPatterns.some(pattern =>
         newPageUrl.toLowerCase().startsWith(pattern),
       );
-      return isNotSupportedDocument || isAlreadyScanned || isBlacklistedUrl || isNotFollowStrategy;
+      const isRobotsDisallowed: boolean = isDisallowedInRobotsTxt(newPageUrl);
+      return isNotSupportedDocument || isAlreadyScanned || isBlacklistedUrl || isNotFollowStrategy || isRobotsDisallowed;
     };
     const setPageListeners = (pageListener: Page): void => {
       // event listener to handle new page popups upon button click
@@ -341,7 +342,7 @@ const crawlDomain = async ({
           } catch (e) {
             consoleLogger.error(e);
           }
-          if (scannedUrlSet.has(req.url)) {
+          if (scannedUrlSet.has(normUrl(req.url))) {
             req.skipNavigation = true;
           }
           if (isDisallowedInRobotsTxt(req.url)) return null;
@@ -481,7 +482,7 @@ const crawlDomain = async ({
           }
 
           const isRedirected = !areLinksEqual(finalUrl, requestLabelUrl);
-          if (isRedirected) {
+          if (isRedirected && !isDisallowedInRobotsTxt(finalUrl)) {
             await enqueueUniqueRequest({ url: finalUrl, label: finalUrl });
           } else {
             request.skipNavigation = false;
@@ -537,7 +538,7 @@ const crawlDomain = async ({
           }
 
           // if URL has already been scanned
-          if (scannedUrlSet.has(request.url)) {
+          if (scannedUrlSet.has(normUrl(request.url))) {
             await enqueueProcess(page, enqueueLinks, browserContext);
             return;
           }
@@ -654,8 +655,33 @@ const crawlDomain = async ({
 
             const results = await runAxeScript({ includeScreenshots, page, randomToken, ruleset });
 
+            // Detect JS redirects that fire during/after axe scan.
+            // Listen for navigation, then give a brief window for pending redirects to complete.
+            try {
+              let navigatedToUrl: string | null = null;
+              const onFrameNavigated = (frame: Frame) => {
+                if (frame === page.mainFrame()) {
+                  navigatedToUrl = frame.url();
+                }
+              };
+              page.on('framenavigated', onFrameNavigated);
+              await page.waitForTimeout(1000);
+              page.off('framenavigated', onFrameNavigated);
+
+              const postScanUrl = navigatedToUrl || page.url();
+              if (postScanUrl && postScanUrl !== 'about:blank' && !isFollowStrategy(postScanUrl, request.url, 'same-hostname')) {
+                urlsCrawled.notScannedRedirects.push({
+                  fromUrl: request.url,
+                  toUrl: postScanUrl,
+                });
+                return;
+              }
+            } catch (_) {
+              // Page/context was destroyed during navigation — handled by outer catch
+            }
+
             if (isRedirected) {
-              const isLoadedUrlInCrawledUrls = scannedResolvedUrlSet.has(actualUrl);
+              const isLoadedUrlInCrawledUrls = scannedResolvedUrlSet.has(normUrl(actualUrl));
 
               if (isLoadedUrlInCrawledUrls) {
                 urlsCrawled.notScannedRedirects.push({
@@ -677,8 +703,8 @@ const crawlDomain = async ({
                   pageTitle: results.pageTitle,
                   actualUrl, // i.e. actualUrl
                 });
-                scannedUrlSet.add(request.url);
-                scannedResolvedUrlSet.add(actualUrl);
+                scannedUrlSet.add(normUrl(request.url));
+                scannedResolvedUrlSet.add(normUrl(actualUrl));
 
                 urlsCrawled.scannedRedirects.push({
                   fromUrl: request.url,
@@ -700,8 +726,8 @@ const crawlDomain = async ({
                 actualUrl: request.url,
                 pageTitle: results.pageTitle,
               });
-              scannedUrlSet.add(request.url);
-              scannedResolvedUrlSet.add(request.url);
+              scannedUrlSet.add(normUrl(request.url));
+              scannedResolvedUrlSet.add(normUrl(request.url));
               await dataset.pushData(results);
             }
           } else {

package/src/crawlers/crawlIntelligentSitemap.ts

@@ -7,7 +7,7 @@ import { consoleLogger, guiInfoLog } from '../logs.js';
 import crawlDomain from './crawlDomain.js';
 import crawlSitemap from './crawlSitemap.js';
 import { ViewportSettingsClass } from '../combine.js';
-import { getPlaywrightLaunchOptions } from '../constants/common.js';
+import { getPlaywrightLaunchOptions, getSitemapsFromRobotsTxt } from '../constants/common.js';
 import { register } from '../utils.js';
 
 const crawlIntelligentSitemap = async (
@@ -100,12 +100,30 @@ const crawlIntelligentSitemap = async (
     }
   };
 
+  // Discover sitemaps from robots.txt first (supports multiple Sitemap: directives)
+  let sitemapUrls: string[] = [];
   try {
-    sitemapUrl = await findSitemap(url, userDataDirectory, extraHTTPHeaders);
+    sitemapUrls = await getSitemapsFromRobotsTxt(url, browser, userDataDirectory, extraHTTPHeaders);
+    if (sitemapUrls.length > 0) {
+      console.log(`Found ${sitemapUrls.length} sitemap(s) in robots.txt: ${sitemapUrls.join(', ')}`);
+      sitemapExist = true;
+    }
   } catch (error) {
     consoleLogger.error(error);
   }
 
+  // Fall back to hardcoded path probing if robots.txt had no sitemaps
+  if (!sitemapExist) {
+    try {
+      sitemapUrl = await findSitemap(url, userDataDirectory, extraHTTPHeaders);
+      if (sitemapExist) {
+        sitemapUrls = [sitemapUrl];
+      }
+    } catch (error) {
+      consoleLogger.error(error);
+    }
+  }
+
   if (!sitemapExist) {
     console.log('Unable to find sitemap. Commencing website crawl instead.');
     return await crawlDomain({
@@ -124,38 +142,53 @@ const crawlIntelligentSitemap = async (
       followRobots,
       extraHTTPHeaders,
       safeMode,
-      scanDuration, // Use full duration since no sitemap
+      scanDuration,
     });
   }
 
-  console.log(`Sitemap found at ${sitemapUrl}`);
-  urlsCrawledFinal = await crawlSitemap({
-    sitemapUrl,
-    randomToken,
-    host,
-    viewportSettings,
-    maxRequestsPerCrawl,
-    browser,
-    userDataDirectory,
-    specifiedMaxConcurrency,
-    fileTypes,
-    blacklistedPatterns,
-    includeScreenshots,
-    extraHTTPHeaders,
-    fromCrawlIntelligentSitemap,
-    userUrlInputFromIntelligent: url,
-    datasetFromIntelligent: dataset,
-    urlsCrawledFromIntelligent: urlsCrawled,
-    crawledFromLocalFile: false,
-    scanDuration,
-  });
+  // Process all discovered sitemaps sequentially, sharing dataset and urlsCrawled
+  for (const currentSitemapUrl of sitemapUrls) {
+    if (urlsCrawled.scanned.length >= maxRequestsPerCrawl) break;
+
+    const elapsed = Date.now() - startTime;
+    const remainingDuration = scanDuration > 0 ? Math.max(scanDuration - elapsed / 1000, 0) : scanDuration;
+    if (scanDuration > 0 && remainingDuration <= 0) {
+      durationExceeded = true;
+      break;
+    }
+
+    console.log(`Processing sitemap: ${currentSitemapUrl}`);
+    urlsCrawledFinal = await crawlSitemap({
+      sitemapUrl: currentSitemapUrl,
+      randomToken,
+      host,
+      viewportSettings,
+      maxRequestsPerCrawl,
+      browser,
+      userDataDirectory,
+      specifiedMaxConcurrency,
+      fileTypes,
+      blacklistedPatterns,
+      includeScreenshots,
+      extraHTTPHeaders,
+      strategy,
+      userUrl: url,
+      fromCrawlIntelligentSitemap,
+      userUrlInputFromIntelligent: url,
+      datasetFromIntelligent: dataset,
+      urlsCrawledFromIntelligent: urlsCrawled,
+      crawledFromLocalFile: false,
+      scanDuration: scanDuration > 0 ? remainingDuration : 0,
+    });
+  }
 
   const elapsed = Date.now() - startTime;
-  const remainingScanDuration = Math.max(scanDuration - elapsed / 1000, 0); // in seconds
+  const remainingScanDuration = scanDuration > 0 ? Math.max(scanDuration - elapsed / 1000, 0) : 0;
+  const hasDurationRemaining = scanDuration === 0 || remainingScanDuration > 0;
 
-  if (urlsCrawledFinal.scanned.length < maxRequestsPerCrawl && remainingScanDuration > 0) {
+  if (urlsCrawled.scanned.length < maxRequestsPerCrawl && hasDurationRemaining) {
     console.log(
-      `Continuing crawl from root website. Remaining scan time: ${remainingScanDuration.toFixed(1)}s`,
+      `Continuing crawl from root website.${scanDuration > 0 ? ` Remaining scan time: ${remainingScanDuration.toFixed(1)}s` : ''}`,
     );
     urlsCrawledFinal = await crawlDomain({
       url,
@@ -175,10 +208,10 @@ const crawlIntelligentSitemap = async (
       safeMode,
       fromCrawlIntelligentSitemap,
       datasetFromIntelligent: dataset,
-      urlsCrawledFromIntelligent: urlsCrawledFinal,
+      urlsCrawledFromIntelligent: urlsCrawled,
       scanDuration: remainingScanDuration,
     });
-  } else if (remainingScanDuration <= 0) {
+  } else if (!hasDurationRemaining) {
     console.log(
       `Crawl duration exceeded before more pages could be found (limit: ${scanDuration}s).`,
     );
@@ -186,7 +219,7 @@ const crawlIntelligentSitemap = async (
   }
 
   guiInfoLog(guiInfoStatusTypes.COMPLETED, {});
-  return { urlsCrawled: urlsCrawledFinal, durationExceeded };
+  return { urlsCrawled, durationExceeded };
 };
 
 export default crawlIntelligentSitemap;

package/src/crawlers/crawlLocalFile.ts

@@ -8,6 +8,7 @@ import constants, {
   UrlsCrawled,
   STATUS_CODE_METADATA,
   FileTypes,
+  RuleFlags,
 } from '../constants/constants.js';
 import { ViewportSettingsClass } from '../combine.js';
 import {
@@ -35,6 +36,7 @@ export const crawlLocalFile = async ({
   includeScreenshots,
   extraHTTPHeaders,
   scanDuration = 0,
+  ruleset = [],
   fromCrawlIntelligentSitemap = false,
   userUrlInputFromIntelligent = null,
   datasetFromIntelligent = null,
@@ -53,6 +55,7 @@ export const crawlLocalFile = async ({
   includeScreenshots: boolean;
   extraHTTPHeaders: Record<string, string>;
   scanDuration?: number;
+  ruleset?: RuleFlags[];
   fromCrawlIntelligentSitemap?: boolean;
   userUrlInputFromIntelligent?: string | null;
   datasetFromIntelligent?: Dataset | null;
@@ -178,7 +181,7 @@ export const crawlLocalFile = async ({
       return urlsCrawled;
     }
 
-    const results = await runAxeScript({ includeScreenshots, page, randomToken });
+    const results = await runAxeScript({ includeScreenshots, page, randomToken, ruleset });
 
     const actualUrl = page.url() || request.loadedUrl || url;
 

package/src/crawlers/crawlSitemap.ts

@@ -1,4 +1,4 @@
-import crawlee, { LaunchContext, Request, RequestList, Dataset } from 'crawlee';
+import crawlee, { EnqueueStrategy, LaunchContext, Request, RequestList, Dataset } from 'crawlee';
 import fs from 'fs';
 import * as path from 'path';
 import fsp from 'fs/promises';
@@ -23,7 +23,7 @@ import {
   waitForPageLoaded,
   isFilePath,
 } from '../constants/common.js';
-import { areLinksEqual, isWhitelistedContentType, register } from '../utils.js';
+import { areLinksEqual, isFollowStrategy, isWhitelistedContentType, normUrl, register } from '../utils.js';
 import {
   handlePdfDownload,
   runPdfScan,
@@ -46,6 +46,8 @@ const crawlSitemap = async ({
   blacklistedPatterns,
   includeScreenshots,
   extraHTTPHeaders,
+  strategy = EnqueueStrategy.All,
+  userUrl = '',
   scanDuration = 0,
   fromCrawlIntelligentSitemap = false,
   userUrlInputFromIntelligent = null,
@@ -65,6 +67,8 @@ const crawlSitemap = async ({
   blacklistedPatterns: string[];
   includeScreenshots: boolean;
   extraHTTPHeaders: Record<string, string>;
+  strategy?: EnqueueStrategy;
+  userUrl?: string;
   scanDuration?: number;
   fromCrawlIntelligentSitemap?: boolean;
   userUrlInputFromIntelligent?: string;
@@ -99,6 +103,8 @@ const crawlSitemap = async ({
     userUrlInputFromIntelligent,
     fromCrawlIntelligentSitemap,
     extraHTTPHeaders,
+    strategy,
+    userUrl || sitemapUrl,
   );
 
   sitemapUrl = encodeURI(sitemapUrl);
@@ -299,7 +305,7 @@ const crawlSitemap = async ({
           if (isScanHtml && status < 300 && isWhitelistedContentType(contentType)) {
             const isRedirected = !areLinksEqual(page.url(), request.url);
             const isLoadedUrlInCrawledUrls = urlsCrawled.scanned.some(
-              item => (item.actualUrl || item.url) === page.url(),
+              item => normUrl(item.actualUrl || item.url) === normUrl(page.url()),
             );
 
             if (isRedirected && isLoadedUrlInCrawledUrls) {
@@ -327,8 +333,49 @@ const crawlSitemap = async ({
               return;
             }
 
+            if (isRedirected && !isFollowStrategy(actualUrl, request.url, 'same-hostname')) {
+              urlsCrawled.notScannedRedirects.push({
+                fromUrl: request.url,
+                toUrl: actualUrl,
+              });
+              guiInfoLog(guiInfoStatusTypes.SKIPPED, {
+                numScanned: urlsCrawled.scanned.length,
+                urlScanned: request.url,
+              });
+              return;
+            }
+
             const results = await runAxeScript({ includeScreenshots, page, randomToken });
 
+            // Detect JS redirects that fire during/after axe scan.
+            // Listen for navigation, then give a brief window for pending redirects to complete.
+            try {
+              let navigatedToUrl: string | null = null;
+              const onFrameNavigated = (frame: any) => {
+                if (frame === page.mainFrame()) {
+                  navigatedToUrl = frame.url();
+                }
+              };
+              page.on('framenavigated', onFrameNavigated);
+              await page.waitForTimeout(1000);
+              page.off('framenavigated', onFrameNavigated);
+
+              const postScanUrl = navigatedToUrl || page.url();
+              if (postScanUrl && postScanUrl !== 'about:blank' && !isFollowStrategy(postScanUrl, request.url, 'same-hostname')) {
+                urlsCrawled.notScannedRedirects.push({
+                  fromUrl: request.url,
+                  toUrl: postScanUrl,
+                });
+                guiInfoLog(guiInfoStatusTypes.SKIPPED, {
+                  numScanned: urlsCrawled.scanned.length,
+                  urlScanned: request.url,
+                });
+                return;
+              }
+            } catch (_) {
+              // Page/context was destroyed during navigation — handled by outer catch
+            }
+
             guiInfoLog(guiInfoStatusTypes.SCANNED, {
               numScanned: urlsCrawled.scanned.length,
               urlScanned: request.url,

package/src/crawlers/custom/extractAndGradeText.ts

@@ -45,7 +45,7 @@ export async function extractAndGradeText(page: Page): Promise<string> {
 
     // Determine the return value
     const result =
-      readabilityScore === 0 || readabilityScore > 50 ? '' : readabilityScore.toString(); // Convert readabilityScore to string
+      readabilityScore <= 0 || readabilityScore > 50 ? '' : readabilityScore.toString();
 
     return result;
   } catch (error) {

package/src/crawlers/custom/getAxeConfiguration.ts

@@ -10,6 +10,12 @@ export function getAxeConfiguration({
   gradingReadabilityFlag?: string;
   disableOobee?: boolean;
 }) {
+  function getReadabilityInterpretation(score: string): string {
+    const num = parseFloat(score);
+    if (Number.isNaN(num)) return '';
+    if (num > 30) return 'It is targeted for junior college (JC) level comprehension and above.';
+    return 'It is targeted for university graduate level comprehension and above.';
+  }
   return {
     branding: {
       application: 'oobee',
@@ -39,7 +45,7 @@ export function getAxeConfiguration({
           return !node.dataset.flagged; // fail any element with a data-flagged attribute set to true
         },
       },
-      ...(enableWcagAaa
+      ...((enableWcagAaa && gradingReadabilityFlag !== '')
         ? [
           {
             id: 'oobee-grading-text-contents',
@@ -47,17 +53,11 @@ export function getAxeConfiguration({
               impact: 'moderate' as ImpactValue,
               messages: {
                 pass: 'The text content is easy to understand.',
-                fail: 'The text content is potentially difficult to understand.',
-                incomplete: `The text content is potentially difficult to read, with a Flesch-Kincaid Reading Ease score of ${gradingReadabilityFlag
-                  }.\nThe target passing score is above 50, indicating content readable by university students and lower grade levels.\nA higher score reflects better readability.`,
+                fail: `Text content is potentially difficult to read.\n  It scored ${gradingReadabilityFlag} out of 50 on the Flesch-Kincaid Readability Test.\n  ${getReadabilityInterpretation(gradingReadabilityFlag)}`,
+                incomplete: `Text content is potentially difficult to read.\n  It scored ${gradingReadabilityFlag} out of 50 on the Flesch-Kincaid Readability Test.\n  ${getReadabilityInterpretation(gradingReadabilityFlag)}`,
               },
             },
-            evaluate: (_node: HTMLElement) => {
-              if (gradingReadabilityFlag === '') {
-                return true; // Pass if no readability issues
-              }
-              // Fail if readability issues are detected
-            },
+            evaluate: (_node: HTMLElement) => false,
           },
         ]
         : []),
@@ -88,19 +88,21 @@ export function getAxeConfiguration({
           helpUrl: 'https://www.deque.com/blog/accessible-aria-buttons',
         },
       },
-      {
-        id: 'oobee-grading-text-contents',
-        selector: 'html',
-        enabled: true,
-        any: ['oobee-grading-text-contents'],
-        tags: ['wcag2aaa', 'wcag315'],
-        metadata: {
-          description:
-            'Text content should be easy to understand for individuals with education levels up to university graduates. If the text content is difficult to understand, provide supplemental content or a version that is easy to understand.',
-          help: 'Text content should be clear and plain to ensure that it is easily understood.',
-          helpUrl: 'https://www.wcag.com/uncategorized/3-1-5-reading-level/',
-        },
-      },
+      ...((enableWcagAaa && gradingReadabilityFlag !== '')
+        ? [{
+            id: 'oobee-grading-text-contents',
+            selector: 'html',
+            enabled: true,
+            any: ['oobee-grading-text-contents'],
+            tags: ['wcag2aaa', 'wcag315'],
+            metadata: {
+              description:
+                'Text content should be easy to understand for individuals with education levels up to university graduates. If the text content is difficult to understand, provide supplemental content or a version that is easy to understand.',
+              help: 'Text content should be clear and plain to ensure that it is easily understood.',
+              helpUrl: 'https://www.wcag.com/uncategorized/3-1-5-reading-level/',
+            },
+          }]
+        : []),
     ]
       .filter(rule => (disableOobee ? !rule.id.startsWith('oobee') : true))
       .concat(

package/src/crawlers/custom/gradeReadability.ts

@@ -20,7 +20,7 @@ export function gradeReadability(sentences: string[]): string {
 
     // Determine the return value
     const result =
-      readabilityScore === 0 || readabilityScore > 50 ? '' : readabilityScore.toString(); // Convert readabilityScore to string
+      readabilityScore <= 0 || readabilityScore > 50 ? '' : readabilityScore.toString();
 
     return result;
   } catch (error) {