@govtechsg/oobee 0.10.51 → 0.10.58

package/src/crawlers/crawlLocalFile.ts

@@ -1,12 +1,15 @@
-import { Request, RequestList } from 'crawlee';
-import printMessage from 'print-message';
+import { Request, RequestList, Dataset } from 'crawlee';
 import fs from 'fs';
 import path from 'path';
 import { createCrawleeSubFolders, runAxeScript, isUrlPdf } from './commonCrawlerFunc.js';
-import constants, { guiInfoStatusTypes, basicAuthRegex } from '../constants/constants.js';
+import constants, {
+  guiInfoStatusTypes,
+  basicAuthRegex,
+  UrlsCrawled,
+} from '../constants/constants.js';
+import { ViewportSettingsClass } from '../combine.js';
 import {
   getPlaywrightLaunchOptions,
-  messageOptions,
   isFilePath,
   convertLocalFileToPath,
   convertPathToLocalFile,
@@ -16,27 +19,47 @@ import { runPdfScan, mapPdfScanResults, doPdfScreenshots } from './pdfScanFunc.j
 import { guiInfoLog } from '../logs.js';
 import crawlSitemap from './crawlSitemap.js';
 
-const crawlLocalFile = async (
-  sitemapUrl: string,
-  randomToken: string,
-  host: string,
-  viewportSettings: any,
-  maxRequestsPerCrawl: number,
-  browser: string,
-  userDataDirectory: string,
-  specifiedMaxConcurrency: number,
-  fileTypes: string,
-  blacklistedPatterns: string[],
-  includeScreenshots: boolean,
-  extraHTTPHeaders: any,
-  fromCrawlIntelligentSitemap: boolean = false, // optional
-  userUrlInputFromIntelligent: any = null, // optional
-  datasetFromIntelligent: any = null, // optional
-  urlsCrawledFromIntelligent: any = null, // optional
-) => {
+export const crawlLocalFile = async ({
+  url,
+  randomToken,
+  host,
+  viewportSettings,
+  maxRequestsPerCrawl,
+  browser,
+  userDataDirectory,
+  specifiedMaxConcurrency,
+  fileTypes,
+  blacklistedPatterns,
+  includeScreenshots,
+  extraHTTPHeaders,
+  scanDuration = 0,
+  fromCrawlIntelligentSitemap = false,
+  userUrlInputFromIntelligent = null,
+  datasetFromIntelligent = null,
+  urlsCrawledFromIntelligent = null,
+}: {
+  url: string;
+  randomToken: string;
+  host: string;
+  viewportSettings: ViewportSettingsClass;
+  maxRequestsPerCrawl: number;
+  browser: string;
+  userDataDirectory: string;
+  specifiedMaxConcurrency: number;
+  fileTypes: string;
+  blacklistedPatterns: string[];
+  includeScreenshots: boolean;
+  extraHTTPHeaders: Record<string, string>;
+  scanDuration?: number;
+  fromCrawlIntelligentSitemap?: boolean;
+  userUrlInputFromIntelligent?: string | null;
+  datasetFromIntelligent?: Dataset | null;
+  urlsCrawledFromIntelligent?: UrlsCrawled | null;
+}) => {
   let dataset: any;
-  let urlsCrawled: any;
+  let urlsCrawled: UrlsCrawled;
   let linksFromSitemap = [];
+  let sitemapUrl = url; 
 
   // Boolean to omit axe scan for basic auth URL
   let isBasicAuth: boolean;
@@ -82,7 +105,7 @@ const crawlLocalFile = async (
     // Non XML file
   } else {
     // Put it to crawlSitemap function to handle xml files
-    const updatedUrlsCrawled = await crawlSitemap(
+    const updatedUrlsCrawled = await crawlSitemap({
       sitemapUrl,
       randomToken,
       host,
@@ -95,12 +118,13 @@ const crawlLocalFile = async (
       blacklistedPatterns,
       includeScreenshots,
       extraHTTPHeaders,
-      (fromCrawlIntelligentSitemap = false), // optional
-      (userUrlInputFromIntelligent = null), // optional
-      (datasetFromIntelligent = null), // optional
-      (urlsCrawledFromIntelligent = null), // optional
-      true,
-    );
+      scanDuration,
+      fromCrawlIntelligentSitemap,
+      userUrlInputFromIntelligent,
+      datasetFromIntelligent,
+      urlsCrawledFromIntelligent,
+      crawledFromLocalFile: true,
+    });
 
     urlsCrawled = { ...urlsCrawled, ...updatedUrlsCrawled };
     return urlsCrawled;
@@ -124,16 +148,12 @@ const crawlLocalFile = async (
 
   const uuidToPdfMapping: Record<string, string> = {}; // key and value of string type
 
-  printMessage(['Fetching URLs. This might take some time...'], { border: false });
-
   finalLinks = [...finalLinks, ...linksFromSitemap];
 
   await RequestList.open({
     sources: finalLinks,
   });
 
-  printMessage(['Fetch URLs completed. Beginning scan'], messageOptions);
-
   const request = linksFromSitemap[0];
   const pdfFileName = path.basename(request.url);
   const trimmedUrl: string = request.url;
@@ -142,6 +162,8 @@ const crawlLocalFile = async (
   fs.writeFileSync(destinationFilePath, data);
   uuidToPdfMapping[pdfFileName] = trimmedUrl;
 
+  let shouldAbort = false;
+
   if (!isUrlPdf(request.url)) {
     await initModifiedUserAgent(browser);
     const browserContext = await constants.launcher.launchPersistentContext('', {
@@ -150,9 +172,24 @@ const crawlLocalFile = async (
       ...playwrightDeviceDetailsObject,
     });
 
+    const timeoutId = scanDuration > 0
+    ? setTimeout(() => {
+        console.log(`Crawl duration of ${scanDuration}s exceeded. Aborting local file scan.`);
+        shouldAbort = true;
+      }, scanDuration * 1000)
+    : null;
+
     const page = await browserContext.newPage();
     request.url = convertPathToLocalFile(request.url);
     await page.goto(request.url);
+
+    if (shouldAbort) {
+      console.warn('Scan aborted due to timeout before page scan.');
+      await dataset.pushData({ scanned: [], scannedRedirects: [] });
+      await browserContext.close().catch(() => {});
+      return urlsCrawled;
+    }
+
     const results = await runAxeScript({ includeScreenshots, page, randomToken });
 
     const actualUrl = page.url() || request.loadedUrl || request.url;
@@ -178,7 +215,11 @@ const crawlLocalFile = async (
 
     await dataset.pushData(results);
   } else {
-    urlsCrawled.scanned.push({ url: trimmedUrl, pageTitle: pdfFileName });
+    urlsCrawled.scanned.push({
+      url: trimmedUrl,
+      pageTitle: pdfFileName,
+      actualUrl: trimmedUrl,
+    });
 
     await runPdfScan(randomToken);
     // transform result format
@@ -192,6 +233,7 @@ const crawlLocalFile = async (
     // push results for each pdf document to key value store
     await Promise.all(pdfResults.map(result => dataset.pushData(result)));
   }
+
   return urlsCrawled;
 };
 export default crawlLocalFile;

package/src/crawlers/crawlSitemap.ts

@@ -1,5 +1,4 @@
-import crawlee, { LaunchContext, Request, RequestList } from 'crawlee';
-import printMessage from 'print-message';
+import crawlee, { LaunchContext, Request, RequestList, Dataset } from 'crawlee';
 import fs from 'fs';
 import {
   createCrawleeSubFolders,
@@ -8,11 +7,15 @@ import {
   isUrlPdf,
 } from './commonCrawlerFunc.js';
 
-import constants, { STATUS_CODE_METADATA, guiInfoStatusTypes, UrlsCrawled } from '../constants/constants.js';
+import constants, {
+  STATUS_CODE_METADATA,
+  guiInfoStatusTypes,
+  UrlsCrawled,
+  disallowedListOfPatterns,
+} from '../constants/constants.js';
 import {
   getLinksFromSitemap,
   getPlaywrightLaunchOptions,
-  messageOptions,
   isSkippedUrl,
   urlWithoutAuth,
   waitForPageLoaded,
@@ -24,25 +27,46 @@ import { handlePdfDownload, runPdfScan, mapPdfScanResults } from './pdfScanFunc.
 import { guiInfoLog } from '../logs.js';
 import { ViewportSettingsClass } from '../combine.js';
 
-const crawlSitemap = async (
-  sitemapUrl: string,
-  randomToken: string,
-  _host: string,
-  viewportSettings: ViewportSettingsClass,
-  maxRequestsPerCrawl: number,
-  browser: string,
-  userDataDirectory: string,
-  specifiedMaxConcurrency: number,
-  fileTypes: string,
-  blacklistedPatterns: string[],
-  includeScreenshots: boolean,
-  extraHTTPHeaders: Record<string, string>,
-  fromCrawlIntelligentSitemap = false, // optional
-  userUrlInputFromIntelligent: string = null, // optional
-  datasetFromIntelligent: crawlee.Dataset = null, // optional
-  urlsCrawledFromIntelligent: UrlsCrawled = null, // optional
-  crawledFromLocalFile = false, // optional
-) => {
+const crawlSitemap = async ({
+  sitemapUrl,
+  randomToken,
+  host,
+  viewportSettings,
+  maxRequestsPerCrawl,
+  browser,
+  userDataDirectory,
+  specifiedMaxConcurrency,
+  fileTypes,
+  blacklistedPatterns,
+  includeScreenshots,
+  extraHTTPHeaders,
+  scanDuration = 0,
+  fromCrawlIntelligentSitemap = false,
+  userUrlInputFromIntelligent = null,
+  datasetFromIntelligent = null,
+  urlsCrawledFromIntelligent = null,
+  crawledFromLocalFile = false,
+}: {
+  sitemapUrl: string;
+  randomToken: string;
+  host: string;
+  viewportSettings: ViewportSettingsClass;
+  maxRequestsPerCrawl: number;
+  browser: string;
+  userDataDirectory: string;
+  specifiedMaxConcurrency: number;
+  fileTypes: string;
+  blacklistedPatterns: string[];
+  includeScreenshots: boolean;
+  extraHTTPHeaders: Record<string, string>;
+  scanDuration?: number;
+  fromCrawlIntelligentSitemap?: boolean;
+  userUrlInputFromIntelligent?: string;
+  datasetFromIntelligent?: Dataset;
+  urlsCrawledFromIntelligent?: UrlsCrawled;
+  crawledFromLocalFile?: boolean;
+}) => {
+  const crawlStartTime = Date.now();
   let dataset: crawlee.Dataset;
   let urlsCrawled: UrlsCrawled;
 
@@ -127,14 +151,11 @@ const crawlSitemap = async (
   const { playwrightDeviceDetailsObject } = viewportSettings;
   const { maxConcurrency } = constants;
 
-  printMessage(['Fetching URLs. This might take some time...'], { border: false });
-
   finalLinks = [...finalLinks, ...linksFromSitemap];
 
   const requestList = await RequestList.open({
     sources: finalLinks,
   });
-  printMessage(['Fetch URLs completed. Beginning scan'], messageOptions);
 
   let userDataDir = '';
   if (userDataDirectory) {
@@ -165,7 +186,6 @@ const crawlSitemap = async (
     },
     requestList,
     postNavigationHooks: [
-
       async ({ page }) => {
         try {
           // Wait for a quiet period in the DOM, but with safeguards
@@ -173,36 +193,35 @@ const crawlSitemap = async (
             return new Promise(resolve => {
               let timeout;
               let mutationCount = 0;
-              const MAX_MUTATIONS     = 250;   // stop if things never quiet down
-              const OBSERVER_TIMEOUT  = 5000;  // hard cap on total wait
-      
+              const MAX_MUTATIONS = 250; // stop if things never quiet down
+              const OBSERVER_TIMEOUT = 5000; // hard cap on total wait
+
               const observer = new MutationObserver(() => {
                 clearTimeout(timeout);
-      
+
                 mutationCount++;
                 if (mutationCount > MAX_MUTATIONS) {
                   observer.disconnect();
                   resolve('Too many mutations, exiting.');
                   return;
                 }
-      
+
                 // restart quiet‑period timer
                 timeout = setTimeout(() => {
                   observer.disconnect();
                   resolve('DOM stabilized.');
                 }, 1000);
               });
-      
+
               // overall timeout in case the page never settles
               timeout = setTimeout(() => {
                 observer.disconnect();
                 resolve('Observer timeout reached.');
               }, OBSERVER_TIMEOUT);
-      
+
               const root = document.documentElement || document.body || document;
               if (!root || typeof observer.observe !== 'function') {
                 resolve('No root node to observe.');
-                return;
               }
             });
           });
@@ -214,27 +233,54 @@ const crawlSitemap = async (
           throw err; // Rethrow unknown errors
         }
       },
-      
     ],
+    preNavigationHooks: [
+      async ({ request, page }, gotoOptions) => {
+        const url = request.url.toLowerCase();
 
-    preNavigationHooks: isBasicAuth
-      ? [
-        async ({ page }) => {
+        const isNotSupportedDocument = disallowedListOfPatterns.some(pattern =>
+          url.startsWith(pattern),
+        );
+
+        if (isNotSupportedDocument) {
+          request.skipNavigation = true;
+          request.userData.isNotSupportedDocument = true;
+
+          // Log for verification (optional, but not required for correctness)
+          // console.log(`[SKIP] Not supported: ${request.url}`);
+
+          return;
+        }
+
+        // Set headers if basic auth
+        if (isBasicAuth) {
           await page.setExtraHTTPHeaders({
             Authorization: authHeader,
             ...extraHTTPHeaders,
           });
-        },
-      ]
-      : [
-        async () => {
+        } else {
           preNavigationHooks(extraHTTPHeaders);
-          // insert other code here
-        },
-      ],
+        }
+      },
+    ],
     requestHandlerTimeoutSecs: 90,
     requestHandler: async ({ page, request, response, sendRequest }) => {
-      await waitForPageLoaded(page, 10000);
+      // Log documents that are not supported
+      if (request.userData?.isNotSupportedDocument) {
+        guiInfoLog(guiInfoStatusTypes.SKIPPED, {
+          numScanned: urlsCrawled.scanned.length,
+          urlScanned: request.url,
+        });
+        urlsCrawled.userExcluded.push({
+          url: request.url,
+          pageTitle: request.url,
+          actualUrl: request.url, // because about:blank is not useful
+          metadata: STATUS_CODE_METADATA[1],
+          httpStatusCode: 0,
+        });
+
+        return;
+      }
 
       // Set basic auth header if needed
       if (isBasicAuth) {
@@ -247,39 +293,48 @@ const crawlSitemap = async (
         request.url = currentUrl.href;
       }
 
+      await waitForPageLoaded(page, 10000);
+
       const actualUrl = page.url() || request.loadedUrl || request.url;
 
-      if (urlsCrawled.scanned.length >= maxRequestsPerCrawl) {
-        crawler.autoscaledPool.abort();
+      const hasExceededDuration =
+        scanDuration > 0 && Date.now() - crawlStartTime > scanDuration * 1000;
+
+      if (urlsCrawled.scanned.length >= maxRequestsPerCrawl || hasExceededDuration) {
+        if (hasExceededDuration) {
+          console.log(`Crawl duration of ${scanDuration}s exceeded. Aborting sitemap crawl.`);
+        }
+        crawler.autoscaledPool.abort(); // stops new requests
         return;
       }
 
-      if (request.skipNavigation && actualUrl === "about:blank") {
-        if (!isScanPdfs) {
-          guiInfoLog(guiInfoStatusTypes.SKIPPED, {
-            numScanned: urlsCrawled.scanned.length,
-            urlScanned: request.url,
-          });
-          urlsCrawled.userExcluded.push({
-            url: request.url,
-            pageTitle: request.url,
-            actualUrl: request.url, // because about:blank is not useful
-            metadata: STATUS_CODE_METADATA[1],
-            httpStatusCode: 0,
-          });
-
+      if (request.skipNavigation && actualUrl === 'about:blank') {
+        if (isScanPdfs) {
+          // pushes download promise into pdfDownloads
+          const { pdfFileName, url } = handlePdfDownload(
+            randomToken,
+            pdfDownloads,
+            request,
+            sendRequest,
+            urlsCrawled,
+          );
+
+          uuidToPdfMapping[pdfFileName] = url;
           return;
         }
-        // pushes download promise into pdfDownloads
-        const { pdfFileName, url } = handlePdfDownload(
-          randomToken,
-          pdfDownloads,
-          request,
-          sendRequest,
-          urlsCrawled,
-        );
 
-        uuidToPdfMapping[pdfFileName] = url;
+        guiInfoLog(guiInfoStatusTypes.SKIPPED, {
+          numScanned: urlsCrawled.scanned.length,
+          urlScanned: request.url,
+        });
+        urlsCrawled.userExcluded.push({
+          url: request.url,
+          pageTitle: request.url,
+          actualUrl: request.url, // because about:blank is not useful
+          metadata: STATUS_CODE_METADATA[1],
+          httpStatusCode: 0,
+        });
+
         return;
       }
 
@@ -303,15 +358,11 @@ const crawlSitemap = async (
         }
 
         // This logic is different from crawlDomain, as it also checks if the pae is redirected before checking if it is excluded using exclusions.txt
-        if (
-          isRedirected &&
-          blacklistedPatterns &&
-          isSkippedUrl(actualUrl, blacklistedPatterns)
-        ) {
+        if (isRedirected && blacklistedPatterns && isSkippedUrl(actualUrl, blacklistedPatterns)) {
           urlsCrawled.userExcluded.push({
             url: request.url,
             pageTitle: request.url,
-            actualUrl: actualUrl,
+            actualUrl,
             metadata: STATUS_CODE_METADATA[0],
             httpStatusCode: 0,
           });
@@ -324,7 +375,7 @@ const crawlSitemap = async (
         }
 
         const results = await runAxeScript({ includeScreenshots, page, randomToken });
-        
+
         guiInfoLog(guiInfoStatusTypes.SCANNED, {
           numScanned: urlsCrawled.scanned.length,
           urlScanned: request.url,
@@ -333,7 +384,7 @@ const crawlSitemap = async (
         urlsCrawled.scanned.push({
           url: urlWithoutAuth(request.url),
           pageTitle: results.pageTitle,
-          actualUrl: actualUrl, // i.e. actualUrl
+          actualUrl, // i.e. actualUrl
         });
 
         urlsCrawled.scannedRedirects.push({
@@ -354,16 +405,17 @@ const crawlSitemap = async (
         if (isScanHtml) {
           // carry through the HTTP status metadata
           const status = response?.status();
-          const metadata = typeof status === 'number'
-          ? (STATUS_CODE_METADATA[status] || STATUS_CODE_METADATA[599])
-          : STATUS_CODE_METADATA[2];
+          const metadata =
+            typeof status === 'number'
+              ? STATUS_CODE_METADATA[status] || STATUS_CODE_METADATA[599]
+              : STATUS_CODE_METADATA[2];
 
-            urlsCrawled.invalid.push({
+          urlsCrawled.invalid.push({
             actualUrl,
             url: request.url,
             pageTitle: request.url,
             metadata,
-            httpStatusCode: typeof status === 'number' ? status : 0
+            httpStatusCode: typeof status === 'number' ? status : 0,
           });
         }
       }
@@ -384,21 +436,31 @@ const crawlSitemap = async (
       });
 
       const status = response?.status();
-      const metadata = typeof status === 'number'
-      ? (STATUS_CODE_METADATA[status] || STATUS_CODE_METADATA[599])
-      : STATUS_CODE_METADATA[2];
+      const metadata =
+        typeof status === 'number'
+          ? STATUS_CODE_METADATA[status] || STATUS_CODE_METADATA[599]
+          : STATUS_CODE_METADATA[2];
 
       urlsCrawled.error.push({
         url: request.url,
         pageTitle: request.url,
         actualUrl: request.url,
         metadata,
-        httpStatusCode: typeof status === 'number' ? status : 0
+        httpStatusCode: typeof status === 'number' ? status : 0,
       });
       crawlee.log.error(`Failed Request - ${request.url}: ${request.errorMessages}`);
     },
     maxRequestsPerCrawl: Infinity,
     maxConcurrency: specifiedMaxConcurrency || maxConcurrency,
+    ...(process.env.OOBEE_FAST_CRAWLER && {
+      autoscaledPoolOptions: {
+        minConcurrency: specifiedMaxConcurrency ? Math.min(specifiedMaxConcurrency, 10) : 10,
+        maxConcurrency: specifiedMaxConcurrency || maxConcurrency,
+        desiredConcurrencyRatio: 0.98, // Increase threshold for scaling up
+        scaleUpStepRatio: 0.99,        // Scale up faster
+        scaleDownStepRatio: 0.1,       // Scale down slower
+      },
+    }),
   });
 
   await crawler.run();
@@ -430,6 +492,11 @@ const crawlSitemap = async (
     guiInfoLog(guiInfoStatusTypes.COMPLETED, {});
   }
 
+  if (scanDuration > 0) {
+    const elapsed = Math.round((Date.now() - crawlStartTime) / 1000);
+    console.log(`Crawl ended after ${elapsed}s (limit: ${scanDuration}s).`);
+  }
+
   return urlsCrawled;
 };
 

package/src/crawlers/pdfScanFunc.ts

@@ -12,6 +12,7 @@ import { consoleLogger, guiInfoLog, silentLogger } from '../logs.js';
 import constants, {
   getExecutablePath,
   guiInfoStatusTypes,
+  STATUS_CODE_METADATA,
   UrlsCrawled,
 } from '../constants/constants.js';
 
@@ -296,6 +297,7 @@ export const handlePdfDownload = (
               url: request.url,
               pageTitle: url,
               actualUrl: url,
+              metadata: STATUS_CODE_METADATA[1],
             });
           }
 

package/src/index.ts

@@ -50,6 +50,7 @@ export type Answers = {
   zip: string;
   ruleset: RuleFlags[];
   generateJsonFiles: boolean;
+  scanDuration?: number;
 };
 
 export type Data = {
@@ -80,6 +81,7 @@ export type Data = {
   zip?: string;
   ruleset: RuleFlags[];
   generateJsonFiles: boolean;
+  scanDuration: number;
 };
 
 const userData = getUserDataTxt();

package/src/logs.ts

@@ -23,8 +23,10 @@ const logFormat = printf(({ timestamp, level, message }) => {
 // All logs in combined.txt, error in errors.txt
 
 const consoleLogger = createLogger({
+  silent: !(process.env.RUNNING_FROM_PH_GUI || process.env.OOBEE_VERBOSE),
   format: combine(timestamp({ format: 'YYYY-MM-DD HH:mm:ss' }), logFormat),
-  transports: [new transports.Console()],
+  transports:
+    process.env.RUNNING_FROM_PH_GUI || process.env.OOBEE_VERBOSE ? [new transports.Console()] : [],
 });
 
 // No display in consoles, this will mostly be used within the interactive script to avoid disrupting the flow
@@ -33,7 +35,7 @@ const consoleLogger = createLogger({
 const silentLogger = createLogger({
   format: combine(timestamp({ format: 'YYYY-MM-DD HH:mm:ss' }), logFormat),
   transports: [
-    process.env.OOBEE_VERBOSE
+    process.env.OOBEE_VERBOSE || process.env.RUNNING_FROM_PH_GUI
       ? new transports.Console({ handleExceptions: true })
       : new transports.File({ filename: 'errors.txt', level: 'warn', handleExceptions: true }),
   ].filter(Boolean),