@govtechsg/oobee 0.10.50 → 0.10.57

package/src/crawlers/crawlIntelligentSitemap.ts

@@ -2,7 +2,7 @@ import fs from 'fs';
 import { chromium, Page } from 'playwright';
 import { createCrawleeSubFolders } from './commonCrawlerFunc.js';
 import constants, { guiInfoStatusTypes, sitemapPaths } from '../constants/constants.js';
-import { silentLogger, guiInfoLog } from '../logs.js';
+import { consoleLogger, guiInfoLog } from '../logs.js';
 import crawlDomain from './crawlDomain.js';
 import crawlSitemap from './crawlSitemap.js';
 import { EnqueueStrategy } from 'crawlee';
@@ -24,46 +24,42 @@ const crawlIntelligentSitemap = async (
   followRobots: boolean,
   extraHTTPHeaders: Record<string, string>,
   safeMode: boolean,
+  scanDuration: number
 ) => {
+  const startTime = Date.now(); // Track start time
+
   let urlsCrawledFinal;
-  let urlsCrawled;
+  let urlsCrawled = { ...constants.urlsCrawledObj };
   let dataset;
   let sitemapExist = false;
   const fromCrawlIntelligentSitemap = true;
   let sitemapUrl;
 
-  urlsCrawled = { ...constants.urlsCrawledObj };
   ({ dataset } = await createCrawleeSubFolders(randomToken));
-
   if (!fs.existsSync(randomToken)) {
     fs.mkdirSync(randomToken);
   }
 
   function getHomeUrl(parsedUrl: string) {
     const urlObject = new URL(parsedUrl);
-    if (urlObject.username !== '' && urlObject.password !== '') {
+    if (urlObject.username && urlObject.password) {
       return `${urlObject.protocol}//${urlObject.username}:${urlObject.password}@${urlObject.hostname}${urlObject.port ? `:${urlObject.port}` : ''}`;
     }
-
     return `${urlObject.protocol}//${urlObject.hostname}${urlObject.port ? `:${urlObject.port}` : ''}`;
   }
 
   async function findSitemap(link: string) {
     const homeUrl = getHomeUrl(link);
-    let sitemapLinkFound = false;
     let sitemapLink = '';
-    const chromiumBrowser = await chromium.launch(
-      {
-        headless: false,
-        channel: 'chrome',
-        args: ['--headless=new', '--no-sandbox']
-      });
-
+    const chromiumBrowser = await chromium.launch({
+      headless: false,
+      channel: 'chrome',
+      args: ['--headless=new', '--no-sandbox'],
+    });
     const page = await chromiumBrowser.newPage();
     for (const path of sitemapPaths) {
       sitemapLink = homeUrl + path;
-      sitemapLinkFound = await checkUrlExists(page, sitemapLink);
-      if (sitemapLinkFound) {
+      if (await checkUrlExists(page, sitemapLink)) {
         sitemapExist = true;
         break;
       }
@@ -75,12 +71,9 @@ const crawlIntelligentSitemap = async (
   const checkUrlExists = async (page: Page, parsedUrl: string) => {
     try {
       const response = await page.goto(parsedUrl);
-      if (response.ok()) {
-        return true;
-      }
-      return false;
+      return response.ok();
     } catch (e) {
-      silentLogger.error(e);
+      consoleLogger.error(e);
       return false;
     }
   };
@@ -88,13 +81,12 @@ const crawlIntelligentSitemap = async (
   try {
     sitemapUrl = await findSitemap(url);
   } catch (error) {
-    silentLogger.error(error);
+    consoleLogger.error(error);
   }
 
   if (!sitemapExist) {
     console.log('Unable to find sitemap. Commencing website crawl instead.');
-    // run crawlDomain as per normal
-    urlsCrawledFinal = await crawlDomain({
+    return await crawlDomain({
       url,
       randomToken,
       host,
@@ -109,12 +101,13 @@ const crawlIntelligentSitemap = async (
       includeScreenshots,
       followRobots,
       extraHTTPHeaders,
+      safeMode,
+      scanDuration, // Use full duration since no sitemap
     });
-    return urlsCrawledFinal;
   }
+
   console.log(`Sitemap found at ${sitemapUrl}`);
-  // run crawlSitemap then crawDomain subsequently if urlsCrawled.scanned.length < maxRequestsPerCrawl
-  urlsCrawledFinal = await crawlSitemap(
+  urlsCrawledFinal = await crawlSitemap({
     sitemapUrl,
     randomToken,
     host,
@@ -128,14 +121,21 @@ const crawlIntelligentSitemap = async (
     includeScreenshots,
     extraHTTPHeaders,
     fromCrawlIntelligentSitemap,
-    url,
-    dataset, // for crawlSitemap to add on to
-    urlsCrawled, // for crawlSitemap to add on to
-    false,
-  );
+    userUrlInputFromIntelligent: url,
+    datasetFromIntelligent: dataset,
+    urlsCrawledFromIntelligent: urlsCrawled,
+    crawledFromLocalFile: false,
+    scanDuration,
+  });
+
+  const elapsed = Date.now() - startTime;
+  const remainingScanDuration = Math.max(scanDuration - elapsed / 1000, 0); // in seconds
 
-  if (urlsCrawled.scanned.length < maxRequestsPerCrawl) {
-    // run crawl domain starting from root website, only on pages not scanned before
+  if (
+    urlsCrawledFinal.scanned.length < maxRequestsPerCrawl &&
+    remainingScanDuration > 0
+  ) {
+    console.log(`Continuing crawl from root website. Remaining scan time: ${remainingScanDuration.toFixed(1)}s`);
     urlsCrawledFinal = await crawlDomain({
       url,
       randomToken,
@@ -153,12 +153,16 @@ const crawlIntelligentSitemap = async (
       extraHTTPHeaders,
       safeMode,
       fromCrawlIntelligentSitemap,
-      datasetFromIntelligent: dataset, // for crawlDomain to add on to
-      urlsCrawledFromIntelligent: urlsCrawledFinal, // urls for crawlDomain to exclude
+      datasetFromIntelligent: dataset,
+      urlsCrawledFromIntelligent: urlsCrawledFinal,
+      scanDuration: remainingScanDuration,
     });
+  } else if (remainingScanDuration <= 0) {
+    console.log(`Crawl duration exceeded before more pages could be found (limit: ${scanDuration}s).`);
   }
 
   guiInfoLog(guiInfoStatusTypes.COMPLETED, {});
   return urlsCrawledFinal;
 };
+
 export default crawlIntelligentSitemap;

package/src/crawlers/crawlLocalFile.ts

@@ -1,12 +1,15 @@
-import { Request, RequestList } from 'crawlee';
-import printMessage from 'print-message';
+import { Request, RequestList, Dataset } from 'crawlee';
 import fs from 'fs';
 import path from 'path';
 import { createCrawleeSubFolders, runAxeScript, isUrlPdf } from './commonCrawlerFunc.js';
-import constants, { guiInfoStatusTypes, basicAuthRegex } from '../constants/constants.js';
+import constants, {
+  guiInfoStatusTypes,
+  basicAuthRegex,
+  UrlsCrawled,
+} from '../constants/constants.js';
+import { ViewportSettingsClass } from '../combine.js';
 import {
   getPlaywrightLaunchOptions,
-  messageOptions,
   isFilePath,
   convertLocalFileToPath,
   convertPathToLocalFile,
@@ -16,27 +19,47 @@ import { runPdfScan, mapPdfScanResults, doPdfScreenshots } from './pdfScanFunc.j
 import { guiInfoLog } from '../logs.js';
 import crawlSitemap from './crawlSitemap.js';
 
-const crawlLocalFile = async (
-  sitemapUrl: string,
-  randomToken: string,
-  host: string,
-  viewportSettings: any,
-  maxRequestsPerCrawl: number,
-  browser: string,
-  userDataDirectory: string,
-  specifiedMaxConcurrency: number,
-  fileTypes: string,
-  blacklistedPatterns: string[],
-  includeScreenshots: boolean,
-  extraHTTPHeaders: any,
-  fromCrawlIntelligentSitemap: boolean = false, // optional
-  userUrlInputFromIntelligent: any = null, // optional
-  datasetFromIntelligent: any = null, // optional
-  urlsCrawledFromIntelligent: any = null, // optional
-) => {
+export const crawlLocalFile = async ({
+  url,
+  randomToken,
+  host,
+  viewportSettings,
+  maxRequestsPerCrawl,
+  browser,
+  userDataDirectory,
+  specifiedMaxConcurrency,
+  fileTypes,
+  blacklistedPatterns,
+  includeScreenshots,
+  extraHTTPHeaders,
+  scanDuration = 0,
+  fromCrawlIntelligentSitemap = false,
+  userUrlInputFromIntelligent = null,
+  datasetFromIntelligent = null,
+  urlsCrawledFromIntelligent = null,
+}: {
+  url: string;
+  randomToken: string;
+  host: string;
+  viewportSettings: ViewportSettingsClass;
+  maxRequestsPerCrawl: number;
+  browser: string;
+  userDataDirectory: string;
+  specifiedMaxConcurrency: number;
+  fileTypes: string;
+  blacklistedPatterns: string[];
+  includeScreenshots: boolean;
+  extraHTTPHeaders: Record<string, string>;
+  scanDuration?: number;
+  fromCrawlIntelligentSitemap?: boolean;
+  userUrlInputFromIntelligent?: string | null;
+  datasetFromIntelligent?: Dataset | null;
+  urlsCrawledFromIntelligent?: UrlsCrawled | null;
+}) => {
   let dataset: any;
-  let urlsCrawled: any;
+  let urlsCrawled: UrlsCrawled;
   let linksFromSitemap = [];
+  let sitemapUrl = url; 
 
   // Boolean to omit axe scan for basic auth URL
   let isBasicAuth: boolean;
@@ -82,7 +105,7 @@ const crawlLocalFile = async (
     // Non XML file
   } else {
     // Put it to crawlSitemap function to handle xml files
-    const updatedUrlsCrawled = await crawlSitemap(
+    const updatedUrlsCrawled = await crawlSitemap({
       sitemapUrl,
       randomToken,
       host,
@@ -95,12 +118,13 @@ const crawlLocalFile = async (
       blacklistedPatterns,
       includeScreenshots,
       extraHTTPHeaders,
-      (fromCrawlIntelligentSitemap = false), // optional
-      (userUrlInputFromIntelligent = null), // optional
-      (datasetFromIntelligent = null), // optional
-      (urlsCrawledFromIntelligent = null), // optional
-      true,
-    );
+      scanDuration,
+      fromCrawlIntelligentSitemap,
+      userUrlInputFromIntelligent,
+      datasetFromIntelligent,
+      urlsCrawledFromIntelligent,
+      crawledFromLocalFile: true,
+    });
 
     urlsCrawled = { ...urlsCrawled, ...updatedUrlsCrawled };
     return urlsCrawled;
@@ -124,16 +148,12 @@ const crawlLocalFile = async (
 
   const uuidToPdfMapping: Record<string, string> = {}; // key and value of string type
 
-  printMessage(['Fetching URLs. This might take some time...'], { border: false });
-
   finalLinks = [...finalLinks, ...linksFromSitemap];
 
   await RequestList.open({
     sources: finalLinks,
   });
 
-  printMessage(['Fetch URLs completed. Beginning scan'], messageOptions);
-
   const request = linksFromSitemap[0];
   const pdfFileName = path.basename(request.url);
   const trimmedUrl: string = request.url;
@@ -142,6 +162,8 @@ const crawlLocalFile = async (
   fs.writeFileSync(destinationFilePath, data);
   uuidToPdfMapping[pdfFileName] = trimmedUrl;
 
+  let shouldAbort = false;
+
   if (!isUrlPdf(request.url)) {
     await initModifiedUserAgent(browser);
     const browserContext = await constants.launcher.launchPersistentContext('', {
@@ -150,9 +172,24 @@ const crawlLocalFile = async (
       ...playwrightDeviceDetailsObject,
     });
 
+    const timeoutId = scanDuration > 0
+    ? setTimeout(() => {
+        console.log(`Crawl duration of ${scanDuration}s exceeded. Aborting local file scan.`);
+        shouldAbort = true;
+      }, scanDuration * 1000)
+    : null;
+
     const page = await browserContext.newPage();
     request.url = convertPathToLocalFile(request.url);
     await page.goto(request.url);
+
+    if (shouldAbort) {
+      console.warn('Scan aborted due to timeout before page scan.');
+      await dataset.pushData({ scanned: [], scannedRedirects: [] });
+      await browserContext.close().catch(() => {});
+      return urlsCrawled;
+    }
+
     const results = await runAxeScript({ includeScreenshots, page, randomToken });
 
     const actualUrl = page.url() || request.loadedUrl || request.url;
@@ -178,7 +215,11 @@ const crawlLocalFile = async (
 
     await dataset.pushData(results);
   } else {
-    urlsCrawled.scanned.push({ url: trimmedUrl, pageTitle: pdfFileName });
+    urlsCrawled.scanned.push({
+      url: trimmedUrl,
+      pageTitle: pdfFileName,
+      actualUrl: trimmedUrl,
+    });
 
     await runPdfScan(randomToken);
     // transform result format
@@ -192,6 +233,7 @@ const crawlLocalFile = async (
     // push results for each pdf document to key value store
     await Promise.all(pdfResults.map(result => dataset.pushData(result)));
   }
+
   return urlsCrawled;
 };
 export default crawlLocalFile;