@govtechsg/oobee 0.10.39 → 0.10.43

package/src/crawlers/crawlDomain.ts

@@ -9,7 +9,6 @@ import https from 'https';
 import type { BatchAddRequestsResult } from '@crawlee/types';
 import {
   createCrawleeSubFolders,
-  preNavigationHooks,
   runAxeScript,
   isUrlPdf,
 } from './commonCrawlerFunc.js';
@@ -19,6 +18,7 @@ import constants, {
   guiInfoStatusTypes,
   cssQuerySelectors,
   RuleFlags,
+  STATUS_CODE_METADATA,
 } from '../constants/constants.js';
 import {
   getPlaywrightLaunchOptions,
@@ -26,7 +26,6 @@ import {
   isSkippedUrl,
   isDisallowedInRobotsTxt,
   getUrlsFromRobotsTxt,
-  getBlackListedPatterns,
   urlWithoutAuth,
   waitForPageLoaded,
   initModifiedUserAgent,
@@ -116,13 +115,12 @@ const crawlDomain = async ({
     fs.mkdirSync(randomToken);
   }
 
-  const pdfDownloads = [];
-  const uuidToPdfMapping = {};
+  const pdfDownloads: Promise<void>[] = [];
+  const uuidToPdfMapping: Record<string, string> = {};
   const isScanHtml = ['all', 'html-only'].includes(fileTypes);
   const isScanPdfs = ['all', 'pdf-only'].includes(fileTypes);
   const { maxConcurrency } = constants;
   const { playwrightDeviceDetailsObject } = viewportSettings;
-  const isBlacklistedUrl = isBlacklisted(url, blacklistedPatterns);
 
   const httpsAgent = new https.Agent({ rejectUnauthorized: false });
 
@@ -167,8 +165,8 @@ const crawlDomain = async ({
   const httpHeadCache = new Map<string, boolean>();
   const isProcessibleUrl = async (url: string): Promise<boolean> => {
     if (httpHeadCache.has(url)) {
-      silentLogger.info('cache hit', url, httpHeadCache.get(url));
-      return false; // return false to avoid processing the url again
+      silentLogger.info(`Skipping request as URL has been processed before ${url}}`);
+      return false; // return false to avoid processing the same url again
     }
 
     try {
@@ -490,56 +488,35 @@ const crawlDomain = async ({
           return new Promise(resolve => {
             let timeout;
             let mutationCount = 0;
-            const MAX_MUTATIONS = 250;
-            const MAX_SAME_MUTATION_LIMIT = 10;
-            const mutationHash = {};
-
-            const observer = new MutationObserver(mutationsList => {
+            const MAX_MUTATIONS     = 250;   // stop if things never quiet down
+            const OBSERVER_TIMEOUT  = 5000;  // hard cap on total wait
+    
+            const observer = new MutationObserver(() => {
               clearTimeout(timeout);
-
-              mutationCount += 1;
-
+    
+              mutationCount++;
               if (mutationCount > MAX_MUTATIONS) {
                 observer.disconnect();
-                resolve('Too many mutations detected');
+                resolve('Too many mutations, exiting.');
+                return;
               }
-
-              // To handle scenario where DOM elements are constantly changing and unable to exit
-              mutationsList.forEach(mutation => {
-                let mutationKey;
-
-                if (mutation.target instanceof Element) {
-                  Array.from(mutation.target.attributes).forEach(attr => {
-                    mutationKey = `${mutation.target.nodeName}-${attr.name}`;
-
-                    if (mutationKey) {
-                      if (!mutationHash[mutationKey]) {
-                        mutationHash[mutationKey] = 1;
-                      } else {
-                        mutationHash[mutationKey]++;
-                      }
-
-                      if (mutationHash[mutationKey] >= MAX_SAME_MUTATION_LIMIT) {
-                        observer.disconnect();
-                        resolve(`Repeated mutation detected for ${mutationKey}`);
-                      }
-                    }
-                  });
-                }
-              });
-
+    
+              // restart quiet‑period timer
               timeout = setTimeout(() => {
                 observer.disconnect();
-                resolve('DOM stabilized after mutations.');
+                resolve('DOM stabilized.');
               }, 1000);
             });
-
+    
+            // overall timeout in case the page never settles
             timeout = setTimeout(() => {
               observer.disconnect();
-              resolve('No mutations detected, exit from idle state');
-            }, 1000);
-
-            observer.observe(document, { childList: true, subtree: true, attributes: true });
+              resolve('Observer timeout reached.');
+            }, OBSERVER_TIMEOUT);
+    
+            // **HERE**: select the real DOM node inside evaluate
+            const root = document.documentElement;
+            observer.observe(root, { childList: true, subtree: true });
           });
         });
 
@@ -635,16 +612,18 @@ const crawlDomain = async ({
         }
 
         // handle pdfs
-        if (request.skipNavigation && isUrlPdf(actualUrl)) {
+        if (request.skipNavigation && actualUrl === "about:blank") {
           if (!isScanPdfs) {
             guiInfoLog(guiInfoStatusTypes.SKIPPED, {
               numScanned: urlsCrawled.scanned.length,
               urlScanned: request.url,
             });
-            urlsCrawled.blacklisted.push({
+            urlsCrawled.userExcluded.push({
               url: request.url,
               pageTitle: request.url,
-              actualUrl: actualUrl, // i.e. actualUrl
+              actualUrl: request.url, // because about:blank is not useful
+              metadata: STATUS_CODE_METADATA[1],
+              httpStatusCode: 0,
             });
 
             return;
@@ -661,33 +640,17 @@ const crawlDomain = async ({
           return;
         }
 
-        const resHeaders = response ? response.headers() : {}; // Safely access response headers
-        const contentType = resHeaders['content-type'] || ''; // Ensure contentType is defined
-
-        // Skip non-HTML and non-PDF URLs
-        if (!contentType.includes('text/html') && !contentType.includes('application/pdf')) {
-          guiInfoLog(guiInfoStatusTypes.SKIPPED, {
-            numScanned: urlsCrawled.scanned.length,
-            urlScanned: request.url,
-          });
-          urlsCrawled.blacklisted.push({
-            url: request.url,
-            pageTitle: request.url,
-            actualUrl: actualUrl, // i.e. actualUrl
-          });
-
-          return;
-        }
-
         if (isBlacklistedFileExtensions(actualUrl, blackListedFileExtensions)) {
           guiInfoLog(guiInfoStatusTypes.SKIPPED, {
             numScanned: urlsCrawled.scanned.length,
             urlScanned: request.url,
           });
-          urlsCrawled.blacklisted.push({
+          urlsCrawled.userExcluded.push({
             url: request.url,
             pageTitle: request.url,
-            actualUrl: actualUrl, // i.e. actualUrl
+            actualUrl: actualUrl, // because about:blank is not useful
+            metadata: STATUS_CODE_METADATA[1],
+            httpStatusCode: 0,
           });
 
           return;
@@ -698,37 +661,16 @@ const crawlDomain = async ({
             url: request.url,
             pageTitle: request.url,
             actualUrl: actualUrl,
+            metadata: STATUS_CODE_METADATA[0],
+            httpStatusCode: 0,
           });
-
-          await enqueueProcess(page, enqueueLinks, browserContext);
-          return;
-        }
-
-        if (response.status() === 403) {
+          
           guiInfoLog(guiInfoStatusTypes.SKIPPED, {
             numScanned: urlsCrawled.scanned.length,
             urlScanned: request.url,
           });
-          urlsCrawled.forbidden.push({
-            url: request.url,
-            pageTitle: request.url,
-            actualUrl: actualUrl, // i.e. actualUrl
-          });
-
-          return;
-        }
-
-        if (response.status() !== 200) {
-          guiInfoLog(guiInfoStatusTypes.SKIPPED, {
-            numScanned: urlsCrawled.scanned.length,
-            urlScanned: request.url,
-          });
-          urlsCrawled.invalid.push({
-            url: request.url,
-            pageTitle: request.url,
-            actualUrl: actualUrl, // i.e. actualUrl
-          });
 
+          await enqueueProcess(page, enqueueLinks, browserContext);
           return;
         }
 
@@ -750,6 +692,22 @@ const crawlDomain = async ({
             return;
           }
 
+          const responseStatus = response?.status();
+            if (responseStatus && responseStatus >= 300) {
+            guiInfoLog(guiInfoStatusTypes.SKIPPED, {
+              numScanned: urlsCrawled.scanned.length,
+              urlScanned: request.url,
+            });
+            urlsCrawled.userExcluded.push({
+              url: request.url,
+              pageTitle: request.url,
+              actualUrl,
+              metadata: STATUS_CODE_METADATA[responseStatus] || STATUS_CODE_METADATA[599],
+              httpStatusCode: responseStatus,
+            });
+            return;
+            }
+
           const results = await runAxeScript({ includeScreenshots, page, randomToken, ruleset });
 
           if (isRedirected) {
@@ -807,10 +765,12 @@ const crawlDomain = async ({
             numScanned: urlsCrawled.scanned.length,
             urlScanned: request.url,
           });
-          urlsCrawled.blacklisted.push({
+          urlsCrawled.userExcluded.push({
             url: request.url,
             pageTitle: request.url,
-            actualUrl: actualUrl, // i.e. actualUrl
+            actualUrl: actualUrl, // because about:blank is not useful
+            metadata: STATUS_CODE_METADATA[1],
+            httpStatusCode: 0,
           });
 
         }
@@ -850,18 +810,39 @@ const crawlDomain = async ({
         // when max pages have been scanned, scan will abort and all relevant pages still opened will close instantly.
         // a browser close error will then be flagged. Since this is an intended behaviour, this error will be excluded.
         if (!isAbortingScanNow) {
-            urlsCrawled.error.push({ url: request.url, pageTitle: request.url, actualUrl: request.url });
+          guiInfoLog(guiInfoStatusTypes.ERROR, {
+            numScanned: urlsCrawled.scanned.length,
+            urlScanned: request.url,
+          });
+
+          urlsCrawled.error.push({ 
+            url: request.url, 
+            pageTitle: request.url, 
+            actualUrl: request.url, 
+            metadata: STATUS_CODE_METADATA[2] 
+          });
         }
       }
     },
-    failedRequestHandler: async ({ request }) => {
+    failedRequestHandler: async ({ request, response }) => {
       guiInfoLog(guiInfoStatusTypes.ERROR, {
         numScanned: urlsCrawled.scanned.length,
         urlScanned: request.url,
       });
-      urlsCrawled.error.push({ url: request.url, pageTitle: request.url, actualUrl: request.url });
-    
-      crawlee.log.error(`Failed Request - ${request.url}: ${request.errorMessages}`);
+
+      const status = response?.status();
+      const metadata = typeof status === 'number'
+      ? (STATUS_CODE_METADATA[status] || STATUS_CODE_METADATA[599])
+      : STATUS_CODE_METADATA[2];
+
+      urlsCrawled.error.push({
+        url: request.url,
+        pageTitle: request.url,
+        actualUrl: request.url,
+        metadata,
+        httpStatusCode: typeof status === 'number' ? status : 0,
+      });
+
     },
     maxRequestsPerCrawl: Infinity,
     maxConcurrency: specifiedMaxConcurrency || maxConcurrency,

package/src/crawlers/crawlIntelligentSitemap.ts

@@ -1,27 +1,29 @@
 import fs from 'fs';
-import { chromium } from 'playwright';
+import { chromium, Page } from 'playwright';
 import { createCrawleeSubFolders } from './commonCrawlerFunc.js';
 import constants, { guiInfoStatusTypes, sitemapPaths } from '../constants/constants.js';
 import { silentLogger, guiInfoLog } from '../logs.js';
 import crawlDomain from './crawlDomain.js';
 import crawlSitemap from './crawlSitemap.js';
+import { EnqueueStrategy } from 'crawlee';
+import { ViewportSettingsClass } from '../combine.js';
 
 const crawlIntelligentSitemap = async (
-  url,
-  randomToken,
-  host,
-  viewportSettings,
-  maxRequestsPerCrawl,
-  browser,
-  userDataDirectory,
-  strategy,
-  specifiedMaxConcurrency,
-  fileTypes,
-  blacklistedPatterns,
-  includeScreenshots,
-  followRobots,
-  extraHTTPHeaders,
-  safeMode,
+  url: string,
+  randomToken: string,
+  host: string,
+  viewportSettings: ViewportSettingsClass,
+  maxRequestsPerCrawl: number,
+  browser: string,
+  userDataDirectory: string,
+  strategy: EnqueueStrategy,
+  specifiedMaxConcurrency: number,
+  fileTypes: string,
+  blacklistedPatterns: string[],
+  includeScreenshots: boolean,
+  followRobots: boolean,
+  extraHTTPHeaders: Record<string, string>,
+  safeMode: boolean,
 ) => {
   let urlsCrawledFinal;
   let urlsCrawled;
@@ -37,7 +39,7 @@ const crawlIntelligentSitemap = async (
     fs.mkdirSync(randomToken);
   }
 
-  function getHomeUrl(parsedUrl) {
+  function getHomeUrl(parsedUrl: string) {
     const urlObject = new URL(parsedUrl);
     if (urlObject.username !== '' && urlObject.password !== '') {
       return `${urlObject.protocol}//${urlObject.username}:${urlObject.password}@${urlObject.hostname}${urlObject.port ? `:${urlObject.port}` : ''}`;
@@ -46,7 +48,7 @@ const crawlIntelligentSitemap = async (
     return `${urlObject.protocol}//${urlObject.hostname}${urlObject.port ? `:${urlObject.port}` : ''}`;
   }
 
-  async function findSitemap(link) {
+  async function findSitemap(link: string) {
     const homeUrl = getHomeUrl(link);
     let sitemapLinkFound = false;
     let sitemapLink = '';
@@ -70,7 +72,7 @@ const crawlIntelligentSitemap = async (
     return sitemapExist ? sitemapLink : '';
   }
 
-  const checkUrlExists = async (page, parsedUrl) => {
+  const checkUrlExists = async (page: Page, parsedUrl: string) => {
     try {
       const response = await page.goto(parsedUrl);
       if (response.ok()) {