@govtechsg/oobee 0.10.42 → 0.10.43

package/src/crawlers/commonCrawlerFunc.ts

@@ -1,4 +1,4 @@
-import crawlee, { CrawlingContext, PlaywrightGotoOptions } from 'crawlee';
+import crawlee, { CrawlingContext, PlaywrightGotoOptions, Request } from 'crawlee';
 import axe, { AxeResults, ImpactValue, NodeResult, Result, resultGroups, TagValue } from 'axe-core';
 import { BrowserContext, Page } from 'playwright';
 import {
@@ -18,7 +18,7 @@ import { framesCheck } from './custom/framesCheck.js';
 import { findElementByCssSelector } from './custom/findElementByCssSelector.js';
 import { getAxeConfiguration } from './custom/getAxeConfiguration.js';
 import { flagUnlabelledClickableElements } from './custom/flagUnlabelledClickableElements.js';
-import { xPathToCss } from './custom/xPathToCss.js';
+import xPathToCss from './custom/xPathToCss.js';
 
 // types
 interface AxeResultsWithScreenshot extends AxeResults {
@@ -118,13 +118,13 @@ export const filterAxeResults = (
 
     if (conformance[0] !== 'best-practice' && !wcagRegex.test(conformance[0])) {
       conformance.sort((a, b) => {
-      if (wcagRegex.test(a) && !wcagRegex.test(b)) {
-        return -1;
-      }
-      if (!wcagRegex.test(a) && wcagRegex.test(b)) {
-        return 1;
-      }
-      return 0;
+        if (wcagRegex.test(a) && !wcagRegex.test(b)) {
+          return -1;
+        }
+        if (!wcagRegex.test(a) && wcagRegex.test(b)) {
+          return 1;
+        }
+        return 0;
       });
     }
 
@@ -166,7 +166,6 @@ export const filterAxeResults = (
     };
 
     nodes.forEach(node => {
-      const { impact } = node;
       const hasWcagA = conformance.some(tag => /^wcag\d*a$/.test(tag));
       const hasWcagAA = conformance.some(tag => /^wcag\d*aa$/.test(tag));
       // const hasWcagAAA = conformance.some(tag => /^wcag\d*aaa$/.test(tag));
@@ -255,7 +254,7 @@ export const runAxeScript = async ({
         let mutationCount = 0;
         const MAX_MUTATIONS = 250;
         const MAX_SAME_MUTATION_LIMIT = 10;
-        const mutationHash = {};
+        const mutationHash: Record<string, number> = {};
 
         const observer = new MutationObserver(mutationsList => {
           clearTimeout(timeout);
@@ -309,6 +308,8 @@ export const runAxeScript = async ({
     silentLogger.warn(`Error while checking for DOM mutations: ${e}`);
   }
 
+  // Omit logging of browser console errors to reduce unnecessary verbosity
+  /*
   page.on('console', msg => {
     const type = msg.type();
     if (type === 'error') {
@@ -317,6 +318,7 @@ export const runAxeScript = async ({
       silentLogger.log({ level: 'info', message: msg.text() });
     }
   });
+  */
 
   const disableOobee = ruleset.includes(RuleFlags.DISABLE_OOBEE);
   const enableWcagAaa = ruleset.includes(RuleFlags.ENABLE_WCAG_AAA);
@@ -399,7 +401,7 @@ export const runAxeScript = async ({
               help: 'Clickable elements (i.e. elements with mouse-click interaction) must have accessible labels.',
               helpUrl: 'https://www.deque.com/blog/accessible-aria-buttons',
               nodes: escapedCssSelectors
-                .map(cssSelector => ({
+                .map((cssSelector: string): NodeResult => ({
                   html: findElementByCssSelector(cssSelector),
                   target: [cssSelector],
                   impact: 'serious' as ImpactValue,
@@ -443,8 +445,7 @@ export const runAxeScript = async ({
       framesCheckFunctionString: framesCheck.toString(),
       findElementByCssSelectorFunctionString: findElementByCssSelector.toString(),
       getAxeConfigurationFunctionString: getAxeConfiguration.toString(),
-      flagUnlabelledClickableElementsFunctionString:
-        flagUnlabelledClickableElements.toString(),
+      flagUnlabelledClickableElementsFunctionString: flagUnlabelledClickableElements.toString(),
       xPathToCssFunctionString: xPathToCss.toString(),
     },
   );
@@ -495,7 +496,7 @@ export const postNavigationHooks = [
   },
 ];
 
-export const failedRequestHandler = async ({ request }) => {
+export const failedRequestHandler = async ({ request }: { request: Request }) => {
   guiInfoLog(guiInfoStatusTypes.ERROR, { numScanned: 0, urlScanned: request.url });
   crawlee.log.error(`Failed Request - ${request.url}: ${request.errorMessages}`);
 };

package/src/crawlers/crawlDomain.ts

@@ -9,7 +9,6 @@ import https from 'https';
 import type { BatchAddRequestsResult } from '@crawlee/types';
 import {
   createCrawleeSubFolders,
-  preNavigationHooks,
   runAxeScript,
   isUrlPdf,
 } from './commonCrawlerFunc.js';
@@ -19,6 +18,7 @@ import constants, {
   guiInfoStatusTypes,
   cssQuerySelectors,
   RuleFlags,
+  STATUS_CODE_METADATA,
 } from '../constants/constants.js';
 import {
   getPlaywrightLaunchOptions,
@@ -26,7 +26,6 @@ import {
   isSkippedUrl,
   isDisallowedInRobotsTxt,
   getUrlsFromRobotsTxt,
-  getBlackListedPatterns,
   urlWithoutAuth,
   waitForPageLoaded,
   initModifiedUserAgent,
@@ -116,13 +115,12 @@ const crawlDomain = async ({
     fs.mkdirSync(randomToken);
   }
 
-  const pdfDownloads = [];
-  const uuidToPdfMapping = {};
+  const pdfDownloads: Promise<void>[] = [];
+  const uuidToPdfMapping: Record<string, string> = {};
   const isScanHtml = ['all', 'html-only'].includes(fileTypes);
   const isScanPdfs = ['all', 'pdf-only'].includes(fileTypes);
   const { maxConcurrency } = constants;
   const { playwrightDeviceDetailsObject } = viewportSettings;
-  const isBlacklistedUrl = isBlacklisted(url, blacklistedPatterns);
 
   const httpsAgent = new https.Agent({ rejectUnauthorized: false });
 
@@ -167,8 +165,8 @@ const crawlDomain = async ({
   const httpHeadCache = new Map<string, boolean>();
   const isProcessibleUrl = async (url: string): Promise<boolean> => {
     if (httpHeadCache.has(url)) {
-      silentLogger.info('cache hit', url, httpHeadCache.get(url));
-      return false; // return false to avoid processing the url again
+      silentLogger.info(`Skipping request as URL has been processed before ${url}}`);
+      return false; // return false to avoid processing the same url again
     }
 
     try {
@@ -490,56 +488,35 @@ const crawlDomain = async ({
           return new Promise(resolve => {
             let timeout;
             let mutationCount = 0;
-            const MAX_MUTATIONS = 250;
-            const MAX_SAME_MUTATION_LIMIT = 10;
-            const mutationHash = {};
-
-            const observer = new MutationObserver(mutationsList => {
+            const MAX_MUTATIONS     = 250;   // stop if things never quiet down
+            const OBSERVER_TIMEOUT  = 5000;  // hard cap on total wait
+    
+            const observer = new MutationObserver(() => {
               clearTimeout(timeout);
-
-              mutationCount += 1;
-
+    
+              mutationCount++;
               if (mutationCount > MAX_MUTATIONS) {
                 observer.disconnect();
-                resolve('Too many mutations detected');
+                resolve('Too many mutations, exiting.');
+                return;
               }
-
-              // To handle scenario where DOM elements are constantly changing and unable to exit
-              mutationsList.forEach(mutation => {
-                let mutationKey;
-
-                if (mutation.target instanceof Element) {
-                  Array.from(mutation.target.attributes).forEach(attr => {
-                    mutationKey = `${mutation.target.nodeName}-${attr.name}`;
-
-                    if (mutationKey) {
-                      if (!mutationHash[mutationKey]) {
-                        mutationHash[mutationKey] = 1;
-                      } else {
-                        mutationHash[mutationKey]++;
-                      }
-
-                      if (mutationHash[mutationKey] >= MAX_SAME_MUTATION_LIMIT) {
-                        observer.disconnect();
-                        resolve(`Repeated mutation detected for ${mutationKey}`);
-                      }
-                    }
-                  });
-                }
-              });
-
+    
+              // restart quiet‑period timer
               timeout = setTimeout(() => {
                 observer.disconnect();
-                resolve('DOM stabilized after mutations.');
+                resolve('DOM stabilized.');
               }, 1000);
             });
-
+    
+            // overall timeout in case the page never settles
             timeout = setTimeout(() => {
               observer.disconnect();
-              resolve('No mutations detected, exit from idle state');
-            }, 1000);
-
-            observer.observe(document, { childList: true, subtree: true, attributes: true });
+              resolve('Observer timeout reached.');
+            }, OBSERVER_TIMEOUT);
+    
+            // **HERE**: select the real DOM node inside evaluate
+            const root = document.documentElement;
+            observer.observe(root, { childList: true, subtree: true });
           });
         });
 
@@ -641,10 +618,12 @@ const crawlDomain = async ({
               numScanned: urlsCrawled.scanned.length,
               urlScanned: request.url,
             });
-            urlsCrawled.blacklisted.push({
+            urlsCrawled.userExcluded.push({
               url: request.url,
               pageTitle: request.url,
-              actualUrl: actualUrl, // i.e. actualUrl
+              actualUrl: request.url, // because about:blank is not useful
+              metadata: STATUS_CODE_METADATA[1],
+              httpStatusCode: 0,
             });
 
             return;
@@ -666,10 +645,12 @@ const crawlDomain = async ({
             numScanned: urlsCrawled.scanned.length,
             urlScanned: request.url,
           });
-          urlsCrawled.blacklisted.push({
+          urlsCrawled.userExcluded.push({
             url: request.url,
             pageTitle: request.url,
-            actualUrl: actualUrl, // i.e. actualUrl
+            actualUrl: actualUrl, // because about:blank is not useful
+            metadata: STATUS_CODE_METADATA[1],
+            httpStatusCode: 0,
           });
 
           return;
@@ -680,38 +661,16 @@ const crawlDomain = async ({
             url: request.url,
             pageTitle: request.url,
             actualUrl: actualUrl,
+            metadata: STATUS_CODE_METADATA[0],
+            httpStatusCode: 0,
           });
-
-          await enqueueProcess(page, enqueueLinks, browserContext);
-          return;
-        }
-
-        if (response && response.status() === 403) {
-          guiInfoLog(guiInfoStatusTypes.SKIPPED, {
-            numScanned: urlsCrawled.scanned.length,
-            urlScanned: request.url,
-          });
-          urlsCrawled.forbidden.push({
-            url: request.url,
-            pageTitle: request.url,
-            actualUrl: actualUrl, // i.e. actualUrl
-          });
-
-          return;
-        }
-
-        if (response && response.status() !== 200) {
-
+          
           guiInfoLog(guiInfoStatusTypes.SKIPPED, {
             numScanned: urlsCrawled.scanned.length,
             urlScanned: request.url,
           });
-          urlsCrawled.invalid.push({
-            url: request.url,
-            pageTitle: request.url,
-            actualUrl: actualUrl, // i.e. actualUrl
-          });
 
+          await enqueueProcess(page, enqueueLinks, browserContext);
           return;
         }
 
@@ -733,6 +692,22 @@ const crawlDomain = async ({
             return;
           }
 
+          const responseStatus = response?.status();
+            if (responseStatus && responseStatus >= 300) {
+            guiInfoLog(guiInfoStatusTypes.SKIPPED, {
+              numScanned: urlsCrawled.scanned.length,
+              urlScanned: request.url,
+            });
+            urlsCrawled.userExcluded.push({
+              url: request.url,
+              pageTitle: request.url,
+              actualUrl,
+              metadata: STATUS_CODE_METADATA[responseStatus] || STATUS_CODE_METADATA[599],
+              httpStatusCode: responseStatus,
+            });
+            return;
+            }
+
           const results = await runAxeScript({ includeScreenshots, page, randomToken, ruleset });
 
           if (isRedirected) {
@@ -790,10 +765,12 @@ const crawlDomain = async ({
             numScanned: urlsCrawled.scanned.length,
             urlScanned: request.url,
           });
-          urlsCrawled.blacklisted.push({
+          urlsCrawled.userExcluded.push({
             url: request.url,
             pageTitle: request.url,
-            actualUrl: actualUrl, // i.e. actualUrl
+            actualUrl: actualUrl, // because about:blank is not useful
+            metadata: STATUS_CODE_METADATA[1],
+            httpStatusCode: 0,
           });
 
         }
@@ -833,18 +810,39 @@ const crawlDomain = async ({
         // when max pages have been scanned, scan will abort and all relevant pages still opened will close instantly.
         // a browser close error will then be flagged. Since this is an intended behaviour, this error will be excluded.
         if (!isAbortingScanNow) {
-            urlsCrawled.error.push({ url: request.url, pageTitle: request.url, actualUrl: request.url });
+          guiInfoLog(guiInfoStatusTypes.ERROR, {
+            numScanned: urlsCrawled.scanned.length,
+            urlScanned: request.url,
+          });
+
+          urlsCrawled.error.push({ 
+            url: request.url, 
+            pageTitle: request.url, 
+            actualUrl: request.url, 
+            metadata: STATUS_CODE_METADATA[2] 
+          });
         }
       }
     },
-    failedRequestHandler: async ({ request }) => {
+    failedRequestHandler: async ({ request, response }) => {
       guiInfoLog(guiInfoStatusTypes.ERROR, {
         numScanned: urlsCrawled.scanned.length,
         urlScanned: request.url,
       });
-      urlsCrawled.error.push({ url: request.url, pageTitle: request.url, actualUrl: request.url });
-    
-      crawlee.log.error(`Failed Request - ${request.url}: ${request.errorMessages}`);
+
+      const status = response?.status();
+      const metadata = typeof status === 'number'
+      ? (STATUS_CODE_METADATA[status] || STATUS_CODE_METADATA[599])
+      : STATUS_CODE_METADATA[2];
+
+      urlsCrawled.error.push({
+        url: request.url,
+        pageTitle: request.url,
+        actualUrl: request.url,
+        metadata,
+        httpStatusCode: typeof status === 'number' ? status : 0,
+      });
+
     },
     maxRequestsPerCrawl: Infinity,
     maxConcurrency: specifiedMaxConcurrency || maxConcurrency,

package/src/crawlers/crawlIntelligentSitemap.ts

@@ -1,27 +1,29 @@
 import fs from 'fs';
-import { chromium } from 'playwright';
+import { chromium, Page } from 'playwright';
 import { createCrawleeSubFolders } from './commonCrawlerFunc.js';
 import constants, { guiInfoStatusTypes, sitemapPaths } from '../constants/constants.js';
 import { silentLogger, guiInfoLog } from '../logs.js';
 import crawlDomain from './crawlDomain.js';
 import crawlSitemap from './crawlSitemap.js';
+import { EnqueueStrategy } from 'crawlee';
+import { ViewportSettingsClass } from '../combine.js';
 
 const crawlIntelligentSitemap = async (
-  url,
-  randomToken,
-  host,
-  viewportSettings,
-  maxRequestsPerCrawl,
-  browser,
-  userDataDirectory,
-  strategy,
-  specifiedMaxConcurrency,
-  fileTypes,
-  blacklistedPatterns,
-  includeScreenshots,
-  followRobots,
-  extraHTTPHeaders,
-  safeMode,
+  url: string,
+  randomToken: string,
+  host: string,
+  viewportSettings: ViewportSettingsClass,
+  maxRequestsPerCrawl: number,
+  browser: string,
+  userDataDirectory: string,
+  strategy: EnqueueStrategy,
+  specifiedMaxConcurrency: number,
+  fileTypes: string,
+  blacklistedPatterns: string[],
+  includeScreenshots: boolean,
+  followRobots: boolean,
+  extraHTTPHeaders: Record<string, string>,
+  safeMode: boolean,
 ) => {
   let urlsCrawledFinal;
   let urlsCrawled;
@@ -37,7 +39,7 @@ const crawlIntelligentSitemap = async (
     fs.mkdirSync(randomToken);
   }
 
-  function getHomeUrl(parsedUrl) {
+  function getHomeUrl(parsedUrl: string) {
     const urlObject = new URL(parsedUrl);
     if (urlObject.username !== '' && urlObject.password !== '') {
       return `${urlObject.protocol}//${urlObject.username}:${urlObject.password}@${urlObject.hostname}${urlObject.port ? `:${urlObject.port}` : ''}`;
@@ -46,7 +48,7 @@ const crawlIntelligentSitemap = async (
     return `${urlObject.protocol}//${urlObject.hostname}${urlObject.port ? `:${urlObject.port}` : ''}`;
   }
 
-  async function findSitemap(link) {
+  async function findSitemap(link: string) {
     const homeUrl = getHomeUrl(link);
     let sitemapLinkFound = false;
     let sitemapLink = '';
@@ -70,7 +72,7 @@ const crawlIntelligentSitemap = async (
     return sitemapExist ? sitemapLink : '';
   }
 
-  const checkUrlExists = async (page, parsedUrl) => {
+  const checkUrlExists = async (page: Page, parsedUrl: string) => {
     try {
       const response = await page.goto(parsedUrl);
       if (response.ok()) {