@govtechsg/oobee 0.10.42 → 0.10.45

package/src/crawlers/crawlSitemap.ts

@@ -1,4 +1,4 @@
-import crawlee, { Request, RequestList } from 'crawlee';
+import crawlee, { LaunchContext, Request, RequestList } from 'crawlee';
 import printMessage from 'print-message';
 import fs from 'fs';
 import {
@@ -8,7 +8,7 @@ import {
   isUrlPdf,
 } from './commonCrawlerFunc.js';
 
-import constants, { guiInfoStatusTypes } from '../constants/constants.js';
+import constants, { STATUS_CODE_METADATA, guiInfoStatusTypes, UrlsCrawled } from '../constants/constants.js';
 import {
   getLinksFromSitemap,
   getPlaywrightLaunchOptions,
@@ -22,31 +22,32 @@ import {
 import { areLinksEqual, isWhitelistedContentType, isFollowStrategy } from '../utils.js';
 import { handlePdfDownload, runPdfScan, mapPdfScanResults } from './pdfScanFunc.js';
 import { guiInfoLog } from '../logs.js';
+import { ViewportSettingsClass } from '../combine.js';
 
 const crawlSitemap = async (
-  sitemapUrl,
-  randomToken,
-  host,
-  viewportSettings,
-  maxRequestsPerCrawl,
-  browser,
-  userDataDirectory,
-  specifiedMaxConcurrency,
-  fileTypes,
-  blacklistedPatterns,
-  includeScreenshots,
-  extraHTTPHeaders,
+  sitemapUrl: string,
+  randomToken: string,
+  _host: string,
+  viewportSettings: ViewportSettingsClass,
+  maxRequestsPerCrawl: number,
+  browser: string,
+  userDataDirectory: string,
+  specifiedMaxConcurrency: number,
+  fileTypes: string,
+  blacklistedPatterns: string[],
+  includeScreenshots: boolean,
+  extraHTTPHeaders: Record<string, string>,
   fromCrawlIntelligentSitemap = false, // optional
-  userUrlInputFromIntelligent = null, // optional
-  datasetFromIntelligent = null, // optional
-  urlsCrawledFromIntelligent = null, // optional
+  userUrlInputFromIntelligent: string = null, // optional
+  datasetFromIntelligent: crawlee.Dataset = null, // optional
+  urlsCrawledFromIntelligent: UrlsCrawled = null, // optional
   crawledFromLocalFile = false, // optional
 ) => {
-  let dataset;
-  let urlsCrawled;
+  let dataset: crawlee.Dataset;
+  let urlsCrawled: UrlsCrawled;
 
   // Boolean to omit axe scan for basic auth URL
-  let isBasicAuth;
+  let isBasicAuth: boolean;
   let basicAuthPage = 0;
   let finalLinks = [];
   let authHeader = '';
@@ -119,8 +120,8 @@ const crawlSitemap = async (
     basicAuthPage = -2;
   }
 
-  const pdfDownloads = [];
-  const uuidToPdfMapping = {};
+  const pdfDownloads: Promise<void>[] = [];
+  const uuidToPdfMapping: Record<string, string> = {};
   const isScanHtml = ['all', 'html-only'].includes(fileTypes);
   const isScanPdfs = ['all', 'pdf-only'].includes(fileTypes);
   const { playwrightDeviceDetailsObject } = viewportSettings;
@@ -152,7 +153,7 @@ const crawlSitemap = async (
     browserPoolOptions: {
       useFingerprints: false,
       preLaunchHooks: [
-        async (pageId, launchContext) => {
+        async (_pageId: string, launchContext: LaunchContext) => {
           launchContext.launchOptions = {
             ...launchContext.launchOptions,
             bypassCSP: true,
@@ -164,39 +165,43 @@ const crawlSitemap = async (
     },
     requestList,
     postNavigationHooks: [
-      async ({ page, request }) => {
+
+      async ({ page }) => {
         try {
           // Wait for a quiet period in the DOM, but with safeguards
           await page.evaluate(() => {
-            return new Promise((resolve) => {
+            return new Promise(resolve => {
               let timeout;
               let mutationCount = 0;
-              const MAX_MUTATIONS = 250; // Prevent infinite mutations
-              const OBSERVER_TIMEOUT = 5000; // Hard timeout to exit
-
+              const MAX_MUTATIONS     = 250;   // stop if things never quiet down
+              const OBSERVER_TIMEOUT  = 5000;  // hard cap on total wait
+      
               const observer = new MutationObserver(() => {
                 clearTimeout(timeout);
-
+      
                 mutationCount++;
                 if (mutationCount > MAX_MUTATIONS) {
                   observer.disconnect();
-                  resolve('Too many mutations detected, exiting.');
+                  resolve('Too many mutations, exiting.');
                   return;
                 }
-
+      
+                // restart quiet‑period timer
                 timeout = setTimeout(() => {
                   observer.disconnect();
-                  resolve('DOM stabilized after mutations.');
+                  resolve('DOM stabilized.');
                 }, 1000);
               });
-
+      
+              // overall timeout in case the page never settles
               timeout = setTimeout(() => {
                 observer.disconnect();
-                resolve('Observer timeout reached, exiting.');
-              }, OBSERVER_TIMEOUT); // Ensure the observer stops after X seconds
-
-              observer.observe(document.documentElement, { childList: true, subtree: true });
-
+                resolve('Observer timeout reached.');
+              }, OBSERVER_TIMEOUT);
+      
+              // **HERE**: select the real DOM node inside evaluate
+              const root = document.documentElement;
+              observer.observe(root, { childList: true, subtree: true });
             });
           });
         } catch (err) {
@@ -207,6 +212,7 @@ const crawlSitemap = async (
           throw err; // Rethrow unknown errors
         }
       },
+      
     ],
 
     preNavigationHooks: isBasicAuth
@@ -252,10 +258,12 @@ const crawlSitemap = async (
             numScanned: urlsCrawled.scanned.length,
             urlScanned: request.url,
           });
-          urlsCrawled.blacklisted.push({
+          urlsCrawled.userExcluded.push({
             url: request.url,
             pageTitle: request.url,
-            actualUrl: actualUrl, // i.e. actualUrl
+            actualUrl: request.url, // because about:blank is not useful
+            metadata: STATUS_CODE_METADATA[1],
+            httpStatusCode: 0,
           });
 
           return;
@@ -276,85 +284,64 @@ const crawlSitemap = async (
       const contentType = response?.headers?.()['content-type'] || '';
       const status = response ? response.status() : 0;
 
-      if (blacklistedPatterns && !isFollowStrategy(actualUrl, request.url, "same-hostname") && isSkippedUrl(actualUrl, blacklistedPatterns)) {
-        urlsCrawled.userExcluded.push({
-          url: request.url,
-          pageTitle: request.url,
-          actualUrl: actualUrl,
-        });
-
-        guiInfoLog(guiInfoStatusTypes.SKIPPED, {
-          numScanned: urlsCrawled.scanned.length,
-          urlScanned: request.url,
-        });
-        return;
-      }
+      if (basicAuthPage < 0) {
+        basicAuthPage += 1;
+      } else if (isScanHtml && status < 300 && isWhitelistedContentType(contentType)) {
+        const isRedirected = !areLinksEqual(page.url(), request.url);
+        const isLoadedUrlInCrawledUrls = urlsCrawled.scanned.some(
+          item => (item.actualUrl || item.url) === page.url(),
+        );
 
-      if (status === 403) {
-        guiInfoLog(guiInfoStatusTypes.SKIPPED, {
-          numScanned: urlsCrawled.scanned.length,
-          urlScanned: request.url,
-        });
-        urlsCrawled.forbidden.push({ url: request.url });
-        return;
-      }
+        if (isRedirected && isLoadedUrlInCrawledUrls) {
+          urlsCrawled.notScannedRedirects.push({
+            fromUrl: request.url,
+            toUrl: actualUrl, // i.e. actualUrl
+          });
+          return;
+        }
 
-      if (status !== 200) {
-        guiInfoLog(guiInfoStatusTypes.SKIPPED, {
-          numScanned: urlsCrawled.scanned.length,
-          urlScanned: request.url,
-        });
-        urlsCrawled.invalid.push({
-          url: request.url,
-          pageTitle: request.url,
-          actualUrl: actualUrl, // i.e. actualUrl
-        });
+        // This logic is different from crawlDomain, as it also checks if the pae is redirected before checking if it is excluded using exclusions.txt
+        if (
+          isRedirected &&
+          blacklistedPatterns &&
+          isSkippedUrl(actualUrl, blacklistedPatterns)
+        ) {
+          urlsCrawled.userExcluded.push({
+            url: request.url,
+            pageTitle: request.url,
+            actualUrl: actualUrl,
+            metadata: STATUS_CODE_METADATA[0],
+            httpStatusCode: 0,
+          });
 
-        return;
-      }
+          guiInfoLog(guiInfoStatusTypes.SKIPPED, {
+            numScanned: urlsCrawled.scanned.length,
+            urlScanned: request.url,
+          });
+          return;
+        }
 
-      if (basicAuthPage < 0) {
-        basicAuthPage += 1;
-      } else if (isScanHtml && status === 200 && isWhitelistedContentType(contentType)) {
         const results = await runAxeScript({ includeScreenshots, page, randomToken });
+        
         guiInfoLog(guiInfoStatusTypes.SCANNED, {
           numScanned: urlsCrawled.scanned.length,
           urlScanned: request.url,
         });
 
-        const isRedirected = !areLinksEqual(page.url(), request.url);
-        if (isRedirected) {
-          const isLoadedUrlInCrawledUrls = urlsCrawled.scanned.some(
-            item => (item.actualUrl || item.url.href) === page,
-          );
-
-          if (isLoadedUrlInCrawledUrls) {
-            urlsCrawled.notScannedRedirects.push({
-              fromUrl: request.url,
-              toUrl: actualUrl, // i.e. actualUrl
-            });
-            return;
-          }
+        urlsCrawled.scanned.push({
+          url: urlWithoutAuth(request.url),
+          pageTitle: results.pageTitle,
+          actualUrl: actualUrl, // i.e. actualUrl
+        });
 
-          urlsCrawled.scanned.push({
-            url: urlWithoutAuth(request.url),
-            pageTitle: results.pageTitle,
-            actualUrl: actualUrl, // i.e. actualUrl
-          });
+        urlsCrawled.scannedRedirects.push({
+          fromUrl: urlWithoutAuth(request.url),
+          toUrl: actualUrl,
+        });
 
-          urlsCrawled.scannedRedirects.push({
-            fromUrl: urlWithoutAuth(request.url),
-            toUrl: actualUrl,
-          });
+        results.url = request.url;
+        results.actualUrl = actualUrl;
 
-          results.url = request.url;
-          results.actualUrl = actualUrl;
-        } else {
-          urlsCrawled.scanned.push({
-            url: urlWithoutAuth(request.url),
-            pageTitle: results.pageTitle,
-          });
-        }
         await dataset.pushData(results);
       } else {
         guiInfoLog(guiInfoStatusTypes.SKIPPED, {
@@ -363,11 +350,23 @@ const crawlSitemap = async (
         });
 
         if (isScanHtml) {
-          urlsCrawled.invalid.push(actualUrl);
+          // carry through the HTTP status metadata
+          const status = response?.status();
+          const metadata = typeof status === 'number'
+          ? (STATUS_CODE_METADATA[status] || STATUS_CODE_METADATA[599])
+          : STATUS_CODE_METADATA[2];
+
+            urlsCrawled.invalid.push({
+            actualUrl,
+            url: request.url,
+            pageTitle: request.url,
+            metadata,
+            httpStatusCode: typeof status === 'number' ? status : 0
+          });
         }
       }
     },
-    failedRequestHandler: async ({ request }) => {
+    failedRequestHandler: async ({ request, response, error }) => {
       if (isBasicAuth && request.url) {
         request.url = `${request.url.split('://')[0]}://${request.url.split('@')[1]}`;
       }
@@ -381,7 +380,19 @@ const crawlSitemap = async (
         numScanned: urlsCrawled.scanned.length,
         urlScanned: request.url,
       });
-      urlsCrawled.error.push(request.url);
+
+      const status = response?.status();
+      const metadata = typeof status === 'number'
+      ? (STATUS_CODE_METADATA[status] || STATUS_CODE_METADATA[599])
+      : STATUS_CODE_METADATA[2];
+
+      urlsCrawled.error.push({
+        url: request.url,
+        pageTitle: request.url,
+        actualUrl: request.url,
+        metadata,
+        httpStatusCode: typeof status === 'number' ? status : 0
+      });
       crawlee.log.error(`Failed Request - ${request.url}: ${request.errorMessages}`);
     },
     maxRequestsPerCrawl: Infinity,

package/src/crawlers/custom/findElementByCssSelector.ts

@@ -16,7 +16,7 @@ export function findElementByCssSelector(cssSelector: string): string | null {
 
   // Handle Shadow DOM if the element is not found
   if (!element) {
-    const shadowRoots = [];
+    const shadowRoots: ShadowRoot[] = [];
     const allElements = document.querySelectorAll('*');
 
     // Look for elements with shadow roots

package/src/crawlers/custom/flagUnlabelledClickableElements.ts

@@ -27,9 +27,9 @@ export async function flagUnlabelledClickableElements() {
   const loggingEnabled = false; // Set to true to enable console warnings
 
   let previousFlaggedXPathsByDocument = {}; // Object to hold previous flagged XPaths
-  const previousAllFlaggedElementsXPaths = []; // Array to store all flagged XPaths
+  const previousAllFlaggedElementsXPaths : {xpath: string, code: string }[] = []; // Array to store all flagged XPaths
 
-  function getXPath(element: Node) {
+  function getXPath(element: Node): string {
     if (!element) return null;
     if (element instanceof HTMLElement && element.id) {
       return `//*[@id="${element.id}"]`;
@@ -297,7 +297,7 @@ function hasPointerCursor(node: Node): boolean {
     return hasAccessibleChildElement || hasDirectAccessibleText;
   }
 
-  function hasAllChildrenAccessible(element: Element) {
+  function hasAllChildrenAccessible(element: Element): boolean {
     // If the element is aria-hidden, consider it accessible
     if (element.getAttribute('aria-hidden') === 'true') {
       return true;
@@ -331,7 +331,7 @@ function hasPointerCursor(node: Node): boolean {
   function hasChildNotANewInteractWithAccessibleText(element: Element) {
 
    // Helper function to check if the element is a link or button
-    const isBuildInInteractable = (child) => {
+    const isBuildInInteractable = (child: Element) => {
         return child.nodeName.toLowerCase() === "a" || child.nodeName.toLowerCase() === "button" || child.nodeName.toLowerCase() === "input" ||
                child.getAttribute('role') === 'link' || child.getAttribute('role') === 'button';
     };
@@ -376,7 +376,7 @@ function hasPointerCursor(node: Node): boolean {
         }
 
         // Recursively check for text content inside child nodes of elements that are not links or buttons
-        if (node.nodeType === Node.ELEMENT_NODE && !isBuildInInteractable(node)) {
+        if (node.nodeType === Node.ELEMENT_NODE && !isBuildInInteractable(node as Element)) {
             return Array.from(node.childNodes).some(innerNode => {
                 if (innerNode.nodeType === Node.TEXT_NODE) {
                     const innerTextContent = getTextContent(innerNode).trim();
@@ -440,7 +440,7 @@ function hasPointerCursor(node: Node): boolean {
     const beforeContent = window.getComputedStyle(element, '::before').getPropertyValue('content');
     const afterContent = window.getComputedStyle(element, '::after').getPropertyValue('content');
 
-    function isAccessibleContent(value) {
+    function isAccessibleContent(value: string) {
         if (!value || value === 'none' || value === 'normal') {
             return false;
         }
@@ -1126,11 +1126,11 @@ function hasPointerCursor(node: Node): boolean {
     });
 
     // Collect XPaths and outerHTMLs of flagged elements per document
-    const flaggedXPathsByDocument = {};
+    const flaggedXPathsByDocument: { [key: string]: { xpath: string; code: string }[] } = {};
 
     for (const docKey in currentFlaggedElementsByDocument) {
       const elements = currentFlaggedElementsByDocument[docKey];
-      const flaggedInfo = []; // Array to hold flagged element info
+      const flaggedInfo: { xpath: string; code: string }[] = []; // Array to hold flagged element info
       elements.forEach(flaggedElement => {
         const parentFlagged = flaggedElement.closest('[data-flagged="true"]');
         if (!parentFlagged || parentFlagged === flaggedElement) {

package/src/crawlers/custom/xPathToCss.ts

@@ -1,12 +1,12 @@
-export function xPathToCss(expr: string) {
-  const isValidXPath = expr =>
+export default function xPathToCss(expr: string) {
+  const isValidXPath = (expr: string) =>
     typeof expr !== 'undefined' &&
     expr.replace(/[\s-_=]/g, '') !== '' &&
     expr.length ===
-    expr.replace(
-      /[-_\w:.]+\(\)\s*=|=\s*[-_\w:.]+\(\)|\sor\s|\sand\s|\[(?:[^\/\]]+[\/\[]\/?.+)+\]|starts-with\(|\[.*last\(\)\s*[-\+<>=].+\]|number\(\)|not\(|count\(|text\(|first\(|normalize-space|[^\/]following-sibling|concat\(|descendant::|parent::|self::|child::|/gi,
-      '',
-    ).length;
+      expr.replace(
+        /[-_\w:.]+\(\)\s*=|=\s*[-_\w:.]+\(\)|\sor\s|\sand\s|\[(?:[^\/\]]+[\/\[]\/?.+)+\]|starts-with\(|\[.*last\(\)\s*[-\+<>=].+\]|number\(\)|not\(|count\(|text\(|first\(|normalize-space|[^\/]following-sibling|concat\(|descendant::|parent::|self::|child::|/gi,
+        '',
+      ).length;
 
   const getValidationRegex = () => {
     let regex =
@@ -30,7 +30,7 @@ export function xPathToCss(expr: string) {
       value: '\\s*[\\w/:][-/\\w\\s,:;.]*',
     };
 
-    Object.keys(subRegexes).forEach(key => {
+    Object.keys(subRegexes).forEach((key: keyof typeof subRegexes) => {
       regex = regex.replace(new RegExp(`%\\(${key}\\)s`, 'gi'), subRegexes[key]);
     });
 
@@ -42,14 +42,14 @@ export function xPathToCss(expr: string) {
     return new RegExp(regex, 'gi');
   };
 
-  const preParseXpath = expr =>
+  const preParseXpath = (expr: string) =>
     expr.replace(
       /contains\s*\(\s*concat\(["']\s+["']\s*,\s*@class\s*,\s*["']\s+["']\)\s*,\s*["']\s+([a-zA-Z0-9-_]+)\s+["']\)/gi,
       '@class="$1"',
     );
 
-  function escapeCssIdSelectors(cssSelector) {
-    return cssSelector.replace(/#([^ >]+)/g, (match, id) => {
+  function escapeCssIdSelectors(cssSelector: string) {
+    return cssSelector.replace(/#([^ >]+)/g, (_match, id) => {
       // Escape special characters in the id part
       return `#${id.replace(/[!"#$%&'()*+,./:;<=>?@[\\\]^`{|}~]/g, '\\$&')}`;
     });

package/src/crawlers/runCustom.ts

@@ -48,7 +48,7 @@ const runCustom = async (
   includeScreenshots: boolean,
 ) => {
   // checks and delete datasets path if it already exists
-  await cleanUp(randomToken);
+  cleanUp(randomToken);
   process.env.CRAWLEE_STORAGE_DIR = randomToken;
 
   const urlsCrawled: UrlsCrawled = { ...constants.urlsCrawledObj };

package/src/index.ts

@@ -1,6 +1,4 @@
 #!/usr/bin/env node
-/* eslint-disable func-names */
-/* eslint-disable no-param-reassign */
 import printMessage from 'print-message';
 import inquirer from 'inquirer';
 import { EnqueueStrategy } from 'crawlee';
@@ -22,6 +20,7 @@ import {
 import questions from './constants/questions.js';
 import combineRun from './combine.js';
 import { BrowserTypes, RuleFlags, ScannerTypes } from './constants/constants.js';
+import { DeviceDescriptor } from './types/types.js';
 
 export type Answers = {
   headless: boolean;
@@ -32,7 +31,7 @@ export type Answers = {
   scanner: ScannerTypes;
   url: string;
   clonedBrowserDataDir: string;
-  playwrightDeviceDetailsObject: object;
+  playwrightDeviceDetailsObject: DeviceDescriptor;
   nameEmail: string;
   fileTypes: string;
   metadata: string;
@@ -61,7 +60,7 @@ export type Data = {
   deviceChosen: string;
   customDevice: string;
   viewportWidth: number;
-  playwrightDeviceDetailsObject: object;
+  playwrightDeviceDetailsObject: DeviceDescriptor;
   maxRequestsPerCrawl: number;
   strategy: EnqueueStrategy;
   isLocalFileScan: boolean;

package/src/logs.ts

@@ -40,7 +40,7 @@ const silentLogger = createLogger({
 });
 
 // guiInfoLogger feeds the gui information via console log and is mainly used for scanning process
-export const guiInfoLog = (status, data) => {
+export const guiInfoLog = (status: string, data: { numScanned?: number; urlScanned?: string }) => {
   if (process.env.RUNNING_FROM_PH_GUI || process.env.OOBEE_VERBOSE) {
     switch (status) {
       case guiInfoStatusTypes.COMPLETED: