@govtechsg/oobee 0.10.51 → 0.10.58

package/src/constants/constants.ts

@@ -7,7 +7,7 @@ import os from 'os';
 import { spawnSync, execSync } from 'child_process';
 import { chromium } from 'playwright';
 import * as Sentry from '@sentry/node';
-import { silentLogger } from '../logs.js';
+import { consoleLogger, silentLogger } from '../logs.js';
 import { PageInfo } from '../mergeAxeResults.js';
 
 const filename = fileURLToPath(import.meta.url);
@@ -128,7 +128,7 @@ export const getDefaultChromiumDataDir = () => {
         defaultChromiumDataDir = '/tmp';
       }
 
-      silentLogger.warn(`Using Chromium support directory at ${defaultChromiumDataDir}`);
+      consoleLogger.info(`Using Chromium support directory at ${defaultChromiumDataDir}`);
     }
 
     if (defaultChromiumDataDir && fs.existsSync(defaultChromiumDataDir)) {
@@ -179,6 +179,7 @@ export const basicAuthRegex = /^.*\/\/.*:.*@.*$/i;
 // for crawlers
 export const axeScript = path.join(dirname, '../../node_modules/axe-core/axe.min.js');
 export class UrlsCrawled {
+  siteName: string;
   toScan: string[] = [];
   scanned: PageInfo[] = [];
   invalid: PageInfo[] = [];
@@ -361,6 +362,7 @@ const wcagLinks = {
   // 'WCAG 1.4.10': 'https://www.w3.org/TR/WCAG22/#reflow', - TODO: review for veraPDF
   'WCAG 1.4.12': 'https://www.w3.org/TR/WCAG22/#text-spacing',
   'WCAG 2.1.1': 'https://www.w3.org/TR/WCAG22/#keyboard',
+  'WCAG 2.1.3': 'https://www.w3.org/WAI/WCAG22/Understanding/keyboard-no-exception.html', // AAA
   'WCAG 2.2.1': 'https://www.w3.org/TR/WCAG22/#timing-adjustable',
   'WCAG 2.2.2': 'https://www.w3.org/TR/WCAG22/#pause-stop-hide',
   'WCAG 2.2.4': 'https://www.w3.org/TR/WCAG22/#interruptions', // AAA
@@ -564,3 +566,46 @@ export const STATUS_CODE_METADATA: Record<number,string> = {
   511: '511 - Network Authentication Required',
 
 };
+
+  // Elements that should not be clicked or enqueued
+  // With reference from https://chromeenterprise.google/policies/url-patterns/
+export const disallowedListOfPatterns = [
+  "#",
+  "mailto:",
+  "tel:",
+  "sms:",
+  "skype:",
+  "zoommtg:",
+  "msteams:",
+  "whatsapp:",
+  "slack:",
+  "viber:",
+  "tg:",
+  "line:",
+  "meet:",
+  "facetime:",
+  "imessage:",
+  "discord:",
+  "sgnl:",
+  "webex:",
+  "intent:",
+  "ms-outlook:",
+  "ms-onedrive:",
+  "ms-word:",
+  "ms-excel:",
+  "ms-powerpoint:",
+  "ms-office:",
+  "onenote:",
+  "vs:",
+  "chrome-extension:",
+  "chrome-search:",
+  "chrome:",
+  "chrome-untrusted:",
+  "devtools:",
+  "isolated-app:"
+];
+
+export const disallowedSelectorPatterns = disallowedListOfPatterns
+  .map(pattern => `a[href^="${pattern}"]`)
+  .join(',')
+  .replace(/\s+/g, '');

package/src/crawlers/commonCrawlerFunc.ts

@@ -1,13 +1,14 @@
 import crawlee, { CrawlingContext, PlaywrightGotoOptions, Request } from 'crawlee';
 import axe, { AxeResults, ImpactValue, NodeResult, Result, resultGroups, TagValue } from 'axe-core';
-import { BrowserContext, Page } from 'playwright';
+import { BrowserContext, ElementHandle, Page } from 'playwright';
 import {
   axeScript,
+  disallowedListOfPatterns,
   guiInfoStatusTypes,
   RuleFlags,
   saflyIconSelector,
 } from '../constants/constants.js';
-import { guiInfoLog, silentLogger } from '../logs.js';
+import { consoleLogger, guiInfoLog, silentLogger } from '../logs.js';
 import { takeScreenshotForHTMLElements } from '../screenshotFunc/htmlScreenshotFunc.js';
 import { isFilePath } from '../constants/common.js';
 import { extractAndGradeText } from './custom/extractAndGradeText.js';
@@ -19,6 +20,7 @@ import { findElementByCssSelector } from './custom/findElementByCssSelector.js';
 import { getAxeConfiguration } from './custom/getAxeConfiguration.js';
 import { flagUnlabelledClickableElements } from './custom/flagUnlabelledClickableElements.js';
 import xPathToCss from './custom/xPathToCss.js';
+import type { Response as PlaywrightResponse } from 'playwright';
 
 // types
 interface AxeResultsWithScreenshot extends AxeResults {
@@ -305,7 +307,7 @@ export const runAxeScript = async ({
       });
     });
   } catch (e) {
-    silentLogger.warn(`Error while checking for DOM mutations: ${e}`);
+    // do nothing, just continue
   }
 
   // Omit logging of browser console errors to reduce unnecessary verbosity
@@ -459,9 +461,9 @@ export const runAxeScript = async ({
   try {
     pageTitle = await page.evaluate(() => document.title);
   } catch (e) {
-    silentLogger.warn(`Error while getting page title: ${e}`);
+    consoleLogger.info(`Error while getting page title: ${e}`);
     if (page.isClosed()) {
-      silentLogger.info(`Page was closed for ${requestUrl}, creating new page`);
+      consoleLogger.info(`Page was closed for ${requestUrl}, creating new page`);
       page = await browserContext.newPage();
       await page.goto(requestUrl, { waitUntil: 'domcontentloaded' });
       pageTitle = await page.evaluate(() => document.title);
@@ -508,3 +510,80 @@ export const isUrlPdf = (url: string) => {
   const parsedUrl = new URL(url);
   return /\.pdf($|\?|#)/i.test(parsedUrl.pathname) || /\.pdf($|\?|#)/i.test(parsedUrl.href);
 };
+
+export async function shouldSkipClickDueToDisallowedHref(
+  page: Page,
+  element: ElementHandle
+): Promise<boolean> {
+  return await page.evaluate(
+    ({ el, disallowedPrefixes }) => {
+      function isDisallowedHref(href: string | null): boolean {
+        if (!href) return false;
+        href = href.toLowerCase();
+        return disallowedPrefixes.some((prefix: string) => href.startsWith(prefix));
+      }
+
+      const castEl = el as HTMLElement;
+
+      // Check descendant <a href="">
+      const descendants = castEl.querySelectorAll('a[href]');
+      for (const a of descendants) {
+        const href = a.getAttribute('href');
+        if (isDisallowedHref(href)) {
+          return true;
+        }
+      }
+
+      // Check self and ancestors for disallowed <a>
+      let current: HTMLElement | null = castEl;
+      while (current) {
+        if (
+          current.tagName === 'A' &&
+          isDisallowedHref(current.getAttribute('href'))
+        ) {
+          return true;
+        }
+        current = current.parentElement;
+      }
+
+      return false;
+    },
+    {
+      el: element,
+      disallowedPrefixes: disallowedListOfPatterns,
+    }
+  );
+}
+
+/**
+ * Check if response should be skipped based on content headers.
+ * @param response - Playwright Response object
+ * @param requestUrl - Optional: request URL for logging
+ * @returns true if the content should be skipped
+ */
+export const shouldSkipDueToUnsupportedContent = (
+  response: PlaywrightResponse,
+  requestUrl: string = ''
+): boolean => {
+  if (!response) return false;
+
+  const headers = response.headers();
+  const contentDisposition = headers['content-disposition'] || '';
+  const contentType = headers['content-type'] || '';
+
+  if (contentDisposition.includes('attachment')) {
+    // consoleLogger.info(`Skipping attachment (content-disposition) at ${requestUrl}`);
+    return true;
+  }
+
+  if (
+    contentType.startsWith('application/') ||
+    contentType.includes('octet-stream') ||
+    (!contentType.startsWith('text/') && !contentType.includes('html'))
+  ) {
+    // consoleLogger.info(`Skipping non-processible content-type "${contentType}" at ${requestUrl}`);
+    return true;
+  }
+
+  return false;
+};