@govtechsg/oobee 0.10.36 → 0.10.39

package/DETAILS.md

@@ -14,12 +14,12 @@ Details of each issue and severity rating provided by the current scan engine.
 
 ## Conformance Covered
 
-#### Definitions of Conformance Level, Must Fix, Good To Fix, Manual Review Required Required
+#### Definitions of Conformance Level, Must Fix, Good To Fix, Manual Review Required
 
 In Oobee, issues are grouped into one of three categories:
 - **Must Fix** issues includes WCAG A & AA success criteria (excluding those requiring review). 
 - **Good To Fix** issues includes WCAG Level AAA success criteria issues and all best practice rules that do not necessarily conform to WCAG success criterion but are industry accepted practices that improve the user experience.
-- **Manual Review Required Required** occurrences could potentially be false positive, requiring human validation for accuracy.
+- **Manual Review Required** occurrences could potentially be false positive, requiring human validation for accuracy.
 
 Note: Level AAA are disabled by default.  Please specify `enable-wcag-aaa` in ruleset flag to enable AAA rules.
 
@@ -158,7 +158,7 @@ Note: Level AAA are disabled by default.  Please specify `enable-wcag-aaa` in ru
 | color-contrast-enhanced      | Ensure the contrast between foreground and background colors meets WCAG 2 AAA enhanced contrast ratio thresholds | Good to Fix | WCAG 1.4.6             |
 | identical-links-same-purpose | Ensure that links with the same accessible name serve a similar purpose                                          | Good to Fix | WCAG 2.4.9             |
 | meta-refresh-no-exceptions   | Ensure <meta http-equiv="refresh"> is not used for delayed refresh                                               | Good to Fix | WCAG 2.2.4, WCAG 3.2.5 |
-| oobee-grading-text-contents               | Text content should be clear and plain to ensure that it is easily understood.                                                                                                              | Manual Review Required Required    | WCAG 3.1.5             |
+| oobee-grading-text-contents               | Text content should be clear and plain to ensure that it is easily understood.                                                                                                              | Manual Review Required    | WCAG 3.1.5             |
 
 ## Best Practice
 

package/INTEGRATION.md

@@ -30,7 +30,7 @@ In order to use this functionality, the testing framework must support:
 
 ### API Reference
 
-#### `async oobeeA11yInit(entryUrl, testLabel, name, email, includeScreenshots, viewportSettings, thresholds, scanAboutMetadata)`
+#### `async oobeeA11yInit({entryUrl, testLabel, name, email, includeScreenshots, viewportSettings, thresholds, scanAboutMetadata, zip, deviceChosen, strategy, ruleset, specifiedMaxConcurrency, followRobots})`
 
 Returns an instance of Oobee
 
@@ -54,6 +54,16 @@ Returns an instance of Oobee
   - Include additional information in the Scan About section of the report by passing in a JSON object.
 - `zip` (optional)
   - Name of the generated zip of Oobee results at the end of scan. Defaults to "oobee-scan-results".
+- `deviceChosen` (optional)
+  - Name of the device to scan on. Example: `iPhone 13 Pro Max`
+- `strategy` (optional)
+  - The EnqueueStrategy to use. Options: `all`, `same-hostname`, `same-domain`, `same-origin`
+- `ruleset` (optional)
+  - The array of rulesets to use. Options: `default`, `disable-oobee`, `enable-wcag-aaa`
+- `specifiedMaxConcurrency` (optional)
+  - The maximum number of concurrent requests to be made. Defaults to 25.
+- `followRobots` (optional)
+  - Whether to follow robots.txt. Defaults to false.
 
 #### Oobee Instance
 
@@ -78,14 +88,15 @@ Unique identifier for the scan instance
 
 `getScripts()`
 
-Get the axe-core script to be injected into the browser
+Get the axe-core script to be injected into the browser, along with other custom Oobee scripts
 
-- `runA11yScan(elementsToScan)`
+- `runA11yScan(elementsToScan, gradingReadabilityFlag)`
   Runs axe scan on the current page.
 
   Parameter(s):
 
   - `elementsToScan`: Specifies which element should and which should not be tested
+  - `gradingReadabilityFlag`: This is the readability score as a string. If it is non-empty, a readability violation will be added.
 
   Returns:
 
@@ -176,17 +187,22 @@ Create <code>cypress.config.js</code> with the following contents, and change yo
     // name of the generated zip of the results at the end of scan
     const resultsZipName = "oobee-scan-results"
 
-    const oobeeA11y = await oobeeA11yInit(
-        "https://govtechsg.github.io", // initial url to start scan
-        "Demo Cypress Scan", // label for test
-        "Your Name",
-        "email@domain.com",
-        true, // include screenshots of affected elements in the report
-        viewportSettings,
-        thresholds,
-        scanAboutMetadata,
-        resultsZipName
-    );
+    const oobeeA11y = await oobeeA11yInit({
+      entryUrl: "https://govtechsg.github.io", // initial url to start scan
+      testLabel: "Demo Cypress Scan", // label for test
+      name: "Your Name",
+      email: "email@domain.com",
+      includeScreenshots: true, // include screenshots of affected elements in the report
+      viewportSettings,
+      thresholds: { mustFix: undefined, goodToFix: undefined },
+      scanAboutMetadata: undefined,
+      zip: resultsZipName,
+      deviceChosen: "",
+      strategy: undefined,
+      ruleset: ["enable-wcag-aaa"], // add "disable-oobee" to disable Oobee custom checks
+      specifiedMaxConcurrency: undefined,
+      followRobots: undefined,
+    });
 
     export default defineConfig({
         taskTimeout: 120000, // need to extend as screenshot function requires some time
@@ -198,6 +214,9 @@ Create <code>cypress.config.js</code> with the following contents, and change yo
                     getPurpleA11yScripts() {
                         return oobeeA11y.getScripts();
                     },
+                    gradeReadability(sentences) {
+                        return oobeeA11y.gradeReadability(sentences);
+                    },
                     async pushPurpleA11yScanResults({res, metadata, elementsToClick}) {
                         return await oobeeA11y.pushScanResults(res, metadata, elementsToClick);
                     },
@@ -229,8 +248,26 @@ Create a sub-folder and file <code>cypress/support/e2e.js</code> with the follow
     Cypress.Commands.add("runPurpleA11yScan", (items={}) => {
         cy.window().then(async (win) => {
             const { elementsToScan, elementsToClick, metadata } = items;
-            const res = await win.runA11yScan(elementsToScan);
-            cy.task("pushPurpleA11yScanResults", {res, metadata, elementsToClick}).then((count) => { return count });
+
+            // extract text from the page for readability grading
+            const sentences = win.extractText();
+            // run readability grading separately as it cannot be done within the browser context
+            cy.task("gradeReadability", sentences).then(
+                async (gradingReadabilityFlag) => {
+                    // passing the grading flag to runA11yScan to inject violation as needed
+                    const res = await win.runA11yScan(
+                        elementsToScan,
+                        gradingReadabilityFlag,
+                    );
+                    cy.task("pushPurpleA11yScanResults", {
+                        res,
+                        metadata,
+                        elementsToClick,
+                    }).then((count) => {
+                         return count;
+                    });
+                },
+            );
             cy.task("finishPurpleA11yTestCase"); // test the accumulated number of issue occurrences against specified thresholds. If exceed, terminate oobeeA11y instance.
         });
     });
@@ -244,7 +281,7 @@ Create <code>cypress/e2e/spec.cy.js</code> with the following contents:
     describe("template spec", () => {
         it("should run oobee A11y", () => {
             cy.visit(
-                "https://govtechsg.github.io/purple-banner-embeds/oobee-integrated-scan-example.htm"
+                "https://govtechsg.github.io/purple-banner-embeds/purple-integrated-scan-example.htm"
             );
             cy.injectPurpleA11yScripts();
             cy.runPurpleA11yScan();
@@ -338,17 +375,22 @@ Create <code>cypress.config.ts</code> with the following contents, and change yo
     // name of the generated zip of the results at the end of scan
     const resultsZipName: string = "oobee-scan-results"
 
-    const oobeeA11y = await oobeeA11yInit(
+    const oobeeA11y = await oobeeA11yInit({
         "https://govtechsg.github.io", // initial url to start scan
         "Demo Cypress Scan", // label for test
         "Your Name",
         "email@domain.com",
         true, // include screenshots of affected elements in the report
         viewportSettings,
-        thresholds,
-        scanAboutMetadata,
-        resultsZipName
-    );
+        thresholds: { mustFix: undefined, goodToFix: undefined },
+        scanAboutMetadata: undefined,
+        zip: resultsZipName,
+        deviceChosen: "",
+        strategy: undefined,
+        ruleset: ["enable-wcag-aaa"], // add "disable-oobee" to disable Oobee custom checks
+        specifiedMaxConcurrency: undefined,
+        followRobots: undefined,
+    });
 
     export default defineConfig({
         taskTimeout: 120000, // need to extend as screenshot function requires some time
@@ -360,6 +402,9 @@ Create <code>cypress.config.ts</code> with the following contents, and change yo
                     getPurpleA11yScripts(): string {
                         return oobeeA11y.getScripts();
                     },
+                    gradeReadability(sentences: string[]): string {
+                        return oobeeA11y.gradeReadability(sentences);
+                    },
                     async pushPurpleA11yScanResults({res, metadata, elementsToClick}: { res: any, metadata: any, elementsToClick: any[] }): Promise<{ mustFix: number, goodToFix: number }> {
                         return await oobeeA11y.pushScanResults(res, metadata, elementsToClick);
                     },
@@ -393,8 +438,26 @@ Create a sub-folder and file <code>src/cypress/support/e2e.ts</code> with the fo
     Cypress.Commands.add("runPurpleA11yScan", (items={}) => {
         cy.window().then(async (win) => {
             const { elementsToScan, elementsToClick, metadata } = items;
-            const res = await win.runA11yScan(elementsToScan);
-            cy.task("pushPurpleA11yScanResults", {res, metadata, elementsToClick}).then((count) => { return count });
+
+            // extract text from the page for readability grading
+            const sentences = win.extractText();
+            // run readability grading separately as it cannot be done within the browser context
+            cy.task("gradeReadability", sentences).then(
+                async (gradingReadabilityFlag: string) => {
+                    // passing the grading flag to runA11yScan to inject violation as needed
+                    const res = await win.runA11yScan(
+                        elementsToScan,
+                        gradingReadabilityFlag,
+                    );
+                    cy.task("pushPurpleA11yScanResults", {
+                        res,
+                        metadata,
+                        elementsToClick,
+                    }).then((count) => {
+                         return count;
+                    });
+                },
+            );
             cy.task("finishPurpleA11yTestCase"); // test the accumulated number of issue occurrences against specified thresholds. If exceed, terminate oobeeA11y instance.
         });
     });
@@ -443,6 +506,7 @@ declare namespace Cypress {
 
 interface Window {
   runA11yScan: (elementsToScan?: string[]) => Promise<any>;
+  extractText: () => string[];
 }
 ```
 
@@ -475,6 +539,7 @@ On your project's root folder, create a Playwright test file <code>oobeeA11y-pla
 
     import { chromium } from "playwright";
     import oobeeA11yInit from "@govtechsg/oobee";
+    import { extractText } from "@govtechsg/oobee/dist/crawlers/custom/extractText.js";
 
     // viewport used in tests to optimise screenshots
     const viewportSettings = { width: 1920, height: 1040 };
@@ -483,16 +548,22 @@ On your project's root folder, create a Playwright test file <code>oobeeA11y-pla
     // additional information to include in the "Scan About" section of the report
     const scanAboutMetadata = { browser: 'Chrome (Desktop)' };
 
-    const oobeeA11y = await oobeeA11yInit(
-        "https://govtechsg.github.io", // initial url to start scan
-        "Demo Playwright Scan", // label for test
-        "Your Name",
-        "email@domain.com",
-        true, // include screenshots of affected elements in the report
+    const oobeeA11y = await oobeeA11yInit({
+        entryUrl: "https://govtechsg.github.io", // initial url to start scan
+        testLabel: "Demo Cypress Scan", // label for test
+        name: "Your Name",
+        email: "email@domain.com",
+        includeScreenshots: true, // include screenshots of affected elements in the report
         viewportSettings,
-        thresholds,
-        scanAboutMetadata,
-    );
+        thresholds: { mustFix: undefined, goodToFix: undefined },
+        scanAboutMetadata: undefined,
+        zip: resultsZipName,
+        deviceChosen: "",
+        strategy: undefined,
+        ruleset: ["enable-wcag-aaa"],
+        specifiedMaxConcurrency: undefined,
+        followRobots: undefined,
+    });
 
     (async () => {
         const browser = await chromium.launch({
@@ -501,18 +572,22 @@ On your project's root folder, create a Playwright test file <code>oobeeA11y-pla
         const context = await browser.newContext();
         const page = await context.newPage();
 
-        const runPurpleA11yScan = async (elementsToScan) => {
+        const runPurpleA11yScan = async (elementsToScan, gradingReadabilityFlag) => {
             const scanRes = await page.evaluate(
-                async elementsToScan => await runA11yScan(elementsToScan),
-                elementsToScan,
+                async ({ elementsToScan, gradingReadabilityFlag }) => await runA11yScan(elementsToScan, gradingReadabilityFlag),
+                { elementsToScan, gradingReadabilityFlag },
             );
             await oobeeA11y.pushScanResults(scanRes);
             oobeeA11y.testThresholds(); // test the accumulated number of issue occurrences against specified thresholds. If exceed, terminate oobeeA11y instance.
         };
 
-        await page.goto('https://govtechsg.github.io/purple-banner-embeds/oobee-integrated-scan-example.htm');
+        await page.goto('https://govtechsg.github.io/purple-banner-embeds/purple-integrated-scan-example.htm');
         await page.evaluate(oobeeA11y.getScripts());
-        await runPurpleA11yScan();
+
+        const sentences = await page.evaluate(() => extractText());
+        const gradingReadabilityFlag = await oobeeA11y.gradeReadability(sentences);
+
+        await runPurpleA11yScan([], gradingReadabilityFlag);;
 
         await page.getByRole('button', { name: 'Click Me' }).click();
         // Run a scan on <input> and <button> elements
@@ -566,8 +641,12 @@ Create a sub-folder and Playwright test file <code>src/oobeeA11y-playwright-demo
 
     import { Browser, BrowserContext, Page, chromium } from "playwright";
     import oobeeA11yInit from "@govtechsg/oobee";
+    import { extractText } from "@govtechsg/oobee/dist/crawlers/custom/extractText.js";
 
-    declare const runA11yScan: (elementsToScan?: string[]) => Promise<any>;
+    declare const runA11yScan: (
+        elementsToScan?: string[],
+        gradingReadabilityFlag?: string,
+    ) => Promise<any>;
 
     interface ViewportSettings {
         width: number;
@@ -590,16 +669,22 @@ Create a sub-folder and Playwright test file <code>src/oobeeA11y-playwright-demo
     // additional information to include in the "Scan About" section of the report
     const scanAboutMetadata: ScanAboutMetadata = { browser: 'Chrome (Desktop)' };
 
-    const oobeeA11y = await oobeeA11yInit(
-        "https://govtechsg.github.io", // initial url to start scan
-        "Demo Playwright Scan", // label for test
-        "Your Name",
-        "email@domain.com",
-        true, // include screenshots of affected elements in the report
+    const oobeeA11y = await oobeeA11yInit({
+        entryUrl: "https://govtechsg.github.io", // initial url to start scan
+        testLabel: "Demo Cypress Scan", // label for test
+        name: "Your Name",
+        email: "email@domain.com",
+        includeScreenshots: true, // include screenshots of affected elements in the report
         viewportSettings,
-        thresholds,
-        scanAboutMetadata,
-    );
+        thresholds: { mustFix: undefined, goodToFix: undefined },
+        scanAboutMetadata: undefined,
+        zip: resultsZipName,
+        deviceChosen: "",
+        strategy: undefined,
+        ruleset: ["enable-wcag-aaa"],
+        specifiedMaxConcurrency: undefined,
+        followRobots: undefined,
+    });
 
     (async () => {
         const browser: Browser = await chromium.launch({
@@ -608,18 +693,22 @@ Create a sub-folder and Playwright test file <code>src/oobeeA11y-playwright-demo
         const context: BrowserContext = await browser.newContext();
         const page: Page = await context.newPage();
 
-        const runPurpleA11yScan = async (elementsToScan?: string[]) => {
+        const runPurpleA11yScan = async (elementsToScan?: string[], gradingReadabilityFlag?: string) => {
             const scanRes = await page.evaluate(
-                async elementsToScan => await runA11yScan(elementsToScan),
-                elementsToScan,
+                async ({ elementsToScan, gradingReadabilityFlag }) => await runA11yScan(elementsToScan, gradingReadabilityFlag),
+                { elementsToScan, gradingReadabilityFlag },
             );
             await oobeeA11y.pushScanResults(scanRes);
             oobeeA11y.testThresholds(); // test the accumulated number of issue occurrences against specified thresholds. If exceed, terminate oobeeA11y instance.
         };
 
-        await page.goto('https://govtechsg.github.io/purple-banner-embeds/oobee-integrated-scan-example.htm');
+        await page.goto('https://govtechsg.github.io/purple-banner-embeds/purple-integrated-scan-example.htm');
         await page.evaluate(oobeeA11y.getScripts());
-        await runPurpleA11yScan();
+
+        const sentences = await page.evaluate(() => extractText());
+        const gradingReadabilityFlag = await oobeeA11y.gradeReadability(sentences);
+
+        await runPurpleA11yScan([], gradingReadabilityFlag);
 
         await page.getByRole('button', { name: 'Click Me' }).click();
         // Run a scan on <input> and <button> elements

package/README.md

@@ -80,6 +80,21 @@ export PATH="<location of verapdf>:$PATH"
 verapdf --version
 ```
 
+#### Environment variables (Optional)
+| Variable Name | Description | Default |
+| ------------- | ----------- | ------- |
+| OOBEE_VERBOSE | When set to `true`, log output goes to console | `false` |
+| RUNNING_FROM_PH_GUI | Legacy, replaced by OOBEE_VERBOSE | `false` |
+| WARN_LEVEL | Only used in tests. |  |
+
+#### Environment variables used internally (Do not set)
+Do not set these environment variables or behaviour might change unexpectedly.
+| Variable Name | Description |
+| ------------- | ----------- |
+| CRAWLEE_LOG_LEVEL | https://crawlee.dev/docs/guides/configuration#crawlee_log_level |
+| CRAWLEE_STORAGE_DIR | https://crawlee.dev/docs/guides/configuration#crawlee_storage_dir |
+| CRAWLEE_HEADLESS | https://crawlee.dev/docs/guides/configuration#crawlee_headless |
+
 #### Facing issues?
 
 Please refer to [Troubleshooting section](#troubleshooting) for more information.

package/exclusions.txt

@@ -1,3 +1,6 @@
 \.*login.singpass.gov.sg\.*
 \.*auth.singpass.gov.sg\.*
-\.*form.gov.sg\.*
+\.*form.gov.sg\.*
+\.*login.microsoftonline.com\.*
+\.*id.atlassian.com\.*
+\.*cloudflareaccess.com\.*

package/package.json

@@ -1,14 +1,14 @@
 {
   "name": "@govtechsg/oobee",
   "main": "dist/npmIndex.js",
-  "version": "0.10.36",
+  "version": "0.10.39",
   "type": "module",
   "author": "Government Technology Agency <info@tech.gov.sg>",
   "dependencies": {
     "@json2csv/node": "^7.0.3",
     "@napi-rs/canvas": "^0.1.53",
     "axe-core": "^4.10.2",
-    "axios": "^1.7.4",
+    "axios": "^1.8.2",
     "base64-stream": "^1.0.0",
     "cheerio": "^1.0.0-rc.12",
     "crawlee": "^3.11.1",

package/src/constants/cliFunctions.ts

@@ -333,10 +333,3 @@ To obtain the JSON files, you need to base64-decode the file followed by gunzip.
   },
 };
 
-export const configureReportSetting = (isEnabled: boolean): void => {
-  if (isEnabled) {
-    process.env.REPORT_BREAKDOWN = '1';
-  } else {
-    process.env.REPORT_BREAKDOWN = '0';
-  }
-};

package/src/constants/common.ts

@@ -461,7 +461,7 @@ const checkUrlConnectivityWithBrowser = async (
 
       res.content = await page.content();
 
-      const contentType = response.headers()['content-type'];
+      const contentType = response?.headers?.()['content-type'] || '';
       if (contentType.includes('xml')) {
         const responseFromUrl = await requestToUrl(res.url, true, extraHTTPHeaders);
 
@@ -1776,6 +1776,44 @@ export const submitForm = async (
     }
   }
 };
+
+export async function initModifiedUserAgent(browser?: string, playwrightDeviceDetailsObject?: object) {
+  const isHeadless = process.env.CRAWLEE_HEADLESS === '1';
+  
+  // If headless mode is enabled, ensure the headless flag is set.
+  if (isHeadless && !constants.launchOptionsArgs.includes('--headless=new')) {
+    constants.launchOptionsArgs.push('--headless=new');
+  }
+  
+  // Build the launch options using your production settings.
+  // headless is forced to false as in your persistent context, and we merge in getPlaywrightLaunchOptions and device details.
+  const launchOptions = {
+    headless: false,
+    ...getPlaywrightLaunchOptions(browser),
+    ...playwrightDeviceDetailsObject,
+  };
+
+  // Launch a temporary persistent context with an empty userDataDir to mimic your production browser setup.
+  const browserContext = await constants.launcher.launchPersistentContext('', launchOptions);
+  const page = await browserContext.newPage();
+
+  // Retrieve the default user agent.
+  const defaultUA = await page.evaluate(() => navigator.userAgent);
+  await browserContext.close();
+
+  // Modify the UA:
+  // Replace "HeadlessChrome" with "Chrome" if present.
+  let modifiedUA = defaultUA.includes('HeadlessChrome')
+    ? defaultUA.replace('HeadlessChrome', 'Chrome')
+    : defaultUA;
+    
+  // Push the modified UA flag into your global launch options.
+  constants.launchOptionsArgs.push(`--user-agent=${modifiedUA}`);
+  // Optionally log the modified UA.
+  // console.log('Modified User Agent:', modifiedUA);
+}
+
+
 /**
  * @param {string} browser browser name ("chrome" or "edge", null for chromium, the default Playwright browser)
  * @returns playwright launch options object. For more details: https://playwright.dev/docs/api/class-browsertype#browser-type-launch

package/src/constants/constants.ts

@@ -7,6 +7,7 @@ import os from 'os';
 import { spawnSync, execSync } from 'child_process';
 import { chromium } from 'playwright';
 import { silentLogger } from '../logs.js';
+import { PageInfo } from '../mergeAxeResults.js';
 
 const filename = fileURLToPath(import.meta.url);
 const dirname = path.dirname(filename);
@@ -177,16 +178,16 @@ export const basicAuthRegex = /^.*\/\/.*:.*@.*$/i;
 export const axeScript = path.join(dirname, '../../node_modules/axe-core/axe.min.js');
 export class UrlsCrawled {
   toScan: string[] = [];
-  scanned: { url: string; actualUrl: string; pageTitle: string }[] = [];
-  invalid: { url: string; actualUrl: string; pageTitle: string }[] = [];
+  scanned: PageInfo[] = [];
+  invalid: PageInfo[] = [];
   scannedRedirects: { fromUrl: string; toUrl: string }[] = [];
   notScannedRedirects: { fromUrl: string; toUrl: string }[] = [];
-  outOfDomain: string[] = [];
-  blacklisted: { url: string; actualUrl: string; pageTitle: string }[] = [];
-  error: { url: string }[] = [];
-  exceededRequests: string[] = [];
-  forbidden: { url: string; actualUrl: string; pageTitle: string }[] = [];
-  userExcluded: { url: string; actualUrl: string; pageTitle: string }[] = [];
+  outOfDomain: PageInfo[] = [];
+  blacklisted: PageInfo[] = [];
+  error: PageInfo[] = [];
+  exceededRequests: PageInfo[] = [];
+  forbidden: PageInfo[] = [];
+  userExcluded: PageInfo[] = [];
   everything: string[] = [];
 
   constructor(urlsCrawled?: Partial<UrlsCrawled>) {