@govtechsg/oobee 0.10.76 → 0.10.77

package/.github/workflows/publish.yml

@@ -11,8 +11,15 @@ jobs:
       # Setup .npmrc file to publish to npm
       - uses: actions/setup-node@v3
         with:
-          node-version: '16.x'
+          node-version: '22.x'
           registry-url: 'https://registry.npmjs.org'
+
+      - run: npm ci
+        continue-on-error: false
+
+      - run: npm run build
+        continue-on-error: false
+
       - run: npm publish
         env:
           NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }}

package/INTEGRATION.md

@@ -106,15 +106,17 @@ Get the Oobee custom functions to run accessibility scan. Call this after `getAx
 
   - Object consisting of the current page url, current page title and axe scan result. `{ pageUrl, pageTitle, axeScanResults }`
 
-`async pushScanResults(res, metadata, elementsToClick)`
+`async pushScanResults(res, metadata, elementsToClick, page, disableScreenshots)`
 
 Process scan results to be included in the report.
 
 Parameter(s):
 
 - `res`: Object consisting of the current page url, current page title and axe scan result. ` {pageUrl, pageTitle, axeScanResults}`
-- `metadata` (optional): Additional information to be displayed for each page scanned in the report
-- `elementsToClick` (optional): Elements clicked during the test to reveal hidden elements. Required to be able identify hidden elements if they were scanned for screenshot purposes. Ensure selectors resolve to a single element.
+- `metadata` (optional): Additional string information to be stored. Useful for distinguishing between different states of the same page. Note: This feature is not implemented in the report HTML currently.
+- `elementsToClick` (optional): A list of CSS selectors to click to reveal hidden elements (like modals or menus) before taking screenshots. This is only used if Oobee launches its own browser for screenshots (i.e., when `page` is not provided and `disableScreenshots` is false). Reproducing the state allows the screenshots to capture the violating elements correctly.
+- `page` (optional): Playwright Page object. If provided, reuses the existing page for taking screenshots of affected elements.
+- `disableScreenshots` (optional): Boolean to disable screenshot capturing. Useful when integrating with tools like Cypress where the browser context cannot be easily shared with Node.js.
 
 Returns:
 
@@ -144,6 +146,8 @@ With reference to an instance of Oobee as `oobeeA11y`:
    - It is possible to run the scan for specific sections or elements in the page. One way to do this is to pass an array of CSS selectors of the elements to be scanned into `runA11yScan`. For example, `runA11yScan(['#my-component', 'button'])`. Other acceptable forms of argument can be found [here](https://github.com/dequelabs/axe-core/blob/develop/doc/API.md#context-parameter).
 4. Pass the scan results back into the NodeJS environment where `oobeeA11y` is in.
 5. Push the results using `await oobeeA11y.pushScanResults(scanResults)`.
+   - For Playwright, pass the `page` object as the 4th argument: `await oobeeA11y.pushScanResults(scanResults, undefined, undefined, page)`. This optimizes performance by reusing the existing browser for screenshots.
+   - For Cypress, to avoid launching a separate browser for screenshots, you must handle screenshot capturing manually and set `disableScreenshots` to `true`. Refer to the provided Cypress examples for the implementation details.
 6. Repeat steps 2-5 as many times as desired.
 7. Terminate Oobee by using `await oobeeA11y.terminate()`. A folder containing the details and report of your scan will be created, under the directory `results` which can be found in your project's root directory.
 

package/dist/cli.js

@@ -0,0 +1,252 @@
+#!/usr/bin/env node
+import _yargs from 'yargs';
+import { hideBin } from 'yargs/helpers';
+import printMessage from 'print-message';
+import { devices } from 'playwright';
+import { fileURLToPath } from 'url';
+import path from 'path';
+import { setHeadlessMode, getVersion, getStoragePath, listenForCleanUp, cleanUpAndExit } from './utils.js';
+import { checkUrl, prepareData, validEmail, validName, getScreenToScan, validateDirPath, validateFilePath, validateCustomFlowLabel, } from './constants/common.js';
+import constants, { ScannerTypes } from './constants/constants.js';
+import { cliOptions, messageOptions } from './constants/cliFunctions.js';
+import combineRun from './combine.js';
+import { consoleLogger } from './logs.js';
+const appVersion = getVersion();
+const yargs = _yargs(hideBin(process.argv));
+const options = yargs
+    .version(false)
+    .usage(`Oobee version: ${appVersion}
+Usage: npm run cli -- -c <crawler> -d <device> -w <viewport> -u <url> OPTIONS`)
+    .strictOptions(true)
+    .options(cliOptions)
+    .example([
+    [
+        `To scan sitemap of website:', 'npm run cli -- -c [ 1 | sitemap ] -u <url_link> [ -d <device> | -w <viewport_width> ]`,
+    ],
+    [
+        `To scan a website', 'npm run cli -- -c [ 2 | website ] -u <url_link> [ -d <device> | -w <viewport_width> ]`,
+    ],
+    [
+        `To start a custom flow scan', 'npm run cli -- -c [ 3 | custom ] -u <url_link> [ -d <device> | -w <viewport_width> ]`,
+    ],
+])
+    .coerce('d', option => {
+    const device = devices[option];
+    if (!device && option !== 'Desktop' && option !== 'Mobile') {
+        printMessage([`Invalid device. Please provide an existing device to start the scan.`], messageOptions);
+        cleanUpAndExit(1);
+    }
+    return option;
+})
+    .coerce('w', option => {
+    if (!option || Number.isNaN(option)) {
+        printMessage([`Invalid viewport width. Please provide a number. `], messageOptions);
+        cleanUpAndExit(1);
+    }
+    else if (option < 320 || option > 1080) {
+        printMessage(['Invalid viewport width! Please provide a viewport width between 320-1080 pixels.'], messageOptions);
+        cleanUpAndExit(1);
+    }
+    return option;
+})
+    .coerce('p', option => {
+    if (!Number.isInteger(option) || Number(option) <= 0) {
+        printMessage([`Invalid maximum number of pages. Please provide a positive integer.`], messageOptions);
+        cleanUpAndExit(1);
+    }
+    return option;
+})
+    .coerce('t', option => {
+    if (!Number.isInteger(option) || Number(option) <= 0) {
+        printMessage([`Invalid number for max concurrency. Please provide a positive integer.`], messageOptions);
+        cleanUpAndExit(1);
+    }
+    return option;
+})
+    .coerce('k', nameEmail => {
+    if (nameEmail.indexOf(':') === -1) {
+        printMessage([`Invalid format. Please provide your name and email address separated by ":"`], messageOptions);
+        cleanUpAndExit(1);
+    }
+    const [name, email] = nameEmail.split(':');
+    if (name === '' || name === undefined || name === null) {
+        printMessage([`Please provide your name.`], messageOptions);
+        cleanUpAndExit(1);
+    }
+    if (!validName(name)) {
+        printMessage([`Invalid name. Please provide a valid name.`], messageOptions);
+        cleanUpAndExit(1);
+    }
+    if (!validEmail(email)) {
+        printMessage([`Invalid email address. Please provide a valid email address.`], messageOptions);
+        cleanUpAndExit(1);
+    }
+    return nameEmail;
+})
+    .coerce('e', option => {
+    const validationErrors = validateDirPath(option);
+    if (validationErrors) {
+        printMessage([`Invalid exportDirectory directory path. ${validationErrors}`], messageOptions);
+        cleanUpAndExit(1);
+    }
+    return option;
+})
+    .coerce('x', option => {
+    const filename = fileURLToPath(import.meta.url);
+    const dirname = `${path.dirname(filename)}/../`; // check in the parent of dist directory
+    try {
+        return validateFilePath(option, dirname);
+    }
+    catch (err) {
+        printMessage([`Invalid blacklistedPatternsFilename file path. ${err}`], messageOptions);
+        cleanUpAndExit(1);
+    }
+})
+    .coerce('i', option => {
+    const { choices } = cliOptions.i;
+    if (!choices.includes(option)) {
+        printMessage([`Invalid value for fileTypes. Please provide valid keywords: ${choices.join(', ')}.`], messageOptions);
+        cleanUpAndExit(1);
+    }
+    return option;
+})
+    .coerce('j', option => {
+    const { isValid, errorMessage } = validateCustomFlowLabel(option);
+    if (!isValid) {
+        printMessage([errorMessage], messageOptions);
+        cleanUpAndExit(1);
+    }
+    return option;
+})
+    .coerce('a', option => {
+    const { choices } = cliOptions.a;
+    if (!choices.includes(option)) {
+        printMessage([`Invalid value for additional. Please provide valid keywords: ${choices.join(', ')}.`], messageOptions);
+        cleanUpAndExit(1);
+    }
+    return option;
+})
+    .coerce('q', option => {
+    try {
+        JSON.parse(option);
+    }
+    catch {
+        // default to empty object
+        return '{}';
+    }
+    return option;
+})
+    .coerce('m', option => {
+    return option;
+})
+    .check(argvs => {
+    if (argvs.scanner === ScannerTypes.CUSTOM && argvs.maxpages) {
+        throw new Error('-p or --maxpages is only available in website, sitemap and local file scans.');
+    }
+    return true;
+})
+    .check(argvs => {
+    if (argvs.scanner !== ScannerTypes.WEBSITE && argvs.strategy) {
+        throw new Error('-s or --strategy is only available in website scans.');
+    }
+    return true;
+})
+    .coerce('l', (option) => {
+    const duration = Number(option);
+    if (isNaN(duration) || duration < 0) {
+        printMessage(['Invalid scan duration. Please provide a positive number of seconds.'], messageOptions);
+        cleanUpAndExit(1);
+    }
+    return duration;
+})
+    .check(argvs => {
+    if (argvs.scanner === ScannerTypes.CUSTOM && typeof argvs.scanDuration === 'number' && argvs.scanDuration > 0) {
+        throw new Error('-l or --scanDuration is not allowed for custom flow scans.');
+    }
+    return true;
+})
+    .conflicts('d', 'w')
+    .parse();
+const scanInit = async (argvs) => {
+    const updatedArgvs = { ...argvs };
+    // Cannot use data.browser and data.isHeadless as the connectivity check comes first before prepareData
+    setHeadlessMode(updatedArgvs.browserToRun, updatedArgvs.headless);
+    const statuses = constants.urlCheckStatuses;
+    let data;
+    try {
+        data = await prepareData(updatedArgvs);
+    }
+    catch (e) {
+        consoleLogger.error(`Error preparing data: ${e.message}\n${e.stack}`);
+        cleanUpAndExit(1);
+    }
+    // Executes cleanUp script if error encountered
+    listenForCleanUp(data.randomToken);
+    const res = await checkUrl(data.type, data.entryUrl, data.browser, data.userDataDirectory, data.playwrightDeviceDetailsObject, data.extraHTTPHeaders, data.fileTypes);
+    if (res.httpStatus)
+        consoleLogger.info(`Connectivity Check HTTP Response Code: ${res.httpStatus}`);
+    if (res.status === statuses.success.code) {
+        data.url = res.url;
+        if (process.env.OOBEE_VALIDATE_URL) {
+            consoleLogger.info('Url is valid');
+            cleanUpAndExit(0, data.randomToken);
+            return;
+        }
+        // fall through (continue normal flow after success)
+    }
+    else {
+        const match = Object.values(statuses).find((s) => s.code === res.status);
+        const msg = match && 'message' in match ? match.message : 'Unknown error';
+        printMessage([msg], messageOptions);
+        consoleLogger.info(msg);
+        cleanUpAndExit(res.status);
+        return;
+    }
+    if (process.env.OOBEE_VERBOSE) {
+        const randomTokenMessage = {
+            type: 'randomToken',
+            payload: `${data.randomToken}`,
+        };
+        if (process.send) {
+            process.send(JSON.stringify(randomTokenMessage));
+        }
+    }
+    const screenToScan = getScreenToScan(data.deviceChosen, data.customDevice, data.viewportWidth);
+    printMessage([`Oobee version: ${appVersion}`, 'Starting scan...'], messageOptions);
+    consoleLogger.info(`Oobee version: ${appVersion}`);
+    await combineRun(data, screenToScan);
+    return getStoragePath(data.randomToken);
+};
+const optionsAnswer = {
+    scanner: options.scanner,
+    header: options.header,
+    browserToRun: options.browserToRun,
+    zip: options.zip,
+    url: options.url,
+    finalUrl: options.finalUrl,
+    headless: options.headless,
+    maxpages: options.maxpages,
+    metadata: options.metadata,
+    safeMode: options.safeMode,
+    strategy: options.strategy,
+    fileTypes: options.fileTypes,
+    nameEmail: options.nameEmail,
+    additional: options.additional,
+    customDevice: options.customDevice,
+    deviceChosen: options.deviceChosen,
+    followRobots: options.followRobots,
+    customFlowLabel: options.customFlowLabel,
+    viewportWidth: options.viewportWidth,
+    isLocalFileScan: options.isLocalFileScan,
+    exportDirectory: options.exportDirectory,
+    clonedBrowserDataDir: options.clonedBrowserDataDir,
+    specifiedMaxConcurrency: options.specifiedMaxConcurrency,
+    blacklistedPatternsFilename: options.blacklistedPatternsFilename,
+    playwrightDeviceDetailsObject: options.playwrightDeviceDetailsObject,
+    ruleset: options.ruleset,
+    generateJsonFiles: options.generateJsonFiles,
+    scanDuration: options.scanDuration,
+};
+await scanInit(optionsAnswer);
+cleanUpAndExit(0);
+export default options;

package/dist/combine.js

@@ -0,0 +1,221 @@
+import printMessage from 'print-message';
+import { pathToFileURL } from 'url';
+import crawlSitemap from './crawlers/crawlSitemap.js';
+import crawlDomain from './crawlers/crawlDomain.js';
+import crawlLocalFile from './crawlers/crawlLocalFile.js';
+import crawlIntelligentSitemap from './crawlers/crawlIntelligentSitemap.js';
+import generateArtifacts from './mergeAxeResults.js';
+import { getHost, createAndUpdateResultsFolders, cleanUpAndExit, getStoragePath } from './utils.js';
+import { ScannerTypes, UrlsCrawled } from './constants/constants.js';
+import { getBlackListedPatterns, submitForm } from './constants/common.js';
+import { consoleLogger } from './logs.js';
+import runCustom from './crawlers/runCustom.js';
+import { alertMessageOptions } from './constants/cliFunctions.js';
+import { isS3UploadEnabled, getS3MetadataFromEnv, getS3UploadPrefix, uploadFolderToS3, } from './services/s3Uploader.js';
+// Class exports
+export class ViewportSettingsClass {
+    constructor(deviceChosen, customDevice, viewportWidth, playwrightDeviceDetailsObject) {
+        this.deviceChosen = deviceChosen;
+        this.customDevice = customDevice;
+        this.viewportWidth = viewportWidth;
+        this.playwrightDeviceDetailsObject = playwrightDeviceDetailsObject;
+    }
+}
+const combineRun = async (details, deviceToScan) => {
+    const envDetails = { ...details };
+    const { type, url, nameEmail, randomToken, deviceChosen, customDevice, viewportWidth, playwrightDeviceDetailsObject, maxRequestsPerCrawl, browser, userDataDirectory, strategy, // Allow subdomains: if checked, = 'same-domain'
+    specifiedMaxConcurrency, // Slow scan mode: if checked, = '1'
+    fileTypes, blacklistedPatternsFilename, includeScreenshots, // Include screenshots: if checked, = 'true'
+    followRobots, // Adhere to robots.txt: if checked, = 'true'
+    metadata, customFlowLabel = 'None', extraHTTPHeaders, safeMode, zip, ruleset, // Enable custom checks, Enable WCAG AAA: if checked, = 'enable-wcag-aaa')
+    generateJsonFiles, scanDuration, } = envDetails;
+    process.env.CRAWLEE_LOG_LEVEL = 'ERROR';
+    process.env.CRAWLEE_STORAGE_DIR = randomToken;
+    if (process.env.CRAWLEE_SYSTEM_INFO_V2 === undefined) {
+        // Set the environment variable to enable system info v2
+        // Resolves issue with when wmic is not installed on Windows
+        process.env.CRAWLEE_SYSTEM_INFO_V2 = '1';
+    }
+    const host = type === ScannerTypes.SITEMAP || type === ScannerTypes.LOCALFILE ? '' : getHost(url);
+    let blacklistedPatterns = null;
+    try {
+        blacklistedPatterns = getBlackListedPatterns(blacklistedPatternsFilename);
+    }
+    catch (error) {
+        consoleLogger.error(error);
+        cleanUpAndExit(1);
+    }
+    // remove basic-auth credentials from URL
+    const finalUrl = !(type === ScannerTypes.SITEMAP || type === ScannerTypes.LOCALFILE)
+        ? new URL(url)
+        : new URL(pathToFileURL(url));
+    // Use the string version of finalUrl to reduce logic at submitForm
+    const finalUrlString = finalUrl.toString();
+    const scanDetails = {
+        startTime: new Date(),
+        endTime: new Date(),
+        crawlType: type,
+        requestUrl: finalUrl,
+        urlsCrawled: new UrlsCrawled(),
+        isIncludeScreenshots: envDetails.includeScreenshots,
+        isAllowSubdomains: envDetails.strategy,
+        isEnableCustomChecks: envDetails.ruleset,
+        isEnableWcagAaa: envDetails.ruleset,
+        isSlowScanMode: envDetails.specifiedMaxConcurrency,
+        isAdhereRobots: envDetails.followRobots,
+        deviceChosen: deviceToScan,
+        nameEmail: undefined,
+    };
+    // Parse nameEmail and add it to scanDetails for use in generateArtifacts
+    if (nameEmail) {
+        const [name, email] = nameEmail.split(':');
+        scanDetails.nameEmail = { name, email };
+    }
+    const viewportSettings = new ViewportSettingsClass(deviceChosen, customDevice, viewportWidth, playwrightDeviceDetailsObject);
+    let urlsCrawledObj;
+    let uiCustomFlowLabel;
+    let durationExceeded = false;
+    switch (type) {
+        case ScannerTypes.CUSTOM:
+            const res = await runCustom(url, randomToken, viewportSettings, blacklistedPatterns, includeScreenshots, customFlowLabel && customFlowLabel !== 'None' ? customFlowLabel : '');
+            urlsCrawledObj = res.urlsCrawled;
+            uiCustomFlowLabel = res.customFlowLabel;
+            break;
+        case ScannerTypes.SITEMAP:
+            const sitemapResult = await crawlSitemap({
+                sitemapUrl: url,
+                randomToken,
+                host,
+                viewportSettings,
+                maxRequestsPerCrawl,
+                browser,
+                userDataDirectory,
+                specifiedMaxConcurrency,
+                fileTypes,
+                blacklistedPatterns,
+                includeScreenshots,
+                extraHTTPHeaders,
+                scanDuration,
+            });
+            urlsCrawledObj = sitemapResult.urlsCrawled;
+            durationExceeded = sitemapResult.durationExceeded;
+            break;
+        case ScannerTypes.LOCALFILE:
+            const localFileResult = await crawlLocalFile({
+                url,
+                randomToken,
+                host,
+                viewportSettings,
+                maxRequestsPerCrawl,
+                browser,
+                userDataDirectory,
+                specifiedMaxConcurrency,
+                fileTypes,
+                blacklistedPatterns,
+                includeScreenshots,
+                extraHTTPHeaders,
+                scanDuration,
+            });
+            if (localFileResult) {
+                if ('urlsCrawled' in localFileResult) {
+                    urlsCrawledObj = localFileResult.urlsCrawled;
+                    durationExceeded = localFileResult.durationExceeded;
+                }
+                else {
+                    urlsCrawledObj = localFileResult;
+                }
+            }
+            break;
+        case ScannerTypes.INTELLIGENT:
+            const intelligentResult = await crawlIntelligentSitemap(url, randomToken, host, viewportSettings, maxRequestsPerCrawl, browser, userDataDirectory, strategy, specifiedMaxConcurrency, fileTypes, blacklistedPatterns, includeScreenshots, followRobots, extraHTTPHeaders, safeMode, scanDuration);
+            urlsCrawledObj = intelligentResult.urlsCrawled;
+            durationExceeded = intelligentResult.durationExceeded;
+            break;
+        case ScannerTypes.WEBSITE:
+            const websiteResult = await crawlDomain({
+                url,
+                randomToken,
+                host,
+                viewportSettings,
+                maxRequestsPerCrawl,
+                browser,
+                userDataDirectory,
+                strategy,
+                specifiedMaxConcurrency,
+                fileTypes,
+                blacklistedPatterns,
+                includeScreenshots,
+                followRobots,
+                extraHTTPHeaders,
+                scanDuration,
+                safeMode,
+                ruleset,
+            });
+            urlsCrawledObj = websiteResult.urlsCrawled;
+            durationExceeded = websiteResult.durationExceeded;
+            break;
+        default:
+            consoleLogger.error(`type: ${type} not defined`);
+            cleanUpAndExit(1);
+    }
+    scanDetails.endTime = new Date();
+    scanDetails.urlsCrawled = urlsCrawledObj;
+    if (scanDetails.urlsCrawled) {
+        if (scanDetails.urlsCrawled.scanned.length > 0) {
+            await createAndUpdateResultsFolders(randomToken);
+            const pagesNotScanned = [
+                ...urlsCrawledObj.error,
+                ...urlsCrawledObj.invalid,
+                ...urlsCrawledObj.forbidden,
+                ...urlsCrawledObj.userExcluded,
+            ];
+            const basicFormHTMLSnippet = await generateArtifacts(randomToken, url, type, deviceToScan, urlsCrawledObj.scanned, pagesNotScanned, uiCustomFlowLabel && uiCustomFlowLabel.length > 0
+                ? uiCustomFlowLabel
+                : customFlowLabel || 'None', undefined, scanDetails, zip, generateJsonFiles);
+            const [name, email] = nameEmail.split(':');
+            // Upload results to S3 if environment variables are set
+            if (isS3UploadEnabled()) {
+                const siteName = (urlsCrawledObj.scanned[0]?.pageTitle ?? '')
+                    .replace(/^\d+\s*:\s*/, '')
+                    .trim();
+                const scanMetadata = getS3MetadataFromEnv(siteName, durationExceeded);
+                const s3Prefix = getS3UploadPrefix();
+                if (scanMetadata && s3Prefix) {
+                    try {
+                        const storagePath = getStoragePath(randomToken);
+                        consoleLogger.info('Starting S3 upload...');
+                        consoleLogger.info(`Upload path: ${s3Prefix}`);
+                        const uploadedFiles = await uploadFolderToS3(storagePath, s3Prefix, scanMetadata);
+                        consoleLogger.info(`Successfully uploaded ${uploadedFiles.length} files to S3`);
+                    }
+                    catch (error) {
+                        const errorMessage = error instanceof Error ? error.message : String(error);
+                        consoleLogger.error(`Failed to upload results to S3: ${errorMessage}`);
+                        // Don't fail the scan if S3 upload fails
+                        consoleLogger.warn('Continuing without S3 upload...');
+                    }
+                }
+                else {
+                    consoleLogger.warn('S3 upload enabled but metadata/prefix not available');
+                }
+            }
+            else {
+                consoleLogger.info('S3 upload not enabled (missing environment variables)');
+            }
+            await submitForm(browser, userDataDirectory, url, // scannedUrl
+            new URL(finalUrlString).href, // entryUrl
+            type, email, name, JSON.stringify(basicFormHTMLSnippet), urlsCrawledObj.scanned.length, urlsCrawledObj.scannedRedirects.length, pagesNotScanned.length, metadata);
+        }
+        else {
+            // No page were scanned because the URL loaded does not meet the crawler requirements
+            printMessage([`No pages were scanned.`], alertMessageOptions);
+            cleanUpAndExit(1, randomToken, true);
+        }
+    }
+    else {
+        // No page were scanned because the URL loaded does not meet the crawler requirements
+        printMessage([`No pages were scanned.`], alertMessageOptions);
+        cleanUpAndExit(1, randomToken, true);
+    }
+};
+export default combineRun;