@govtechsg/oobee 0.10.20

package/src/cli.ts

@@ -0,0 +1,401 @@
+#!/usr/bin/env node
+import _yargs from 'yargs';
+import { hideBin } from 'yargs/helpers';
+import printMessage from 'print-message';
+import { devices } from 'playwright';
+import { fileURLToPath } from 'url';
+import path from 'path';
+import { cleanUp, setHeadlessMode, getVersion, getStoragePath } from './utils.js';
+import {
+  checkUrl,
+  prepareData,
+  getFileSitemap,
+  validEmail,
+  validName,
+  getBrowserToRun,
+  getPlaywrightDeviceDetailsObject,
+  deleteClonedProfiles,
+  getScreenToScan,
+  getClonedProfilesWithRandomToken,
+  validateDirPath,
+  validateFilePath,
+  validateCustomFlowLabel,
+  parseHeaders,
+} from './constants/common.js';
+import constants, { ScannerTypes } from './constants/constants.js';
+import { cliOptions, messageOptions } from './constants/cliFunctions.js';
+import combineRun from './combine.js';
+import { Answers } from './index.js';
+
+const appVersion = getVersion();
+const yargs = _yargs(hideBin(process.argv));
+
+const options = yargs
+  .version(false)
+  .usage(
+    `Oobee version: ${appVersion}
+Usage: npm run cli -- -c <crawler> -d <device> -w <viewport> -u <url> OPTIONS`,
+  )
+  .strictOptions(true)
+  .options(cliOptions)
+  .example([
+    [
+      `To scan sitemap of website:', 'npm run cli -- -c [ 1 | sitemap ] -u <url_link> [ -d <device> | -w <viewport_width> ]`,
+    ],
+    [
+      `To scan a website', 'npm run cli -- -c [ 2 | website ] -u <url_link> [ -d <device> | -w <viewport_width> ]`,
+    ],
+    [
+      `To start a custom flow scan', 'npm run cli -- -c [ 3 | custom ] -u <url_link> [ -d <device> | -w <viewport_width> ]`,
+    ],
+  ])
+  .coerce('d', option => {
+    const device = devices[option];
+    if (!device && option !== 'Desktop' && option !== 'Mobile') {
+      printMessage(
+        [`Invalid device. Please provide an existing device to start the scan.`],
+        messageOptions,
+      );
+      process.exit(1);
+    }
+    return option;
+  })
+  .coerce('w', option => {
+    if (!option || Number.isNaN(option)) {
+      printMessage([`Invalid viewport width. Please provide a number. `], messageOptions);
+      process.exit(1);
+    } else if (option < 320 || option > 1080) {
+      printMessage(
+        ['Invalid viewport width! Please provide a viewport width between 320-1080 pixels.'],
+        messageOptions,
+      );
+      process.exit(1);
+    }
+    return option;
+  })
+  .coerce('p', option => {
+    if (!Number.isInteger(option) || Number(option) <= 0) {
+      printMessage(
+        [`Invalid maximum number of pages. Please provide a positive integer.`],
+        messageOptions,
+      );
+      process.exit(1);
+    }
+    return option;
+  })
+  .coerce('t', option => {
+    if (!Number.isInteger(option) || Number(option) <= 0) {
+      printMessage(
+        [`Invalid number for max concurrency. Please provide a positive integer.`],
+        messageOptions,
+      );
+      process.exit(1);
+    }
+    return option;
+  })
+  .coerce('k', nameEmail => {
+    if (nameEmail.indexOf(':') === -1) {
+      printMessage(
+        [`Invalid format. Please provide your name and email address separated by ":"`],
+        messageOptions,
+      );
+      process.exit(1);
+    }
+    const [name, email] = nameEmail.split(':');
+    if (name === '' || name === undefined || name === null) {
+      printMessage([`Please provide your name.`], messageOptions);
+      process.exit(1);
+    }
+    if (!validName(name)) {
+      printMessage([`Invalid name. Please provide a valid name.`], messageOptions);
+      process.exit(1);
+    }
+    if (!validEmail(email)) {
+      printMessage(
+        [`Invalid email address. Please provide a valid email address.`],
+        messageOptions,
+      );
+      process.exit(1);
+    }
+    return nameEmail;
+  })
+  .coerce('e', option => {
+    const validationErrors = validateDirPath(option);
+    if (validationErrors) {
+      printMessage([`Invalid exportDirectory directory path. ${validationErrors}`], messageOptions);
+      process.exit(1);
+    }
+    return option;
+  })
+  .coerce('x', option => {
+    const filename = fileURLToPath(import.meta.url);
+    const dirname = `${path.dirname(filename)}/../`; // check in the parent of dist directory
+
+    try {
+      return validateFilePath(option, dirname);
+    } catch (err) {
+      printMessage([`Invalid blacklistedPatternsFilename file path. ${err}`], messageOptions);
+      process.exit(1);
+    }
+
+    // eslint-disable-next-line no-unreachable
+    return null;
+  })
+  .coerce('i', option => {
+    const { choices } = cliOptions.i;
+    if (!choices.includes(option)) {
+      printMessage(
+        [`Invalid value for fileTypes. Please provide valid keywords: ${choices.join(', ')}.`],
+        messageOptions,
+      );
+      process.exit(1);
+    }
+    return option;
+  })
+  .coerce('j', option => {
+    const { isValid, errorMessage } = validateCustomFlowLabel(option);
+    if (!isValid) {
+      printMessage([errorMessage], messageOptions);
+      process.exit(1);
+    }
+    return option;
+  })
+  .coerce('a', option => {
+    const { choices } = cliOptions.a;
+    if (!choices.includes(option)) {
+      printMessage(
+        [`Invalid value for additional. Please provide valid keywords: ${choices.join(', ')}.`],
+        messageOptions,
+      );
+      process.exit(1);
+    }
+    return option;
+  })
+  .coerce('q', option => {
+    try {
+      JSON.parse(option);
+    } catch {
+      // default to empty object
+      return '{}';
+    }
+    return option;
+  })
+  .coerce('m', option => {
+    return option;
+  })
+  .check(argvs => {
+    if (
+      (argvs.scanner === ScannerTypes.CUSTOM || argvs.scanner === ScannerTypes.LOCALFILE) &&
+      argvs.maxpages
+    ) {
+      throw new Error('-p or --maxpages is only available in website and sitemap scans.');
+    }
+    return true;
+  })
+  .check(argvs => {
+    if (argvs.scanner !== ScannerTypes.WEBSITE && argvs.strategy) {
+      throw new Error('-s or --strategy is only available in website scans.');
+    }
+    return true;
+  })
+  .conflicts('d', 'w')
+  .parse() as unknown as Answers;
+
+const scanInit = async (argvs: Answers): Promise<string> => {
+  let isCustomFlow = false;
+  if (argvs.scanner === ScannerTypes.CUSTOM) {
+    isCustomFlow = true;
+  }
+
+  const updatedArgvs = { ...argvs };
+
+  // let chromeDataDir = null;
+  // let edgeDataDir = null;
+  // Empty string for profile directory will use incognito mode in playwright
+  let clonedDataDir = '';
+  const statuses = constants.urlCheckStatuses;
+
+  const { browserToRun, clonedBrowserDataDir } = getBrowserToRun(updatedArgvs.browserToRun, true);
+  updatedArgvs.browserToRun = browserToRun;
+  clonedDataDir = clonedBrowserDataDir;
+
+  if (updatedArgvs.customDevice === 'Desktop' || updatedArgvs.customDevice === 'Mobile') {
+    updatedArgvs.deviceChosen = argvs.customDevice;
+    delete updatedArgvs.customDevice;
+  }
+
+  // Creating the playwrightDeviceDetailObject
+  // for use in crawlDomain & crawlSitemap's preLaunchHook
+  updatedArgvs.playwrightDeviceDetailsObject = getPlaywrightDeviceDetailsObject(
+    updatedArgvs.deviceChosen,
+    updatedArgvs.customDevice,
+    updatedArgvs.viewportWidth,
+  );
+
+  const res = await checkUrl(
+    updatedArgvs.scanner,
+    updatedArgvs.url,
+    updatedArgvs.browserToRun,
+    clonedDataDir,
+    updatedArgvs.playwrightDeviceDetailsObject,
+    isCustomFlow,
+    updatedArgvs.header,
+  );
+  switch (res.status) {
+    case statuses.success.code: {
+      updatedArgvs.finalUrl = res.url;
+      if (process.env.VALIDATE_URL_PH_GUI) {
+        console.log('Url is valid');
+        process.exit(0);
+      }
+      break;
+    }
+    case statuses.unauthorised.code: {
+      printMessage([statuses.unauthorised.message], messageOptions);
+      process.exit(res.status);
+      break;
+    }
+    case statuses.cannotBeResolved.code: {
+      printMessage([statuses.cannotBeResolved.message], messageOptions);
+      process.exit(res.status);
+      break;
+    }
+    case statuses.systemError.code: {
+      printMessage([statuses.systemError.message], messageOptions);
+      process.exit(res.status);
+      break;
+    }
+    case statuses.invalidUrl.code: {
+      if (
+        updatedArgvs.scanner !== ScannerTypes.SITEMAP &&
+        updatedArgvs.scanner !== ScannerTypes.LOCALFILE
+      ) {
+        printMessage([statuses.invalidUrl.message], messageOptions);
+        process.exit(res.status);
+      }
+
+      const finalFilePath = getFileSitemap(updatedArgvs.url);
+      if (finalFilePath) {
+        updatedArgvs.isLocalFileScan = true;
+        updatedArgvs.finalUrl = finalFilePath;
+        if (process.env.VALIDATE_URL_PH_GUI) {
+          console.log('Url is valid');
+          process.exit(0);
+        }
+      } else if (updatedArgvs.scanner === ScannerTypes.LOCALFILE) {
+        printMessage([statuses.notALocalFile.message], messageOptions);
+        process.exit(statuses.notALocalFile.code);
+      } else if (updatedArgvs.scanner !== ScannerTypes.SITEMAP) {
+        printMessage([statuses.notASitemap.message], messageOptions);
+        process.exit(statuses.notASitemap.code);
+      }
+      break;
+    }
+    case statuses.notASitemap.code: {
+      printMessage([statuses.notASitemap.message], messageOptions);
+      process.exit(res.status);
+      break;
+    }
+    case statuses.notALocalFile.code: {
+      printMessage([statuses.notALocalFile.message], messageOptions);
+      process.exit(res.status);
+      break;
+    }
+    case statuses.browserError.code: {
+      printMessage([statuses.browserError.message], messageOptions);
+      process.exit(res.status);
+      break;
+    }
+    default:
+      break;
+  }
+
+  if (updatedArgvs.scanner === ScannerTypes.WEBSITE && !updatedArgvs.strategy) {
+    updatedArgvs.strategy = 'same-domain';
+  }
+
+  const data = await prepareData(updatedArgvs);
+
+  // File clean up after url check
+  // files will clone a second time below if url check passes
+  if (process.env.OOBEE_VERBOSE) {
+    deleteClonedProfiles(data.browser, data.randomToken);
+  } else {
+    deleteClonedProfiles(data.browser); // first deletion
+  }
+
+  if (updatedArgvs.exportDirectory) {
+    constants.exportDirectory = updatedArgvs.exportDirectory;
+  }
+
+  if (process.env.RUNNING_FROM_PH_GUI || process.env.OOBEE_VERBOSE) {
+    const randomTokenMessage = {
+      type: 'randomToken',
+      payload: `${data.randomToken}`,
+    };
+    if (process.send) {
+      process.send(JSON.stringify(randomTokenMessage));
+    }
+  }
+
+  setHeadlessMode(data.browser, data.isHeadless);
+
+  const screenToScan = getScreenToScan(
+    updatedArgvs.deviceChosen,
+    updatedArgvs.customDevice,
+    updatedArgvs.viewportWidth,
+  );
+
+  // Clone profiles a second time
+  clonedDataDir = getClonedProfilesWithRandomToken(data.browser, data.randomToken);
+  data.userDataDirectory = clonedDataDir;
+
+  printMessage([`Oobee version: ${appVersion}`, 'Starting scan...'], messageOptions);
+
+  await combineRun(data, screenToScan);
+
+  // Delete cloned directory
+  if (process.env.OOBEE_VERBOSE) {
+    deleteClonedProfiles(data.browser, data.randomToken);
+  } else {
+    deleteClonedProfiles(data.browser); // second deletion
+  }
+
+  // Delete dataset and request queues
+  await cleanUp(data.randomToken);
+
+  return getStoragePath(data.randomToken);
+};
+
+const optionsAnswer: Answers = {
+  scanner: options.scanner,
+  header: options.header,
+  browserToRun: options.browserToRun,
+  zip: options.zip,
+  url: options.url,
+  finalUrl: options.finalUrl,
+  headless: options.headless,
+  maxpages: options.maxpages,
+  metadata: options.metadata,
+  safeMode: options.safeMode,
+  strategy: options.strategy,
+  fileTypes: options.fileTypes,
+  nameEmail: options.nameEmail,
+  additional: options.additional,
+  customDevice: options.customDevice,
+  deviceChosen: options.deviceChosen,
+  followRobots: options.followRobots,
+  customFlowLabel: options.customFlowLabel,
+  viewportWidth: options.viewportWidth,
+  isLocalFileScan: options.isLocalFileScan,
+  exportDirectory: options.exportDirectory,
+  clonedBrowserDataDir: options.clonedBrowserDataDir,
+  specifiedMaxConcurrency: options.specifiedMaxConcurrency,
+  blacklistedPatternsFilename: options.blacklistedPatternsFilename,
+  playwrightDeviceDetailsObject: options.playwrightDeviceDetailsObject,
+  ruleset: options.ruleset,
+};
+await scanInit(optionsAnswer);
+process.exit(0);
+
+export default options;

package/src/combine.ts

@@ -0,0 +1,240 @@
+import printMessage from 'print-message';
+import { pathToFileURL } from 'url';
+import crawlSitemap from './crawlers/crawlSitemap.js';
+import crawlDomain from './crawlers/crawlDomain.js';
+import crawlLocalFile from './crawlers/crawlLocalFile.js';
+import crawlIntelligentSitemap from './crawlers/crawlIntelligentSitemap.js';
+import generateArtifacts from './mergeAxeResults.js';
+import { getHost, createAndUpdateResultsFolders, createDetailsAndLogs } from './utils.js';
+import { ScannerTypes, UrlsCrawled } from './constants/constants.js';
+import { getBlackListedPatterns, submitForm, urlWithoutAuth } from './constants/common.js';
+import { consoleLogger, silentLogger } from './logs.js';
+import runCustom from './crawlers/runCustom.js';
+import { alertMessageOptions } from './constants/cliFunctions.js';
+import { Data } from './index.js';
+
+// Class exports
+export class ViewportSettingsClass {
+  deviceChosen: string;
+  customDevice: string;
+  viewportWidth: number;
+  playwrightDeviceDetailsObject: any; // You can replace 'any' with a more specific type if possible
+
+  constructor(
+    deviceChosen: string,
+    customDevice: string,
+    viewportWidth: number,
+    playwrightDeviceDetailsObject: any,
+  ) {
+    this.deviceChosen = deviceChosen;
+    this.customDevice = customDevice;
+    this.viewportWidth = viewportWidth;
+    this.playwrightDeviceDetailsObject = playwrightDeviceDetailsObject;
+  }
+}
+
+const combineRun = async (details: Data, deviceToScan: string) => {
+  const envDetails = { ...details };
+
+  const {
+    type,
+    url,
+    nameEmail,
+    randomToken,
+    deviceChosen,
+    customDevice,
+    viewportWidth,
+    playwrightDeviceDetailsObject,
+    maxRequestsPerCrawl,
+    browser,
+    userDataDirectory,
+    strategy,
+    specifiedMaxConcurrency,
+    fileTypes,
+    blacklistedPatternsFilename,
+    includeScreenshots,
+    followRobots,
+    metadata,
+    customFlowLabel = 'Custom Flow',
+    extraHTTPHeaders,
+    safeMode,
+    zip,
+    ruleset,
+  } = envDetails;
+
+  process.env.CRAWLEE_LOG_LEVEL = 'ERROR';
+  process.env.CRAWLEE_STORAGE_DIR = randomToken;
+
+  const host = type === ScannerTypes.SITEMAP || type === ScannerTypes.LOCALFILE ? '' : getHost(url);
+
+  let blacklistedPatterns: string[] | null = null;
+  try {
+    blacklistedPatterns = getBlackListedPatterns(blacklistedPatternsFilename);
+  } catch (error) {
+    consoleLogger.error(error);
+    silentLogger.error(error);
+    process.exit(1);
+  }
+
+  // remove basic-auth credentials from URL
+  const finalUrl = !(type === ScannerTypes.SITEMAP || type === ScannerTypes.LOCALFILE)
+    ? urlWithoutAuth(url)
+    : new URL(pathToFileURL(url));
+
+  // Use the string version of finalUrl to reduce logic at submitForm
+  const finalUrlString = finalUrl.toString();
+
+  const scanDetails = {
+    startTime: new Date(),
+    endTime: new Date(),
+    crawlType: type,
+    requestUrl: finalUrl,
+    urlsCrawled: new UrlsCrawled(),
+  };
+
+  const viewportSettings: ViewportSettingsClass = new ViewportSettingsClass(
+    deviceChosen,
+    customDevice,
+    viewportWidth,
+    playwrightDeviceDetailsObject,
+  );
+
+  let urlsCrawledObj: UrlsCrawled;
+  switch (type) {
+    case ScannerTypes.CUSTOM:
+      urlsCrawledObj = await runCustom(
+        url,
+        randomToken,
+        viewportSettings,
+        blacklistedPatterns,
+        includeScreenshots,
+      );
+      break;
+
+    case ScannerTypes.SITEMAP:
+      urlsCrawledObj = await crawlSitemap(
+        url,
+        randomToken,
+        host,
+        viewportSettings,
+        maxRequestsPerCrawl,
+        browser,
+        userDataDirectory,
+        specifiedMaxConcurrency,
+        fileTypes,
+        blacklistedPatterns,
+        includeScreenshots,
+        extraHTTPHeaders,
+      );
+      break;
+
+    case ScannerTypes.LOCALFILE:
+      urlsCrawledObj = await crawlLocalFile(
+        url,
+        randomToken,
+        host,
+        viewportSettings,
+        maxRequestsPerCrawl,
+        browser,
+        userDataDirectory,
+        specifiedMaxConcurrency,
+        fileTypes,
+        blacklistedPatterns,
+        includeScreenshots,
+        extraHTTPHeaders,
+      );
+      break;
+
+    case ScannerTypes.INTELLIGENT:
+      urlsCrawledObj = await crawlIntelligentSitemap(
+        url,
+        randomToken,
+        host,
+        viewportSettings,
+        maxRequestsPerCrawl,
+        browser,
+        userDataDirectory,
+        strategy,
+        specifiedMaxConcurrency,
+        fileTypes,
+        blacklistedPatterns,
+        includeScreenshots,
+        followRobots,
+        extraHTTPHeaders,
+        safeMode,
+      );
+      break;
+
+    case ScannerTypes.WEBSITE:
+      urlsCrawledObj = await crawlDomain({
+        url,
+        randomToken,
+        host,
+        viewportSettings,
+        maxRequestsPerCrawl,
+        browser,
+        userDataDirectory,
+        strategy,
+        specifiedMaxConcurrency,
+        fileTypes,
+        blacklistedPatterns,
+        includeScreenshots,
+        followRobots,
+        extraHTTPHeaders,
+        safeMode,
+        ruleset,
+      });
+      break;
+
+    default:
+      consoleLogger.error(`type: ${type} not defined`);
+      silentLogger.error(`type: ${type} not defined`);
+      process.exit(1);
+  }
+
+  scanDetails.endTime = new Date();
+  scanDetails.urlsCrawled = urlsCrawledObj;
+  await createDetailsAndLogs(randomToken);
+  if (scanDetails.urlsCrawled) {
+    if (scanDetails.urlsCrawled.scanned.length > 0) {
+      await createAndUpdateResultsFolders(randomToken);
+      const pagesNotScanned = [
+        ...urlsCrawledObj.error,
+        ...urlsCrawledObj.invalid,
+        ...urlsCrawledObj.forbidden,
+      ];
+      const basicFormHTMLSnippet = await generateArtifacts(
+        randomToken,
+        url,
+        type,
+        deviceToScan,
+        urlsCrawledObj.scanned,
+        pagesNotScanned,
+        customFlowLabel,
+        undefined,
+        scanDetails,
+        zip,
+      );
+      const [name, email] = nameEmail.split(':');
+
+      await submitForm(
+        browser,
+        userDataDirectory,
+        url, // scannedUrl
+        new URL(finalUrlString).href, // entryUrl
+        type,
+        email,
+        name,
+        JSON.stringify(basicFormHTMLSnippet),
+        urlsCrawledObj.scanned.length,
+        urlsCrawledObj.scannedRedirects.length,
+        pagesNotScanned.length,
+        metadata,
+      );
+    }
+  } else {
+    printMessage([`No pages were scanned.`], alertMessageOptions);
+  }
+};
+
+export default combineRun;

package/src/constants/__tests__/common.test.ts

@@ -0,0 +1,44 @@
+import { jest } from '@jest/globals';
+import axios from 'axios';
+import * as sampleData from '../sampleData';
+import { getLinksFromSitemap } from '../common';
+import constants from '../constants';
+
+jest.mock('axios');
+
+describe('test getLinksFromSitemap', () => {
+  const maxRequestsPerCrawl = constants.maxRequestsPerCrawl;
+
+  test('should only get links from loc tags in an XML sitemap and not include namespace links or links in comments', async () => {
+    axios.get = jest.fn().mockResolvedValue({ data: sampleData.sampleXmlSitemap });
+    // URL passed to getLinksFromSitemap here doesn't matter because the response from any get requests is mocked
+    const links = await getLinksFromSitemap('http://mockUrl/sitemap.xml', maxRequestsPerCrawl);
+    expect(links).toEqual(sampleData.sampleXmlSitemapLinks.slice(0, maxRequestsPerCrawl));
+  });
+
+  test('should only get links from link tags in a RSS feed sitemap, and duplicate links should only be added once', async () => {
+    axios.get = jest.fn().mockResolvedValue({ data: sampleData.sampleRssFeed });
+    const links = await getLinksFromSitemap('http://mockUrl/rssfeed.xml', maxRequestsPerCrawl);
+    expect(links).toEqual(sampleData.sampleRssFeedLinks.slice(0, maxRequestsPerCrawl));
+  });
+
+  test('should only get links from the href property in link tags in an Atom feed sitemap', async () => {
+    axios.get = jest.fn().mockResolvedValue({ data: sampleData.sampleAtomFeed });
+    const links = await getLinksFromSitemap('http://mockUrl/atomfeed.xml', maxRequestsPerCrawl);
+    expect(links).toEqual(sampleData.sampleAtomFeedLinks.slice(0, maxRequestsPerCrawl));
+  });
+
+  test('should get all links from a txt sitemap', async () => {
+    axios.get = jest.fn().mockResolvedValue({ data: sampleData.sampleTxtSitemap });
+    const links = await getLinksFromSitemap('http://mockUrl/sitemap.txt', maxRequestsPerCrawl);
+    expect(links).toEqual(sampleData.sampleTxtSitemapLinks.slice(0, maxRequestsPerCrawl));
+  });
+
+  test('should get all links from a non standard XML sitemap', async () => {
+    axios.get = jest.fn().mockResolvedValue({ data: sampleData.sampleNonStandardXmlSitemap });
+    const links = await getLinksFromSitemap('http://mockUrl/weirdSitemap.xml', maxRequestsPerCrawl);
+    expect(links).toEqual(
+      sampleData.sampleNonStandardXmlSitemapLinks.slice(0, maxRequestsPerCrawl),
+    );
+  });
+});