@govtechsg/oobee 0.10.20

package/src/index.ts

@@ -0,0 +1,173 @@
+#!/usr/bin/env node
+/* eslint-disable func-names */
+/* eslint-disable no-param-reassign */
+import printMessage from 'print-message';
+import inquirer from 'inquirer';
+import { EnqueueStrategy } from 'crawlee';
+import {
+  getVersion,
+  cleanUp,
+  setHeadlessMode,
+  getUserDataTxt,
+  writeToUserDataTxt,
+} from './utils.js';
+import {
+  prepareData,
+  messageOptions,
+  getPlaywrightDeviceDetailsObject,
+  getBrowserToRun,
+  getScreenToScan,
+  getClonedProfilesWithRandomToken,
+  deleteClonedProfiles,
+} from './constants/common.js';
+import questions from './constants/questions.js';
+import combineRun from './combine.js';
+import { BrowserTypes, RuleFlags, ScannerTypes } from './constants/constants.js';
+
+export type Answers = {
+  headless: boolean;
+  deviceChosen: string;
+  customDevice: string;
+  viewportWidth: number;
+  browserToRun: BrowserTypes;
+  scanner: ScannerTypes;
+  url: string;
+  clonedBrowserDataDir: string;
+  playwrightDeviceDetailsObject: object;
+  nameEmail: string;
+  fileTypes: string;
+  metadata: string;
+  maxpages: number;
+  strategy: string;
+  isLocalFileScan: boolean;
+  finalUrl: string;
+  customFlowLabel: string;
+  specifiedMaxConcurrency: number;
+  blacklistedPatternsFilename: string;
+  additional: string;
+  followRobots: boolean;
+  header: string;
+  safeMode: boolean;
+  exportDirectory: string;
+  zip: string;
+  ruleset: RuleFlags[];
+};
+
+export type Data = {
+  type: ScannerTypes;
+  url: string;
+  entryUrl: string;
+  isHeadless: boolean;
+  deviceChosen: string;
+  customDevice: string;
+  viewportWidth: number;
+  playwrightDeviceDetailsObject: object;
+  maxRequestsPerCrawl: number;
+  strategy: EnqueueStrategy;
+  isLocalFileScan: boolean;
+  browser: string;
+  nameEmail: string;
+  customFlowLabel: string;
+  specifiedMaxConcurrency: number;
+  randomToken: string;
+  fileTypes: string;
+  blacklistedPatternsFilename: string;
+  includeScreenshots: boolean;
+  metadata: string;
+  followRobots: boolean;
+  extraHTTPHeaders: Record<string, string>;
+  safeMode: boolean;
+  userDataDirectory?: string;
+  zip?: string;
+  ruleset: RuleFlags[];
+};
+
+const userData = getUserDataTxt();
+
+const runScan = async (answers: Answers) => {
+  const screenToScan = getScreenToScan(
+    answers.deviceChosen,
+    answers.customDevice,
+    answers.viewportWidth,
+  );
+  answers.playwrightDeviceDetailsObject = getPlaywrightDeviceDetailsObject(
+    answers.deviceChosen,
+    answers.customDevice,
+    answers.viewportWidth,
+  );
+  const { browserToRun } = getBrowserToRun(BrowserTypes.CHROME);
+  deleteClonedProfiles(browserToRun);
+  answers.browserToRun = browserToRun;
+
+  if (!answers.nameEmail) {
+    answers.nameEmail = `${userData.name}:${userData.email}`;
+  }
+
+  answers.fileTypes = 'html-only';
+  answers.metadata = '{}';
+
+  const data: Data = await prepareData(answers);
+  data.userDataDirectory = getClonedProfilesWithRandomToken(data.browser, data.randomToken);
+
+  setHeadlessMode(data.browser, data.isHeadless);
+  printMessage(['Scanning website...'], messageOptions);
+
+  await combineRun(data, screenToScan);
+
+  // Delete cloned directory
+  deleteClonedProfiles(data.browser);
+
+  // Delete dataset and request queues
+  cleanUp(data.randomToken);
+
+  process.exit(0);
+};
+
+if (userData) {
+  printMessage(
+    [
+      `Oobee (ver ${getVersion()})`,
+      'We recommend using Chrome browser for the best experience.',
+      '',
+      `Welcome back ${userData.name}!`,
+      `(Refer to readme.txt on how to change your profile)`,
+    ],
+    {
+      // Note that the color is based on kleur NPM package
+      border: true,
+      borderColor: 'magenta',
+    },
+  );
+
+  inquirer.prompt(questions).then(async answers => {
+    await runScan(answers);
+  });
+} else {
+  printMessage(
+    [`Oobee (ver ${getVersion()})`, 'We recommend using Chrome browser for the best experience.'],
+    {
+      // Note that the color is based on kleur NPM package
+      border: true,
+      borderColor: 'magenta',
+    },
+  );
+
+  printMessage(
+    [
+      `To personalise your experience, we will be collecting your name, email address and app usage data.`,
+      `Your information fully complies with GovTech's Privacy Policy.`,
+    ],
+    {
+      border: false,
+    },
+  );
+
+  inquirer.prompt(questions).then(async answers => {
+    const { name, email } = answers;
+    answers.nameEmail = `${name}:${email}`;
+    await writeToUserDataTxt('name', name);
+    await writeToUserDataTxt('email', email);
+
+    await runScan(answers);
+  });
+}

package/src/logs.ts

@@ -0,0 +1,66 @@
+/* eslint-disable no-console */
+/* eslint-disable no-shadow */
+import { createLogger, format, transports } from 'winston';
+import { guiInfoStatusTypes } from './constants/constants.js';
+import { urlWithoutAuth } from './constants/common.js';
+
+const { combine, timestamp, printf } = format;
+
+// Sample output
+// {"timestamp":"2020-11-25 17:29:07","level":"error","message":"hello world"}
+const logFormat = printf(({ timestamp, level, message }) => {
+  const log = {
+    timestamp: `${timestamp}`,
+    level: `${level}`,
+    message: `${message}`,
+  };
+
+  return JSON.stringify(log);
+});
+
+// transport: storage device for logs
+// Enabled for console and storing into files; Files are overwritten each time
+// All logs in combined.txt, error in errors.txt
+
+const consoleLogger = createLogger({
+  format: combine(timestamp({ format: 'YYYY-MM-DD HH:mm:ss' }), logFormat),
+  transports: [new transports.Console()],
+});
+
+// No display in consoles, this will mostly be used within the interactive script to avoid disrupting the flow
+// Also used in common functions to not link internal information
+// if running from mass scanner, log out errors in console
+const silentLogger = createLogger({
+  format: combine(timestamp({ format: 'YYYY-MM-DD HH:mm:ss' }), logFormat),
+  transports: [
+    process.env.OOBEE_VERBOSE
+      ? new transports.Console({ handleExceptions: true })
+      : new transports.File({ filename: 'errors.txt', level: 'warn', handleExceptions: true }),
+  ].filter(Boolean),
+});
+
+// guiInfoLogger feeds the gui information via console log and is mainly used for scanning process
+export const guiInfoLog = (status, data) => {
+  if (process.env.RUNNING_FROM_PH_GUI || process.env.OOBEE_VERBOSE) {
+    switch (status) {
+      case guiInfoStatusTypes.COMPLETED:
+        console.log('Scan completed');
+        break;
+      case guiInfoStatusTypes.SCANNED:
+      case guiInfoStatusTypes.SKIPPED:
+      case guiInfoStatusTypes.ERROR:
+      case guiInfoStatusTypes.DUPLICATE:
+        console.log(
+          `crawling::${data.numScanned || 0}::${status}::${
+            urlWithoutAuth(data.urlScanned) || 'no url provided'
+          }`,
+        );
+        break;
+      default:
+        console.log(`Status provided to gui info log not recognized: ${status}`);
+        break;
+    }
+  }
+};
+
+export { logFormat, consoleLogger, silentLogger };