@elastic/synthetics 1.0.0-beta.4 → 1.0.0-beta.41

package/package.json

@@ -1,13 +1,14 @@
 {
   "name": "@elastic/synthetics",
-  "version": "1.0.0-beta.4",
+  "version": "1.0.0-beta.41",
   "description": "Elastic synthetic monitoring agent",
   "main": "dist/index.js",
   "types": "dist/index.d.ts",
   "scripts": {
-    "clean": "rm -rf dist",
+    "clean": "rimraf dist",
     "prepublish": "npm run clean && npm run build",
     "build": "tsc",
+    "build:locations": "sh utils/update_locations.sh",
     "watch": "tsc -w",
     "lint": "eslint . --rulesdir utils/eslint-rules",
     "lint:fix": "npm run lint -- --fix",
@@ -22,7 +23,8 @@
   },
   "files": [
     "dist",
-    "src"
+    "src",
+    "templates"
   ],
   "lint-staged": {
     "*.{js,ts}": [
@@ -38,32 +40,45 @@
   "author": "",
   "license": "MIT",
   "dependencies": {
-    "commander": "^7.0.0",
+    "archiver": "^5.3.1",
+    "commander": "^9.4.1",
     "deepmerge": "^4.2.2",
-    "expect": "^27.0.2",
+    "enquirer": "^2.3.6",
+    "esbuild": "^0.16.10",
+    "expect": "^28.1.3",
     "http-proxy": "^1.18.1",
-    "kleur": "^4.1.3",
-    "micromatch": "^4.0.4",
-    "playwright-chromium": "=1.11.0",
-    "sharp": "^0.28.3",
-    "snakecase-keys": "^3.2.1",
-    "sonic-boom": "^1.3.0",
-    "source-map-support": "^0.5.19",
-    "ts-node": "^9.1.1",
-    "typescript": "^4.1.3"
+    "kleur": "^4.1.5",
+    "micromatch": "^4.0.5",
+    "pirates": "^4.0.5",
+    "playwright-chromium": "=1.31.1",
+    "playwright-core": "=1.31.1",
+    "semver": "^7.3.8",
+    "sharp": "^0.31.1",
+    "snakecase-keys": "^4.0.1",
+    "sonic-boom": "^3.2.0",
+    "source-map-support": "^0.5.21",
+    "typescript": "^4.8.4",
+    "undici": "^5.11.0",
+    "yaml": "^2.1.3"
   },
   "devDependencies": {
-    "@types/jest": "^26.0.19",
-    "@types/micromatch": "^4.0.1",
-    "@types/node": "^14.14.14",
-    "@types/sharp": "^0.28.2",
-    "@typescript-eslint/eslint-plugin": "^3.10.1",
-    "@typescript-eslint/parser": "^3.10.1",
-    "eslint": "^7.15.0",
+    "@types/jest": "^28.1.8",
+    "@types/micromatch": "^4.0.2",
+    "@types/node": "^16.11.59",
+    "@types/sharp": "^0.28.6",
+    "@typescript-eslint/eslint-plugin": "^5.38.0",
+    "@typescript-eslint/parser": "^5.38.0",
+    "eslint": "^8.23.1",
     "husky": "^4.3.6",
-    "jest": "^26.6.3",
+    "is-positive": "3.1.0",
+    "jest": "^28.1.3",
     "lint-staged": "^10.5.3",
-    "prettier": "^2.2.1",
-    "ts-jest": "^26.4.4"
+    "prettier": "^2.7.1",
+    "rimraf": "^3.0.2",
+    "ts-jest": "^28.0.8",
+    "unzipper": "^0.10.11"
+  },
+  "engines": {
+    "node": ">=18.12.0"
   }
 }

package/src/cli.ts

@@ -25,168 +25,236 @@
  *
  */
 
-import { stdin, cwd } from 'process';
-import { resolve } from 'path';
-import merge from 'deepmerge';
-import { step, journey } from './core';
-import { log } from './core/logger';
-import program, { options } from './parse_args';
+import { program, Option } from 'commander';
+import { cwd } from 'process';
+import { CliArgs, PushOptions } from './common_types';
+import { reporters } from './reporters';
 import {
-  findPkgJsonByTraversing,
-  isDepInstalled,
-  isDirectory,
-  totalist,
-} from './helpers';
+  normalizeOptions,
+  parseThrottling,
+  getCommonCommandOpts,
+} from './options';
+import { globalSetup } from './loader';
 import { run } from './';
-import { readConfig } from './config';
-
-const resolvedCwd = cwd();
-/**
- * Set debug based on DEBUG ENV and -d flags
- * namespace - synthetics
- */
-const namespace = 'synthetics';
-if (process.env.DEBUG === namespace || Boolean(options.debug)) {
-  process.env.DEBUG = '1';
-}
-
-const loadInlineScript = source => {
-  const scriptFn = new Function('step', 'page', 'browser', 'params', source);
-  journey('inline', async ({ page, browser, params }) => {
-    scriptFn.apply(null, [step, page, browser, params]);
-  });
-};
-
-async function readStdin() {
-  const chunks = [];
-  stdin.resume();
-  stdin.setEncoding('utf-8');
-  for await (const chunk of stdin) {
-    chunks.push(chunk);
-  }
-  return chunks.join();
-}
+import { runner } from './core';
+import { SyntheticsLocations } from './dsl/monitor';
+import {
+  push,
+  loadSettings,
+  validateSettings,
+  catchIncorrectSettings,
+} from './push';
+import {
+  formatLocations,
+  getLocations,
+  renderLocations,
+  LocationCmdOptions,
+} from './locations';
+import { resolve } from 'path';
+import { Generator } from './generator';
+import { error } from './helpers';
+import { LocationsMap } from './locations/public-locations';
+import { createLightweightMonitors } from './push/monitor';
 
-function requireSuites(suites: Iterable<string>) {
-  for (const suite of suites) {
-    require(suite);
-  }
-}
+/* eslint-disable-next-line @typescript-eslint/no-var-requires */
+const { name, version } = require('../package.json');
 
-/**
- * Handle both directory and files that are passed through TTY
- * and add them to suites
- */
-async function prepareSuites(inputs: string[]) {
-  const suites = new Set<string>();
-  const addSuite = absPath => {
-    log(`Processing file: ${absPath}`);
-    suites.add(require.resolve(absPath));
-  };
-  /**
-   * Match all files inside the directory with the
-   * .journey.{mjs|cjs|js|ts) extensions
-   */
-  const pattern = options.pattern
-    ? new RegExp(options.pattern, 'i')
-    : /.+\.journey\.([mc]js|[jt]s?)$/;
-  /**
-   * Ignore node_modules by default when running suites
-   */
-  const ignored = /node_modules/i;
+const { params, pattern, tags, match, playwrightOpts } = getCommonCommandOpts();
 
-  for (const input of inputs) {
-    const absPath = resolve(resolvedCwd, input);
-    /**
-     * Validate for package.json file before running
-     * the suites
-     */
-    findPkgJsonByTraversing(absPath, resolvedCwd);
-    if (isDirectory(absPath)) {
-      await totalist(absPath, (rel, abs) => {
-        if (pattern.test(rel) && !ignored.test(rel)) {
-          addSuite(abs);
+program
+  .name(`npx ${name}`)
+  .usage('[options] [dir] [files] file')
+  .option(
+    '-c, --config <path>',
+    'configuration path (default: synthetics.config.js)'
+  )
+  .addOption(pattern)
+  .addOption(tags)
+  .addOption(match)
+  .addOption(params)
+  .addOption(
+    new Option('--reporter <value>', `output reporter format`).choices(
+      Object.keys(reporters)
+    )
+  )
+  .option('--inline', 'Run inline journeys from heartbeat')
+  .option('-r, --require <modules...>', 'module(s) to preload')
+  .option('--no-headless', 'run browser in headful mode')
+  .option('--sandbox', 'enable chromium sandboxing')
+  .option('--rich-events', 'Mimics a heartbeat run')
+  .option(
+    '--capability <features...>',
+    'Enable capabilities through feature flags'
+  )
+  .addOption(
+    new Option('--screenshots [flag]', 'take screenshots at end of each step')
+      .choices(['on', 'off', 'only-on-failure'])
+      .default('on')
+  )
+  .option(
+    '--dry-run',
+    "don't actually execute anything, report only registered journeys"
+  )
+  .option(
+    '--outfd <fd>',
+    'specify a file descriptor for logs. Default is stdout',
+    parseInt
+  )
+  .option(
+    '--ws-endpoint <endpoint>',
+    'Browser WebSocket endpoint to connect to'
+  )
+  .option(
+    '--pause-on-error',
+    'pause on error until a keypress is made in the console. Useful during development'
+  )
+  .option(
+    '--ignore-https-errors',
+    'ignores any HTTPS errors in sites being tested, including ones related to unrecognized certs or signatures. This can be insecure!'
+  )
+  .option(
+    '--quiet-exit-code',
+    'always return 0 as an exit code status, regardless of test pass / fail. Only return > 0 exit codes on internal errors where the suite could not be run'
+  )
+  .option(
+    '--throttling <config>',
+    'JSON object to throttle network conditions for download and upload throughput in megabits/second and latency in milliseconds. Ex: { "download": 10, "upload": 5, "latency": 200 }.',
+    parseThrottling,
+    {}
+  )
+  .option('--no-throttling', 'Turns off default network throttling.')
+  .addOption(playwrightOpts)
+  .version(version)
+  .description('Run synthetic tests')
+  .action(async (cliArgs: CliArgs) => {
+    const tearDown = await globalSetup(cliArgs, program.args);
+    try {
+      const options = normalizeOptions(cliArgs);
+      const results = await run(options);
+      /**
+       * Exit with error status if any journey fails
+       */
+      if (!options.quietExitCode) {
+        for (const result of Object.values(results)) {
+          if (result.status === 'failed') {
+            process.exit(1);
+          }
         }
-      });
-    } else {
-      addSuite(absPath);
-    }
-  }
-  return suites.values();
-}
-
-(async () => {
-  if (options.inline) {
-    const source = await readStdin();
-    loadInlineScript(source);
-  } else {
-    /**
-     * Preload modules before running the suites
-     */
-    const modules = [].concat(options.require || []).filter(Boolean);
-    for (const name of modules) {
-      if (isDepInstalled(name)) {
-        require(name);
-      } else {
-        throw new Error(`cannot find module '${name}'`);
       }
+    } catch (e) {
+      console.error(e);
+      process.exit(1);
+    } finally {
+      tearDown();
     }
-    /**
-     * Transform `.ts` files out of the box by invoking
-     * the `ts-node` via `transpile-only` mode that compiles
-     * TS files without doing any extensive type checks
-     */
-    /* eslint-disable-next-line @typescript-eslint/no-var-requires */
-    require('ts-node').register({
-      transpileOnly: true,
-      compilerOptions: {
-        esModuleInterop: true,
-        allowJs: true,
-        target: 'es2018',
-      },
-    });
-    /**
-     * Handle piped files by reading the STDIN
-     * ex: ls example/suites/*.js | npx @elastic/synthetics
-     */
-    const files =
-      program.args.length > 0
-        ? program.args
-        : (await readStdin()).split('\n').filter(Boolean);
-    const suites = await prepareSuites(files);
-    requireSuites(suites);
-  }
+  });
 
-  /**
-   * Use the NODE_ENV variable to control the environment
-   */
-  const environment = process.env['NODE_ENV'] || 'development';
-  /**
-   * Validate and handle configs
-   */
-  const config = readConfig(environment, options.config);
-  const params = merge(config.params, JSON.parse(options.suiteParams));
-  /**
-   * use JSON reporter if json flag is enabled
-   */
-  const reporter = options.json ? 'json' : options.reporter;
+// Push command
+program
+  .command('push')
+  .description(
+    'Push all journeys in the current directory to create monitors within the Kibana monitor management UI'
+  )
+  .addOption(
+    new Option(
+      '--auth <auth>',
+      'API key used for Kibana authentication(https://www.elastic.co/guide/en/kibana/master/api-keys.html).'
+    )
+      .env('SYNTHETICS_API_KEY')
+      .makeOptionMandatory(true)
+  )
+  .option(
+    '--schedule <time-in-minutes>',
+    "schedule in minutes for the pushed monitors. Setting `10`, for example, configures monitors which don't have an interval defined to run every 10 minutes.",
+    parseInt
+  )
+  .addOption(
+    new Option(
+      '--locations <locations...>',
+      'default list of locations from which your monitors will run.'
+    ).choices(SyntheticsLocations)
+  )
+  .option(
+    '--private-locations <locations...>',
+    'default list of private locations from which your monitors will run.'
+  )
+  .option('--url <url>', 'Kibana URL to upload the monitors')
+  .option(
+    '--id <id>',
+    'project id that will be used for logically grouping monitors'
+  )
+  .option(
+    '--space <space>',
+    'the target Kibana spaces for the pushed monitors — spaces help you organise pushed monitors.'
+  )
+  .option('-y, --yes', 'skip all questions and run non-interactively')
+  .addOption(pattern)
+  .addOption(tags)
+  .addOption(match)
+  .addOption(params)
+  .addOption(playwrightOpts)
+  .action(async (cmdOpts: PushOptions) => {
+    const workDir = cwd();
+    const tearDown = await globalSetup({ inline: false, ...program.opts() }, [
+      workDir,
+    ]);
+    try {
+      const settings = await loadSettings();
+      const options = normalizeOptions({
+        ...program.opts(),
+        ...settings,
+        ...cmdOpts,
+      }) as PushOptions;
+      validateSettings(options);
+      await catchIncorrectSettings(settings, options);
+      const monitors = runner.buildMonitors(options);
+      monitors.push(...(await createLightweightMonitors(workDir, options)));
+      await push(monitors, options);
+    } catch (e) {
+      e && console.error(e);
+      process.exit(1);
+    } finally {
+      tearDown();
+    }
+  });
 
-  const results = await run({
-    params: Object.freeze(params),
-    environment,
-    reporter,
-    ...options,
+// Init command
+program
+  .command('init [dir]')
+  .description('Initialize Elastic synthetics project')
+  .action(async (dir = '') => {
+    try {
+      const generator = await new Generator(resolve(process.cwd(), dir));
+      await generator.setup();
+    } catch (e) {
+      e && error(e);
+      process.exit(1);
+    }
   });
 
-  /**
-   * Exit with error status if any journey fails
-   */
-  for (const result of Object.values(results)) {
-    if (result.status === 'failed') {
+// Locations command
+program
+  .command('locations')
+  .description(
+    `List all locations to run the synthetics monitors. Pass optional '--url' and '--auth' to list private locations.`
+  )
+  .option('--url <url>', 'Kibana URL to fetch all public and private locations')
+  .option(
+    '--auth <auth>',
+    'API key used for Kibana authentication(https://www.elastic.co/guide/en/kibana/master/api-keys.html).'
+  )
+  .action(async (cmdOpts: LocationCmdOptions) => {
+    try {
+      let locations = Object.keys(LocationsMap);
+      if (cmdOpts.auth && cmdOpts.url) {
+        const allLocations = await getLocations(cmdOpts);
+        locations = formatLocations(allLocations);
+      }
+      renderLocations(locations);
+    } catch (e) {
+      e && error(e);
       process.exit(1);
     }
-  }
-})().catch(e => {
-  console.error(e);
-  process.exit(1);
-});
+  });
+
+program.parse(process.argv);