@elastic/synthetics 1.0.0-beta.22 → 1.0.0-beta.25

package/src/cli.ts

@@ -25,200 +25,126 @@
  *
  */
 
-import { stdin, cwd } from 'process';
-import { resolve } from 'path';
-import merge from 'deepmerge';
-import { step, journey } from './core';
-import { log } from './core/logger';
-import program, { options } from './parse_args';
-import { expect } from './core/expect';
-import {
-  findPkgJsonByTraversing,
-  isDepInstalled,
-  isDirectory,
-  totalist,
-  parseNetworkConditions,
-} from './helpers';
+import { program, Option } from 'commander';
+import { CliArgs } from './common_types';
+import { reporters } from './reporters';
+import { DEFAULT_NETWORK_CONDITIONS_ARG } from './helpers';
+import { normalizeOptions } from './options';
+import { loadTestFiles } from './loader';
 import { run } from './';
-import { readConfig } from './config';
 
-const resolvedCwd = cwd();
-/**
- * Set debug based on DEBUG ENV and -d flags
- * namespace - synthetics
- */
-const namespace = 'synthetics';
-if (process.env.DEBUG === namespace || Boolean(options.debug)) {
-  process.env.DEBUG = '1';
-}
-
-const loadInlineScript = source => {
-  const scriptFn = new Function(
-    'step',
-    'page',
-    'context',
-    'browser',
-    'params',
-    'expect',
-    source
-  );
-  journey('inline', ({ page, context, browser, params }) => {
-    scriptFn.apply(null, [step, page, context, browser, params, expect]);
-  });
-};
-
-async function readStdin() {
-  const chunks = [];
-  stdin.resume();
-  stdin.setEncoding('utf-8');
-  for await (const chunk of stdin) {
-    chunks.push(chunk);
-  }
-  return chunks.join();
-}
-
-function requireSuites(suites: Iterable<string>) {
-  for (const suite of suites) {
-    require(suite);
-  }
-}
-
-/**
- * Handle both directory and files that are passed through TTY
- * and add them to suites
- */
-async function prepareSuites(inputs: string[]) {
-  const suites = new Set<string>();
-  const addSuite = absPath => {
-    log(`Processing file: ${absPath}`);
-    suites.add(require.resolve(absPath));
-  };
-  /**
-   * Match all files inside the directory with the
-   * .journey.{mjs|cjs|js|ts) extensions
-   */
-  const pattern = options.pattern
-    ? new RegExp(options.pattern, 'i')
-    : /.+\.journey\.([mc]js|[jt]s?)$/;
-  /**
-   * Ignore node_modules by default when running suites
-   */
-  const ignored = /node_modules/i;
+/* eslint-disable-next-line @typescript-eslint/no-var-requires */
+const { name, version } = require('../package.json');
 
-  for (const input of inputs) {
-    const absPath = resolve(resolvedCwd, input);
-    /**
-     * Validate for package.json file before running
-     * the suites
-     */
-    findPkgJsonByTraversing(absPath, resolvedCwd);
-    if (isDirectory(absPath)) {
-      await totalist(absPath, (rel, abs) => {
-        if (pattern.test(rel) && !ignored.test(rel)) {
-          addSuite(abs);
+program
+  .name(`npx ${name}`)
+  .usage('[options] [dir] [files] file')
+  .option(
+    '-c, --config <path>',
+    'configuration path (default: synthetics.config.js)'
+  )
+  .option(
+    '-p, --params <jsonstring>',
+    'JSON object that gets injected to all journeys',
+    JSON.parse
+  )
+  .addOption(
+    new Option('--reporter <value>', `output repoter format`).choices(
+      Object.keys(reporters)
+    )
+  )
+  .option(
+    '--pattern <pattern>',
+    'RegExp file patterns to search inside directory'
+  )
+  .option('--inline', 'Run inline journeys from heartbeat')
+  .option('-r, --require <modules...>', 'module(s) to preload')
+  .option('--no-headless', 'run browser in headful mode')
+  .option('--sandbox', 'enable chromium sandboxing', false)
+  .option('--rich-events', 'Mimics a heartbeat run')
+  .option(
+    '--capability <features...>',
+    'Enable capabilities through feature flags'
+  )
+  .addOption(
+    new Option('--screenshots [flag]', 'take screenshots at end of each step')
+      .choices(['on', 'off', 'only-on-failure'])
+      .default('on')
+  )
+  .option(
+    '--dry-run',
+    "don't actually execute anything, report only registered journeys"
+  )
+  .option(
+    '--match <name>',
+    'run only journeys with a name or tags that matches the glob'
+  )
+  .option(
+    '--tags <name...>',
+    'run only journeys with the given tag(s), or globs'
+  )
+  .option(
+    '--outfd <fd>',
+    'specify a file descriptor for logs. Default is stdout',
+    parseInt
+  )
+  .option(
+    '--ws-endpoint <endpoint>',
+    'Browser WebSocket endpoint to connect to'
+  )
+  .option(
+    '--pause-on-error',
+    'pause on error until a keypress is made in the console. Useful during development'
+  )
+  .option(
+    '--ignore-https-errors',
+    'ignores any HTTPS errors in sites being tested, including ones related to unrecognized certs or signatures. This can be insecure!'
+  )
+  .option(
+    '--quiet-exit-code',
+    'always return 0 as an exit code status, regardless of test pass / fail. Only return > 0 exit codes on internal errors where the suite could not be run'
+  )
+  .option(
+    '--throttling <d/u/l>',
+    'List of options to throttle network conditions for download throughput (d) in megabits/second, upload throughput (u) in megabits/second and latency (l) in milliseconds.',
+    DEFAULT_NETWORK_CONDITIONS_ARG
+  )
+  .option('--no-throttling', 'Turns off default network throttling.')
+  .option(
+    '--playwright-options <jsonstring>',
+    'JSON object to pass in custom Playwright options for the agent. Options passed will be merged with Playwright options defined in your synthetics.config.js file. Options defined via --playwright-options take precedence.',
+    JSON.parse
+  )
+  .version(version)
+  .description('Run synthetic tests')
+  .action(async (cliArgs: CliArgs) => {
+    try {
+      await loadTestFiles(cliArgs, program.args);
+      const options = normalizeOptions(cliArgs);
+      const results = await run(options);
+      /**
+       * Exit with error status if any journey fails
+       */
+      if (!options.quietExitCode) {
+        for (const result of Object.values(results)) {
+          if (result.status === 'failed') {
+            process.exit(1);
+          }
         }
-      });
-    } else {
-      addSuite(absPath);
-    }
-  }
-  return suites.values();
-}
-
-(async () => {
-  /**
-   * Transform `.ts` files out of the box by invoking
-   * the `ts-node` via `transpile-only` mode that compiles
-   * TS files without doing any extensive type checks.
-   *
-   * We must register `ts-node` _before_ loading inline
-   * scripts too because otherwise we will not be able to
-   * require `.ts` configuration files.
-   */
-  /* eslint-disable-next-line @typescript-eslint/no-var-requires */
-  require('ts-node').register({
-    transpileOnly: true,
-    compilerOptions: {
-      esModuleInterop: true,
-      allowJs: true,
-      target: 'es2018',
-    },
-  });
-
-  if (options.inline) {
-    const source = await readStdin();
-    loadInlineScript(source);
-  } else {
-    /**
-     * Preload modules before running the suites
-     */
-    const modules = [].concat(options.require || []).filter(Boolean);
-    for (const name of modules) {
-      if (isDepInstalled(name)) {
-        require(name);
-      } else {
-        throw new Error(`cannot find module '${name}'`);
       }
+    } catch (e) {
+      console.error(e);
+      process.exit(1);
     }
-    /**
-     * Handle piped files by reading the STDIN
-     * ex: ls example/suites/*.js | npx @elastic/synthetics
-     */
-    const files =
-      program.args.length > 0
-        ? program.args
-        : (await readStdin()).split('\n').filter(Boolean);
-    const suites = await prepareSuites(files);
-    requireSuites(suites);
-  }
-
-  /**
-   * Use the NODE_ENV variable to control the environment
-   */
-  const environment = process.env['NODE_ENV'] || 'development';
-  /**
-   * Validate and handle configs
-   */
-  const config =
-    options.config || !options.inline
-      ? readConfig(environment, options.config)
-      : {};
-  const params = merge(config.params, options.params || {});
-
-  /**
-   * Favor playwright options passed via cli to inline playwright options
-   */
-  const playwrightOptions = merge.all([
-    config.playwrightOptions || {},
-    options.playwrightOptions || {},
-    {
-      headless: options.headless,
-      chromiumSandbox: options.sandbox,
-      ignoreHTTPSErrors: options.ignoreHttpsErrors,
-    },
-  ]);
+  });
 
-  const results = await run({
-    params: Object.freeze(params),
-    networkConditions: options.throttling
-      ? parseNetworkConditions(options.throttling as string)
-      : undefined,
-    environment,
-    playwrightOptions,
-    ...options,
+program
+  .command('push')
+  .description(
+    'Push monitors to create new montors with Kibana monitor management UI'
+  )
+  .action(() => {
+    throw new Error('TODO: Implement me');
   });
 
-  if (!options.quietExitCode) {
-    /**
-     * Exit with error status if any journey fails
-     */
-    for (const result of Object.values(results)) {
-      if (result.status === 'failed') {
-        process.exit(1);
-      }
-    }
-  }
-})().catch(e => {
-  console.error(e);
-  process.exit(1);
-});
+program.parse(process.argv);

package/src/common_types.ts

@@ -31,18 +31,17 @@ import {
   ChromiumBrowserContext,
   Page,
 } from 'playwright-chromium';
-import { Protocol } from 'playwright-chromium/types/protocol';
 import { Step } from './dsl';
-import { reporters } from './reporters';
+import { Reporter, reporters } from './reporters';
 
 export type VoidCallback = () => void;
-export type Params = Record<string, any>;
-export type NetworkConditions = {
-  offline: boolean;
-  downloadThroughput: number;
-  uploadThroughput: number;
-  latency: number;
+export type Location = {
+  file: string;
+  line: number;
+  column: number;
 };
+
+export type Params = Record<string, any>;
 export type HooksArgs = {
   env: string;
   params: Params;
@@ -51,6 +50,13 @@ export type HooksCallback = (args: HooksArgs) => void;
 export type StatusValue = 'succeeded' | 'failed' | 'skipped';
 export type Reporters = keyof typeof reporters;
 
+export type NetworkConditions = {
+  offline: boolean;
+  downloadThroughput: number;
+  uploadThroughput: number;
+  latency: number;
+};
+
 export type Driver = {
   browser: ChromiumBrowser;
   context: ChromiumBrowserContext;
@@ -123,11 +129,9 @@ export type Request = {
 
 export type Response = {
   url?: string;
-  protocol?: string;
   status: number;
   statusText?: string;
-  mimeType: string;
-  httpVersion?: string;
+  mimeType?: string;
   headers: Record<string, string>;
   // Total size in bytes of the response (body and headers)
   bytes?: number;
@@ -141,7 +145,6 @@ export type Response = {
   remoteIPAddress?: string;
   remotePort?: number;
   fromServiceWorker?: boolean;
-  timing?: Protocol.Network.ResourceTiming;
 };
 
 export type NetworkInfo = {
@@ -149,19 +152,17 @@ export type NetworkInfo = {
   browser: BrowserInfo;
   type: string;
   request: Request;
-  response?: Response;
+  response: Response;
   isNavigationRequest: boolean;
   requestSentTime: number;
   loadEndTime: number;
   responseReceivedTime: number;
   resourceSize: number;
   transferSize: number;
-  timings?: {
+  timings: {
     blocked: number;
-    queueing: number;
     dns: number;
     ssl: number;
-    proxy: number;
     connect: number;
     send: number;
     wait: number;
@@ -186,34 +187,44 @@ export type PluginOutput = {
 
 export type ScreenshotOptions = 'on' | 'off' | 'only-on-failure';
 
-export type CliArgs = {
-  capability?: Array<string>;
-  config?: string;
-  outfd?: number;
-  headless?: boolean;
+type BaseArgs = {
+  params?: Params;
   screenshots?: ScreenshotOptions;
-  ssblocks?: boolean;
-  metrics?: boolean;
-  filmstrips?: boolean;
-  trace?: boolean;
   dryRun?: boolean;
-  network?: boolean;
+  match?: string;
+  tags?: Array<string>;
+  outfd?: number;
+  wsEndpoint?: string;
   pauseOnError?: boolean;
+  ignoreHttpsErrors?: boolean;
+  playwrightOptions?: PlaywrightOptions;
   quietExitCode?: boolean;
+};
+
+export type CliArgs = BaseArgs & {
+  config?: string;
   reporter?: Reporters;
-  wsEndpoint?: string;
-  sandbox?: boolean;
   pattern?: string;
-  inline: boolean;
-  match?: string;
-  tags?: Array<string>;
-  require: Array<string>;
-  debug?: boolean;
+  inline?: boolean;
+  require?: Array<string>;
+  headless?: boolean;
+  sandbox?: boolean;
+  richEvents?: boolean;
+  capability?: Array<string>;
   ignoreHttpsErrors?: boolean;
-  params?: Params;
   throttling?: boolean | string;
-  playwrightOptions?: LaunchOptions & BrowserContextOptions;
-  richEvents?: boolean;
+};
+
+export type RunOptions = BaseArgs & {
+  metrics?: boolean;
+  ssblocks?: boolean;
+  network?: boolean;
+  trace?: boolean;
+  filmstrips?: boolean;
+  environment?: string;
+  playwrightOptions?: PlaywrightOptions;
+  networkConditions?: NetworkConditions;
+  reporter?: CliArgs['reporter'] | Reporter;
 };
 
 export type PlaywrightOptions = LaunchOptions & BrowserContextOptions;

package/src/core/gatherer.ts

@@ -25,9 +25,8 @@
 
 import { chromium, ChromiumBrowser, BrowserContext } from 'playwright-chromium';
 import { PluginManager } from '../plugins';
-import { RunOptions } from './runner';
 import { log } from './logger';
-import { Driver } from '../common_types';
+import { Driver, RunOptions } from '../common_types';
 
 /**
  * Purpose of the Gatherer is to set up the necessary browser driver
@@ -58,7 +57,7 @@ export class Gatherer {
       ...playwrightOptions,
       userAgent: await Gatherer.getUserAgent(playwrightOptions?.userAgent),
     });
-    await Gatherer.setNetworkConditions(context, networkConditions);
+    Gatherer.setNetworkConditions(context, networkConditions);
 
     const page = await context.newPage();
     const client = await context.newCDPSession(page);
@@ -74,18 +73,26 @@ export class Gatherer {
     return userAgent + syntheticsIdentifier;
   }
 
-  static async setNetworkConditions(
+  static setNetworkConditions(
     context: BrowserContext,
     networkConditions: RunOptions['networkConditions']
   ) {
     if (networkConditions) {
-      context.on('page', async page => {
+      context.on('page', page => {
         const context = page.context();
-        const client = await context.newCDPSession(page);
-        await client.send(
-          'Network.emulateNetworkConditions',
-          networkConditions
-        );
+        const emulatePromise = context
+          .newCDPSession(page)
+          .then(client =>
+            client.send('Network.emulateNetworkConditions', networkConditions)
+          );
+        /**
+         * Guard against pages that gets closed before the emulation kicks to capture
+         * unhandled rejections from accessing the CDP session of closed page
+         */
+        Promise.race([
+          new Promise<void>(resolve => page.on('close', () => resolve())),
+          emulatePromise,
+        ]);
       });
     }
   }

package/src/core/index.ts

@@ -25,7 +25,8 @@
 
 import { Journey, JourneyCallback, JourneyOptions } from '../dsl';
 import Runner from './runner';
-import { VoidCallback, HooksCallback } from '../common_types';
+import { VoidCallback, HooksCallback, Location } from '../common_types';
+import { wrapFnWithLocation } from '../helpers';
 import { log } from './logger';
 
 /**
@@ -39,23 +40,28 @@ if (!global[SYNTHETICS_RUNNER]) {
 
 export const runner: Runner = global[SYNTHETICS_RUNNER];
 
-export const journey = (
-  options: JourneyOptions | string,
-  callback: JourneyCallback
-) => {
-  log(`register journey: ${JSON.stringify(options)}`);
-  if (typeof options === 'string') {
-    options = { name: options, id: options };
+export const journey = wrapFnWithLocation(
+  (
+    location: Location,
+    options: JourneyOptions | string,
+    callback: JourneyCallback
+  ) => {
+    log(`register journey: ${JSON.stringify(options)}`);
+    if (typeof options === 'string') {
+      options = { name: options, id: options };
+    }
+    const j = new Journey(options, callback, location);
+    runner.addJourney(j);
+    return j;
   }
-  const j = new Journey(options, callback);
-  runner.addJourney(j);
-  return j;
-};
+);
 
-export const step = (name: string, callback: VoidCallback) => {
-  log(`register step: ${name}`);
-  return runner.currentJourney?.addStep(name, callback);
-};
+export const step = wrapFnWithLocation(
+  (location: Location, name: string, callback: VoidCallback) => {
+    log(`register step: ${name}`);
+    return runner.currentJourney?.addStep(name, callback, location);
+  }
+);
 
 export const beforeAll = (callback: HooksCallback) => {
   runner.addHook('beforeAll', callback);

package/src/core/runner.ts

@@ -28,7 +28,7 @@ import { join } from 'path';
 import { rm } from 'fs/promises';
 import { Journey } from '../dsl/journey';
 import { Step } from '../dsl/step';
-import { reporters, Reporter } from '../reporters';
+import { reporters } from '../reporters';
 import {
   CACHE_PATH,
   monotonicTimeInSeconds,
@@ -44,36 +44,16 @@ import {
   Params,
   NetworkConditions,
   PluginOutput,
-  CliArgs,
   HooksArgs,
-  PlaywrightOptions,
   Driver,
   Screenshot,
+  RunOptions,
 } from '../common_types';
 import { PageMetrics, PluginManager } from '../plugins';
 import { PerformanceManager } from '../plugins';
 import { Gatherer } from './gatherer';
 import { log } from './logger';
 
-export type RunOptions = Omit<
-  CliArgs,
-  | 'debug'
-  | 'pattern'
-  | 'inline'
-  | 'require'
-  | 'reporter'
-  | 'richEvents'
-  | 'capability'
-  | 'sandbox'
-  | 'headless'
-  | 'throttling'
-> & {
-  environment?: string;
-  playwrightOptions?: PlaywrightOptions;
-  networkConditions?: NetworkConditions;
-  reporter?: CliArgs['reporter'] | Reporter;
-};
-
 type HookType = 'beforeAll' | 'afterAll';
 export type SuiteHooks = Record<HookType, Array<HooksCallback>>;
 
@@ -352,7 +332,7 @@ export default class Runner extends EventEmitter {
   ) {
     const end = monotonicTimeInSeconds();
     const { pluginManager, start, status, error } = result;
-    const pluginOutput = pluginManager.output();
+    const pluginOutput = await pluginManager.output();
     this.emit('journey:end', {
       journey,
       status,

package/src/dsl/journey.ts

@@ -26,7 +26,7 @@
 import { Browser, Page, BrowserContext, CDPSession } from 'playwright-chromium';
 import micromatch, { isMatch } from 'micromatch';
 import { Step } from './step';
-import { VoidCallback, HooksCallback, Params } from '../common_types';
+import { VoidCallback, HooksCallback, Params, Location } from '../common_types';
 
 export type JourneyOptions = {
   name: string;
@@ -49,18 +49,24 @@ export class Journey {
   id?: string;
   tags?: string[];
   callback: JourneyCallback;
+  location?: Location;
   steps: Step[] = [];
   hooks: Hooks = { before: [], after: [] };
 
-  constructor(options: JourneyOptions, callback: JourneyCallback) {
+  constructor(
+    options: JourneyOptions,
+    callback: JourneyCallback,
+    location?: Location
+  ) {
     this.name = options.name;
     this.id = options.id || options.name;
     this.tags = options.tags;
     this.callback = callback;
+    this.location = location;
   }
 
-  addStep(name: string, callback: VoidCallback) {
-    const step = new Step(name, this.steps.length + 1, callback);
+  addStep(name: string, callback: VoidCallback, location?: Location) {
+    const step = new Step(name, this.steps.length + 1, callback, location);
     this.steps.push(step);
     return step;
   }

package/src/dsl/step.ts

@@ -23,16 +23,23 @@
  *
  */
 
-import { VoidCallback } from '../common_types';
+import { Location, VoidCallback } from '../common_types';
 
 export class Step {
   name: string;
   index: number;
   callback: VoidCallback;
+  location?: Location;
 
-  constructor(name: string, index: number, callback: VoidCallback) {
+  constructor(
+    name: string,
+    index: number,
+    callback: VoidCallback,
+    location: Location
+  ) {
     this.name = name;
     this.index = index;
     this.callback = callback;
+    this.location = location;
   }
 }