@elastic/synthetics 1.0.0-beta.2 → 1.0.0-beta.20

package/src/parse_args.ts

@@ -23,15 +23,14 @@
  *
  */
 
-import { program } from 'commander';
+import { program, Option } from 'commander';
+import { CliArgs } from './common_types';
 import { reporters } from './reporters';
-
-const availableReporters = Object.keys(reporters)
-  .map(r => String(r))
-  .join();
+import { DEFAULT_NETWORK_CONDITIONS_ARG } from './helpers';
 
 /* eslint-disable-next-line @typescript-eslint/no-var-requires */
 const { name, version } = require('../package.json');
+
 program
   .name(`npx ${name}`)
   .usage('[options] [dir] [files] file')
@@ -39,12 +38,15 @@ program
     '-c, --config <path>',
     'configuration path (default: synthetics.config.js)'
   )
-  .option('-s, --suite-params <jsonstring>', 'suite variables', '{}')
-  .option('-e, --environment <envname>', 'e.g. production', 'development')
-  .option('-j, --json', 'output newline delimited JSON')
   .option(
-    '--reporter <reporter>',
-    `output repoter format, can be one of ${availableReporters}`
+    '-p, --params <jsonstring>',
+    'JSON object that gets injected to all journeys',
+    JSON.parse
+  )
+  .addOption(
+    new Option('--reporter <value>', `output repoter format`).choices(
+      Object.keys(reporters)
+    )
   )
   .option('-d, --debug', 'print debug logs info')
   .option(
@@ -54,30 +56,103 @@ program
   .option('--inline', 'Run inline journeys from heartbeat')
   .option('-r, --require <modules...>', 'module(s) to preload')
   .option('--no-headless', 'run browser in headful mode')
-  .option('--sandbox', 'enable chromium sandboxing')
+  .option('--sandbox', 'enable chromium sandboxing', false)
+  .option('--rich-events', 'Mimics a heartbeat run')
   .option(
-    '--ws-endpoint <endpoint>',
-    'Browser WebSocket endpoint to connect to'
+    '--capability <features...>',
+    'Enable capabilities through feature flags'
   )
-  .option(
-    '--pause-on-error',
-    'pause on error until a keypress is made in the console. Useful during development'
+  .addOption(
+    new Option('--screenshots [flag]', 'take screenshots at end of each step')
+      .choices(['on', 'off', 'only-on-failure'])
+      .default('on')
   )
-  .option('--screenshots', 'take screenshot for each step')
-  .option('--filmstrips', 'record detailed filmstrip info for all journeys')
-  .option('--network', 'capture network information for all journeys')
-  .option('--metrics', 'capture performance metrics for each step')
   .option(
     '--dry-run',
     "don't actually execute anything, report only registered journeys"
   )
-  .option('--journey-name <name>', 'only run the journey with the given name')
+  .option(
+    '--match <name>',
+    'run only journeys with a name or tags that matches the glob'
+  )
+  .option(
+    '--tags <name...>',
+    'run only journeys with the given tag(s), or globs'
+  )
   .option(
     '--outfd <fd>',
     'specify a file descriptor for logs. Default is stdout',
     parseInt
   )
+  .option(
+    '--ws-endpoint <endpoint>',
+    'Browser WebSocket endpoint to connect to'
+  )
+  .option(
+    '--pause-on-error',
+    'pause on error until a keypress is made in the console. Useful during development'
+  )
+  .option(
+    '--ignore-https-errors',
+    'ignores any HTTPS errors in sites being tested, including ones related to unrecognized certs or signatures. This can be insecure!'
+  )
+  .option(
+    '--quiet-exit-code',
+    'always return 0 as an exit code status, regardless of test pass / fail. Only return > 0 exit codes on internal errors where the suite could not be run'
+  )
+  .addOption(
+    new Option(
+      '--throttling <d/u/l>',
+      'List of options to throttle network conditions for download throughput (d) in megabits/second, upload throughput (u) in megabits/second and latency (l) in milliseconds.'
+    ).default(DEFAULT_NETWORK_CONDITIONS_ARG)
+  )
+  .option('--no-throttling', 'Turns off default network throttling.')
+  .option('--playwright-options <jsonstring>', 'JSON object to pass in custom Playwright options for the agent. Options passed will be merged with Playwright options defined in your synthetics.config.js file. Options defined via --playwright-options take precedence.', JSON.parse)
   .version(version)
   .description('Run synthetic tests');
 
-export default program.parse(process.argv);
+const command = program.parse(process.argv);
+const options = command.opts() as CliArgs;
+
+/**
+ * Group all events that can be consumed by heartbeat and
+ * eventually by the Synthetics UI.
+ */
+if (options.richEvents) {
+  options.reporter = options.reporter ?? 'json';
+  options.ssblocks = true;
+  options.network = true;
+  options.quietExitCode = true;
+  options.trace = true;
+}
+
+if (options.capability) {
+  const supportedCapabilities = [
+    'trace',
+    'network',
+    'filmstrips',
+    'metrics',
+    'ssblocks',
+  ];
+  /**
+   * trace - record chrome trace events(LCP, FCP, CLS, etc.) for all journeys
+   * network - capture network information for all journeys
+   * filmstrips - record detailed filmstrips for all journeys
+   * metrics - capture performance metrics (DOM Nodes, Heap size, etc.) for each step
+   * ssblocks - Dedupes the screenshots in to blocks to save storage space
+   */
+  for (const flag of options.capability) {
+    if (supportedCapabilities.includes(flag)) {
+      options[flag] = true;
+    } else {
+      console.warn(
+        `Missing capability "${flag}", current supported capabilities are ${supportedCapabilities.join(
+          ', '
+        )}`
+      );
+    }
+  }
+}
+
+export { options };
+export default command;

package/src/plugins/browser-console.ts

@@ -23,22 +23,18 @@
  *
  */
 
-import { Page } from 'playwright-chromium';
-import { DefaultPluginOutput } from '../common_types';
+import { BrowserMessage, Driver } from '../common_types';
 import { Step } from '../dsl';
 import { getTimestamp } from '../helpers';
 
-export type BrowserMessage = {
-  text: string;
-  type: string;
-} & DefaultPluginOutput;
-
 const defaultMessageLimit = 1000;
 
 export class BrowserConsole {
   private messages: BrowserMessage[] = [];
   _currentStep: Partial<Step> = null;
 
+  constructor(private driver: Driver) {}
+
   private consoleEventListener = msg => {
     if (!this._currentStep) {
       return;
@@ -52,20 +48,41 @@ export class BrowserConsole {
         type,
         step: { name, index },
       });
-      if (this.messages.length > defaultMessageLimit) {
-        this.messages.splice(0, 1);
-      }
+
+      this.enforceMessagesLimit();
     }
   };
 
-  constructor(private page: Page) {}
+  private pageErrorEventListener = (error: Error) => {
+    if (!this._currentStep) {
+      return;
+    }
+    const { name, index } = this._currentStep;
+    this.messages.push({
+      timestamp: getTimestamp(),
+      text: error.message,
+      type: 'error',
+      step: { name, index },
+      error,
+    });
+
+    this.enforceMessagesLimit();
+  };
+
+  private enforceMessagesLimit() {
+    if (this.messages.length > defaultMessageLimit) {
+      this.messages.splice(0, 1);
+    }
+  }
 
   start() {
-    this.page.on('console', this.consoleEventListener);
+    this.driver.page.on('console', this.consoleEventListener);
+    this.driver.page.on('pageerror', this.pageErrorEventListener);
   }
 
   stop() {
-    this.page.off('console', this.consoleEventListener);
+    this.driver.page.off('console', this.consoleEventListener);
+    this.driver.page.off('pageerror', this.pageErrorEventListener);
     return this.messages;
   }
 }

package/src/plugins/network.ts

@@ -23,9 +23,8 @@
  *
  */
 
-import { CDPSession } from 'playwright-chromium';
 import { Protocol } from 'playwright-chromium/types/protocol';
-import { NetworkInfo, BrowserInfo } from '../common_types';
+import { NetworkInfo, BrowserInfo, Driver } from '../common_types';
 import { Step } from '../dsl';
 import { getTimestamp } from '../helpers';
 
@@ -35,7 +34,10 @@ export class NetworkManager {
   results: Array<NetworkInfo> = [];
   _currentStep: Partial<Step> = null;
 
-  async start(client: CDPSession) {
+  constructor(private driver: Driver) {}
+
+  async start() {
+    const { client } = this.driver;
     const { product } = await client.send('Browser.getVersion');
     const [name, version] = product.split('/');
     this._browser = { name, version };
@@ -194,7 +196,7 @@ export class NetworkManager {
       record.transferSize = encodedDataLength;
     }
     record.loadEndTime = endTime;
-    record.timings = this.calculateTimings(record);
+    record.timings = calculateTimings(record);
     this.results.push(record);
     /**
      * When there is DNS failure, browsers keeps retrying requests which
@@ -207,159 +209,150 @@ export class NetworkManager {
    * Account for missing response received event and also adjust the
    * response received event based on when first byte event was recorded
    */
-  getResponseReceivedTime(
-    timing: Protocol.Network.Response['timing'],
-    responseReceivedTime: number
-  ) {
-    if (timing == null) {
-      return responseReceivedTime;
-    }
-    const startTime = timing.requestTime;
-    const headersReceivedTime = startTime + timing.receiveHeadersEnd / 1000;
-    if (
-      responseReceivedTime < 0 ||
-      responseReceivedTime > headersReceivedTime
-    ) {
-      responseReceivedTime = headersReceivedTime;
-    }
-    if (startTime > responseReceivedTime) {
-      responseReceivedTime = startTime;
-    }
+
+  stop() {
+    return this.results;
+  }
+}
+
+function getResponseReceivedTime(
+  timing: Protocol.Network.Response['timing'],
+  responseReceivedTime: number
+) {
+  if (timing == null) {
     return responseReceivedTime;
   }
+  const startTime = timing.requestTime;
+  const headersReceivedTime = startTime + timing.receiveHeadersEnd / 1000;
+  if (responseReceivedTime < 0 || responseReceivedTime > headersReceivedTime) {
+    responseReceivedTime = headersReceivedTime;
+  }
+  if (startTime > responseReceivedTime) {
+    responseReceivedTime = startTime;
+  }
+  return responseReceivedTime;
+}
+
+/**
+ * The timing calculations are based on the chrome devtools frontend
+ * https://github.com/ChromeDevTools/devtools-frontend/blob/7f5478d8ceb7586f23f4073ab5c2085dac1ec26a/front_end/network/RequestTimingView.js#L98-L193
+ */
+export function calculateTimings(record: NetworkInfo) {
+  const result: NetworkInfo['timings'] = {
+    blocked: -1,
+    queueing: -1,
+    proxy: -1,
+    dns: -1,
+    ssl: -1,
+    connect: -1,
+    send: -1,
+    wait: -1,
+    receive: -1,
+    total: -1,
+  };
+  const { requestSentTime, loadEndTime, responseReceivedTime } = record;
 
   /**
-   * The timing calculations are based on the chrome devtools frontend
-   * https://github.com/ChromeDevTools/devtools-frontend/blob/7f5478d8ceb7586f23f4073ab5c2085dac1ec26a/front_end/network/RequestTimingView.js#L98-L193
+   * Handle when request failed and no responseReceived event was
+   * fired for that particular request
+   * Eg: connection refused by remote host
    */
-  calculateTimings(record: NetworkInfo) {
-    const result: NetworkInfo['timings'] = {
-      blocked: -1,
-      queueing: -1,
-      proxy: -1,
-      dns: -1,
-      ssl: -1,
-      connect: -1,
-      send: -1,
-      wait: -1,
-      receive: -1,
-      total: -1,
-    };
-    const { requestSentTime, loadEndTime, responseReceivedTime } = record;
-
-    /**
-     * Handle when request failed and no responseReceived event was
-     * fired for that particular request
-     * Eg: connection refused by remote host
-     */
-    const response = record.response || {
-      timing: null,
-      fromServiceWorker: false,
-    };
+  const response = record.response || {
+    timing: null,
+    fromServiceWorker: false,
+  };
 
-    const toMilliseconds = (time: number) => (time === -1 ? -1 : time * 1000);
-    const calculateDiffInMs = (
-      name: keyof NetworkInfo['timings'],
-      start: number,
-      end: number
-    ) => {
-      if (start < Number.MAX_VALUE && start <= end) {
-        result[name] = toMilliseconds(end - start);
-      }
-    };
-    const firstPositive = (numbers: number[]) => {
-      for (let i = 0; i < numbers.length; ++i) {
-        if (numbers[i] > 0) {
-          return numbers[i];
-        }
-      }
-      return null;
-    };
-    /**
-     * requestTime is baseline in seconds, rest of the timing data are ticks in milliseconds
-     * from the requestTime, so we calculate the offset from that
-     */
-    const addOffsetRange = (
-      name: keyof NetworkInfo['timings'],
-      start: number,
-      end: number
-    ) => {
-      if (start >= 0 && end >= 0) {
-        calculateDiffInMs(
-          name,
-          startTime + start / 1000,
-          startTime + end / 1000
-        );
-      }
-    };
-    const timing = response.timing;
-    const actResRcvdTime = this.getResponseReceivedTime(
-      timing,
-      responseReceivedTime
-    );
-    const issueTime = requestSentTime;
-    const startTime = timing == null ? -1 : timing.requestTime;
-    const endTime = firstPositive([loadEndTime, actResRcvdTime]) || startTime;
-
-    if (timing == null) {
-      const start =
-        issueTime !== -1 ? issueTime : startTime !== -1 ? startTime : 0;
-      const middle = actResRcvdTime === -1 ? Number.MAX_VALUE : actResRcvdTime;
-      const end = endTime === -1 ? Number.MAX_VALUE : endTime;
-      calculateDiffInMs('total', start, end);
-      calculateDiffInMs('blocked', start, middle);
-      calculateDiffInMs('receive', middle, end);
-      // Check if the request is blocked/stalled for the whole timeframe and
-      // calculate timings appropriately for that request
-      if (!isFinite(result['blocked'])) {
-        result['blocked'] = result['total'];
+  const toMilliseconds = (time: number) => (time === -1 ? -1 : time * 1000);
+  const calculateDiffInMs = (
+    name: keyof NetworkInfo['timings'],
+    start: number,
+    end: number
+  ) => {
+    if (start < Number.MAX_VALUE && start <= end) {
+      result[name] = toMilliseconds(end - start);
+    }
+  };
+  const firstPositive = (numbers: number[]) => {
+    for (let i = 0; i < numbers.length; ++i) {
+      if (numbers[i] > 0) {
+        return numbers[i];
       }
-      return result;
     }
-    calculateDiffInMs(
-      'total',
-      issueTime < startTime ? issueTime : startTime,
-      endTime
-    );
-    if (issueTime < startTime) {
-      calculateDiffInMs('queueing', issueTime, startTime);
+    return null;
+  };
+  /**
+   * requestTime is baseline in seconds, rest of the timing data are ticks in milliseconds
+   * from the requestTime, so we calculate the offset from that
+   */
+  const addOffsetRange = (
+    name: keyof NetworkInfo['timings'],
+    start: number,
+    end: number
+  ) => {
+    if (start >= 0 && end >= 0) {
+      calculateDiffInMs(name, startTime + start / 1000, startTime + end / 1000);
     }
-    const responseReceived = toMilliseconds(actResRcvdTime - startTime);
+  };
+  const timing = response.timing;
+  const actResRcvdTime = getResponseReceivedTime(timing, responseReceivedTime);
+  const issueTime = requestSentTime;
+  const startTime = timing == null ? -1 : timing.requestTime;
+  const endTime = firstPositive([loadEndTime, actResRcvdTime]) || startTime;
 
-    if (response.fromServiceWorker) {
-      addOffsetRange('blocked', 0, timing.workerStart);
-      addOffsetRange('wait', timing.sendEnd, responseReceived);
-    } else if (!timing.pushStart) {
-      const blockingEnd =
-        firstPositive([
-          timing.dnsStart,
-          timing.connectStart,
-          timing.sendStart,
-          responseReceived,
-        ]) || 0;
-      addOffsetRange('blocked', 0, blockingEnd);
-      addOffsetRange('proxy', timing.proxyStart, timing.proxyEnd);
-      addOffsetRange('dns', timing.dnsStart, timing.dnsEnd);
-      addOffsetRange('connect', timing.connectStart, timing.connectEnd);
-      addOffsetRange('ssl', timing.sslStart, timing.sslEnd);
-      addOffsetRange('send', timing.sendStart, timing.sendEnd);
-      addOffsetRange(
-        'wait',
-        Math.max(
-          timing.sendEnd,
-          timing.connectEnd,
-          timing.dnsEnd,
-          timing.proxyEnd,
-          blockingEnd
-        ),
-        responseReceived
-      );
+  if (timing == null) {
+    const start =
+      issueTime !== -1 ? issueTime : startTime !== -1 ? startTime : 0;
+    const middle = actResRcvdTime === -1 ? Number.MAX_VALUE : actResRcvdTime;
+    const end = endTime === -1 ? Number.MAX_VALUE : endTime;
+    calculateDiffInMs('total', start, end);
+    calculateDiffInMs('blocked', start, middle);
+    calculateDiffInMs('receive', middle, end);
+    // Check if the request is blocked/stalled for the whole timeframe and
+    // calculate timings appropriately for that request
+    if (!isFinite(result['blocked'])) {
+      result['blocked'] = result['total'];
     }
-    calculateDiffInMs('receive', actResRcvdTime, endTime);
     return result;
   }
+  calculateDiffInMs(
+    'total',
+    issueTime < startTime ? issueTime : startTime,
+    endTime
+  );
+  if (issueTime < startTime) {
+    calculateDiffInMs('queueing', issueTime, startTime);
+  }
+  const responseReceived = toMilliseconds(actResRcvdTime - startTime);
 
-  stop() {
-    return this.results;
+  if (response.fromServiceWorker) {
+    addOffsetRange('blocked', 0, timing.workerStart);
+    addOffsetRange('wait', timing.sendEnd, responseReceived);
+  } else if (!timing.pushStart) {
+    const blockingEnd =
+      firstPositive([
+        timing.dnsStart,
+        timing.connectStart,
+        timing.sendStart,
+        responseReceived,
+      ]) || 0;
+    addOffsetRange('blocked', 0, blockingEnd);
+    addOffsetRange('proxy', timing.proxyStart, timing.proxyEnd);
+    addOffsetRange('dns', timing.dnsStart, timing.dnsEnd);
+    addOffsetRange('connect', timing.connectStart, timing.connectEnd);
+    addOffsetRange('ssl', timing.sslStart, timing.sslEnd);
+    addOffsetRange('send', timing.sendStart, timing.sendEnd);
+    addOffsetRange(
+      'wait',
+      Math.max(
+        timing.sendEnd,
+        timing.connectEnd,
+        timing.dnsEnd,
+        timing.proxyEnd,
+        blockingEnd
+      ),
+      responseReceived
+    );
   }
+  calculateDiffInMs('receive', actResRcvdTime, endTime);
+  return result;
 }

package/src/plugins/performance.ts

@@ -23,7 +23,7 @@
  *
  */
 
-import { CDPSession } from 'playwright-chromium';
+import { Driver } from '../common_types';
 
 const supportedMetrics = new Set<string>([
   'Timestamp',
@@ -41,7 +41,7 @@ const supportedMetrics = new Set<string>([
   'JSHeapTotalSize',
 ]);
 
-export interface Metrics {
+export interface PageMetrics {
   Timestamp?: number;
   Documents?: number;
   Frames?: number;
@@ -58,22 +58,22 @@ export interface Metrics {
 }
 
 export class PerformanceManager {
-  constructor(private client: CDPSession) {}
+  constructor(private driver: Driver) {}
 
   async start() {
-    await this.client.send('Performance.enable');
+    await this.driver.client.send('Performance.enable');
   }
 
   async stop() {
-    await this.client.send('Performance.disable');
+    await this.driver.client.send('Performance.disable');
   }
 
   async getMetrics() {
-    const { metrics } = await this.client.send('Performance.getMetrics');
+    const { metrics } = await this.driver.client.send('Performance.getMetrics');
     return this.buildMetricsObject(metrics);
   }
 
-  private buildMetricsObject(metrics = []): Metrics {
+  private buildMetricsObject(metrics = []): PageMetrics {
     const result = {};
     for (const metric of metrics) {
       if (supportedMetrics.has(metric.name)) result[metric.name] = metric.value;