@elastic/synthetics 1.4.0 → 1.5.1

package/src/dsl/journey.ts

@@ -29,7 +29,7 @@ import {
   BrowserContext,
   CDPSession,
   APIRequestContext,
-} from 'playwright-chromium';
+} from 'playwright-core';
 import { Step } from './step';
 import { VoidCallback, HooksCallback, Params, Location } from '../common_types';
 import { Monitor, MonitorConfig } from './monitor';

package/src/dsl/monitor.ts

@@ -72,6 +72,10 @@ export type MonitorConfig = {
   params?: Params;
   playwrightOptions?: PlaywrightOptions;
   alert?: AlertConfig;
+  /**
+   * By default, the monitor will be retested on failure
+   */
+  retestOnFailure?: boolean;
 };
 
 type MonitorFilter = {

package/src/helpers.ts

@@ -234,30 +234,6 @@ export function rewriteErrorStack(stack: string, indexes: [number, number]) {
   return stack;
 }
 
-// formatError prefers receiving proper Errors, but since at runtime
-// non Error exceptions can be thrown, it tolerates though. The
-// redundant type Error | any expresses that.
-export function formatError(error: Error | any) {
-  if (error === undefined || error === null) {
-    return;
-  }
-
-  if (!(error instanceof Error)) {
-    return {
-      message: `Error "${error}" received, with type "${typeof error}". (Do not throw exceptions without using \`new Error("my message")\`)`,
-      name: '',
-      stack: '',
-    };
-  }
-  const { name, message, stack } = error;
-  const indexes = findPWLogsIndexes(message);
-  return {
-    name,
-    message: rewriteErrorMessage(message, indexes[0]),
-    stack: rewriteErrorStack(stack, indexes),
-  };
-}
-
 const cwd = process.cwd();
 /**
  * All the settings that are related to the Synthetics is stored

package/src/index.ts

@@ -55,7 +55,7 @@ export type {
   CDPSession,
   APIRequestContext,
   devices,
-} from 'playwright-chromium';
+} from 'playwright-core';
 
 /**
  * Export the types necessary to write custom reporters

package/src/options.ts

@@ -31,10 +31,10 @@ import { THROTTLING_WARNING_MSG, error, warn } from './helpers';
 
 type Mode = 'run' | 'push';
 
-export function normalizeOptions(
+export async function normalizeOptions(
   cliArgs: CliArgs,
   mode: Mode = 'run'
-): RunOptions {
+): Promise<RunOptions> {
   const options: RunOptions = {
     ...cliArgs,
     environment: process.env['NODE_ENV'] || 'development',
@@ -85,7 +85,7 @@ export function normalizeOptions(
    */
   const config =
     cliArgs.config || !cliArgs.inline
-      ? readConfig(options.environment, cliArgs.config)
+      ? await readConfig(options.environment, cliArgs.config)
       : {};
 
   /**
@@ -106,7 +106,7 @@ export function normalizeOptions(
   );
   options.playwrightOptions = {
     ...playwrightOpts,
-    headless: cliArgs.headless ?? playwrightOpts?.headless,
+    headless: getHeadlessFlag(cliArgs.headless, playwrightOpts?.headless),
     chromiumSandbox: cliArgs.sandbox ?? playwrightOpts?.chromiumSandbox,
     ignoreHTTPSErrors:
       cliArgs.ignoreHttpsErrors ?? playwrightOpts?.ignoreHTTPSErrors,
@@ -140,6 +140,18 @@ export function normalizeOptions(
   return options;
 }
 
+export function getHeadlessFlag(
+  cliHeadless: boolean,
+  configHeadless?: boolean
+) {
+  // if cliHeadless is false, then we don't care about configHeadless
+  if (!cliHeadless) {
+    return false;
+  }
+  // default is headless
+  return configHeadless ?? true;
+}
+
 export function validatePushOptions(opts: PushOptions) {
   if (opts.tags || opts.match) {
     throw error(`Aborted. Invalid CLI flags.

package/src/plugins/browser-console.ts

@@ -23,13 +23,16 @@
  *
  */
 
-import type { ConsoleMessage } from 'playwright-core';
-import { BrowserMessage, Driver } from '../common_types';
+import type { ConsoleMessage, WebError } from 'playwright-core';
+import { BrowserMessage, Driver, StatusValue } from '../common_types';
 import { log } from '../core/logger';
 import { Step } from '../dsl';
 import { getTimestamp } from '../helpers';
 
-const defaultMessageLimit = 1000;
+const DEFAULT_MSG_LIMIt = 1000;
+const SUCCESSFUL_MSG_LIMIT = 100;
+
+const allowedTypes = ['error', 'warning', 'log'];
 
 export class BrowserConsole {
   private messages: BrowserMessage[] = [];
@@ -42,7 +45,7 @@ export class BrowserConsole {
       return;
     }
     const type = msg.type();
-    if (type === 'error' || type === 'warning') {
+    if (allowedTypes.includes(type)) {
       const { name, index } = this._currentStep;
       this.messages.push({
         timestamp: getTimestamp(),
@@ -55,11 +58,12 @@ export class BrowserConsole {
     }
   };
 
-  private pageErrorEventListener = (error: Error) => {
+  private webErrorListener = (webErr: WebError) => {
     if (!this._currentStep) {
       return;
     }
     const { name, index } = this._currentStep;
+    const error = webErr.error();
     this.messages.push({
       timestamp: getTimestamp(),
       text: error.message,
@@ -72,7 +76,7 @@ export class BrowserConsole {
   };
 
   private enforceMessagesLimit() {
-    if (this.messages.length > defaultMessageLimit) {
+    if (this.messages.length > DEFAULT_MSG_LIMIt) {
       this.messages.splice(0, 1);
     }
   }
@@ -80,13 +84,40 @@ export class BrowserConsole {
   start() {
     log(`Plugins: started collecting console events`);
     this.driver.context.on('console', this.consoleEventListener);
-    this.driver.page.on('pageerror', this.pageErrorEventListener);
+    this.driver.context.on('weberror', this.webErrorListener);
   }
 
   stop() {
     this.driver.context.off('console', this.consoleEventListener);
-    this.driver.page.off('pageerror', this.pageErrorEventListener);
+    this.driver.context.off('weberror', this.webErrorListener);
     log(`Plugins: stopped collecting console events`);
     return this.messages;
   }
 }
+
+export function filterBrowserMessages(
+  messages: BrowserMessage[],
+  status: StatusValue
+) {
+  if (status == 'skipped') {
+    return [];
+  } else if (status === 'failed' || messages.length <= SUCCESSFUL_MSG_LIMIT) {
+    return messages;
+  }
+  // collect 100 messages from the browser console when the test is successful,
+  // giving priority to errors and warnings
+  const result = messages.filter(msg => msg.type === 'error');
+  if (result.length >= SUCCESSFUL_MSG_LIMIT) {
+    return result.slice(-SUCCESSFUL_MSG_LIMIT);
+  }
+
+  // collect warnings
+  result.push(...messages.filter(msg => msg.type === 'warning'));
+  if (result.length >= SUCCESSFUL_MSG_LIMIT) {
+    return result.slice(-SUCCESSFUL_MSG_LIMIT);
+  }
+
+  // collect logs
+  result.push(...messages.filter(msg => msg.type === 'log'));
+  return result.slice(-SUCCESSFUL_MSG_LIMIT);
+}

package/src/plugins/network.ts

@@ -23,7 +23,7 @@
  *
  */
 
-import { Page, Request, Response } from 'playwright-chromium';
+import { Frame, Page, Request, Response } from 'playwright-core';
 import { NetworkInfo, BrowserInfo, Driver } from '../common_types';
 import { log } from '../core/logger';
 import { Step } from '../dsl';
@@ -93,6 +93,17 @@ export class NetworkManager {
     race.then(() => this._barrierPromises.delete(race));
   }
 
+  private _nullableFrameBarrier(req: Request): Frame | null {
+    try {
+      return req.frame();
+    } catch (_) {
+      // frame might be unavailable for certain requests if they are issued
+      // before the frame is created - true for navigation requests
+      // https://playwright.dev/docs/api/class-request#request-frame
+    }
+    return null;
+  }
+
   async start() {
     log(`Plugins: started collecting network events`);
     const { client, context } = this.driver;
@@ -226,7 +237,8 @@ export class NetworkManager {
     };
     this._calcTotalTime(networkEntry, timing);
 
-    const page = request.frame().page();
+    const frame = this._nullableFrameBarrier(request);
+    const page = frame?.page();
     this._addBarrier(
       page,
       request.allHeaders().then(reqHeaders => {
@@ -278,7 +290,8 @@ export class NetworkManager {
       return;
     }
 
-    const page = request.frame().page();
+    const frame = this._nullableFrameBarrier(request);
+    const page = frame?.page();
     this._addBarrier(
       page,
       request.sizes().then(sizes => {

package/src/push/kibana_api.ts

@@ -124,8 +124,8 @@ export async function bulkDeleteMonitors(
 }
 
 type StatusResponse = {
-  version: {
-    number: string;
+  kibana: {
+    version: string;
   };
 };
 
@@ -135,7 +135,8 @@ export async function getVersion(options: PushOptions) {
     method: 'GET',
     auth: options.auth,
   });
-  return data.version.number;
+
+  return data.kibana.version;
 }
 
 export type LegacyAPISchema = {

package/src/push/monitor.ts

@@ -242,12 +242,15 @@ export function buildMonitorFromYaml(
     config['private_locations'] ||
     config.privateLocations ||
     options.privateLocations;
-  delete config['private_locations'];
+  const retestOnFailure =
+    config['retest_on_failure'] ?? options.retestOnFailure;
   const alertConfig = parseAlertConfig(config, options.alert);
+
   const mon = new Monitor({
     locations: options.locations,
     tags: options.tags,
-    ...config,
+    ...normalizeConfig(config),
+    retestOnFailure,
     privateLocations,
     schedule: schedule || options.schedule,
     alert: alertConfig,
@@ -263,6 +266,14 @@ export function buildMonitorFromYaml(
   return mon;
 }
 
+// Deletes unncessary fields from the lightweight monitor config
+//  that is not supported by the Kibana API
+function normalizeConfig(config: MonitorConfig) {
+  delete config['private_locations'];
+  delete config['retest_on_failure'];
+  return config;
+}
+
 export const parseAlertConfig = (
   config: MonitorConfig,
   gConfig?: AlertConfig

package/src/push/request.ts

@@ -48,7 +48,8 @@ export async function sendRequest(options: APIRequestOptions) {
       'user-agent': `Elastic/Synthetics ${version}`,
       'kbn-xsrf': 'true',
     },
-    headersTimeout: 60 * 1000,
+    // align with the default timeout of the kibana route
+    headersTimeout: 2 * 60 * 1000,
   });
 }
 
@@ -56,9 +57,16 @@ export async function sendReqAndHandleError<T>(
   options: APIRequestOptions
 ): Promise<T> {
   const { statusCode, body } = await sendRequest(options);
-  return await (await handleError(statusCode, options.url, body)).json();
+  return (
+    await handleError(statusCode, options.url, body)
+  ).json() as Promise<T>;
 }
 
+type APIError = {
+  error: string;
+  message: string;
+};
+
 // Handle bad status code errors from Kibana API and format the
 // error message to be displayed to the user.
 // returns the response stream if no error is found
@@ -70,9 +78,9 @@ export async function handleError(
   if (statusCode === 404) {
     throw formatNotFoundError(url, await body.text());
   } else if (!ok(statusCode)) {
-    let parsed: { error: string; message: string };
+    let parsed: APIError;
     try {
-      parsed = await body.json();
+      parsed = (await body.json()) as APIError;
     } catch (e) {
       throw formatAPIError(
         statusCode,

package/src/push/utils.ts

@@ -64,7 +64,7 @@ export function generateURL(options: PushOptions, operation: Operation) {
   const url = removeTrailingSlash(options.url);
   switch (operation) {
     case 'status':
-      return `${url}/s/${options.space}/api/status`;
+      return `${url}/s/${options.space}/api/stats`;
     case 'bulk_get':
     case 'bulk_update':
     case 'bulk_delete':

package/src/reporters/base.ts

@@ -24,14 +24,9 @@
  */
 
 import SonicBoom from 'sonic-boom';
-import { green, red, cyan } from 'kleur/colors';
-import {
-  symbols,
-  indent,
-  now,
-  findPWLogsIndexes,
-  rewriteErrorStack,
-} from '../helpers';
+import { green, red, cyan, gray } from 'kleur/colors';
+import { renderError, serializeError } from './reporter-util';
+import { symbols, indent, now } from '../helpers';
 import { Reporter, ReporterOptions } from '.';
 import {
   JourneyEndResult,
@@ -40,25 +35,6 @@ import {
 } from '../common_types';
 import { Journey, Step } from '../dsl';
 
-function renderError(error) {
-  let output = '';
-  const outer = indent('');
-  const inner = indent(outer);
-  const container = outer + '---\n';
-  output += container;
-  let stack = error.stack;
-  if (stack) {
-    output += inner + 'stack: |-\n';
-    stack = rewriteErrorStack(stack, findPWLogsIndexes(stack));
-    const lines = String(stack).split('\n');
-    for (const line of lines) {
-      output += inner + '  ' + line + '\n';
-    }
-  }
-  output += container;
-  return red(output);
-}
-
 function renderDuration(durationMs) {
   return parseInt(durationMs);
 }
@@ -66,14 +42,17 @@ function renderDuration(durationMs) {
 export default class BaseReporter implements Reporter {
   stream: SonicBoom;
   fd: number;
+  dryRun: boolean;
   metrics = {
     succeeded: 0,
     failed: 0,
     skipped: 0,
+    registered: 0,
   };
 
   constructor(options: ReporterOptions = {}) {
     this.fd = options.fd || process.stdout.fd;
+    this.dryRun = options.dryRun ?? false;
     /**
      * minLength is set to 1 byte to make sure we flush the
      * content even if its the last byte on the stream buffer
@@ -82,17 +61,25 @@ export default class BaseReporter implements Reporter {
     this.stream = new SonicBoom({ fd: this.fd, sync: true, minLength: 1 });
   }
 
+  onJourneyRegister(journey: Journey): void {
+    this.write(`\nJourney: ${journey.name}`);
+    this.metrics.registered++;
+  }
+
   onJourneyStart(journey: Journey, {}: JourneyStartResult) {
     this.write(`\nJourney: ${journey.name}`);
   }
 
   onStepEnd(_: Journey, step: Step, result: StepEndResult) {
     const { status, end, start, error } = result;
-    const message = `${symbols[status]}  Step: '${
-      step.name
-    }' ${status} (${renderDuration((end - start) * 1000)} ms)`;
+    const message = `${symbols[status]}  Step: '${step.name}' ${status} ${gray(
+      '(' + renderDuration((end - start) * 1000) + ' ms)'
+    )}`;
     this.write(indent(message));
-    error && this.write(renderError(error));
+    if (error) {
+      const message = renderError(serializeError(error));
+      this.write(indent(message) + '\n');
+    }
     this.metrics[status]++;
   }
 
@@ -105,14 +92,20 @@ export default class BaseReporter implements Reporter {
      * be executed which means the error happened in one of the hooks
      */
     if (total === 0 && error) {
-      this.write(renderError(error));
+      const message = renderError(serializeError(error));
+      this.write(indent(message) + '\n');
     }
   }
 
   onEnd() {
-    const { failed, succeeded, skipped } = this.metrics;
+    const { failed, succeeded, skipped, registered } = this.metrics;
     const total = failed + succeeded + skipped;
 
+    if (this.dryRun) {
+      this.write(`\n${registered} journey(s) registered`);
+      return;
+    }
+
     let message = '\n';
     if (total === 0) {
       message = 'No tests found!';

package/src/reporters/index.ts

@@ -34,7 +34,11 @@ import {
   StepEndResult,
 } from '../common_types';
 
-export type ReporterOptions = { fd?: number; colors?: boolean };
+export type ReporterOptions = {
+  fd?: number;
+  colors?: boolean;
+  dryRun?: boolean;
+};
 export type BuiltInReporterName = 'default' | 'json' | 'junit';
 export type ReporterInstance = new (opts: ReporterOptions) => Reporter;
 export const reporters: {

package/src/reporters/json.ts

@@ -28,8 +28,8 @@ import { join } from 'path';
 import sharp from 'sharp';
 import { createHash } from 'crypto';
 import BaseReporter from './base';
+import { renderError, serializeError, stripAnsiCodes } from './reporter-util';
 import {
-  formatError,
   getTimestamp,
   CACHE_PATH,
   totalist,
@@ -52,6 +52,7 @@ import {
   JourneyEndResult,
   PageMetrics,
 } from '../common_types';
+import { inspect } from 'util';
 
 /* eslint-disable @typescript-eslint/no-var-requires */
 const { version, name } = require('../../package.json');
@@ -189,6 +190,41 @@ export function formatNetworkFields(network: NetworkInfo) {
   return { ecs, payload };
 }
 
+/**
+ * formatJSONError formats the error in a structured format
+ * we restructure the error with code frame and stack trace for test errors
+ */
+function formatJSONError(error: Error | any, type: OutputType) {
+  if (error == null) {
+    return;
+  }
+
+  // Early exit for non Error objects
+  if (!(error instanceof Error) || !error.stack) {
+    return { message: `thrown: ${inspect(error)}` };
+  }
+
+  /**
+   * Do not process browser errors - console.error and unhandled exceptions
+   */
+  if (type != null && type === 'journey/browserconsole') {
+    return {
+      name: error.name,
+      message: error.message,
+      stack: error.stack,
+    };
+  }
+
+  /**
+   * Do not highlight source for these errors and strip ANSI codes
+   */
+  return {
+    name: error.name,
+    message: stripAnsiCodes(error.message.split('\n')[0]),
+    stack: stripAnsiCodes(renderError(serializeError(error, false))),
+  };
+}
+
 function journeyInfo(
   journey: OutputFields['journey'],
   type: OutputFields['type'],
@@ -317,7 +353,7 @@ export default class JSONReporter extends BaseReporter {
     });
   }
 
-  onJourneyRegister(journey: Journey): void {
+  override onJourneyRegister(journey: Journey): void {
     this.writeJSON({
       type: 'journey/register',
       journey,
@@ -550,7 +586,7 @@ export default class JSONReporter extends BaseReporter {
       payload,
       blob,
       blob_mime,
-      error: formatError(error),
+      error: formatJSONError(error, type),
       url,
       package_version: version,
     });

package/src/reporters/junit.ts

@@ -31,8 +31,9 @@ import {
   StepEndResult,
 } from '../common_types';
 import { Journey, Step } from '../dsl';
-import { formatError, indent, now } from '../helpers';
+import { indent, now } from '../helpers';
 import BaseReporter from './base';
+import { serializeError, stripAnsiCodes } from './reporter-util';
 
 type XMLEntry = {
   name: string;
@@ -97,14 +98,13 @@ export default class JUnitReporter extends BaseReporter {
 
     entry.attributes.tests++;
     if (status === 'failed') {
-      const { name, message, stack } = formatError(error);
       caseEntry.children.push({
         name: 'failure',
         attributes: {
-          message,
-          type: name,
+          message: error.message,
+          type: error.name,
         },
-        text: stack,
+        text: stripAnsiCodes(serializeError(error).stack),
       });
       entry.attributes.failures++;
     } else if (status === 'skipped') {