@elastic/synthetics 1.4.0 → 1.5.0

package/package.json

@@ -1,14 +1,15 @@
 {
   "name": "@elastic/synthetics",
-  "version": "1.4.0",
+  "version": "1.5.0",
   "description": "Elastic synthetic monitoring agent",
   "main": "dist/index.js",
   "types": "dist/index.d.ts",
   "scripts": {
     "clean": "rimraf dist",
     "prepublish": "npm run clean && npm run build",
-    "build": "tsc",
+    "build:lib": "tsc",
     "build:locations": "sh utils/update_locations.sh",
+    "build": "node utils/build.js",
     "watch": "tsc -w",
     "lint": "eslint . --rulesdir utils/eslint-rules",
     "lint:fix": "npm run lint -- --fix",
@@ -40,32 +41,38 @@
   "author": "",
   "license": "MIT",
   "dependencies": {
+    "@babel/code-frame": "^7.22.13",
     "archiver": "^5.3.1",
     "commander": "^10.0.1",
     "deepmerge": "^4.3.1",
     "enquirer": "^2.3.6",
     "esbuild": "^0.18.11",
-    "expect": "^28.1.3",
     "http-proxy": "^1.18.1",
     "kleur": "^4.1.5",
     "micromatch": "^4.0.5",
     "pirates": "^4.0.5",
-    "playwright-chromium": "=1.37.0",
-    "playwright-core": "=1.37.0",
+    "playwright": "=1.38.0",
+    "playwright-chromium": "=1.38.1",
+    "playwright-core": "=1.38.1",
     "semver": "^7.5.2",
-    "sharp": "^0.32.0",
+    "sharp": "^0.32.6",
     "snakecase-keys": "^4.0.1",
     "sonic-boom": "^3.3.0",
     "source-map-support": "^0.5.21",
+    "stack-utils": "^2.0.6",
     "undici": "^5.21.2",
     "yaml": "^2.2.2"
   },
   "devDependencies": {
+    "@commitlint/cli": "^17.7.1",
+    "@commitlint/config-conventional": "^17.7.0",
+    "@semantic-release/exec": "^6.0.3",
+    "@types/babel__code-frame": "^7.0.3",
     "@types/jest": "^28.1.8",
     "@types/micromatch": "^4.0.2",
     "@types/node": "^16.11.59",
-    "@types/sharp": "^0.28.6",
     "@types/semver": "^7",
+    "@types/stack-utils": "^2.0.1",
     "@typescript-eslint/eslint-plugin": "^5.38.0",
     "@typescript-eslint/parser": "^5.38.0",
     "eslint": "^8.23.1",
@@ -76,6 +83,7 @@
     "lint-staged": "^10.5.3",
     "prettier": "^2.7.1",
     "rimraf": "^3.0.2",
+    "semantic-release": "^21.1.1",
     "ts-jest": "^29.1.1",
     "typescript": "^5.1.6",
     "unzipper": "^0.10.11"

package/src/cli.ts

@@ -134,7 +134,7 @@ program
   .action(async (cliArgs: CliArgs) => {
     const tearDown = await globalSetup(cliArgs, program.args);
     try {
-      const options = normalizeOptions(cliArgs, 'run');
+      const options = await normalizeOptions(cliArgs, 'run');
       const results = await run(options);
       /**
        * Exit with error status if any journey fails
@@ -196,14 +196,14 @@ program
     ]);
     try {
       const settings = await loadSettings();
-      const options = normalizeOptions(
+      const options = (await normalizeOptions(
         {
           ...program.opts(),
           ...settings,
           ...cmdOpts,
         },
         'push'
-      ) as PushOptions;
+      )) as PushOptions;
       await validatePush(options, settings);
       const monitors = runner.buildMonitors(options);
       if ((options as CliArgs).throttling == null) {

package/src/common_types.ts

@@ -31,7 +31,7 @@ import {
   ChromiumBrowserContext,
   Page,
   APIRequestContext,
-} from 'playwright-chromium';
+} from 'playwright-core';
 import { Step } from './dsl';
 import { BuiltInReporterName, ReporterInstance } from './reporters';
 import { AlertConfig, MonitorConfig } from './dsl/monitor';
@@ -43,6 +43,10 @@ export type Location = {
   column: number;
 };
 
+export type StackFrame = Location & {
+  function?: string;
+};
+
 export type Params = Record<string, any>;
 export type HooksArgs = {
   env: string;
@@ -213,7 +217,6 @@ type BaseArgs = {
   schedule?: MonitorConfig['schedule'];
   locations?: MonitorConfig['locations'];
   privateLocations?: MonitorConfig['privateLocations'];
-  alert?: AlertConfig;
 };
 
 export type CliArgs = BaseArgs & {
@@ -244,6 +247,8 @@ export type PushOptions = Partial<ProjectSettings> &
     auth: string;
     kibanaVersion?: string;
     yes?: boolean;
+    alert?: AlertConfig;
+    retestOnFailure?: MonitorConfig['retestOnFailure'];
   };
 
 export type ProjectSettings = {
@@ -274,6 +279,14 @@ export type JourneyResult = {
   browserconsole?: PluginOutput['browserconsole'];
 };
 
+export type TestError = {
+  message: string;
+  stack?: string;
+  location?: Location;
+  // source location highlighting the error source
+  source?: string;
+};
+
 export type StepResult = {
   status: StatusValue;
   url?: string;

package/src/config.ts

@@ -27,7 +27,10 @@ import { isAbsolute, resolve, dirname } from 'path';
 import { SyntheticsConfig } from './common_types';
 import { isFile } from './helpers';
 
-export function readConfig(env: string, config?: string): SyntheticsConfig {
+export async function readConfig(
+  env: string,
+  config?: string
+): Promise<SyntheticsConfig> {
   let options = {};
   const cwd = process.cwd();
   /**
@@ -47,6 +50,14 @@ export function readConfig(env: string, config?: string): SyntheticsConfig {
   }
   if (typeof options === 'function') {
     options = options(env);
+    const optionsPromise = options as Promise<SyntheticsConfig>;
+    if (
+      optionsPromise != null &&
+      typeof optionsPromise.then === 'function' &&
+      typeof optionsPromise.catch === 'function'
+    ) {
+      options = await optionsPromise;
+    }
   }
   return options;
 }

package/src/core/browser-service.ts

@@ -26,7 +26,7 @@
 import { URL } from 'url';
 import { createServer } from 'http';
 import { createProxyServer } from 'http-proxy';
-import { chromium } from 'playwright-chromium';
+import { chromium } from 'playwright-core';
 import { log } from './logger';
 
 export class BrowserService {

package/src/core/expect.ts

@@ -23,199 +23,21 @@
  *
  */
 
-import expectLib from 'expect';
-import type {
-  AsymmetricMatcher,
-  MatcherState,
-  ExpectedAssertionsErrors,
-  Inverse,
-  PromiseMatchers,
-} from 'expect/build/types';
+/* eslint-disable @typescript-eslint/no-var-requires */
+const expectLib = require('../../dist/bundles/lib/index').expect;
 
-// Copied source code from expect/build/types.d.ts
-export declare type Expect = {
-  <T = unknown>(actual: T): synthetics.Matchers<T>;
-  assertions(arg0: number): void;
-  extend(arg0: any): void;
-  extractExpectedAssertionsErrors: () => ExpectedAssertionsErrors;
-  getState(): MatcherState;
-  hasAssertions(): void;
-  setState(state: Partial<MatcherState>): void;
-  any(expectedObject: any): AsymmetricMatcher;
-  anything(): AsymmetricMatcher;
-  arrayContaining(sample: Array<unknown>): AsymmetricMatcher;
-  objectContaining(sample: Record<string, unknown>): AsymmetricMatcher;
-  stringContaining(expected: string): AsymmetricMatcher;
-  stringMatching(expected: string | RegExp): AsymmetricMatcher;
-};
+function notSupported(name: string) {
+  throw new Error(`expect.${name} is not supported in @elastic/synthetics.`);
+}
 
 /**
- * Override matchers from expect in our synthetic namespace to
- * reduce support for spy and snapshot matchers which is
- * impelemed via jest matchers
- *
- * We declare the types in global synthetics namespace which allows
- * users to extend expect functionality without type errors
+ * Exclude toHaveScreenshot and toMatchSnapshot from our custom expect
+ * since they are expected to be running inside PW test runner for it to work properly.
  */
-declare global {
-  export namespace synthetics {
-    export interface Matchers<R> {
-      /**
-       * If you know how to test something, `.not` lets you test its opposite.
-       */
-      not: Matchers<R> & Inverse<Matchers<R>> & PromiseMatchers;
-      /**
-       * Use resolves to unwrap the value of a fulfilled promise so any other
-       * matcher can be chained. If the promise is rejected the assertion fails.
-       */
-      resolves: Matchers<R> & Inverse<Matchers<R>> & PromiseMatchers;
-      /**
-       * Unwraps the reason of a rejected promise so any other matcher can be chained.
-       * If the promise is fulfilled the assertion fails.
-       */
-      rejects: Matchers<R> & Inverse<Matchers<R>> & PromiseMatchers;
-      /**
-       * Checks that a value is what you expect. It uses `===` to check strict equality.
-       * Don't use `toBe` with floating-point numbers.
-       */
-      toBe(
-        expected: unknown
-      ): Matchers<R> & Inverse<Matchers<R>> & PromiseMatchers;
-      /**
-       * Using exact equality with floating point numbers is a bad idea.
-       * Rounding means that intuitive things fail.
-       * The default for numDigits is 2.
-       */
-      toBeCloseTo(
-        expected: number,
-        numDigits?: number
-      ): Matchers<R> & Inverse<Matchers<R>> & PromiseMatchers;
-      /**
-       * Ensure that a variable is not undefined.
-       */
-      toBeDefined(): Matchers<R> & Inverse<Matchers<R>> & PromiseMatchers;
-      /**
-       * When you don't care what a value is, you just want to
-       * ensure a value is false in a boolean context.
-       */
-      toBeFalsy(): Matchers<R> & Inverse<Matchers<R>> & PromiseMatchers;
-      /**
-       * For comparing floating point numbers.
-       */
-      toBeGreaterThan(
-        expected: number | bigint
-      ): Matchers<R> & Inverse<Matchers<R>> & PromiseMatchers;
-      /**
-       * For comparing floating point numbers.
-       */
-      toBeGreaterThanOrEqual(
-        expected: number | bigint
-      ): Matchers<R> & Inverse<Matchers<R>> & PromiseMatchers;
-      /**
-       * Ensure that an object is an instance of a class.
-       * This matcher uses `instanceof` underneath.
-       */
-      /* eslint-disable-next-line */
-      toBeInstanceOf(
-        expected: unknown
-      ): Matchers<R> & Inverse<Matchers<R>> & PromiseMatchers;
-      /**
-       * For comparing floating point numbers.
-       */
-      toBeLessThan(
-        expected: number | bigint
-      ): Matchers<R> & Inverse<Matchers<R>> & PromiseMatchers;
-      /**
-       * For comparing floating point numbers.
-       */
-      toBeLessThanOrEqual(
-        expected: number | bigint
-      ): Matchers<R> & Inverse<Matchers<R>> & PromiseMatchers;
-      /**
-       * This is the same as `.toBe(null)` but the error messages are a bit nicer.
-       * So use `.toBeNull()` when you want to check that something is null.
-       */
-      toBeNull(): Matchers<R> & Inverse<Matchers<R>> & PromiseMatchers;
-      /**
-       * Use when you don't care what a value is, you just want to ensure a value
-       * is true in a boolean context. In JavaScript, there are six falsy values:
-       * `false`, `0`, `''`, `null`, `undefined`, and `NaN`. Everything else is truthy.
-       */
-      toBeTruthy(): Matchers<R> & Inverse<Matchers<R>> & PromiseMatchers;
-      /**
-       * Used to check that a variable is undefined.
-       */
-      toBeUndefined(): Matchers<R> & Inverse<Matchers<R>> & PromiseMatchers;
-      /**
-       * Used to check that a variable is NaN.
-       */
-      toBeNaN(): Matchers<R> & Inverse<Matchers<R>> & PromiseMatchers;
-      /**
-       * Used when you want to check that an item is in a list.
-       * For testing the items in the list, this uses `===`, a strict equality check.
-       */
-      toContain(
-        expected: unknown
-      ): Matchers<R> & Inverse<Matchers<R>> & PromiseMatchers;
-      /**
-       * Used when you want to check that an item is in a list.
-       * For testing the items in the list, this  matcher recursively checks the
-       * equality of all fields, rather than checking for object identity.
-       */
-      toContainEqual(
-        expected: unknown
-      ): Matchers<R> & Inverse<Matchers<R>> & PromiseMatchers;
-      /**
-       * Used when you want to check that two objects have the same value.
-       * This matcher recursively checks the equality of all fields, rather than checking for object identity.
-       */
-      toEqual(
-        expected: unknown
-      ): Matchers<R> & Inverse<Matchers<R>> & PromiseMatchers;
-      /**
-       * Used to check that an object has a `.length` property
-       * and it is set to a certain numeric value.
-       */
-      toHaveLength(
-        expected: number
-      ): Matchers<R> & Inverse<Matchers<R>> & PromiseMatchers;
-      /**
-       * Use to check if property at provided reference keyPath exists for an object.
-       * For checking deeply nested properties in an object you may use dot notation or an array containing
-       * the keyPath for deep references.
-       *
-       * Optionally, you can provide a value to check if it's equal to the value present at keyPath
-       * on the target object. This matcher uses 'deep equality' (like `toEqual()`) and recursively checks
-       * the equality of all fields.
-       *
-       * @example
-       *
-       * expect(houseForSale).toHaveProperty('kitchen.area', 20);
-       */
-      toHaveProperty(
-        keyPath: string | Array<string>,
-        value?: unknown
-      ): Matchers<R> & Inverse<Matchers<R>> & PromiseMatchers;
-      /**
-       * Check that a string matches a regular expression.
-       */
-      toMatch(
-        expected: string | RegExp
-      ): Matchers<R> & Inverse<Matchers<R>> & PromiseMatchers;
-      /**
-       * Used to check that a JavaScript object matches a subset of the properties of an object
-       */
-      toMatchObject(
-        expected: Record<string, unknown> | Array<unknown>
-      ): Matchers<R> & Inverse<Matchers<R>> & PromiseMatchers;
-      /**
-       * Use to test that objects have the same types as well as structure.
-       */
-      toStrictEqual(
-        expected: unknown
-      ): Matchers<R> & Inverse<Matchers<R>> & PromiseMatchers;
-    }
-  }
-}
+expectLib.extend({
+  toHaveScreenshot: () => notSupported('toHaveScreenshot'),
+  toMatchSnapshot: () => notSupported('toMatchSnapshot'),
+});
 
-export const expect: Expect = expectLib;
+export const expect: typeof import('../../bundles/node_modules/playwright/types/test').expect =
+  expectLib;

package/src/core/gatherer.ts

@@ -29,7 +29,7 @@ import {
   BrowserContext,
   request as apiRequest,
   selectors,
-} from 'playwright-chromium';
+} from 'playwright-core';
 import { PluginManager } from '../plugins';
 import { log } from './logger';
 import { Driver, NetworkConditions, RunOptions } from '../common_types';

package/src/core/runner.ts

@@ -46,8 +46,11 @@ import {
   StepResult,
   PushOptions,
 } from '../common_types';
-import { PluginManager } from '../plugins';
-import { PerformanceManager } from '../plugins';
+import {
+  PluginManager,
+  PerformanceManager,
+  filterBrowserMessages,
+} from '../plugins';
 import { Gatherer } from './gatherer';
 import { log } from './logger';
 import { Monitor, MonitorConfig } from '../dsl/monitor';
@@ -149,12 +152,12 @@ export default class Runner {
      * Set up the corresponding reporter and fallback
      * to default reporter if not provided
      */
-    const { reporter, outfd } = options;
+    const { reporter, outfd, dryRun } = options;
     const Reporter =
       typeof reporter === 'function'
         ? reporter
         : reporters[reporter] || reporters['default'];
-    this.#reporter = new Reporter({ fd: outfd });
+    this.#reporter = new Reporter({ fd: outfd, dryRun });
   }
 
   async runBeforeAllHook(args: HooksArgs) {
@@ -209,8 +212,9 @@ export default class Runner {
        */
       pluginManager.onStep(step);
       traceEnabled && (await pluginManager.start('trace'));
-      // call the step definition
-      await step.callback();
+      // invoke the step callback by extracting to a variable to get better stack trace
+      const cb = step.callback;
+      await cb();
     } catch (error) {
       data.status = 'failed';
       data.error = error;
@@ -288,8 +292,7 @@ export default class Runner {
       params,
     });
     /**
-     * Exeucute the journey callback which would
-     * register the steps for the current journey
+     * Exeucute the journey callback which registers the steps for current journey
      */
     journey.callback({ ...context.driver, params });
   }
@@ -310,7 +313,10 @@ export default class Runner {
       timestamp: getTimestamp(),
       options,
       ...pluginOutput,
-      browserconsole: status == 'failed' ? pluginOutput.browserconsole : [],
+      browserconsole: filterBrowserMessages(
+        pluginOutput.browserconsole,
+        status
+      ),
     });
     // clear screenshots cache after each journey
     await rm(Runner.screenshotPath, { recursive: true, force: true });

package/src/dsl/journey.ts

@@ -29,7 +29,7 @@ import {
   BrowserContext,
   CDPSession,
   APIRequestContext,
-} from 'playwright-chromium';
+} from 'playwright-core';
 import { Step } from './step';
 import { VoidCallback, HooksCallback, Params, Location } from '../common_types';
 import { Monitor, MonitorConfig } from './monitor';

package/src/dsl/monitor.ts

@@ -72,6 +72,10 @@ export type MonitorConfig = {
   params?: Params;
   playwrightOptions?: PlaywrightOptions;
   alert?: AlertConfig;
+  /**
+   * By default, the monitor will be retested on failure
+   */
+  retestOnFailure?: boolean;
 };
 
 type MonitorFilter = {

package/src/helpers.ts

@@ -234,30 +234,6 @@ export function rewriteErrorStack(stack: string, indexes: [number, number]) {
   return stack;
 }
 
-// formatError prefers receiving proper Errors, but since at runtime
-// non Error exceptions can be thrown, it tolerates though. The
-// redundant type Error | any expresses that.
-export function formatError(error: Error | any) {
-  if (error === undefined || error === null) {
-    return;
-  }
-
-  if (!(error instanceof Error)) {
-    return {
-      message: `Error "${error}" received, with type "${typeof error}". (Do not throw exceptions without using \`new Error("my message")\`)`,
-      name: '',
-      stack: '',
-    };
-  }
-  const { name, message, stack } = error;
-  const indexes = findPWLogsIndexes(message);
-  return {
-    name,
-    message: rewriteErrorMessage(message, indexes[0]),
-    stack: rewriteErrorStack(stack, indexes),
-  };
-}
-
 const cwd = process.cwd();
 /**
  * All the settings that are related to the Synthetics is stored

package/src/index.ts

@@ -55,7 +55,7 @@ export type {
   CDPSession,
   APIRequestContext,
   devices,
-} from 'playwright-chromium';
+} from 'playwright-core';
 
 /**
  * Export the types necessary to write custom reporters

package/src/options.ts

@@ -31,10 +31,10 @@ import { THROTTLING_WARNING_MSG, error, warn } from './helpers';
 
 type Mode = 'run' | 'push';
 
-export function normalizeOptions(
+export async function normalizeOptions(
   cliArgs: CliArgs,
   mode: Mode = 'run'
-): RunOptions {
+): Promise<RunOptions> {
   const options: RunOptions = {
     ...cliArgs,
     environment: process.env['NODE_ENV'] || 'development',
@@ -85,7 +85,7 @@ export function normalizeOptions(
    */
   const config =
     cliArgs.config || !cliArgs.inline
-      ? readConfig(options.environment, cliArgs.config)
+      ? await readConfig(options.environment, cliArgs.config)
       : {};
 
   /**
@@ -106,7 +106,7 @@ export function normalizeOptions(
   );
   options.playwrightOptions = {
     ...playwrightOpts,
-    headless: cliArgs.headless ?? playwrightOpts?.headless,
+    headless: getHeadlessFlag(cliArgs.headless, playwrightOpts?.headless),
     chromiumSandbox: cliArgs.sandbox ?? playwrightOpts?.chromiumSandbox,
     ignoreHTTPSErrors:
       cliArgs.ignoreHttpsErrors ?? playwrightOpts?.ignoreHTTPSErrors,
@@ -140,6 +140,18 @@ export function normalizeOptions(
   return options;
 }
 
+export function getHeadlessFlag(
+  cliHeadless: boolean,
+  configHeadless?: boolean
+) {
+  // if cliHeadless is false, then we don't care about configHeadless
+  if (!cliHeadless) {
+    return false;
+  }
+  // default is headless
+  return configHeadless ?? true;
+}
+
 export function validatePushOptions(opts: PushOptions) {
   if (opts.tags || opts.match) {
     throw error(`Aborted. Invalid CLI flags.

package/src/plugins/browser-console.ts

@@ -23,13 +23,16 @@
  *
  */
 
-import type { ConsoleMessage } from 'playwright-core';
-import { BrowserMessage, Driver } from '../common_types';
+import type { ConsoleMessage, WebError } from 'playwright-core';
+import { BrowserMessage, Driver, StatusValue } from '../common_types';
 import { log } from '../core/logger';
 import { Step } from '../dsl';
 import { getTimestamp } from '../helpers';
 
-const defaultMessageLimit = 1000;
+const DEFAULT_MSG_LIMIt = 1000;
+const SUCCESSFUL_MSG_LIMIT = 100;
+
+const allowedTypes = ['error', 'warning', 'log'];
 
 export class BrowserConsole {
   private messages: BrowserMessage[] = [];
@@ -42,7 +45,7 @@ export class BrowserConsole {
       return;
     }
     const type = msg.type();
-    if (type === 'error' || type === 'warning') {
+    if (allowedTypes.includes(type)) {
       const { name, index } = this._currentStep;
       this.messages.push({
         timestamp: getTimestamp(),
@@ -55,11 +58,12 @@ export class BrowserConsole {
     }
   };
 
-  private pageErrorEventListener = (error: Error) => {
+  private webErrorListener = (webErr: WebError) => {
     if (!this._currentStep) {
       return;
     }
     const { name, index } = this._currentStep;
+    const error = webErr.error();
     this.messages.push({
       timestamp: getTimestamp(),
       text: error.message,
@@ -72,7 +76,7 @@ export class BrowserConsole {
   };
 
   private enforceMessagesLimit() {
-    if (this.messages.length > defaultMessageLimit) {
+    if (this.messages.length > DEFAULT_MSG_LIMIt) {
       this.messages.splice(0, 1);
     }
   }
@@ -80,13 +84,40 @@ export class BrowserConsole {
   start() {
     log(`Plugins: started collecting console events`);
     this.driver.context.on('console', this.consoleEventListener);
-    this.driver.page.on('pageerror', this.pageErrorEventListener);
+    this.driver.context.on('weberror', this.webErrorListener);
   }
 
   stop() {
     this.driver.context.off('console', this.consoleEventListener);
-    this.driver.page.off('pageerror', this.pageErrorEventListener);
+    this.driver.context.off('weberror', this.webErrorListener);
     log(`Plugins: stopped collecting console events`);
     return this.messages;
   }
 }
+
+export function filterBrowserMessages(
+  messages: BrowserMessage[],
+  status: StatusValue
+) {
+  if (status == 'skipped') {
+    return [];
+  } else if (status === 'failed' || messages.length <= SUCCESSFUL_MSG_LIMIT) {
+    return messages;
+  }
+  // collect 100 messages from the browser console when the test is successful,
+  // giving priority to errors and warnings
+  const result = messages.filter(msg => msg.type === 'error');
+  if (result.length >= SUCCESSFUL_MSG_LIMIT) {
+    return result.slice(-SUCCESSFUL_MSG_LIMIT);
+  }
+
+  // collect warnings
+  result.push(...messages.filter(msg => msg.type === 'warning'));
+  if (result.length >= SUCCESSFUL_MSG_LIMIT) {
+    return result.slice(-SUCCESSFUL_MSG_LIMIT);
+  }
+
+  // collect logs
+  result.push(...messages.filter(msg => msg.type === 'log'));
+  return result.slice(-SUCCESSFUL_MSG_LIMIT);
+}