@elastic/synthetics 1.0.0-beta.2 → 1.0.0-beta.22

package/src/core/runner.ts

@@ -23,47 +23,63 @@
  *
  */
 
-import { EventEmitter } from 'events';
+import { once, EventEmitter } from 'events';
+import { join } from 'path';
+import { rm } from 'fs/promises';
 import { Journey } from '../dsl/journey';
 import { Step } from '../dsl/step';
 import { reporters, Reporter } from '../reporters';
-import { getMonotonicTime, getTimestamp, runParallel } from '../helpers';
+import {
+  CACHE_PATH,
+  monotonicTimeInSeconds,
+  getTimestamp,
+  runParallel,
+  generateUniqueId,
+  mkdirAsync,
+  writeFileAsync,
+} from '../helpers';
 import {
   StatusValue,
-  FilmStrip,
-  NetworkInfo,
-  VoidCallback,
+  HooksCallback,
+  Params,
+  NetworkConditions,
+  PluginOutput,
   CliArgs,
+  HooksArgs,
+  PlaywrightOptions,
+  Driver,
+  Screenshot,
 } from '../common_types';
-import { BrowserMessage, PluginManager } from '../plugins';
-import { PerformanceManager, Metrics } from '../plugins';
-import { Driver, Gatherer } from './gatherer';
+import { PageMetrics, PluginManager } from '../plugins';
+import { PerformanceManager } from '../plugins';
+import { Gatherer } from './gatherer';
 import { log } from './logger';
 
 export type RunOptions = Omit<
   CliArgs,
   | 'debug'
-  | 'json'
   | 'pattern'
   | 'inline'
   | 'require'
-  | 'suiteParams'
   | 'reporter'
-  | 'environment'
+  | 'richEvents'
+  | 'capability'
+  | 'sandbox'
+  | 'headless'
+  | 'throttling'
 > & {
-  params?: RunParamaters;
+  environment?: string;
+  playwrightOptions?: PlaywrightOptions;
+  networkConditions?: NetworkConditions;
   reporter?: CliArgs['reporter'] | Reporter;
 };
 
-type RunParamaters = { environment: string } & Record<string, unknown>;
+type HookType = 'beforeAll' | 'afterAll';
+export type SuiteHooks = Record<HookType, Array<HooksCallback>>;
 
-type BaseContext = {
-  params?: RunParamaters;
+type JourneyContext = {
+  params?: Params;
   start: number;
-  end?: number;
-};
-
-type JourneyContext = BaseContext & {
   driver: Driver;
   pluginManager: PluginManager;
 };
@@ -71,59 +87,85 @@ type JourneyContext = BaseContext & {
 type StepResult = {
   status: StatusValue;
   url?: string;
-  metrics?: Metrics;
   error?: Error;
-  screenshot?: string;
+  pagemetrics?: PageMetrics;
+  filmstrips?: PluginOutput['filmstrips'];
+  metrics?: PluginOutput['metrics'];
+  traces?: PluginOutput['traces'];
 };
 
 type JourneyResult = {
   status: StatusValue;
   error?: Error;
+  networkinfo?: PluginOutput['networkinfo'];
+  browserconsole?: PluginOutput['browserconsole'];
 };
 
 type RunResult = Record<string, JourneyResult>;
 
-type HookType = 'beforeAll' | 'afterAll';
-export type SuiteHooks = Record<HookType, Array<VoidCallback>>;
+interface StepEvent {
+  step: Step;
+  journey: Journey;
+}
+
+interface JourneyEvent {
+  journey: Journey;
+  timestamp: number;
+  params?: Params;
+}
 
 interface Events {
-  start: { numJourneys: number };
-  'journey:register': {
-    journey: Journey;
+  start: {
+    numJourneys: number;
+    networkConditions?: NetworkConditions;
   };
-  'journey:start': {
+  'journey:register': {
     journey: Journey;
-    timestamp: number;
-    params: RunParamaters;
   };
-  'journey:end': BaseContext &
+  'journey:start': JourneyEvent;
+  'journey:end': JourneyEvent &
     JourneyResult & {
-      journey: Journey;
-      filmstrips?: Array<FilmStrip>;
-      networkinfo?: Array<NetworkInfo>;
-      browserconsole?: Array<BrowserMessage>;
+      start: number;
+      end: number;
+      options: RunOptions;
+      timestamp: number;
+    };
+  'journey:end:reported': unknown;
+  'step:start': StepEvent;
+  'step:end': StepEvent &
+    StepResult & {
+      start: number;
+      end: number;
     };
-  'step:start': { journey: Journey; step: Step };
-  'step:end': StepResult & {
-    start: number;
-    end: number;
-    journey: Journey;
-    step: Step;
-  };
   end: unknown;
 }
 
-export default class Runner {
+export default interface Runner {
+  on<K extends keyof Events>(name: K, listener: (v: Events[K]) => void): this;
+  emit<K extends keyof Events>(event: K, args: Events[K]): boolean;
+}
+
+export default class Runner extends EventEmitter {
   active = false;
-  eventEmitter = new EventEmitter();
   currentJourney?: Journey = null;
   journeys: Journey[] = [];
   hooks: SuiteHooks = { beforeAll: [], afterAll: [] };
+  hookError: Error | undefined;
+  static screenshotPath = join(CACHE_PATH, 'screenshots');
 
   static async createContext(options: RunOptions): Promise<JourneyContext> {
-    const start = getMonotonicTime();
     const driver = await Gatherer.setupDriver(options);
+    /**
+     * Do not include browser launch/context creation duration
+     * as part of journey duration
+     */
+    const start = monotonicTimeInSeconds();
     const pluginManager = await Gatherer.beginRecording(driver, options);
+    /**
+     * For each journey we create the screenshots folder for
+     * caching all screenshots and clear them at end of each journey
+     */
+    await mkdirAsync(this.screenshotPath, { recursive: true });
     return {
       start,
       params: options.params,
@@ -132,7 +174,34 @@ export default class Runner {
     };
   }
 
-  addHook(type: HookType, callback: VoidCallback) {
+  async captureScreenshot(page: Driver['page'], step: Step) {
+    await page.waitForLoadState('load');
+    const buffer = await page
+      .screenshot({
+        type: 'jpeg',
+        quality: 80,
+      })
+      .catch(() => {});
+    /**
+     * Write the screenshot image buffer with additional details (step
+     * information) which could be extracted at the end of
+     * each journey without impacting the step timing information
+     */
+    if (buffer) {
+      const fileName = `${generateUniqueId()}.json`;
+      const screenshot: Screenshot = {
+        step,
+        timestamp: getTimestamp(),
+        data: buffer.toString('base64'),
+      };
+      await writeFileAsync(
+        join(Runner.screenshotPath, fileName),
+        JSON.stringify(screenshot)
+      );
+    }
+  }
+
+  addHook(type: HookType, callback: HooksCallback) {
     this.hooks[type].push(callback);
   }
 
@@ -141,33 +210,24 @@ export default class Runner {
     this.currentJourney = journey;
   }
 
-  emit<K extends keyof Events>(e: K, v: Events[K]) {
-    log(`Runner: emit> ${e}`);
-    this.eventEmitter.emit(e, v);
-  }
-
-  on<K extends keyof Events>(e: K, cb: (v: Events[K]) => void) {
-    this.eventEmitter.on(e, cb);
-  }
-
-  async runBeforeAllHook() {
+  async runBeforeAllHook(args: HooksArgs) {
     log(`Runner: beforeAll hooks`);
-    await runParallel(this.hooks.beforeAll);
+    await runParallel(this.hooks.beforeAll, args);
   }
 
-  async runAfterAllHook() {
+  async runAfterAllHook(args: HooksArgs) {
     log(`Runner: afterAll hooks`);
-    await runParallel(this.hooks.afterAll);
+    await runParallel(this.hooks.afterAll, args);
   }
 
-  async runBeforeHook(journey: Journey) {
+  async runBeforeHook(journey: Journey, args: HooksArgs) {
     log(`Runner: before hooks for (${journey.name})`);
-    await runParallel(journey.hooks.before);
+    await runParallel(journey.hooks.before, args);
   }
 
-  async runAfterHook(journey: Journey) {
+  async runAfterHook(journey: Journey, args: HooksArgs) {
     log(`Runner: after hooks for (${journey.name})`);
-    await runParallel(journey.hooks.after);
+    await runParallel(journey.hooks.after, args);
   }
 
   async runStep(
@@ -179,7 +239,7 @@ export default class Runner {
       status: 'succeeded',
     };
     log(`Runner: start step (${step.name})`);
-    const { metrics, screenshots } = options;
+    const { metrics, screenshots, filmstrips, trace } = options;
     const { driver, pluginManager } = context;
     /**
      * URL needs to be the first navigation request of any step
@@ -190,28 +250,49 @@ export default class Runner {
       if (!data.url && req.isNavigationRequest()) {
         data.url = req.url();
       }
-      driver.page.off('request', captureUrl);
+      driver.context.off('request', captureUrl);
     };
-    driver.page.on('request', captureUrl);
+    driver.context.on('request', captureUrl);
+
     try {
+      /**
+       * Set up plugin manager context and also register
+       * step level plugins
+       */
+      const traceEnabled = trace || filmstrips;
       pluginManager.onStep(step);
+      traceEnabled && (await pluginManager.start('trace'));
+      // call the step definition
       await step.callback();
+      /**
+       * Collect all step level metrics
+       */
       if (metrics) {
-        data.metrics = await pluginManager.get(PerformanceManager).getMetrics();
+        data.pagemetrics = await (
+          pluginManager.get('performance') as PerformanceManager
+        ).getMetrics();
+      }
+      if (traceEnabled) {
+        const traceOutput = await pluginManager.stop('trace');
+        Object.assign(data, traceOutput);
       }
     } catch (error) {
       data.status = 'failed';
       data.error = error;
     } finally {
-      data.url ??= driver.page.url();
-      if (screenshots) {
-        await driver.page.waitForLoadState('load');
-        data.screenshot = (
-          await driver.page.screenshot({
-            type: 'jpeg',
-            quality: 80,
-          })
-        ).toString('base64');
+      /**
+       * Capture screenshot for the newly created pages
+       * via popup or new windows/tabs
+       *
+       * Last open page will get us the correct screenshot
+       */
+      const pages = driver.context.pages();
+      const page = pages[pages.length - 1];
+      if (page) {
+        data.url ??= page.url();
+        if (screenshots && screenshots !== 'off') {
+          await this.captureScreenshot(page, step);
+        }
       }
     }
     log(`Runner: end step (${step.name})`);
@@ -226,7 +307,7 @@ export default class Runner {
     const results: Array<StepResult> = [];
     let skipStep = false;
     for (const step of journey.steps) {
-      const start = getMonotonicTime();
+      const start = monotonicTimeInSeconds();
       this.emit('step:start', { journey, step });
       let data: StepResult = { status: 'succeeded' };
       if (skipStep) {
@@ -242,7 +323,7 @@ export default class Runner {
         journey,
         step,
         start,
-        end: getMonotonicTime(),
+        end: monotonicTimeInSeconds(),
         ...data,
       });
       if (options.pauseOnError && data.error) {
@@ -264,35 +345,77 @@ export default class Runner {
     journey.callback({ ...context.driver, params });
   }
 
-  async endJourney(journey, result: JourneyContext & JourneyResult) {
-    const { pluginManager, start, params, status, error } = result;
-    const {
-      filmstrips,
-      networkinfo,
-      browserconsole,
-    } = await pluginManager.output();
+  async endJourney(
+    journey,
+    result: JourneyContext & JourneyResult,
+    options: RunOptions
+  ) {
+    const end = monotonicTimeInSeconds();
+    const { pluginManager, start, status, error } = result;
+    const pluginOutput = pluginManager.output();
     this.emit('journey:end', {
       journey,
       status,
       error,
-      params,
       start,
-      end: getMonotonicTime(),
-      filmstrips,
-      networkinfo,
-      browserconsole: status == 'failed' ? browserconsole : null,
+      end,
+      timestamp: getTimestamp(),
+      options,
+      ...pluginOutput,
+      browserconsole: status == 'failed' ? pluginOutput.browserconsole : [],
+    });
+    /**
+     * Wait for the all the reported events to be consumed aschronously
+     * by reporter.
+     */
+    if (options.reporter === 'json') {
+      await once(this, 'journey:end:reported');
+    }
+    // clear screenshots cache after each journey
+    await rm(Runner.screenshotPath, { recursive: true, force: true });
+  }
+
+  /**
+   * Simulate a journey run to capture errors in the beforeAll hook
+   */
+  async runFakeJourney(journey: Journey, options: RunOptions) {
+    const start = monotonicTimeInSeconds();
+    this.emit('journey:start', {
+      journey,
+      timestamp: getTimestamp(),
+      params: options.params,
+    });
+    const result: JourneyResult = {
+      status: 'failed',
+      error: this.hookError,
+    };
+    this.emit('journey:end', {
+      journey,
+      timestamp: getTimestamp(),
+      start,
+      options,
+      end: monotonicTimeInSeconds(),
+      ...result,
     });
+    if (options.reporter === 'json') {
+      await once(this, 'journey:end:reported');
+    }
+    return result;
   }
 
   async runJourney(journey: Journey, options: RunOptions) {
     const result: JourneyResult = {
       status: 'succeeded',
     };
-    log(`Runner: start journey (${journey.name})`);
     const context = await Runner.createContext(options);
+    log(`Runner: start journey (${journey.name})`);
     try {
       this.registerJourney(journey, context);
-      await this.runBeforeHook(journey);
+      const hookArgs = {
+        env: options.environment,
+        params: options.params,
+      };
+      await this.runBeforeHook(journey, hookArgs);
       const stepResults = await this.runSteps(journey, context, options);
       /**
        * Mark journey as failed if any intermediate step fails
@@ -303,26 +426,20 @@ export default class Runner {
           result.error = stepResult.error;
         }
       }
-      await this.runAfterHook(journey);
+      await this.runAfterHook(journey, hookArgs);
     } catch (e) {
       result.status = 'failed';
       result.error = e;
     } finally {
-      await this.endJourney(journey, { ...context, ...result });
+      await this.endJourney(journey, { ...context, ...result }, options);
       await Gatherer.dispose(context.driver);
     }
     log(`Runner: end journey (${journey.name})`);
     return result;
   }
 
-  async run(options: RunOptions) {
-    const result: RunResult = {};
-    if (this.active) {
-      return result;
-    }
-    this.active = true;
-    log(`Runner: run ${this.journeys.length} journeys`);
-    const { reporter, journeyName, outfd } = options;
+  async init(options: RunOptions) {
+    const { reporter, outfd, networkConditions } = options;
     /**
      * Set up the corresponding reporter and fallback
      */
@@ -331,34 +448,64 @@ export default class Runner {
         ? reporter
         : reporters[reporter] || reporters['default'];
     new Reporter(this, { fd: outfd });
-    this.emit('start', { numJourneys: this.journeys.length });
-    await this.runBeforeAllHook();
+    this.emit('start', {
+      numJourneys: this.journeys.length,
+      networkConditions,
+    });
+    /**
+     * Set up the directory for caching screenshots
+     */
+    await mkdirAsync(CACHE_PATH, { recursive: true });
+  }
+
+  async run(options: RunOptions) {
+    const result: RunResult = {};
+    if (this.active) {
+      return result;
+    }
+    this.active = true;
+    log(`Runner: run ${this.journeys.length} journeys`);
+    this.init(options);
+    await this.runBeforeAllHook({
+      env: options.environment,
+      params: options.params,
+    }).catch(e => (this.hookError = e));
+
+    const { dryRun, match, tags } = options;
     for (const journey of this.journeys) {
       /**
        * Used by heartbeat to gather all registered journeys
        */
-      if (options.dryRun) {
+      if (dryRun) {
         this.emit('journey:register', { journey });
         continue;
       }
-      // TODO: Replace functionality with journey.only
-      if (journeyName && journey.name != journeyName) {
+      if (!journey.isMatch(match, tags)) {
         continue;
       }
-      const journeyResult = await this.runJourney(journey, options);
+      const journeyResult: JourneyResult = this.hookError
+        ? await this.runFakeJourney(journey, options)
+        : await this.runJourney(journey, options);
       result[journey.name] = journeyResult;
     }
     await Gatherer.stop();
-    await this.runAfterAllHook();
-    this.reset();
-    this.emit('end', {});
+    await this.runAfterAllHook({
+      env: options.environment,
+      params: options.params,
+    });
+    await this.reset();
     return result;
   }
 
-  reset() {
-    log('Runner: reset');
+  async reset() {
+    /**
+     * Clear all cache data stored for post processing by
+     * the current synthetic agent run
+     */
+    await rm(CACHE_PATH, { recursive: true, force: true });
     this.currentJourney = null;
     this.journeys = [];
     this.active = false;
+    this.emit('end', {});
   }
 }

package/src/dsl/journey.ts

@@ -24,35 +24,38 @@
  */
 
 import { Browser, Page, BrowserContext, CDPSession } from 'playwright-chromium';
+import micromatch, { isMatch } from 'micromatch';
 import { Step } from './step';
-import { VoidCallback } from '../common_types';
+import { VoidCallback, HooksCallback, Params } from '../common_types';
 
 export type JourneyOptions = {
   name: string;
   id?: string;
+  tags?: string[];
 };
 
 type HookType = 'before' | 'after';
-export type Hooks = Record<HookType, Array<VoidCallback>>;
-
+export type Hooks = Record<HookType, Array<HooksCallback>>;
 export type JourneyCallback = (options: {
   page: Page;
   context: BrowserContext;
   browser: Browser;
   client: CDPSession;
-  params: Record<string, any>;
+  params: Params;
 }) => void;
 
 export class Journey {
   name: string;
   id?: string;
+  tags?: string[];
   callback: JourneyCallback;
   steps: Step[] = [];
   hooks: Hooks = { before: [], after: [] };
 
   constructor(options: JourneyOptions, callback: JourneyCallback) {
     this.name = options.name;
-    this.id = options.id;
+    this.id = options.id || options.name;
+    this.tags = options.tags;
     this.callback = callback;
   }
 
@@ -62,7 +65,26 @@ export class Journey {
     return step;
   }
 
-  addHook(type: HookType, callback: VoidCallback) {
+  addHook(type: HookType, callback: HooksCallback) {
     this.hooks[type].push(callback);
   }
+  /**
+   * Matches journeys based on the provided args. Proitize tags over match
+   * - tags pattern that matches only tags
+   * - match pattern that matches both name and tags
+   */
+  isMatch(matchPattern: string, tagsPattern: Array<string>) {
+    if (tagsPattern) {
+      return this.tagsMatch(tagsPattern);
+    }
+    if (matchPattern) {
+      return isMatch(this.name, matchPattern) || this.tagsMatch(matchPattern);
+    }
+    return true;
+  }
+
+  tagsMatch(pattern) {
+    const matchess = micromatch(this.tags || ['*'], pattern);
+    return matchess.length > 0;
+  }
 }