@elastic/synthetics 1.0.0-beta.4 → 1.0.0-beta.41

package/src/reporters/json.ts

@@ -40,12 +40,18 @@ import { Journey, Step } from '../dsl';
 import snakeCaseKeys from 'snakecase-keys';
 import {
   NetworkInfo,
+  SecurityDetails,
+  NetworkConditions,
   TraceOutput,
   StatusValue,
   PerfMetrics,
+  Screenshot,
+  StartEvent,
+  JourneyStartResult,
+  StepEndResult,
+  JourneyEndResult,
+  PageMetrics,
 } from '../common_types';
-import { Protocol } from 'playwright-chromium/types/protocol';
-import { Metrics } from '../plugins';
 
 /* eslint-disable @typescript-eslint/no-var-requires */
 const { version, name } = require('../../package.json');
@@ -57,11 +63,11 @@ type OutputType =
   | 'screenshot/block'
   | 'step/screenshot_ref'
   | 'step/screenshot'
+  | 'step/metrics'
+  | 'step/filmstrips'
   | 'step/end'
   | 'journey/network_info'
-  | 'journey/filmstrips'
   | 'journey/browserconsole'
-  | 'journey/metrics'
   | 'journey/end';
 
 type Payload = {
@@ -69,12 +75,12 @@ type Payload = {
   start?: number;
   end?: number;
   url?: string;
-  status?: StatusValue | number;
-  metrics?: Metrics;
-  params?: Record<string, unknown>;
+  status?: StatusValue;
+  pagemetrics?: PageMetrics;
   type?: OutputType;
   text?: string;
   index?: number;
+  network_conditions?: NetworkConditions;
 };
 
 type OutputFields = {
@@ -90,7 +96,7 @@ type OutputFields = {
   };
   error?: Error;
   root_fields?: Record<string, unknown>;
-  payload?: Payload | NetworkInfo;
+  payload?: Payload;
   blob?: string;
   blob_mime?: string;
 };
@@ -112,13 +118,6 @@ type ScreenshotReference = {
   }>;
 };
 
-type ScreenshotOutput = {
-  step: Step;
-  blob_mime: string;
-  blocks: Array<ScreenshotBlob>;
-  reference: ScreenshotReference;
-};
-
 function getMetadata() {
   return {
     os: {
@@ -131,56 +130,12 @@ function getMetadata() {
   };
 }
 
-function formatVersion(protocol: string | undefined) {
-  if (!protocol) {
-    return;
-  }
-  if (protocol === 'h2') {
-    return 2;
-  } else if (protocol === 'http/1.1') {
-    return 1.1;
-  } else if (protocol === 'http/1.0') {
-    return 1.0;
-  } else if (protocol.startsWith('h3')) {
-    return 3;
-  }
-}
-
-function formatRequest(request: Protocol.Network.Request) {
-  const postData = request.postData ? request.postData : '';
-  return {
-    ...request,
-    body: {
-      bytes: postData.length,
-      content: postData,
-    },
-    referrer: request.headers?.Referer,
-  };
-}
-
-function formatResponse(response: Protocol.Network.Response) {
-  if (!response) {
+function formatTLS(tls: SecurityDetails) {
+  if (!tls || !tls.protocol) {
     return;
   }
-  return {
-    ...response,
-    body: {
-      bytes: response.encodedDataLength,
-    },
-    status_code: response.status,
-  };
-}
-
-function formatTLS(tls: Protocol.Network.SecurityDetails) {
-  if (!tls) {
-    return;
-  }
-  const cipher = `${tls.keyExchange ? tls.keyExchange + '_' : ''}${
-    tls.cipher
-  }_${tls.keyExchangeGroup}`;
   const [name, version] = tls.protocol.toLowerCase().split(' ');
   return {
-    cipher,
     server: {
       x509: {
         issuer: {
@@ -194,7 +149,7 @@ function formatTLS(tls: Protocol.Network.SecurityDetails) {
       },
     },
     version_protocol: name,
-    version: version,
+    version,
   };
 }
 
@@ -209,17 +164,14 @@ export function formatNetworkFields(network: NetworkInfo) {
       original: request.headers?.['User-Agent'],
     },
     http: {
-      version: formatVersion(response?.protocol),
-      request: formatRequest(request),
-      response: formatResponse(response),
+      request,
+      response,
     },
     tls: formatTLS(response?.securityDetails),
   };
 
   const pickItems: Array<keyof NetworkInfo> = [
     'browser',
-    'status',
-    'method',
     'type',
     'isNavigationRequest',
     'requestSentTime',
@@ -270,8 +222,7 @@ function stepInfo(
 }
 
 export async function getScreenshotBlocks(screenshot: Buffer) {
-  const img = sharp(screenshot, { sequentialRead: true });
-  const { width, height } = await img.metadata();
+  const { width, height } = await sharp(screenshot).metadata();
   /**
    * Chop the screenshot image (1280*720) which is the default
    * viewport size in to 64 equal blocks for a given image
@@ -281,8 +232,8 @@ export async function getScreenshotBlocks(screenshot: Buffer) {
    * optimal for caching in the ES and proved out to be bad, so we stick to 8
    */
   const divisions = 8;
-  const blockWidth = width / divisions;
-  const blockHeight = height / divisions;
+  const blockWidth = Math.floor(width / divisions);
+  const blockHeight = Math.floor(height / divisions);
   const reference: ScreenshotReference = {
     width,
     height,
@@ -294,7 +245,9 @@ export async function getScreenshotBlocks(screenshot: Buffer) {
     const top = row * blockHeight;
     for (let col = 0; col < divisions; col++) {
       const left = col * blockWidth;
-      const buf = await img
+      // We create a new sharp instance for each block to avoid
+      // running in to extraction/orientation issues
+      const buf = await sharp(screenshot, { sequentialRead: true })
         .extract({ top, left, width: blockWidth, height: blockHeight })
         .jpeg()
         .toBuffer();
@@ -330,14 +283,14 @@ export async function getScreenshotBlocks(screenshot: Buffer) {
  */
 export async function gatherScreenshots(
   screenshotsPath: string,
-  callback: (step: Step, data: string) => Promise<void>
+  callback: (data: Screenshot) => Promise<void>
 ) {
   if (isDirectory(screenshotsPath)) {
     await totalist(screenshotsPath, async (_, absPath) => {
       try {
         const content = readFileSync(absPath, 'utf8');
-        const { step, data } = JSON.parse(content);
-        await callback(step, data);
+        const screenshot: Screenshot = JSON.parse(content);
+        await callback(screenshot);
       } catch (_) {
         // TODO: capture progarammatic synthetic errors under different type
       }
@@ -346,166 +299,191 @@ export async function gatherScreenshots(
 }
 
 export default class JSONReporter extends BaseReporter {
-  _registerListeners() {
+  onStart(event: StartEvent) {
     /**
      * report the number of journeys that exists on a suite which
      * could be used for better sharding
      */
-    this.runner.on('start', ({ numJourneys }) => {
-      this.writeJSON({
-        type: 'synthetics/metadata',
-        root_fields: {
-          num_journeys: numJourneys,
-        },
-      });
+    this.writeJSON({
+      type: 'synthetics/metadata',
+      root_fields: {
+        num_journeys: event.numJourneys,
+      },
+      payload: event.networkConditions
+        ? {
+            network_conditions: event.networkConditions,
+          }
+        : undefined,
     });
+  }
 
-    this.runner.on('journey:register', ({ journey }) => {
-      this.writeJSON({
-        type: 'journey/register',
-        journey,
-      });
+  onJourneyRegister(journey: Journey): void {
+    this.writeJSON({
+      type: 'journey/register',
+      journey,
     });
+  }
 
-    this.runner.on('journey:start', ({ journey, timestamp, params }) => {
-      this.writeJSON({
-        type: 'journey/start',
-        journey,
-        timestamp,
-        payload: { params, source: journey.callback.toString() },
-      });
+  override onJourneyStart(journey: Journey, { timestamp }: JourneyStartResult) {
+    this.writeJSON({
+      type: 'journey/start',
+      journey,
+      timestamp,
+      payload: { source: journey.callback.toString() },
     });
+  }
 
-    this.runner.on(
-      'step:end',
-      ({ journey, step, start, end, error, url, status, metrics }) => {
+  override onStepEnd(
+    journey: Journey,
+    step: Step,
+    {
+      start,
+      end,
+      error,
+      url,
+      status,
+      pagemetrics,
+      traces,
+      metrics,
+      filmstrips,
+    }: StepEndResult
+  ) {
+    this.writeMetrics(journey, step, 'relative_trace', traces);
+    this.writeMetrics(journey, step, 'experience', metrics);
+    if (filmstrips) {
+      // Write each filmstrip separately so that we don't get documents that are too large
+      filmstrips.forEach((strip, index) => {
         this.writeJSON({
-          type: 'step/end',
+          type: 'step/filmstrips',
           journey,
-          step: {
-            ...step,
-            duration: {
-              us: getDurationInUs(end - start),
-            },
-          },
-          url,
-          error,
-          payload: {
-            source: step.callback.toString(),
-            url,
-            status,
-            metrics,
+          step,
+          payload: { index },
+          root_fields: {
+            browser: { relative_trace: { start: strip.start } },
           },
+          blob: strip.blob,
+          blob_mime: strip.mime,
         });
-      }
-    );
+      });
+    }
 
-    this.runner.on(
-      'journey:end',
-      async ({
-        journey,
-        start,
-        end,
-        filmstrips,
-        networkinfo,
-        browserconsole,
-        traces,
-        metrics,
+    this.writeJSON({
+      type: 'step/end',
+      journey,
+      step: {
+        ...step,
+        duration: {
+          us: getDurationInUs(end - start),
+        },
+      },
+      url,
+      error,
+      payload: {
+        source: step.callback.toString(),
+        url,
         status,
-        error,
-        options,
-      }) => {
-        const { ssblocks, screenshots } = options;
-        const writeScreenshots =
-          screenshots === 'on' ||
-          (screenshots === 'only-on-failure' && status === 'failed');
-        if (writeScreenshots) {
-          await gatherScreenshots(
-            join(CACHE_PATH, 'screenshots'),
-            async (step, data) => {
-              if (!data) {
-                return;
-              }
-              if (ssblocks) {
-                await this.writeScreenshotBlocks(journey, step, data);
-              } else {
-                this.writeJSON({
-                  type: 'step/screenshot',
-                  journey,
-                  step,
-                  blob: data,
-                  blob_mime: 'image/jpeg',
-                });
-              }
-            }
-          );
-        }
+        pagemetrics,
+      },
+    });
+  }
 
-        if (networkinfo) {
-          networkinfo.forEach(ni => {
-            const { ecs, payload } = formatNetworkFields(ni);
-            this.writeJSON({
-              type: 'journey/network_info',
-              journey,
-              timestamp: ni.timestamp,
-              root_fields: snakeCaseKeys(ecs),
-              step: ni.step,
-              payload: snakeCaseKeys(payload),
-            });
-          });
-        }
-        if (filmstrips) {
-          // Write each filmstrip separately so that we don't get documents that are too large
-          filmstrips.forEach((strip, index) => {
-            this.writeJSON({
-              type: 'journey/filmstrips',
-              journey,
-              payload: { index },
-              root_fields: {
-                browser: { relative_trace: { start: strip.start } },
-              },
-              blob: strip.blob,
-              blob_mime: strip.mime,
-            });
-          });
-        }
-        if (browserconsole) {
-          browserconsole.forEach(({ timestamp, text, type, step }) => {
+  override async onJourneyEnd(
+    journey: Journey,
+    {
+      timestamp,
+      start,
+      end,
+      networkinfo,
+      browserconsole,
+      status,
+      error,
+      options,
+    }: JourneyEndResult
+  ) {
+    const { ssblocks, screenshots } = options;
+    const writeScreenshots =
+      screenshots === 'on' ||
+      (screenshots === 'only-on-failure' && status === 'failed');
+    if (writeScreenshots) {
+      await gatherScreenshots(
+        join(CACHE_PATH, 'screenshots'),
+        async screenshot => {
+          const { data, timestamp, step } = screenshot;
+          if (!data) {
+            return;
+          }
+          if (ssblocks) {
+            await this.writeScreenshotBlocks(journey, screenshot);
+          } else {
             this.writeJSON({
-              type: 'journey/browserconsole',
-              journey,
+              type: 'step/screenshot',
               timestamp,
+              journey,
               step,
-              payload: { text, type } as Payload,
+              blob: data,
+              blob_mime: 'image/jpeg',
             });
-          });
+          }
         }
-        this.writeMetrics(journey, 'relative_trace', traces);
-        this.writeMetrics(journey, 'experience', metrics);
+      );
+    }
+
+    if (networkinfo) {
+      networkinfo.forEach(ni => {
+        const { ecs, payload } = formatNetworkFields(ni);
+        this.writeJSON({
+          type: 'journey/network_info',
+          journey,
+          timestamp: ni.timestamp,
+          root_fields: snakeCaseKeys(ecs),
+          step: ni.step,
+          payload: snakeCaseKeys(payload),
+        });
+      });
+    }
 
+    if (browserconsole) {
+      browserconsole.forEach(({ timestamp, text, type, step, error }) => {
         this.writeJSON({
-          type: 'journey/end',
+          type: 'journey/browserconsole',
           journey,
+          timestamp,
+          step,
           error,
           payload: {
-            start,
-            end,
-            status,
-          },
+            text,
+            type,
+          } as Payload,
         });
-        process.nextTick(() => this.runner.emit('journey:end:reported', {}));
-      }
-    );
+      });
+    }
+
+    this.writeJSON({
+      type: 'journey/end',
+      journey,
+      timestamp,
+      error,
+      payload: {
+        start,
+        end,
+        status,
+      },
+    });
+  }
+
+  override onEnd() {
+    this.stream.flushSync();
   }
 
-  async writeScreenshotBlocks(journey: Journey, step: Step, data: string) {
+  async writeScreenshotBlocks(journey: Journey, screenshot: Screenshot) {
     const { blob_mime, blocks, reference } = await getScreenshotBlocks(
-      Buffer.from(data, 'base64')
+      Buffer.from(screenshot.data, 'base64')
     );
     for (let i = 0; i < blocks.length; i++) {
       const block = blocks[i];
       this.writeJSON({
         type: 'screenshot/block',
+        timestamp: screenshot.timestamp,
         _id: block.id,
         blob: block.blob,
         blob_mime,
@@ -513,8 +491,9 @@ export default class JSONReporter extends BaseReporter {
     }
     this.writeJSON({
       type: 'step/screenshot_ref',
+      timestamp: screenshot.timestamp,
       journey,
-      step,
+      step: screenshot.step,
       root_fields: {
         screenshot_ref: reference,
       },
@@ -523,6 +502,7 @@ export default class JSONReporter extends BaseReporter {
 
   writeMetrics(
     journey: Journey,
+    step: Step,
     type: string,
     events: Array<TraceOutput> | PerfMetrics
   ) {
@@ -530,8 +510,9 @@ export default class JSONReporter extends BaseReporter {
     metrics.forEach(event => {
       event &&
         this.writeJSON({
-          type: 'journey/metrics',
+          type: 'step/metrics',
           journey,
+          step,
           root_fields: {
             browser: {
               [type]: event,
@@ -541,7 +522,7 @@ export default class JSONReporter extends BaseReporter {
     });
   }
 
-  // Writes a structered synthetics event
+  // Writes a structured synthetics event
   // Note that blob is ultimately stored in ES as a base64 encoded string. You must base 64 encode
   // it before passing it into this function!
   // The payload field is an un-indexed field with no ES mapping, so users can put arbitary structured