@elastic/synthetics 1.0.0-beta.24 → 1.0.0-beta.25

package/src/plugins/network.ts

@@ -23,360 +23,284 @@
  *
  */
 
-import { Protocol } from 'playwright-core/types/protocol';
+import { Page, Request, Response } from 'playwright-chromium';
 import { NetworkInfo, BrowserInfo, Driver } from '../common_types';
 import { Step } from '../dsl';
 import { getTimestamp } from '../helpers';
 
+/**
+ * Kibana UI expects the requestStartTime and loadEndTime to be baseline
+ * in seconds as they have the logic to convert it to milliseconds before
+ * using for offset calculation
+ */
+function epochTimeInSeconds() {
+  return getTimestamp() / 1e6;
+}
+
+/**
+ * Used as a key in each Network Request to identify the
+ * associated request across distinct lifecycle events
+ */
+export const NETWORK_ENTRY_SUMBOL = Symbol.for('NetworkEntry');
+
+type RequestWithEntry = Request & {
+  NETWORK_ENTRY_SUMBOL?: symbol;
+};
+
 export class NetworkManager {
   private _browser: BrowserInfo;
-  private _InflightRequestMap = new Map<string, NetworkInfo>();
+  private _barrierPromises = new Set<Promise<void>>();
   results: Array<NetworkInfo> = [];
   _currentStep: Partial<Step> = null;
 
   constructor(private driver: Driver) {}
 
+  /**
+   * Adds a protection barrier aganist all asynchronous extract operations from
+   * request/response object that are happening during page lifecycle, If the
+   * page is closed during the extraction, the barrier enforces those operations
+   * to not result in exception
+   */
+  private _addBarrier(page: Page, promise: Promise<void>) {
+    if (!page) return;
+    const race = Promise.race([
+      new Promise<void>(resolve =>
+        page.on('close', () => {
+          this._barrierPromises.delete(race);
+          resolve();
+        })
+      ),
+      promise,
+    ]);
+    this._barrierPromises.add(race);
+    race.then(() => this._barrierPromises.delete(race));
+  }
+
   async start() {
-    const { client } = this.driver;
+    const { client, context } = this.driver;
     const { product } = await client.send('Browser.getVersion');
     const [name, version] = product.split('/');
     this._browser = { name, version };
     await client.send('Network.enable');
     /**
-     * Listen for all network events
+     * Listen for all network events from PW context
      */
-    client.on(
-      'Network.requestWillBeSent',
-      this._onRequestWillBeSent.bind(this)
-    );
-    client.on(
-      'Network.requestWillBeSentExtraInfo',
-      this._onRequestWillBeSentExtraInfo.bind(this)
-    );
-    client.on('Network.dataReceived', this._onDataReceived.bind(this));
-    client.on('Network.responseReceived', this._onResponseReceived.bind(this));
-    client.on('Network.loadingFinished', this._onLoadingFinished.bind(this));
-    client.on('Network.loadingFailed', this._onLoadingFailed.bind(this));
+    context.on('request', this._onRequest.bind(this));
+    context.on('response', this._onResponse.bind(this));
+    context.on('requestfinished', this._onRequestCompleted.bind(this));
+    context.on('requestfailed', this._onRequestCompleted.bind(this));
   }
 
-  _findNetworkRecord(requestId: string) {
-    return this._InflightRequestMap.get(requestId);
+  private _findNetworkEntry(
+    request: RequestWithEntry
+  ): NetworkInfo | undefined {
+    return request[NETWORK_ENTRY_SUMBOL];
   }
 
-  _onRequestWillBeSent(event: Protocol.Network.requestWillBeSentPayload) {
-    const { url, method, postData, headers } = event.request;
+  private _onRequest(request: Request) {
+    const url = request.url();
     /**
      * Data URI should not show up as network requests
      */
     if (url.startsWith('data:')) {
       return;
     }
-    const { requestId, timestamp, type, loaderId } = event;
-    const isNavigationRequest = requestId == loaderId && type === 'Document';
-    const record = this._findNetworkRecord(requestId);
-    /**
-     * On redirects, another `requestWillBeSent` event will be fired for the
-     * same requestId. We calculate the timings for the redirect request using
-     * the `redirectedResponse` from the redirected request.
-     */
-    if (record) {
-      if (event.redirectResponse) {
-        record.response.redirectURL = url;
-        const response = event.redirectResponse;
-        const data = Object.assign(event, {
-          type: event.type,
-          response,
-          encodedDataLength: response.encodedDataLength,
-          hasExtraInfo: false,
-        });
-        this._onResponseReceived(data);
-        this._onLoadingFinished(data);
-      } else {
-        /**
-         * Edge case, we handle it to proceed with next navigation
-         */
-        this._onLoadingFailed(
-          Object.assign(event, {
-            type: event.type,
-            errorText: 'redirectResponse data is not available',
-          })
-        );
-      }
-      const redirectedRecord = this._findNetworkRecord(requestId);
-      /**
-       * Rewrite the map with new redirect id to not reset
-       * the redirect request with original request
-       */
-      this._InflightRequestMap.delete(requestId);
-      this._InflightRequestMap.set(
-        `redirect:${timestamp}:${requestId}`,
-        redirectedRecord
-      );
-    }
 
-    this._InflightRequestMap.set(requestId, {
+    const timestamp = getTimestamp();
+    const networkEntry: NetworkInfo = {
       browser: this._browser,
       step: this._currentStep,
-      timestamp: getTimestamp(),
+      timestamp,
       url,
-      type,
+      type: request.resourceType(),
       request: {
         url,
-        method,
-        headers,
-        bytes: postData?.length || 0,
+        method: request.method(),
+        headers: {},
+        body: {
+          bytes: request.postDataBuffer()?.length || 0,
+        },
       },
       response: {
         status: -1,
-        mimeType: 'x-unknown',
         headers: {},
-        timing: null,
+        mimeType: 'x-unknown',
+        redirectURL: '',
       },
-      isNavigationRequest,
-      requestSentTime: timestamp,
-      responseReceivedTime: -1,
+      isNavigationRequest: request.isNavigationRequest(),
+      requestSentTime: epochTimeInSeconds(),
       loadEndTime: -1,
+      responseReceivedTime: -1,
       resourceSize: 0,
       transferSize: 0,
       timings: null,
-    });
-  }
-
-  _onRequestWillBeSentExtraInfo(
-    event: Protocol.Network.requestWillBeSentExtraInfoPayload
-  ) {
-    const { requestId, headers } = event;
-    const record = this._findNetworkRecord(requestId);
-    if (!record) {
-      return;
-    }
-    /**
-     * Enhance request headers with additional information
-     */
-    record.request.headers = {
-      ...record.request.headers,
-      ...headers,
     };
-    record.request.referrer = record.request.headers?.Referer;
-  }
-
-  _onDataReceived(event: Protocol.Network.dataReceivedPayload) {
-    const { requestId, dataLength, encodedDataLength } = event;
-    const record = this._findNetworkRecord(requestId);
-    if (!record) {
-      return;
-    }
-    record.resourceSize += dataLength;
-    if (encodedDataLength >= 0) record.transferSize += encodedDataLength;
-  }
 
-  _onResponseReceived(event: Protocol.Network.responseReceivedPayload) {
-    const { requestId, response, timestamp } = event;
-    const record = this._findNetworkRecord(requestId);
-    if (!record) {
-      return;
+    if (request.redirectedFrom()) {
+      const fromEntry = this._findNetworkEntry(request.redirectedFrom());
+      if (fromEntry) fromEntry.response.redirectURL = request.url();
     }
 
-    record.responseReceivedTime = timestamp;
-    record.transferSize = response.encodedDataLength;
-    record.response = {
-      url: response.url,
-      status: response.status,
-      protocol: response.protocol,
-      statusText: response.statusText,
-      headers: response.headers,
-      mimeType: response.mimeType,
-      remoteIPAddress: response.remoteIPAddress,
-      remotePort: response.remotePort,
-      fromServiceWorker: response.fromServiceWorker,
-      redirectURL: record.response.redirectURL,
-      timing: response.timing,
-    };
-    if (response.securityDetails) {
-      record.response.securityDetails = {
-        protocol: response.securityDetails.protocol,
-        subjectName: response.securityDetails.subjectName,
-        issuer: response.securityDetails.issuer,
-        validFrom: response.securityDetails.validFrom,
-        validTo: response.securityDetails.validTo,
-      };
-    }
-  }
-
-  _onLoadingFinished(event: Protocol.Network.loadingFinishedPayload) {
-    const { requestId, timestamp, encodedDataLength } = event;
-    this._requestCompleted(requestId, timestamp, encodedDataLength);
+    request[NETWORK_ENTRY_SUMBOL] = networkEntry;
+    this.results.push(networkEntry);
   }
 
-  _onLoadingFailed(event: Protocol.Network.loadingFailedPayload) {
-    const { requestId, timestamp } = event;
-    this._requestCompleted(requestId, timestamp);
-  }
+  private async _onResponse(response: Response) {
+    const request = response.request();
+    const networkEntry = this._findNetworkEntry(request);
+    if (!networkEntry) return;
 
-  _requestCompleted(
-    requestId: string,
-    endTime: number,
-    encodedDataLength?: number
-  ) {
-    const record = this._findNetworkRecord(requestId);
-    if (!record) {
-      return;
-    }
-    if (encodedDataLength >= 0) {
-      record.transferSize = encodedDataLength;
-      record.response.body = {
-        bytes: encodedDataLength,
-      };
-    }
-    record.loadEndTime = endTime;
-    record.timings = calculateTimings(record);
-    this.results.push(record);
-    /**
-     * When there is DNS failure, browsers keeps retrying requests which
-     * might result in incorrect timings so we get remove them to measure only once
-     */
-    this._InflightRequestMap.delete(requestId);
-  }
+    networkEntry.responseReceivedTime = epochTimeInSeconds();
+    networkEntry.response = {
+      url: response.url(),
+      status: response.status(),
+      statusText: response.statusText(),
+      headers: {},
+      mimeType: 'x-unknown',
+      redirectURL: networkEntry.response.redirectURL,
+    };
 
-  /**
-   * Account for missing response received event and also adjust the
-   * response received event based on when first byte event was recorded
-   */
+    const page = request.frame().page();
+    this._addBarrier(
+      page,
+      request.allHeaders().then(reqHeaders => {
+        networkEntry.request.headers = reqHeaders;
+        networkEntry.request.referrer = reqHeaders?.referer;
+      })
+    );
+    this._addBarrier(
+      page,
+      response.serverAddr().then(server => {
+        networkEntry.response.remoteIPAddress = server?.ipAddress;
+        networkEntry.response.remotePort = server?.port;
+      })
+    );
+    this._addBarrier(
+      page,
+      response.securityDetails().then(details => {
+        if (details) networkEntry.response.securityDetails = details;
+      })
+    );
+    this._addBarrier(
+      page,
+      response.allHeaders().then(resHeaders => {
+        networkEntry.response.headers = resHeaders;
 
-  stop() {
-    return this.results;
+        const contentType = resHeaders['content-type'];
+        if (contentType)
+          networkEntry.response.mimeType = contentType.split(';')[0];
+      })
+    );
   }
-}
 
-function getResponseReceivedTime(
-  timing: Protocol.Network.Response['timing'],
-  responseReceivedTime: number
-) {
-  if (timing == null) {
-    return responseReceivedTime;
-  }
-  const startTime = timing.requestTime;
-  const headersReceivedTime = startTime + timing.receiveHeadersEnd / 1000;
-  if (responseReceivedTime < 0 || responseReceivedTime > headersReceivedTime) {
-    responseReceivedTime = headersReceivedTime;
-  }
-  if (startTime > responseReceivedTime) {
-    responseReceivedTime = startTime;
-  }
-  return responseReceivedTime;
-}
+  private async _onRequestCompleted(request: Request) {
+    const networkEntry = this._findNetworkEntry(request);
+    if (!networkEntry) return;
 
-/**
- * The timing calculations are based on the chrome devtools frontend
- * https://github.com/ChromeDevTools/devtools-frontend/blob/7f5478d8ceb7586f23f4073ab5c2085dac1ec26a/front_end/network/RequestTimingView.js#L98-L193
- */
-export function calculateTimings(record: NetworkInfo) {
-  const result: NetworkInfo['timings'] = {
-    blocked: -1,
-    queueing: -1,
-    proxy: -1,
-    dns: -1,
-    ssl: -1,
-    connect: -1,
-    send: -1,
-    wait: -1,
-    receive: -1,
-    total: -1,
-  };
+    networkEntry.loadEndTime = epochTimeInSeconds();
+    const timing = request.timing();
+    const { loadEndTime, requestSentTime } = networkEntry;
+    networkEntry.timings = {
+      blocked: -1,
+      dns: -1,
+      ssl: -1,
+      connect: -1,
+      send: -1,
+      wait: -1,
+      receive: -1,
+      total: -1,
+    };
 
-  const toMilliseconds = (time: number) => (time === -1 ? -1 : time * 1000);
-  const calculateDiffInMs = (
-    name: keyof NetworkInfo['timings'],
-    start: number,
-    end: number
-  ) => {
-    if (start < Number.MAX_VALUE && start <= end) {
-      result[name] = toMilliseconds(end - start);
-    }
-  };
-  const firstPositive = (numbers: number[]) => {
-    for (let i = 0; i < numbers.length; ++i) {
-      if (numbers[i] > 0) {
-        return numbers[i];
+    const firstPositive = (numbers: number[]) => {
+      for (let i = 0; i < numbers.length; ++i) {
+        if (numbers[i] > 0) {
+          return numbers[i];
+        }
       }
-    }
-    return null;
-  };
-  /**
-   * requestTime is baseline in seconds, rest of the timing data are ticks in milliseconds
-   * from the requestTime, so we calculate the offset from that
-   */
-  const addOffsetRange = (
-    name: keyof NetworkInfo['timings'],
-    start: number,
-    end: number
-  ) => {
-    if (start >= 0 && end >= 0) {
-      calculateDiffInMs(name, startTime + start / 1000, startTime + end / 1000);
-    }
-  };
-  const { requestSentTime, loadEndTime, responseReceivedTime, response } =
-    record;
-  const timing = response.timing;
-  const actResRcvdTime = getResponseReceivedTime(timing, responseReceivedTime);
-  const issueTime = requestSentTime;
-  const startTime = timing == null ? -1 : timing.requestTime;
-  const endTime = firstPositive([loadEndTime, actResRcvdTime]) || startTime;
+      return null;
+    };
+    const roundMilliSecs = (value: number): number => {
+      return Math.floor(value * 1000) / 1000;
+    };
 
-  if (timing == null) {
-    const start =
-      issueTime !== -1 ? issueTime : startTime !== -1 ? startTime : 0;
-    const middle = actResRcvdTime === -1 ? Number.MAX_VALUE : actResRcvdTime;
-    const end = endTime === -1 ? Number.MAX_VALUE : endTime;
-    calculateDiffInMs('total', start, end);
-    calculateDiffInMs('blocked', start, middle);
-    calculateDiffInMs('receive', middle, end);
-    // Check if the request is blocked/stalled for the whole timeframe and
-    // calculate timings appropriately for that request
-    if (!isFinite(result['blocked'])) {
-      result['blocked'] = result['total'];
+    if (timing.startTime === 0) {
+      // Convert to milliseconds before round off
+      const total = roundMilliSecs((loadEndTime - requestSentTime) * 1000);
+      networkEntry.timings.total = total;
+      return;
     }
-    return result;
-  }
-  calculateDiffInMs(
-    'total',
-    issueTime < startTime ? issueTime : startTime,
-    endTime
-  );
-  if (issueTime < startTime) {
-    calculateDiffInMs('queueing', issueTime, startTime);
-  }
-  const responseReceived = toMilliseconds(actResRcvdTime - startTime);
 
-  if (response.fromServiceWorker) {
-    addOffsetRange('blocked', 0, timing.workerStart);
-    addOffsetRange('wait', timing.sendEnd, responseReceived);
-  } else if (!timing.pushStart) {
-    const blockingEnd =
-      firstPositive([
-        timing.dnsStart,
-        timing.connectStart,
-        timing.sendStart,
-        responseReceived,
-      ]) || 0;
-    addOffsetRange('blocked', 0, blockingEnd);
-    addOffsetRange('proxy', timing.proxyStart, timing.proxyEnd);
-    addOffsetRange('dns', timing.dnsStart, timing.dnsEnd);
-    addOffsetRange('connect', timing.connectStart, timing.connectEnd);
-    addOffsetRange('ssl', timing.sslStart, timing.sslEnd);
-    addOffsetRange('send', timing.sendStart, timing.sendEnd);
-    addOffsetRange(
-      'wait',
-      Math.max(
-        timing.sendEnd,
-        timing.connectEnd,
-        timing.dnsEnd,
-        timing.proxyEnd,
-        blockingEnd
-      ),
-      responseReceived
+    const blocked =
+      roundMilliSecs(
+        firstPositive([
+          timing.domainLookupStart,
+          timing.connectStart,
+          timing.requestStart,
+        ])
+      ) || -1;
+    const dns =
+      timing.domainLookupEnd !== -1
+        ? roundMilliSecs(timing.domainLookupEnd - timing.domainLookupStart)
+        : -1;
+    const connect =
+      timing.connectEnd !== -1
+        ? roundMilliSecs(timing.connectEnd - timing.connectStart)
+        : -1;
+    const ssl =
+      timing.secureConnectionStart !== -1
+        ? roundMilliSecs(timing.connectEnd - timing.secureConnectionStart)
+        : -1;
+    const wait =
+      timing.responseStart !== -1
+        ? roundMilliSecs(timing.responseStart - timing.requestStart)
+        : -1;
+    const receive =
+      timing.responseEnd !== -1
+        ? roundMilliSecs(timing.responseEnd - timing.responseStart)
+        : -1;
+    const total = [blocked, dns, connect, wait, receive].reduce(
+      (pre, cur) => (cur > 0 ? cur + pre : pre),
+      0
+    );
+    networkEntry.timings = {
+      blocked,
+      dns,
+      connect,
+      ssl,
+      wait,
+      send: 0, // not exposed via RT api
+      receive,
+      total,
+    };
+
+    const page = request.frame().page();
+    // For aborted/failed requests sizes will not be present
+    this._addBarrier(
+      page,
+      request.sizes().then(sizes => {
+        networkEntry.request.bytes =
+          sizes.requestHeadersSize + sizes.requestBodySize;
+        networkEntry.request.body = {
+          bytes: sizes.requestBodySize,
+        };
+        networkEntry.response.bytes =
+          sizes.responseHeadersSize + sizes.responseBodySize;
+        networkEntry.response.body = {
+          bytes: sizes.responseBodySize,
+        };
+        networkEntry.transferSize = sizes.responseBodySize;
+      })
     );
   }
-  calculateDiffInMs('receive', actResRcvdTime, endTime);
-  return result;
+
+  async stop() {
+    await Promise.all(this._barrierPromises);
+    const context = this.driver.context;
+    context.off('request', this._onRequest.bind(this));
+    context.off('response', this._onResponse.bind(this));
+    context.off('requestfinished', this._onRequestCompleted.bind(this));
+    context.off('requestfailed', this._onRequestCompleted.bind(this));
+    this._barrierPromises.clear();
+    return this.results;
+  }
 }

package/src/plugins/plugin-manager.ts

@@ -102,11 +102,11 @@ export class PluginManager {
     (this.get('network') as NetworkManager)._currentStep = step;
   }
 
-  output() {
+  async output() {
     const data: PluginOutput = {};
     for (const [, plugin] of this.plugins) {
       if (plugin instanceof NetworkManager) {
-        data.networkinfo = plugin.stop();
+        data.networkinfo = await plugin.stop();
       } else if (plugin instanceof BrowserConsole) {
         data.browserconsole = plugin.stop();
       }

package/src/reporters/json.ts

@@ -128,23 +128,8 @@ function getMetadata() {
   };
 }
 
-function formatVersion(protocol: string | undefined) {
-  if (!protocol) {
-    return;
-  }
-  if (protocol === 'h2') {
-    return 2;
-  } else if (protocol === 'http/1.1') {
-    return 1.1;
-  } else if (protocol === 'http/1.0') {
-    return 1.0;
-  } else if (protocol.startsWith('h3')) {
-    return 3;
-  }
-}
-
 function formatTLS(tls: SecurityDetails) {
-  if (!tls) {
+  if (!tls || !tls.protocol) {
     return;
   }
   const [name, version] = tls.protocol.toLowerCase().split(' ');
@@ -177,7 +162,6 @@ export function formatNetworkFields(network: NetworkInfo) {
       original: request.headers?.['User-Agent'],
     },
     http: {
-      version: formatVersion(response?.protocol),
       request,
       response,
     },