@elastic/synthetics 1.24.4 → 1.26.0

package/src/plugins/api-network.ts

@@ -0,0 +1,252 @@
+/**
+ * MIT License
+ *
+ * Copyright (c) 2020-present, Elastic NV
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ *
+ */
+
+import { APIRequestContext, APIResponse } from 'playwright-core';
+import { NetworkInfo, APIDriver } from '../common_types';
+import { log } from '../core/logger';
+import { Step } from '../dsl';
+import { getTimestamp, now } from '../helpers';
+import {
+  calcTotalTime,
+  getResourceTimings,
+  roundMilliSecs,
+} from '../network-timings';
+
+/**
+ * Kibana UI expects the requestStartTime and loadEndTime to be baseline
+ * in seconds as they have the logic to convert it to milliseconds before
+ * using for offset calculation
+ */
+function epochTimeInSeconds() {
+  return getTimestamp() / 1e6;
+}
+
+// All `APIRequestContext` verbs funnel into `fetch`, so patching `fetch`
+// alone captures every request exactly once.
+type RequestLike = string | { url(): string };
+
+function normalizeUrl(urlOrRequest: RequestLike): string {
+  if (typeof urlOrRequest === 'string') return urlOrRequest;
+  try {
+    return urlOrRequest.url();
+  } catch {
+    return String(urlOrRequest);
+  }
+}
+
+// Best-effort request body size; returns 0 for shapes we can't measure.
+function bodyBytes(body: unknown): number {
+  if (body == null) return 0;
+  if (typeof body === 'string') return Buffer.byteLength(body);
+  if (Buffer.isBuffer(body)) return body.byteLength;
+  if (body instanceof Uint8Array) return body.byteLength;
+  if (typeof body === 'object') {
+    try {
+      return Buffer.byteLength(JSON.stringify(body));
+    } catch {
+      return 0;
+    }
+  }
+  return 0;
+}
+
+// Wire size of headers (`name: value\r\n`) for ECS `*.bytes`.
+function headersBytes(headers: Record<string, string>): number {
+  let total = 0;
+  for (const [k, v] of Object.entries(headers ?? {})) {
+    total += Buffer.byteLength(k) + Buffer.byteLength(String(v)) + 4; // ": " + CRLF
+  }
+  return total;
+}
+
+export class APINetworkManager {
+  results: Array<NetworkInfo> = [];
+  _currentStep: Partial<Step> | null = null;
+  private _originalFetch?: APIRequestContext['fetch'];
+  private _patched = false;
+
+  constructor(private driver: APIDriver) {}
+
+  async start() {
+    if (this._patched) return;
+    log(`Plugins: started collecting API network events`);
+    const request = this.driver.request;
+    this._originalFetch = request.fetch.bind(request);
+    (request as any).fetch = (urlOrRequest: RequestLike, options?: any) =>
+      this._interceptRequest(urlOrRequest, options);
+    this._patched = true;
+  }
+
+  private async _interceptRequest(
+    urlOrRequest: RequestLike,
+    options?: any
+  ): Promise<APIResponse> {
+    const url = normalizeUrl(urlOrRequest);
+    const timestamp = getTimestamp();
+    const requestSentTime = epochTimeInSeconds();
+    const httpMethod = (options?.method ?? 'GET').toUpperCase();
+
+    log(`API network: ${httpMethod} ${url}`);
+
+    const requestBody = options?.postData ?? options?.data;
+    const requestBodyBytes = bodyBytes(requestBody);
+    const requestHeaders = options?.headers ?? {};
+
+    const entry: NetworkInfo = {
+      step: this._currentStep,
+      timestamp,
+      url,
+      type: 'fetch',
+      isNavigationRequest: false,
+      browser: { name: 'api', version: '' },
+      request: {
+        url,
+        method: httpMethod,
+        headers: requestHeaders,
+        bytes: headersBytes(requestHeaders) + requestBodyBytes,
+        body: requestBodyBytes > 0 ? { bytes: requestBodyBytes } : undefined,
+      },
+      response: {
+        status: -1,
+        headers: {},
+        mimeType: 'x-unknown',
+      },
+      requestSentTime,
+      loadEndTime: -1,
+      responseReceivedTime: -1,
+      resourceSize: 0,
+      transferSize: 0,
+      timings: {
+        blocked: -1,
+        dns: -1,
+        ssl: -1,
+        connect: -1,
+        send: -1,
+        wait: -1,
+        receive: -1,
+        total: -1,
+      },
+    };
+    this.results.push(entry);
+
+    const startTime = now();
+    try {
+      const response = (await this._originalFetch(
+        urlOrRequest as any,
+        options
+      )) as APIResponse;
+      const headers = response.headers();
+
+      // Prefer `Content-Length`; fall back to the body buffer (chunked
+      // responses omit the header).
+      const contentLength = parseContentLength(headers['content-length']);
+      let responseBodyBytes = contentLength;
+      if (responseBodyBytes < 0) {
+        try {
+          const buf = await response.body();
+          responseBodyBytes = buf?.byteLength ?? 0;
+        } catch {
+          responseBodyBytes = 0;
+        }
+      }
+      const responseHeaderBytes = headersBytes(headers);
+      const transferBytes = responseHeaderBytes + responseBodyBytes;
+      entry.response = {
+        url: response.url(),
+        status: response.status(),
+        statusText: response.statusText(),
+        headers,
+        mimeType: headers['content-type'] ?? 'x-unknown',
+        bytes: transferBytes,
+        body: { bytes: responseBodyBytes },
+      };
+      entry.transferSize = transferBytes;
+      entry.resourceSize = responseBodyBytes;
+      entry.responseReceivedTime = epochTimeInSeconds();
+      entry.loadEndTime = entry.responseReceivedTime;
+
+      // Per-phase Resource Timing (Playwright >= 1.62). Phases that did not
+      // happen (reused keep-alive socket) or are unknown (HAR replay) come
+      // back as `-1`; `calcTotalTime` then falls back to the wall-clock span.
+      const timing = response.timing();
+      entry.timings = getResourceTimings(timing);
+      entry.timings.total = calcTotalTime(entry, timing);
+
+      // Native (Playwright >= 1.61) TLS/socket info for the final hop;
+      // both resolve to `null` for non-HTTPS or unknown addresses.
+      const [serverAddr, securityDetails] = await Promise.all([
+        response.serverAddr(),
+        response.securityDetails(),
+      ]);
+      if (serverAddr) {
+        entry.response.remoteIPAddress = serverAddr.ipAddress;
+        entry.response.remotePort = serverAddr.port;
+      }
+      if (securityDetails) {
+        entry.response.securityDetails = {
+          ...securityDetails,
+          protocol: normalizeTLSProtocol(securityDetails.protocol),
+        };
+      }
+      return response;
+    } catch (error) {
+      entry.responseReceivedTime = epochTimeInSeconds();
+      entry.loadEndTime = entry.responseReceivedTime;
+      entry.timings.total = roundMilliSecs(now() - startTime);
+      throw error;
+    }
+  }
+
+  // Drop the own-property `fetch` so the prototype method is reachable
+  // again, keeping the context usable after the plugin stops.
+  private _restore() {
+    if (!this._patched) return;
+    delete (this.driver.request as any).fetch;
+    this._originalFetch = undefined;
+    this._patched = false;
+  }
+
+  async stop() {
+    this._restore();
+    log(`Plugins: stopped collecting API network events`);
+    return this.results;
+  }
+}
+
+function parseContentLength(raw: string | undefined): number {
+  if (!raw) return -1;
+  const n = Number(raw);
+  return Number.isFinite(n) && n >= 0 ? n : -1;
+}
+
+/**
+ * `securityDetails()` reports `"TLSv1.3"` but the reporter splits on a
+ * space to derive `tls.version_protocol`/`tls.version`. Normalize to the
+ * spaced `"TLS 1.3"` shape so both journey types emit consistent fields.
+ */
+function normalizeTLSProtocol(raw?: string): string | undefined {
+  if (!raw) return undefined;
+  return raw.replace(/\s*v(?=\d)/i, ' ');
+}

package/src/plugins/network.ts

@@ -28,6 +28,7 @@ import { NetworkInfo, BrowserInfo, Driver } from '../common_types';
 import { log } from '../core/logger';
 import { Step } from '../dsl';
 import { getTimestamp } from '../helpers';
+import { calcTotalTime, getResourceTimings } from '../network-timings';
 
 /**
  * Kibana UI expects the requestStartTime and loadEndTime to be baseline
@@ -38,22 +39,6 @@ function epochTimeInSeconds() {
   return getTimestamp() / 1e6;
 }
 
-/**
- * Find the first positive number in an array for Resource timing data
- */
-const firstPositive = (numbers: number[]) => {
-  for (let i = 0; i < numbers.length; ++i) {
-    if (numbers[i] > 0) {
-      return numbers[i];
-    }
-  }
-  return null;
-};
-
-const roundMilliSecs = (value: number): number => {
-  return Math.floor(value * 1000) / 1000;
-};
-
 /**
  * Used as a key in each Network Request to identify the
  * associated request across distinct lifecycle events
@@ -200,42 +185,8 @@ export class NetworkManager {
     // Gather all resource timing information up until the
     // TTFB(Time to first byte) is received
     const timing = request.timing();
-    const blocked =
-      roundMilliSecs(
-        firstPositive([
-          timing.domainLookupStart,
-          timing.connectStart,
-          timing.requestStart,
-        ])
-      ) || -1;
-    const dns =
-      timing.domainLookupEnd !== -1
-        ? roundMilliSecs(timing.domainLookupEnd - timing.domainLookupStart)
-        : -1;
-    const connect =
-      timing.connectEnd !== -1
-        ? roundMilliSecs(timing.connectEnd - timing.connectStart)
-        : -1;
-    const ssl =
-      timing.secureConnectionStart !== -1
-        ? roundMilliSecs(timing.connectEnd - timing.secureConnectionStart)
-        : -1;
-    const wait =
-      timing.responseStart !== -1
-        ? roundMilliSecs(timing.responseStart - timing.requestStart)
-        : -1;
-
-    networkEntry.timings = {
-      blocked,
-      dns,
-      ssl,
-      connect,
-      send: 0, // not exposed via RT api
-      wait,
-      receive: -1, // will be available after full response is received
-      total: -1,
-    };
-    this._calcTotalTime(networkEntry, timing);
+    networkEntry.timings = getResourceTimings(timing);
+    networkEntry.timings.total = calcTotalTime(networkEntry, timing);
 
     const frame = this._nullableFrameBarrier(request);
     const page = frame?.page();
@@ -278,12 +229,8 @@ export class NetworkManager {
     networkEntry.loadEndTime = epochTimeInSeconds();
     // responseEnd is fired after the last byte of the response is received.
     const timing = request.timing();
-    const receive =
-      timing.responseEnd !== -1
-        ? roundMilliSecs(timing.responseEnd - timing.responseStart)
-        : -1;
-    networkEntry.timings.receive = receive;
-    this._calcTotalTime(networkEntry, timing);
+    networkEntry.timings = getResourceTimings(timing);
+    networkEntry.timings.total = calcTotalTime(networkEntry, timing);
 
     // For aborted/failed requests sizes will not be present
     if (timing.startTime <= 0) {
@@ -310,35 +257,6 @@ export class NetworkManager {
     );
   }
 
-  /**
-   * Calculates the total time for the network request based on the ResourceTiming
-   * data from Playwright. Fallbacks to the event timings if ResourceTiming data
-   * is not available.
-   */
-  private _calcTotalTime(
-    entry: NetworkInfo,
-    rtiming: ReturnType<Request['timing']>
-  ) {
-    const timings = entry.timings;
-    entry.timings.total = [
-      timings.blocked,
-      timings.dns,
-      timings.connect,
-      timings.wait,
-      timings.receive,
-    ].reduce((pre, cur) => ((cur || -1) > 0 ? cur + pre : pre), 0);
-
-    // fallback when ResourceTiming data is not available
-    if (rtiming.startTime <= 0) {
-      const end =
-        entry.loadEndTime ||
-        entry.responseReceivedTime ||
-        entry.requestSentTime;
-      const total = roundMilliSecs((end - entry.requestSentTime) * 1000);
-      entry.timings.total = total <= 0 ? -1 : total;
-    }
-  }
-
   async stop() {
     /**
      * Waiting for all network events is error prone and might hang the tests

package/src/plugins/plugin-manager.ts

@@ -23,7 +23,7 @@
  *
  */
 
-import { PluginOutput, Driver } from '../common_types';
+import { PluginOutput, Driver, APIDriver, NetworkInfo } from '../common_types';
 import {
   BrowserConsole,
   NetworkManager,
@@ -32,11 +32,39 @@ import {
   TraceOptions,
 } from './';
 import { Step } from '../dsl';
+import { APINetworkManager } from './api-network';
 
 type PluginType = 'network' | 'trace' | 'performance' | 'browserconsole';
-type Plugin = NetworkManager | Tracing | PerformanceManager | BrowserConsole;
+
+// Shared shape for `NetworkManager` and `APINetworkManager` so the manager
+// stays agnostic to the driver type.
+export type NetworkPlugin = {
+  results: Array<NetworkInfo>;
+  _currentStep: Partial<Step> | null;
+  start(): Promise<void>;
+  stop(): Promise<Array<NetworkInfo>>;
+};
+
+type Plugin =
+  | NetworkManager
+  | Tracing
+  | PerformanceManager
+  | BrowserConsole
+  | APINetworkManager;
 type PluginOptions = TraceOptions;
 
+function isBrowserDriver(driver: Driver | APIDriver): driver is Driver {
+  return 'context' in driver;
+}
+
+function isNetworkPlugin(
+  plugin: Plugin
+): plugin is NetworkManager | APINetworkManager {
+  return (
+    plugin instanceof NetworkManager || plugin instanceof APINetworkManager
+  );
+}
+
 export class PluginManager {
   protected plugins = new Map<PluginType, Plugin>();
   public PLUGIN_TYPES: Array<PluginType> = [
@@ -45,23 +73,31 @@ export class PluginManager {
     'performance',
     'browserconsole',
   ];
-  constructor(private driver: Driver) {}
+  constructor(private driver: Driver | APIDriver) {}
 
   register(type: PluginType, options: PluginOptions) {
     let instance: Plugin;
-    switch (type) {
-      case 'network':
-        instance = new NetworkManager(this.driver);
-        break;
-      case 'trace':
-        instance = new Tracing(this.driver, options);
-        break;
-      case 'performance':
-        instance = new PerformanceManager(this.driver);
-        break;
-      case 'browserconsole':
-        instance = new BrowserConsole(this.driver);
-        break;
+
+    if (isBrowserDriver(this.driver)) {
+      switch (type) {
+        case 'network':
+          instance = new NetworkManager(this.driver);
+          break;
+        case 'trace':
+          instance = new Tracing(this.driver, options);
+          break;
+        case 'performance':
+          instance = new PerformanceManager(this.driver);
+          break;
+        case 'browserconsole':
+          instance = new BrowserConsole(this.driver);
+          break;
+      }
+    } else {
+      // API journeys only record network; skip browser-only plugins.
+      if (type === 'network') {
+        instance = new APINetworkManager(this.driver);
+      }
     }
     instance && this.plugins.set(type, instance);
     return instance;
@@ -97,15 +133,20 @@ export class PluginManager {
     return this.plugins.get(type);
   }
 
+  // Propagate the active step to step-scoped plugins (no-op when absent).
   onStep(step: Step) {
-    (this.get('browserconsole') as BrowserConsole)._currentStep = step;
-    (this.get('network') as NetworkManager)._currentStep = step;
+    const browserConsole = this.plugins.get('browserconsole') as
+      | BrowserConsole
+      | undefined;
+    if (browserConsole) browserConsole._currentStep = step;
+    const network = this.plugins.get('network');
+    if (network && isNetworkPlugin(network)) network._currentStep = step;
   }
 
   async output() {
     const data: PluginOutput = {};
     for (const [, plugin] of this.plugins) {
-      if (plugin instanceof NetworkManager) {
+      if (isNetworkPlugin(plugin)) {
         data.networkinfo = await plugin.stop();
       } else if (plugin instanceof BrowserConsole) {
         data.browserconsole = plugin.stop();

package/src/push/index.ts

@@ -56,10 +56,12 @@ import {
 import {
   getBatches,
   getSizedBatches,
+  isAPIMonitorSupported,
   isBulkAPISupported,
   isLightweightMonitorSupported,
   logDiff,
   logGroups,
+  MIN_API_MONITOR_VERSION,
   printBytes,
 } from './utils';
 import { runLocal } from './run-local';
@@ -76,6 +78,15 @@ export async function push(monitors: Monitor[], options: PushOptions) {
     throw error(formatDuplicateError(duplicates));
   }
 
+  if (
+    monitors.some(m => m.type === 'api') &&
+    !isAPIMonitorSupported(options.kibanaVersion)
+  ) {
+    throw error(
+      `Aborted: API journey monitors are only supported in Kibana ${MIN_API_MONITOR_VERSION} or above (current: ${options.kibanaVersion}). Please upgrade Kibana or remove API journeys before pushing.`
+    );
+  }
+
   progress(
     `Pushing monitors for '${options.id}' project in kibana '${options.space}' space`
   );

package/src/push/monitor.ts

@@ -38,7 +38,11 @@ import {
   MonitorConfig,
 } from '../dsl/monitor';
 import { PushOptions } from '../common_types';
-import { isParamOptionSupported, normalizeMonitorName } from './utils';
+import {
+  isBundledMonitorType,
+  isParamOptionSupported,
+  normalizeMonitorName,
+} from './utils';
 
 // Allowed extensions for lightweight monitor files
 const ALLOWED_LW_EXTENSIONS = ['.yml', '.yaml'];
@@ -143,7 +147,7 @@ export async function buildMonitorSchema(monitors: Monitor[], isV2: boolean) {
       locations: translateLocation(config.locations),
     };
 
-    if (type === 'browser') {
+    if (isBundledMonitorType(type)) {
       const outPath = join(
         bundlePath,
         normalizeMonitorName(config.name) + '.zip'

package/src/push/run-local.ts

@@ -32,6 +32,7 @@ import { red } from 'kleur/colors';
 import { join } from 'path';
 import { pathToFileURL } from 'url';
 import { MonitorSchema } from './monitor';
+import { isBundledMonitorType } from './utils';
 
 async function unzipFile(zipPath, destination) {
   return new Promise<void>((resolve, reject) => {
@@ -73,21 +74,30 @@ async function runNpmInstall(directory) {
 
 async function runTest(directory, schema: MonitorSchema) {
   return new Promise<void>((resolve, reject) => {
-    const runTest = spawn(
-      'npx',
-      [
-        '@elastic/synthetics',
-        '.',
-        '--playwright-options',
-        JSON.stringify(schema.playwrightOptions),
-        '--params',
-        JSON.stringify(schema.params),
-      ],
-      {
-        cwd: directory,
-        stdio: 'inherit',
+    const args = [
+      '@elastic/synthetics',
+      '.',
+      '--playwright-options',
+      JSON.stringify(schema.playwrightOptions),
+      '--params',
+      JSON.stringify(schema.params),
+    ];
+    const allowlist = schema.certificateErrorSpkiAllowlist;
+    if (allowlist != null) {
+      const entries = Array.isArray(allowlist) ? allowlist : [allowlist];
+      const pemEntries = entries
+        .map(entry =>
+          Buffer.isBuffer(entry) ? entry.toString('utf-8') : entry
+        )
+        .filter((entry): entry is string => Boolean(entry));
+      if (pemEntries.length > 0) {
+        args.push('--certificate-error-spki-allowlist', ...pemEntries);
       }
-    );
+    }
+    const runTest = spawn('npx', args, {
+      cwd: directory,
+      stdio: 'inherit',
+    });
 
     runTest.on('close', resolve);
     runTest.on('error', err => {
@@ -118,7 +128,7 @@ async function extract(
   zipPath: string,
   unzipPath: string
 ) {
-  if (schema.type !== 'browser') {
+  if (!isBundledMonitorType(schema.type)) {
     return;
   }
   const content = schema.content;

package/src/push/utils.ts

@@ -185,6 +185,21 @@ export function isParamOptionSupported(version: string) {
   return semver.satisfies(version, '>=8.7.2');
 }
 
+// API journey monitors are supported only from 9.6.0 and above.
+// Coerce so pre-release/SNAPSHOT builds (e.g. `9.6.0-SNAPSHOT`) count as 9.6.0.
+export const MIN_API_MONITOR_VERSION = '9.6.0';
+export function isAPIMonitorSupported(version: string) {
+  const coerced = semver.coerce(version);
+  return coerced ? semver.gte(coerced, MIN_API_MONITOR_VERSION) : false;
+}
+
+/**
+ * Project monitors ship bundled journey code (browser + api), unlike
+ * lightweight YAML monitors (http/tcp/icmp).
+ */
+export const isBundledMonitorType = (type?: string) =>
+  type === 'browser' || type === 'api';
+
 /**
  * Helper that replaces url paths traversal issues when bundling
  */

package/src/reporters/base.ts

@@ -30,6 +30,7 @@ import { symbols, indent, now } from '../helpers';
 import { Reporter, ReporterOptions } from '.';
 import { Journey, Step } from '../dsl';
 import {
+  APIJourneyEndResult,
   JourneyEndResult,
   JourneyStartResult,
   StepResult,
@@ -84,7 +85,7 @@ export default class BaseReporter implements Reporter {
   }
 
   /* eslint-disable @typescript-eslint/no-unused-vars */
-  onJourneyEnd(journey: Journey, {}: JourneyEndResult) {
+  onJourneyEnd(journey: Journey, {}: JourneyEndResult | APIJourneyEndResult) {
     const { failed, succeeded, skipped } = this.metrics;
     const total = failed + succeeded + skipped;
     /**

package/src/reporters/index.ts

@@ -32,6 +32,7 @@ import {
   JourneyEndResult,
   JourneyStartResult,
   StepEndResult,
+  APIJourneyEndResult,
 } from '../common_types';
 import BuildKiteCLIReporter from './build_kite_cli';
 
@@ -63,7 +64,7 @@ export interface Reporter {
   onStepEnd?(journey: Journey, step: Step, result: StepEndResult): void;
   onJourneyEnd?(
     journey: Journey,
-    result: JourneyEndResult
+    result: JourneyEndResult | APIJourneyEndResult
   ): void | Promise<void>;
   onEnd?(): void | Promise<void>;
 }