@elastic/synthetics 1.4.0 → 1.5.0

package/src/plugins/network.ts

@@ -23,7 +23,7 @@
  *
  */
 
-import { Page, Request, Response } from 'playwright-chromium';
+import { Frame, Page, Request, Response } from 'playwright-core';
 import { NetworkInfo, BrowserInfo, Driver } from '../common_types';
 import { log } from '../core/logger';
 import { Step } from '../dsl';
@@ -93,6 +93,17 @@ export class NetworkManager {
     race.then(() => this._barrierPromises.delete(race));
   }
 
+  private _nullableFrameBarrier(req: Request): Frame | null {
+    try {
+      return req.frame();
+    } catch (_) {
+      // frame might be unavailable for certain requests if they are issued
+      // before the frame is created - true for navigation requests
+      // https://playwright.dev/docs/api/class-request#request-frame
+    }
+    return null;
+  }
+
   async start() {
     log(`Plugins: started collecting network events`);
     const { client, context } = this.driver;
@@ -226,7 +237,8 @@ export class NetworkManager {
     };
     this._calcTotalTime(networkEntry, timing);
 
-    const page = request.frame().page();
+    const frame = this._nullableFrameBarrier(request);
+    const page = frame?.page();
     this._addBarrier(
       page,
       request.allHeaders().then(reqHeaders => {
@@ -278,7 +290,8 @@ export class NetworkManager {
       return;
     }
 
-    const page = request.frame().page();
+    const frame = this._nullableFrameBarrier(request);
+    const page = frame?.page();
     this._addBarrier(
       page,
       request.sizes().then(sizes => {

package/src/push/kibana_api.ts

@@ -124,8 +124,8 @@ export async function bulkDeleteMonitors(
 }
 
 type StatusResponse = {
-  version: {
-    number: string;
+  kibana: {
+    version: string;
   };
 };
 
@@ -135,7 +135,8 @@ export async function getVersion(options: PushOptions) {
     method: 'GET',
     auth: options.auth,
   });
-  return data.version.number;
+
+  return data.kibana.version;
 }
 
 export type LegacyAPISchema = {

package/src/push/monitor.ts

@@ -242,12 +242,15 @@ export function buildMonitorFromYaml(
     config['private_locations'] ||
     config.privateLocations ||
     options.privateLocations;
-  delete config['private_locations'];
+  const retestOnFailure =
+    config['retest_on_failure'] ?? options.retestOnFailure;
   const alertConfig = parseAlertConfig(config, options.alert);
+
   const mon = new Monitor({
     locations: options.locations,
     tags: options.tags,
-    ...config,
+    ...normalizeConfig(config),
+    retestOnFailure,
     privateLocations,
     schedule: schedule || options.schedule,
     alert: alertConfig,
@@ -263,6 +266,14 @@ export function buildMonitorFromYaml(
   return mon;
 }
 
+// Deletes unncessary fields from the lightweight monitor config
+//  that is not supported by the Kibana API
+function normalizeConfig(config: MonitorConfig) {
+  delete config['private_locations'];
+  delete config['retest_on_failure'];
+  return config;
+}
+
 export const parseAlertConfig = (
   config: MonitorConfig,
   gConfig?: AlertConfig

package/src/push/utils.ts

@@ -64,7 +64,7 @@ export function generateURL(options: PushOptions, operation: Operation) {
   const url = removeTrailingSlash(options.url);
   switch (operation) {
     case 'status':
-      return `${url}/s/${options.space}/api/status`;
+      return `${url}/s/${options.space}/api/stats`;
     case 'bulk_get':
     case 'bulk_update':
     case 'bulk_delete':

package/src/reporters/base.ts

@@ -24,14 +24,9 @@
  */
 
 import SonicBoom from 'sonic-boom';
-import { green, red, cyan } from 'kleur/colors';
-import {
-  symbols,
-  indent,
-  now,
-  findPWLogsIndexes,
-  rewriteErrorStack,
-} from '../helpers';
+import { green, red, cyan, gray } from 'kleur/colors';
+import { renderError, serializeError } from './reporter-util';
+import { symbols, indent, now } from '../helpers';
 import { Reporter, ReporterOptions } from '.';
 import {
   JourneyEndResult,
@@ -40,25 +35,6 @@ import {
 } from '../common_types';
 import { Journey, Step } from '../dsl';
 
-function renderError(error) {
-  let output = '';
-  const outer = indent('');
-  const inner = indent(outer);
-  const container = outer + '---\n';
-  output += container;
-  let stack = error.stack;
-  if (stack) {
-    output += inner + 'stack: |-\n';
-    stack = rewriteErrorStack(stack, findPWLogsIndexes(stack));
-    const lines = String(stack).split('\n');
-    for (const line of lines) {
-      output += inner + '  ' + line + '\n';
-    }
-  }
-  output += container;
-  return red(output);
-}
-
 function renderDuration(durationMs) {
   return parseInt(durationMs);
 }
@@ -66,14 +42,17 @@ function renderDuration(durationMs) {
 export default class BaseReporter implements Reporter {
   stream: SonicBoom;
   fd: number;
+  dryRun: boolean;
   metrics = {
     succeeded: 0,
     failed: 0,
     skipped: 0,
+    registered: 0,
   };
 
   constructor(options: ReporterOptions = {}) {
     this.fd = options.fd || process.stdout.fd;
+    this.dryRun = options.dryRun ?? false;
     /**
      * minLength is set to 1 byte to make sure we flush the
      * content even if its the last byte on the stream buffer
@@ -82,17 +61,25 @@ export default class BaseReporter implements Reporter {
     this.stream = new SonicBoom({ fd: this.fd, sync: true, minLength: 1 });
   }
 
+  onJourneyRegister(journey: Journey): void {
+    this.write(`\nJourney: ${journey.name}`);
+    this.metrics.registered++;
+  }
+
   onJourneyStart(journey: Journey, {}: JourneyStartResult) {
     this.write(`\nJourney: ${journey.name}`);
   }
 
   onStepEnd(_: Journey, step: Step, result: StepEndResult) {
     const { status, end, start, error } = result;
-    const message = `${symbols[status]}  Step: '${
-      step.name
-    }' ${status} (${renderDuration((end - start) * 1000)} ms)`;
+    const message = `${symbols[status]}  Step: '${step.name}' ${status} ${gray(
+      '(' + renderDuration((end - start) * 1000) + ' ms)'
+    )}`;
     this.write(indent(message));
-    error && this.write(renderError(error));
+    if (error) {
+      const message = renderError(serializeError(error));
+      this.write(indent(message) + '\n');
+    }
     this.metrics[status]++;
   }
 
@@ -105,14 +92,20 @@ export default class BaseReporter implements Reporter {
      * be executed which means the error happened in one of the hooks
      */
     if (total === 0 && error) {
-      this.write(renderError(error));
+      const message = renderError(serializeError(error));
+      this.write(indent(message) + '\n');
     }
   }
 
   onEnd() {
-    const { failed, succeeded, skipped } = this.metrics;
+    const { failed, succeeded, skipped, registered } = this.metrics;
     const total = failed + succeeded + skipped;
 
+    if (this.dryRun) {
+      this.write(`\n${registered} journey(s) registered`);
+      return;
+    }
+
     let message = '\n';
     if (total === 0) {
       message = 'No tests found!';

package/src/reporters/index.ts

@@ -34,7 +34,11 @@ import {
   StepEndResult,
 } from '../common_types';
 
-export type ReporterOptions = { fd?: number; colors?: boolean };
+export type ReporterOptions = {
+  fd?: number;
+  colors?: boolean;
+  dryRun?: boolean;
+};
 export type BuiltInReporterName = 'default' | 'json' | 'junit';
 export type ReporterInstance = new (opts: ReporterOptions) => Reporter;
 export const reporters: {

package/src/reporters/json.ts

@@ -28,8 +28,8 @@ import { join } from 'path';
 import sharp from 'sharp';
 import { createHash } from 'crypto';
 import BaseReporter from './base';
+import { renderError, serializeError, stripAnsiCodes } from './reporter-util';
 import {
-  formatError,
   getTimestamp,
   CACHE_PATH,
   totalist,
@@ -189,6 +189,36 @@ export function formatNetworkFields(network: NetworkInfo) {
   return { ecs, payload };
 }
 
+/**
+ * formatJSONError formats the error in a structured format
+ * we restructure the error with code frame and stack trace for test errors
+ */
+function formatJSONError(error: Error | any, type: OutputType) {
+  if (error == null) {
+    return;
+  }
+
+  /**
+   * Do not process browser errors - console.error and unhandled exceptions
+   */
+  if (type != null && type === 'journey/browserconsole') {
+    return {
+      name: error.name,
+      message: error.message,
+      stack: error.stack,
+    };
+  }
+
+  /**
+   * Do not highlight source for these errors and strip ANSI codes
+   */
+  return {
+    name: error.name,
+    message: stripAnsiCodes(error.message.split('\n')[0]),
+    stack: stripAnsiCodes(renderError(serializeError(error, false))),
+  };
+}
+
 function journeyInfo(
   journey: OutputFields['journey'],
   type: OutputFields['type'],
@@ -317,7 +347,7 @@ export default class JSONReporter extends BaseReporter {
     });
   }
 
-  onJourneyRegister(journey: Journey): void {
+  override onJourneyRegister(journey: Journey): void {
     this.writeJSON({
       type: 'journey/register',
       journey,
@@ -550,7 +580,7 @@ export default class JSONReporter extends BaseReporter {
       payload,
       blob,
       blob_mime,
-      error: formatError(error),
+      error: formatJSONError(error, type),
       url,
       package_version: version,
     });

package/src/reporters/junit.ts

@@ -31,8 +31,9 @@ import {
   StepEndResult,
 } from '../common_types';
 import { Journey, Step } from '../dsl';
-import { formatError, indent, now } from '../helpers';
+import { indent, now } from '../helpers';
 import BaseReporter from './base';
+import { serializeError, stripAnsiCodes } from './reporter-util';
 
 type XMLEntry = {
   name: string;
@@ -97,14 +98,13 @@ export default class JUnitReporter extends BaseReporter {
 
     entry.attributes.tests++;
     if (status === 'failed') {
-      const { name, message, stack } = formatError(error);
       caseEntry.children.push({
         name: 'failure',
         attributes: {
-          message,
-          type: name,
+          message: error.message,
+          type: error.name,
         },
-        text: stack,
+        text: stripAnsiCodes(serializeError(error).stack),
       });
       entry.attributes.failures++;
     } else if (status === 'skipped') {

package/src/reporters/reporter-util.ts

@@ -0,0 +1,217 @@
+/**
+ * MIT License
+ *
+ * Copyright (c) 2020-present, Elastic NV
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ *
+ */
+
+import { fileURLToPath } from 'url';
+import { resolve, sep } from 'path';
+import { readFileSync } from 'fs';
+import { inspect } from 'util';
+import { gray } from 'kleur/colors';
+import { codeFrameColumns } from '@babel/code-frame';
+import StackUtils from 'stack-utils';
+import { Location, StackFrame, TestError } from '../common_types';
+
+const stackUtils = new StackUtils({ internals: StackUtils.nodeInternals() });
+
+const SYNTHETICS_PATH = resolve(__dirname, '..', '..');
+const PW_CORE_PATH = require.resolve('playwright-core');
+
+function prepareStackFrame(line: string): StackFrame {
+  const frame = stackUtils.parseLine(line);
+  if (!frame) {
+    return;
+  }
+  // ignore node internals
+  if (frame.file?.startsWith('internal') || frame.file?.startsWith('node:')) {
+    return;
+  }
+
+  // handle relative URLs
+  let file = frame.file?.startsWith('file://')
+    ? fileURLToPath(frame.file)
+    : resolve(process.cwd(), frame.file);
+
+  // ignore node_modules
+  if (file.includes(`${sep}node_modules${sep}`)) {
+    return;
+  }
+
+  // filter library and PW files
+  if (!filterLibInternals(file)) {
+    return;
+  }
+
+  // When we bundle the journeys, we store the absolute path of the journey
+  // files and write the sourcemap relative to these files. When we try to
+  // extract the sourcemap file location, the stacktrace will be relatively
+  // resolved to these files which would be under `journeys` folder. So we strip
+  // these extra `journeys` from the path to get the correct file location.
+  if (frame.file?.includes('journeys/journeys')) {
+    file = file.replace('journeys/journeys', 'journeys');
+  }
+
+  return {
+    file,
+    line: frame.line || 0,
+    column: frame.column || 0,
+    function: frame.function,
+  };
+}
+
+function filterLibInternals(file: string) {
+  // To ignore filtering the stack trace on tests
+  if (process.env.TEST_OVERRIDE) {
+    return true;
+  }
+  if (file.startsWith(SYNTHETICS_PATH)) {
+    return false;
+  }
+  // should have been filtered by node_modules, but just in case
+  if (file.startsWith(PW_CORE_PATH)) {
+    return false;
+  }
+}
+
+function constructStackFromFrames(frames: StackFrame[]) {
+  const stackLines: string[] = [];
+  for (const frame of frames) {
+    stackLines.push(
+      `    at ${frame.function ? frame.function + ' ' : ''}(${frame.file}:${
+        frame.line
+      }:${frame.column})`
+    );
+  }
+  return stackLines;
+}
+
+export function prepareError(error: Error) {
+  const stack = error.stack;
+  const lines = stack.split('\n');
+  let startAt = lines.findIndex(line => line.startsWith('    at '));
+  if (startAt === -1) {
+    startAt = lines.length;
+  }
+  const message = lines.slice(0, startAt).join('\n');
+  const stackLines = lines.slice(startAt);
+  // figure out the location of the journey/step that throws the error and
+  // correct stack line corresponding to that error
+  const stackFrames: StackFrame[] = [];
+  for (const line of stackLines) {
+    const frame = prepareStackFrame(line);
+    if (!frame || !frame.file) {
+      continue;
+    }
+    stackFrames.push(frame);
+  }
+
+  let location: Location | undefined;
+  if (stackFrames.length) {
+    const frame = stackFrames[0];
+    location = {
+      file: frame.file,
+      column: frame.column || 0,
+      line: frame.line || 0,
+    };
+  }
+
+  return {
+    message,
+    stack: constructStackFromFrames(stackFrames).join('\n'),
+    location,
+  };
+}
+
+export function highLightSource(location: Location): string {
+  if (!location) {
+    return;
+  }
+
+  try {
+    const source = readFileSync(location.file, 'utf-8');
+    const code = codeFrameColumns(
+      source,
+      { start: location },
+      { highlightCode: true }
+    );
+    return code;
+  } catch (_) {
+    // ignore error
+  }
+}
+
+// serializeError prefers receiving proper Errors, but since at runtime
+// non Error exceptions can be thrown, it tolerates though. The
+// redundant type Error | any expresses that.
+export function serializeError(
+  error: Error | any,
+  highlightCode = true
+): TestError {
+  if (!(error instanceof Error) || !error.stack) {
+    return { message: `thrown: ${inspect(error)}` };
+  }
+
+  const testErr: TestError = { message: error.message };
+  const { message, stack, location } = prepareError(error);
+  testErr.message = message;
+  if (stack) {
+    testErr.stack = stack;
+  }
+  if (location && highlightCode) {
+    testErr.location = location;
+    testErr.source = highLightSource(testErr.location);
+  }
+  return testErr;
+}
+
+export function renderError(error: TestError) {
+  const summary = [];
+  // push empty string to add a new line before rendering error
+  summary.push('');
+  summary.push(error.message);
+
+  if (error.source) {
+    summary.push('');
+    summary.push(error.source);
+  }
+
+  if (error.stack) {
+    summary.push('');
+    summary.push(gray(error.stack));
+  }
+  summary.join('');
+
+  return summary.join('\n');
+}
+
+// strip color codes and ansi escape codes
+const ansiRegex = new RegExp(
+  [
+    '[\\u001B\\u009B][[\\]()#;?]*(?:(?:(?:(?:;[-a-zA-Z\\d\\/#&.:=?%@~_]+)*|[a-zA-Z\\d]+(?:;[-a-zA-Z\\d\\/#&.:=?%@~_]*)*)?\\u0007)',
+    '(?:(?:\\d{1,4}(?:;\\d{0,4})*)?[\\dA-PR-TZcf-nq-uy=><~]))',
+  ].join('|'),
+  'g'
+);
+export function stripAnsiCodes(msg = '') {
+  return msg.replace(ansiRegex, '');
+}