@elastic/synthetics 1.17.2 → 1.18.0

package/src/cli.ts

@@ -278,12 +278,17 @@ program
 // TOTP command
 program
   .command('totp <secret>')
-  .description('Generate a Time-based One-Time token using the provided secret.')
-  .option('--issuer <issuer>', 'Provider or Service the secret is associated with.')
+  .description(
+    'Generate a Time-based One-Time token using the provided secret.'
+  )
+  .option(
+    '--issuer <issuer>',
+    'Provider or Service the secret is associated with.'
+  )
   .option('--label <label>', 'Account Identifier (default: SyntheticsTOTP)')
   .action((secret, cmdOpts: TOTPCmdOptions) => {
     try {
-      const token = totp(secret, cmdOpts)
+      const token = totp(secret, cmdOpts);
       write(bold(`OTP Token: ${token}`));
     } catch (e) {
       error(e);

package/src/common_types.ts

@@ -324,4 +324,4 @@ export type JourneyEndResult = JourneyStartResult &
     options: RunOptions;
   };
 
-export type StepEndResult = StepResult
+export type StepEndResult = StepResult;

package/src/core/globals.ts

@@ -34,4 +34,16 @@ if (!global[SYNTHETICS_RUNNER]) {
   global[SYNTHETICS_RUNNER] = new Runner();
 }
 
+/**
+ * Set debug based on DEBUG ENV and namespace - synthetics
+ */
+if (process.env.DEBUG && process.env.DEBUG.includes('synthetics')) {
+  process.env['__SYNTHETICS__DEBUG__'] = '1';
+}
+
 export const runner: Runner = global[SYNTHETICS_RUNNER];
+
+// is Debug mode enabled
+export function inDebugMode() {
+  return !!process.env['__SYNTHETICS__DEBUG__'];
+}

package/src/core/logger.ts

@@ -25,16 +25,10 @@
 
 import { grey, cyan, dim, italic } from 'kleur/colors';
 import { now } from '../helpers';
-
-/**
- * Set debug based on DEBUG ENV and namespace - synthetics
- */
-if (process.env.DEBUG && process.env.DEBUG.includes('synthetics')) {
-  process.env['__SYNTHETICS__DEBUG__'] = '1';
-}
+import { inDebugMode } from './globals';
 
 export function log(msg) {
-  if (!process.env['__SYNTHETICS__DEBUG__'] || !msg) {
+  if (!inDebugMode() || !msg) {
     return;
   }
   if (typeof msg === 'object') {

package/src/core/mfa.ts

@@ -23,48 +23,48 @@
  *
  */
 
-import { TOTP } from "otpauth";
+import { TOTP } from 'otpauth';
 
 type TOTPOptions = {
   /**
    * Provider or Service the secret is associated with
    */
-  issuer?: string
+  issuer?: string;
   /**
    * Account Identifier.
    * @default 'SyntheticsTOTP'
    */
-  label?: string
+  label?: string;
   /**
    * Include issuer prefix in label.
    */
-  issuerInLabel?: boolean
+  issuerInLabel?: boolean;
   /**
    * The encoded secret key used to generate the TOTP.
    */
-  secret?: string
+  secret?: string;
   /**
    * The algorithm used to generate the TOTP.
    * @default 'SHA1'
    */
-  algorithm?: string
+  algorithm?: string;
   /**
    * Number of digits in the generated token.
    * @default 6
    */
-  digits?: number
+  digits?: number;
   /**
    * Validity period in seconds for the token.
    * @default 30
    */
-  period?: number
+  period?: number;
 };
 
 export type TOTPCmdOptions = {
-  issuer?: string
-  label?: string
+  issuer?: string;
+  label?: string;
 };
 
 export function totp(secret?: string, options: TOTPOptions = {}) {
-  return new TOTP({ label: "SyntheticsTOTP", secret, ...options }).generate();
+  return new TOTP({ label: 'SyntheticsTOTP', secret, ...options }).generate();
 }

package/src/dsl/monitor.ts

@@ -158,6 +158,13 @@ export class Monitor {
       .digest('base64');
   }
 
+  /**
+   * Returns the size of the monitor in bytes which is sent as payload to Kibana
+   */
+  size() {
+    return JSON.stringify(this).length;
+  }
+
   validate() {
     const schedule = this.config.schedule;
     if (ALLOWED_SCHEDULES.includes(schedule)) {

package/src/formatter/javascript.ts

@@ -199,7 +199,9 @@ export class SyntheticsGenerator extends JavaScriptLanguageGenerator {
     const offset = this.isProject ? 2 + stepIndent : 0 + stepIndent;
     const formatter = new JavaScriptFormatter(offset);
 
-    const locators = actionInContext.frame.framePath.map(selector => `.${super._asLocator(selector)}.contentFrame()`);
+    const locators = actionInContext.frame.framePath.map(
+      selector => `.${super._asLocator(selector)}.contentFrame()`
+    );
     const subject = `${pageAlias}${locators.join('')}`;
     const signals = toSignalMap(action);
 

package/src/helpers.ts

@@ -41,7 +41,7 @@ import micromatch from 'micromatch';
 
 const SEPARATOR = '\n';
 
-export function noop() { }
+export function noop() {}
 
 export function indent(lines: string, tab = '   ') {
   return lines.replace(/^/gm, tab);
@@ -152,8 +152,8 @@ export function findPkgJsonByTraversing(resolvePath, cwd) {
   if (resolvePath === parentDirectory) {
     throw red(
       `Could not find package.json file in: "${cwd}"\n` +
-      `It is recommended to run the agent in an NPM project.\n` +
-      `You can create one by running "npm init -y" in the project folder.`
+        `It is recommended to run the agent in an NPM project.\n` +
+        `You can create one by running "npm init -y" in the project folder.`
     );
   }
   return findPkgJsonByTraversing(parentDirectory, cwd);

package/src/loader.ts

@@ -29,7 +29,7 @@ import { CliArgs } from './common_types';
 import { step, journey } from './core';
 import { log } from './core/logger';
 import { expect } from './core/expect';
-import * as mfa from "./core/mfa";
+import * as mfa from './core/mfa';
 import {
   isDepInstalled,
   isDirectory,

package/src/plugins/tracing.ts

@@ -38,7 +38,7 @@ export type TraceOptions = {
  * https://chromedevtools.github.io/devtools-protocol/tot/Tracing/
  */
 export class Tracing {
-  constructor(private driver: Driver, private options: TraceOptions) { }
+  constructor(private driver: Driver, private options: TraceOptions) {}
 
   async start() {
     log(`Plugins: started collecting trace events`);

package/src/push/bundler.ts

@@ -24,25 +24,19 @@
  */
 
 import path from 'path';
-import { stat, unlink, readFile } from 'fs/promises';
+import { unlink, readFile } from 'fs/promises';
 import { createWriteStream } from 'fs';
 import * as esbuild from 'esbuild';
 import archiver from 'archiver';
 import { commonOptions } from '../core/transform';
 import { SyntheticsBundlePlugin } from './plugin';
 
-// 1500KB Max Gzipped limit for bundled code to be pushed as Kibana project monitors.
-const SIZE_LIMIT_KB = 1500;
-
 function relativeToCwd(entry: string) {
   return path.relative(process.cwd(), entry);
 }
 
 export class Bundler {
-  moduleMap = new Map<string, string>();
-  constructor() {}
-
-  async prepare(absPath: string) {
+  async bundle(absPath: string) {
     const options: esbuild.BuildOptions = {
       ...commonOptions(),
       ...{
@@ -59,56 +53,42 @@ export class Bundler {
     if (result.errors.length > 0) {
       throw result.errors;
     }
-    this.moduleMap.set(absPath, result.outputFiles[0].text);
+    return result.outputFiles[0].text;
   }
 
-  async zip(outputPath: string) {
+  async zip(source: string, code: string, dest: string) {
     return new Promise((fulfill, reject) => {
-      const output = createWriteStream(outputPath);
+      const output = createWriteStream(dest);
       const archive = archiver('zip', {
         zlib: { level: 9 },
       });
       archive.on('error', reject);
       output.on('close', fulfill);
       archive.pipe(output);
-      for (const [path, content] of this.moduleMap.entries()) {
-        const relativePath = relativeToCwd(path);
-        // Date is fixed to Unix epoch so the file metadata is
-        // not modified everytime when files are bundled
-        archive.append(content, {
-          name: relativePath,
-          date: new Date('1970-01-01'),
-        });
-      }
+      const relativePath = relativeToCwd(source);
+      // Date is fixed to Unix epoch so the file metadata is
+      // not modified everytime when files are bundled
+      archive.append(code, {
+        name: relativePath,
+        date: new Date('1970-01-01'),
+      });
       archive.finalize();
     });
   }
 
   async build(entry: string, output: string) {
-    await this.prepare(entry);
-    await this.zip(output);
-    const data = await this.encode(output);
-    await this.checkSize(output);
+    const code = await this.bundle(entry);
+    await this.zip(entry, code, output);
+    const content = await this.encode(output);
     await this.cleanup(output);
-    return data;
+    return content;
   }
 
   async encode(outputPath: string) {
     return await readFile(outputPath, 'base64');
   }
 
-  async checkSize(outputPath: string) {
-    const { size } = await stat(outputPath);
-    const sizeKb = size / 1024;
-    if (sizeKb > SIZE_LIMIT_KB) {
-      throw new Error(
-        `Bundled monitor code exceeds the recommended ${SIZE_LIMIT_KB}KB limit. Please check your dependencies and try again.`
-      );
-    }
-  }
-
   async cleanup(outputPath: string) {
-    this.moduleMap = new Map<string, string>();
     await unlink(outputPath);
   }
 }

package/src/push/index.ts

@@ -24,7 +24,7 @@
  */
 import { readFile, writeFile } from 'fs/promises';
 import { prompt } from 'enquirer';
-import { bold, grey } from 'kleur/colors';
+import { bold, underline } from 'kleur/colors';
 import {
   getLocalMonitors,
   buildMonitorSchema,
@@ -57,8 +57,11 @@ import {
   isBulkAPISupported,
   isLightweightMonitorSupported,
   logDiff,
+  logGroups,
+  printBytes,
 } from './utils';
-import { log } from '../core/logger';
+import { runLocal } from './run-local';
+import { inDebugMode } from '../core/globals';
 
 export async function push(monitors: Monitor[], options: PushOptions) {
   if (parseInt(process.env.CHUNK_SIZE) > 250) {
@@ -84,7 +87,7 @@ export async function push(monitors: Monitor[], options: PushOptions) {
   const { monitors: remote } = await bulkGetMonitors(options);
 
   progress(`preparing ${monitors.length} monitors`);
-  const schemas = await buildMonitorSchema(monitors, true);
+  const { schemas, sizes } = await buildMonitorSchema(monitors, true);
   const local = getLocalMonitors(schemas);
 
   const { newIDs, changedIDs, removedIDs, unchangedIDs } = diffMonitorHashIDs(
@@ -92,6 +95,22 @@ export async function push(monitors: Monitor[], options: PushOptions) {
     remote
   );
   logDiff(newIDs, changedIDs, removedIDs, unchangedIDs);
+  if (inDebugMode()) {
+    logGroups(sizes, newIDs, changedIDs, removedIDs, unchangedIDs);
+    // show bundle size for the whole project
+    let totalSize = 0;
+    for (const value of sizes.values()) {
+      totalSize += value;
+    }
+    progress('total size of the monitors payload is ' + printBytes(totalSize));
+  }
+
+  if (options.dryRun) {
+    progress('Running browser monitors in dry run mode');
+    await runLocal(schemas);
+    progress('Dry run completed');
+    return;
+  }
 
   const updatedMonitors = new Set<string>([...changedIDs, ...newIDs]);
   if (updatedMonitors.size > 0) {
@@ -106,7 +125,6 @@ export async function push(monitors: Monitor[], options: PushOptions) {
   }
 
   if (removedIDs.size > 0) {
-    log(`deleting monitor ids: ${Array.from(removedIDs.keys()).join(', ')}`);
     if (updatedMonitors.size === 0 && unchangedIDs.size === 0) {
       await confirmDelete(
         `Pushing without any monitors will delete all monitors associated with the project.\n Do you want to continue?`,
@@ -126,8 +144,7 @@ export async function push(monitors: Monitor[], options: PushOptions) {
       );
     }
   }
-
-  done(`Pushed: ${grey(getMonitorManagementURL(options.url))}`);
+  done(`Pushed: ${underline(getMonitorManagementURL(options.url))} `);
 }
 
 async function confirmDelete(message: string, skip: boolean) {
@@ -223,8 +240,9 @@ export function validateSettings(opts: PushOptions) {
   - CLI '--schedule <mins>'
   - Config file 'monitors.schedule' field`;
   } else if (opts.schedule && !ALLOWED_SCHEDULES.includes(opts.schedule)) {
-    reason = `Set default schedule(${opts.schedule
-      }) to one of the allowed values - ${ALLOWED_SCHEDULES.join(',')}`;
+    reason = `Set default schedule(${
+      opts.schedule
+    }) to one of the allowed values - ${ALLOWED_SCHEDULES.join(',')}`;
   } else if (
     (opts.locations ?? []).length > 0 &&
     (opts?.playwrightOptions?.clientCertificates ?? []).filter(cert => {
@@ -301,7 +319,7 @@ export async function pushLegacy(monitors: Monitor[], options: PushOptions) {
   let schemas: MonitorSchema[] = [];
   if (monitors.length > 0) {
     progress(`preparing ${monitors.length} monitors`);
-    schemas = await buildMonitorSchema(monitors, false);
+    ({ schemas } = await buildMonitorSchema(monitors, false));
     const chunks = getChunks(schemas, 10);
     for (const chunk of chunks) {
       await liveProgress(
@@ -320,7 +338,7 @@ export async function pushLegacy(monitors: Monitor[], options: PushOptions) {
     `deleting all stale monitors`
   );
 
-  done(`Pushed: ${grey(getMonitorManagementURL(options.url))}`);
+  done(`Pushed: ${underline(getMonitorManagementURL(options.url))}`);
 }
 
 // prints warning if any of the monitors has throttling settings enabled during push

package/src/push/monitor.ts

@@ -42,6 +42,8 @@ import { isParamOptionSupported, normalizeMonitorName } from './utils';
 
 // Allowed extensions for lightweight monitor files
 const ALLOWED_LW_EXTENSIONS = ['.yml', '.yaml'];
+// 1500kB Max Gzipped limit for bundled monitor code to be pushed as Kibana project monitors.
+const SIZE_LIMIT_KB = 1500;
 
 export type MonitorSchema = Omit<MonitorConfig, 'locations'> & {
   locations: string[];
@@ -131,6 +133,7 @@ export async function buildMonitorSchema(monitors: Monitor[], isV2: boolean) {
   await mkdir(bundlePath, { recursive: true });
   const bundler = new Bundler();
   const schemas: MonitorSchema[] = [];
+  const sizes: Map<string, number> = new Map();
 
   for (const monitor of monitors) {
     const { source, config, filter, type } = monitor;
@@ -148,6 +151,17 @@ export async function buildMonitorSchema(monitors: Monitor[], isV2: boolean) {
       monitor.setContent(content);
       Object.assign(schema, { content, filter });
     }
+    const size = monitor.size();
+    const sizeKB = Math.round(size / 1000);
+    if (sizeKB > SIZE_LIMIT_KB) {
+      let outer = bold(
+        `Aborted: Bundled code ${sizeKB}kB exceeds the recommended ${SIZE_LIMIT_KB}kB limit. Please check the dependencies imported.\n`
+      );
+      const inner = `* ${config.id} - ${source.file}:${source.line}:${source.column}\n`;
+      outer += indent(inner);
+      throw red(outer);
+    }
+    sizes.set(config.id, size);
     /**
      * Generate hash only after the bundled content is created
      * to capture code changes in imported files
@@ -159,7 +173,7 @@ export async function buildMonitorSchema(monitors: Monitor[], isV2: boolean) {
   }
 
   await rm(bundlePath, { recursive: true });
-  return schemas;
+  return { schemas, sizes };
 }
 
 export async function createLightweightMonitors(
@@ -225,9 +239,7 @@ export async function createLightweightMonitors(
       const monitor = mergedConfig[i];
       // Skip browser monitors from the YML files
       if (monitor['type'] === 'browser') {
-        warn(
-          `Browser monitors from ${file} are skipped.`
-        );
+        warn(`Browser monitors from ${file} are skipped.`);
         continue;
       }
       const { line, col } = lineCounter.linePos(offsets[i]);

package/src/push/run-local.ts

@@ -0,0 +1,155 @@
+/**
+ * MIT License
+ *
+ * Copyright (c) 2020-present, Elastic NV
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ *
+ */
+
+import { execFileSync, spawn } from 'child_process';
+import { rm, writeFile } from 'fs/promises';
+import { createReadStream } from 'fs';
+import { tmpdir } from 'os';
+import { Extract } from 'unzip-stream';
+import { red } from 'kleur/colors';
+import { join } from 'path';
+import { pathToFileURL } from 'url';
+import { MonitorSchema } from './monitor';
+
+async function unzipFile(zipPath, destination) {
+  return new Promise<void>((resolve, reject) => {
+    createReadStream(zipPath)
+      .pipe(Extract({ path: destination }))
+      .on('close', resolve)
+      .on('error', err =>
+        reject(new Error(`failed to extract zip ${zipPath} : ${err.message}`))
+      );
+  });
+}
+
+async function runNpmInstall(directory) {
+  return new Promise<void>((resolve, reject) => {
+    const flags = [
+      '--no-audit', // Prevent audit checks
+      '--no-update-notifier', // Prevent update checks
+      '--no-fund', // No need for package funding messages here
+      '--package-lock=false', // no need to write package lock here
+      '--progress=false', // no need to display progress
+    ];
+
+    const npmInstall = spawn('npm', ['install', ...flags], {
+      cwd: directory,
+      stdio: 'ignore',
+    });
+    npmInstall.on('close', code => {
+      if (code === 0) {
+        resolve();
+      } else {
+        reject(new Error(`npm install failed with exit code ${code}`));
+      }
+    });
+    npmInstall.on('error', err =>
+      reject(new Error(`failed to setup: ${err.message}`))
+    );
+  });
+}
+
+async function runTest(directory, schema: MonitorSchema) {
+  return new Promise<void>((resolve, reject) => {
+    const runTest = spawn(
+      'npx',
+      [
+        '@elastic/synthetics',
+        '.',
+        '--playwright-options',
+        JSON.stringify(schema.playwrightOptions),
+        '--params',
+        JSON.stringify(schema.params),
+      ],
+      {
+        cwd: directory,
+        stdio: 'inherit',
+      }
+    );
+
+    runTest.on('close', resolve);
+    runTest.on('error', err => {
+      reject(
+        new Error(`Failed to execute @elastic/synthetics : ${err.message}`)
+      );
+    });
+  });
+}
+
+async function writePkgJSON(dir: string, synthPath: string) {
+  const packageJsonContent = {
+    name: 'project-journey',
+    private: 'true',
+    dependencies: {
+      '@elastic/synthetics': pathToFileURL(synthPath),
+    },
+  };
+  await writeFile(
+    join(dir, 'package.json'),
+    JSON.stringify(packageJsonContent, null, 2),
+    'utf-8'
+  );
+}
+
+async function extract(
+  schema: MonitorSchema,
+  zipPath: string,
+  unzipPath: string
+) {
+  if (schema.type !== 'browser') {
+    return;
+  }
+  const content = schema.content;
+  await writeFile(zipPath, content, 'base64');
+  await unzipFile(zipPath, unzipPath);
+}
+
+export async function runLocal(schemas: MonitorSchema[]) {
+  // lookup installed bin path of a node module
+  const resolvedPath = execFileSync('which', ['elastic-synthetics'], {
+    encoding: 'utf8',
+  }).trim();
+  const synthPath = resolvedPath.replace(
+    join('bin', 'elastic-synthetics'),
+    join('lib', 'node_modules', '@elastic/synthetics')
+  );
+  const rand = Date.now();
+  const zipPath = join(tmpdir(), `synthetics-zip-${rand}.zip`);
+  const unzipPath = join(tmpdir(), `synthetics-unzip-${rand}`);
+  try {
+    for (const schema of schemas) {
+      await extract(schema, zipPath, unzipPath);
+    }
+    await writePkgJSON(unzipPath, synthPath);
+    await runNpmInstall(unzipPath);
+    // TODO: figure out a way to collect all params and Playwright options
+    await runTest(unzipPath, schemas[0]);
+  } catch (e) {
+    throw red(`Aborted: ${e.message}`);
+  } finally {
+    await rm(zipPath, { recursive: true, force: true });
+    await rm(unzipPath, { recursive: true, force: true });
+  }
+}

package/src/push/utils.ts

@@ -25,7 +25,7 @@
 
 import semver from 'semver';
 import { progress, removeTrailingSlash } from '../helpers';
-import { green, red, grey, yellow } from 'kleur/colors';
+import { green, red, grey, yellow, Colorize, bold } from 'kleur/colors';
 import { PushOptions } from '../common_types';
 import { Monitor } from '../dsl/monitor';
 
@@ -37,13 +37,59 @@ export function logDiff<T extends Set<string>>(
 ) {
   progress(
     'Monitor Diff: ' +
-    green(`Added(${newIDs.size}) `) +
-    yellow(`Updated(${changedIDs.size}) `) +
-    red(`Removed(${removedIDs.size}) `) +
-    grey(`Unchanged(${unchangedIDs.size})`)
+      green(`Added(${newIDs.size}) `) +
+      yellow(`Updated(${changedIDs.size}) `) +
+      red(`Removed(${removedIDs.size}) `) +
+      grey(`Unchanged(${unchangedIDs.size})`)
   );
 }
 
+export function logGroups<T extends Set<string>>(
+  sizes: Map<string, number>,
+  newIDs: T,
+  changedIDs: T,
+  removedIDs: T,
+  unchangedIDs: T
+) {
+  console.groupCollapsed();
+  logGroup(sizes, 'Added', newIDs, green);
+  logGroup(sizes, 'Updated', changedIDs, yellow);
+  logGroup(sizes, 'Removed', removedIDs, red);
+  logGroup(sizes, 'Unchanged', unchangedIDs, grey);
+  console.groupEnd();
+}
+
+function logGroup(
+  sizes: Map<string, number>,
+  name: string,
+  ids: Set<string>,
+  color: Colorize
+) {
+  if (ids.size === 0) return;
+  // under collapsed group, so giving 2 space for padding
+  printLine(process.stdout.columns - 2);
+  console.groupCollapsed(color(bold(name)));
+  [...ids].forEach(id => {
+    console.log(grey(`- ${id} (${printBytes(sizes.get(id))})`));
+  });
+  console.groupEnd();
+}
+
+function printLine(length: number = process.stdout.columns) {
+  console.log(grey('-').repeat(length));
+}
+
+const BYTE_UNITS = ['B', 'kB', 'MB', 'GB', 'TB', 'PB'];
+export function printBytes(bytes: number) {
+  if (bytes <= 0) return '0 B';
+  const exponent = Math.min(
+    Math.floor(Math.log10(bytes) / 3),
+    BYTE_UNITS.length - 1
+  );
+  bytes /= 1000 ** exponent;
+  return `${bytes.toFixed(1)} ${BYTE_UNITS[exponent]}`;
+}
+
 export function getChunks<T>(arr: Array<T>, size: number): Array<T[]> {
   const chunks = [];
   for (let i = 0; i < arr.length; i += size) {