@elastic/synthetics 1.0.0-beta.8 → 1.0.0

package/src/plugins/plugin-manager.ts

@@ -23,7 +23,7 @@
  *
  */
 
-import { PluginOutput } from '../common_types';
+import { PluginOutput, Driver } from '../common_types';
 import {
   BrowserConsole,
   NetworkManager,
@@ -31,7 +31,6 @@ import {
   Tracing,
   TraceOptions,
 } from './';
-import { Driver } from '../core/gatherer';
 import { Step } from '../dsl';
 
 type PluginType = 'network' | 'trace' | 'performance' | 'browserconsole';
@@ -39,59 +38,78 @@ type Plugin = NetworkManager | Tracing | PerformanceManager | BrowserConsole;
 type PluginOptions = TraceOptions;
 
 export class PluginManager {
-  protected plugins = new Map<string, Plugin>();
-
+  protected plugins = new Map<PluginType, Plugin>();
+  public PLUGIN_TYPES: Array<PluginType> = [
+    'network',
+    'trace',
+    'performance',
+    'browserconsole',
+  ];
   constructor(private driver: Driver) {}
 
-  async start(type: PluginType, options?: PluginOptions) {
+  register(type: PluginType, options: PluginOptions) {
     let instance: Plugin;
     switch (type) {
       case 'network':
-        instance = new NetworkManager();
-        await instance.start(this.driver.client);
+        instance = new NetworkManager(this.driver);
         break;
       case 'trace':
-        instance = new Tracing(options);
-        await instance.start(this.driver.client);
+        instance = new Tracing(this.driver, options);
         break;
       case 'performance':
-        instance = new PerformanceManager(this.driver.client);
-        instance.start();
+        instance = new PerformanceManager(this.driver);
         break;
       case 'browserconsole':
-        instance = new BrowserConsole(this.driver.page);
-        instance.start();
+        instance = new BrowserConsole(this.driver);
         break;
     }
+    instance && this.plugins.set(type, instance);
+    return instance;
+  }
+
+  registerAll(options: PluginOptions) {
+    for (const type of this.PLUGIN_TYPES) {
+      this.register(type, options);
+    }
+  }
+
+  unregisterAll() {
+    for (const type of this.PLUGIN_TYPES) {
+      this.plugins.delete(type);
+    }
+  }
+
+  async stop(type: PluginType) {
+    const instance = this.plugins.get(type);
+    if (instance) {
+      return await instance.stop();
+    }
+    return {};
+  }
 
-    this.plugins.set(instance.constructor.name, instance);
+  async start(type: PluginType) {
+    const instance = this.plugins.get(type);
+    instance && (await instance.start());
     return instance;
   }
 
-  get<T extends Plugin>(Type: new (...args: any[]) => T): T {
-    return this.plugins.get(Type.name) as T;
+  get(type: PluginType) {
+    return this.plugins.get(type);
   }
 
   onStep(step: Step) {
-    this.get(BrowserConsole) && (this.get(BrowserConsole)._currentStep = step);
-    this.get(NetworkManager) && (this.get(NetworkManager)._currentStep = step);
+    (this.get('browserconsole') as BrowserConsole)._currentStep = step;
+    (this.get('network') as NetworkManager)._currentStep = step;
   }
 
-  async output(): Promise<PluginOutput> {
+  async output() {
     const data: PluginOutput = {};
-    try {
-      for (const [, plugin] of this.plugins) {
-        if (plugin instanceof NetworkManager) {
-          data.networkinfo = plugin.stop();
-        } else if (plugin instanceof Tracing) {
-          const result = await plugin.stop(this.driver.client);
-          Object.assign(data, { ...result });
-        } else if (plugin instanceof BrowserConsole) {
-          data.browserconsole = plugin.stop();
-        }
+    for (const [, plugin] of this.plugins) {
+      if (plugin instanceof NetworkManager) {
+        data.networkinfo = await plugin.stop();
+      } else if (plugin instanceof BrowserConsole) {
+        data.browserconsole = plugin.stop();
       }
-    } catch (e) {
-      console.error('Error capturing data', e.message);
     }
     return data;
   }

package/src/plugins/tracing.ts

@@ -23,8 +23,8 @@
  *
  */
 
-import { CDPSession } from 'playwright-chromium';
-import { PluginOutput } from '../common_types';
+import { Driver, PluginOutput } from '../common_types';
+import { log } from '../core/logger';
 import { Filmstrips } from '../sdk/trace-metrics';
 import { TraceEvent, TraceProcessor } from '../sdk/trace-processor';
 
@@ -38,9 +38,10 @@ export type TraceOptions = {
  * https://chromedevtools.github.io/devtools-protocol/tot/Tracing/
  */
 export class Tracing {
-  constructor(public options: TraceOptions) {}
+  constructor(private driver: Driver, private options: TraceOptions) {}
 
-  async start(client: CDPSession) {
+  async start() {
+    log(`Plugins: started collecting trace events`);
     const includedCategories = [
       // exclude all default categories
       '-*',
@@ -63,21 +64,21 @@ export class Tracing {
         'disabled-by-default-devtools.timeline'
       );
     }
-    await client.send('Tracing.start', {
+    await this.driver.client.send('Tracing.start', {
       /**
        * Using `ReportEvents` makes gathering trace events
        * much faster as opposed to using `ReturnAsStream` mode
        */
       transferMode: 'ReportEvents',
-      traceConfig: {
-        includedCategories,
-      },
+      categories: includedCategories.join(','),
+      options: 'sampling-frequency=10000', // 1000 is default
     });
   }
 
-  async stop(client: CDPSession) {
+  async stop() {
     const events = [];
     const collectListener = payload => events.push(...payload.value);
+    const { client } = this.driver;
     client.on('Tracing.dataCollected', collectListener);
 
     const [traceEvents] = await Promise.all([
@@ -98,6 +99,7 @@ export class Tracing {
         ...TraceProcessor.computeTrace(traceEvents as Array<TraceEvent>),
       });
     }
+    log(`Plugins: stopped collecting trace events`);
     return output;
   }
 }

package/src/push/bundler.ts

@@ -0,0 +1,112 @@
+/**
+ * MIT License
+ *
+ * Copyright (c) 2020-present, Elastic NV
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ *
+ */
+
+import path from 'path';
+import { stat, unlink, readFile } from 'fs/promises';
+import { createWriteStream } from 'fs';
+import * as esbuild from 'esbuild';
+import archiver from 'archiver';
+import { commonOptions } from '../core/transform';
+import { SyntheticsBundlePlugin } from './plugin';
+
+const SIZE_LIMIT_KB = 800;
+
+function relativeToCwd(entry: string) {
+  return path.relative(process.cwd(), entry);
+}
+
+export class Bundler {
+  moduleMap = new Map<string, string>();
+  constructor() {}
+
+  async prepare(absPath: string) {
+    const options: esbuild.BuildOptions = {
+      ...commonOptions(),
+      ...{
+        entryPoints: [absPath],
+        bundle: true,
+        write: false,
+        sourcemap: 'inline',
+        external: ['@elastic/synthetics'],
+        plugins: [SyntheticsBundlePlugin()],
+      },
+    };
+    const result = await esbuild.build(options);
+    if (result.errors.length > 0) {
+      throw result.errors;
+    }
+    this.moduleMap.set(absPath, result.outputFiles[0].text);
+  }
+
+  async zip(outputPath: string) {
+    return new Promise((fulfill, reject) => {
+      const output = createWriteStream(outputPath);
+      const archive = archiver('zip', {
+        zlib: { level: 9 },
+      });
+      archive.on('error', reject);
+      output.on('close', fulfill);
+      archive.pipe(output);
+      for (const [path, content] of this.moduleMap.entries()) {
+        const relativePath = relativeToCwd(path);
+        // Date is fixed to Unix epoch so the file metadata is
+        // not modified everytime when files are bundled
+        archive.append(content, {
+          name: relativePath,
+          date: new Date('1970-01-01'),
+        });
+      }
+      archive.finalize();
+    });
+  }
+
+  async build(entry: string, output: string) {
+    await this.prepare(entry);
+    await this.zip(output);
+    const data = await this.encode(output);
+    await this.checkSize(output);
+    await this.cleanup(output);
+    return data;
+  }
+
+  async encode(outputPath: string) {
+    return await readFile(outputPath, 'base64');
+  }
+
+  async checkSize(outputPath: string) {
+    const { size } = await stat(outputPath);
+    const sizeKb = size / 1024;
+    if (sizeKb > SIZE_LIMIT_KB) {
+      throw new Error(
+        `You have monitors whose size exceeds the ${SIZE_LIMIT_KB}KB limit.`
+      );
+    }
+  }
+
+  async cleanup(outputPath: string) {
+    this.moduleMap = new Map<string, string>();
+    await unlink(outputPath);
+  }
+}

package/src/push/index.ts

@@ -0,0 +1,291 @@
+/**
+ * MIT License
+ *
+ * Copyright (c) 2020-present, Elastic NV
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ *
+ */
+import semver from 'semver';
+import { readFile, writeFile } from 'fs/promises';
+import { prompt } from 'enquirer';
+import { bold, grey } from 'kleur/colors';
+import {
+  getLocalMonitors,
+  buildMonitorSchema,
+  diffMonitors as diffMonitorHashIDs,
+  MonitorSchema,
+} from './monitor';
+import { ALLOWED_SCHEDULES, Monitor } from '../dsl/monitor';
+import {
+  progress,
+  liveProgress,
+  write,
+  error,
+  warn,
+  indent,
+  done,
+  getMonitorManagementURL,
+  THROTTLING_WARNING_MSG,
+} from '../helpers';
+import type { PushOptions, ProjectSettings } from '../common_types';
+import { findSyntheticsConfig, readConfig } from '../config';
+import {
+  bulkDeleteMonitors,
+  bulkGetMonitors,
+  bulkPutMonitors,
+  getVersion,
+  createMonitorsLegacy,
+  CHUNK_SIZE,
+} from './kibana_api';
+import { getChunks, logDiff } from './utils';
+
+export async function push(monitors: Monitor[], options: PushOptions) {
+  const duplicates = trackDuplicates(monitors);
+  if (duplicates.size > 0) {
+    throw error(formatDuplicateError(duplicates));
+  }
+  progress(`Pushing monitors for project: ${options.id}`);
+
+  const stackVersion = await getVersion(options);
+  const isV2 = semver.satisfies(stackVersion, '>=8.6.0');
+  if (!isV2) {
+    return await pushLegacy(monitors, options, stackVersion);
+  }
+
+  const local = getLocalMonitors(monitors);
+  const { monitors: remote } = await bulkGetMonitors(options);
+  const { newIDs, changedIDs, removedIDs, unchangedIDs } = diffMonitorHashIDs(
+    local,
+    remote
+  );
+  logDiff(newIDs, changedIDs, removedIDs, unchangedIDs);
+
+  const updatedMonitors = new Set<string>([...changedIDs, ...newIDs]);
+  if (updatedMonitors.size > 0) {
+    const toBundle = monitors.filter(m => updatedMonitors.has(m.config.id));
+    progress(`bundling ${toBundle.length} monitors`);
+    const schemas = await buildMonitorSchema(toBundle, true);
+    const chunks = getChunks(schemas, CHUNK_SIZE);
+    for (const chunk of chunks) {
+      await liveProgress(
+        bulkPutMonitors(options, chunk),
+        `creating or updating ${chunk.length} monitors`
+      );
+    }
+  }
+
+  if (removedIDs.size > 0) {
+    if (updatedMonitors.size === 0 && unchangedIDs.size === 0) {
+      await promptConfirmDeleteAll(options);
+    }
+    const chunks = getChunks(Array.from(removedIDs), CHUNK_SIZE);
+    for (const chunk of chunks) {
+      await liveProgress(
+        bulkDeleteMonitors(options, chunk),
+        `deleting ${chunk.length} monitors`
+      );
+    }
+  }
+
+  done(`Pushed: ${grey(getMonitorManagementURL(options.url))}`);
+}
+
+async function promptConfirmDeleteAll(options: PushOptions) {
+  write('');
+  const { deleteAll } = await prompt<{ deleteAll: boolean }>({
+    type: 'confirm',
+    skip() {
+      if (options.yes) {
+        this.initial = process.env.TEST_OVERRIDE ?? true;
+        return true;
+      }
+      return false;
+    },
+    name: 'deleteAll',
+    message: `Pushing without any monitors will delete all monitors associated with the project.\n Do you want to continue?`,
+    initial: false,
+  });
+  if (!deleteAll) {
+    throw warn('Push command Aborted');
+  }
+}
+
+function trackDuplicates(monitors: Monitor[]) {
+  const monitorMap: Map<string, Monitor> = new Map();
+  const duplicates: Set<Monitor> = new Set();
+  for (const monitor of monitors) {
+    const id = monitor.config.id;
+    if (monitorMap.has(id)) {
+      duplicates.add(monitorMap.get(id));
+      duplicates.add(monitor);
+    }
+    monitorMap.set(id, monitor);
+  }
+  return duplicates;
+}
+
+export function formatDuplicateError(monitors: Set<Monitor>) {
+  let outer = bold(`Aborted: Duplicate monitors found\n`);
+
+  let inner = '';
+  for (const monitor of monitors) {
+    const { config, source } = monitor;
+    inner += `* ${config.id} - ${source.file}:${source.line}:${source.column}\n`;
+  }
+  outer += indent(inner);
+  return outer;
+}
+
+const INSTALLATION_HELP = `Run 'npx @elastic/synthetics init' to create project with default settings.`;
+
+export async function loadSettings() {
+  try {
+    const config = await readConfig(process.env['NODE_ENV'] || 'development');
+    // Missing config file, fake throw to capture as missing file
+    if (Object.keys(config).length === 0) {
+      throw '';
+    }
+    return config.project || ({} as ProjectSettings);
+  } catch (e) {
+    throw error(`Aborted (missing synthetics config file), Project not set up correctly.
+
+${INSTALLATION_HELP}`);
+  }
+}
+
+export function validateSettings(opts: PushOptions) {
+  const INVALID = 'Aborted. Invalid synthetics project settings.';
+
+  let reason = '';
+  if (!opts.id) {
+    reason = `Set project id via
+  - CLI '--id <id>'
+  - Config file 'project.id' field`;
+  } else if (!opts.locations && !opts.privateLocations) {
+    reason = `Set default location for all monitors via
+  - CLI '--locations <values...> or --privateLocations <values...>'
+  - Config file 'monitors.locations' | 'monitors.privateLocations' field`;
+  } else if (!opts.schedule) {
+    reason = `Set default schedule in minutes for all monitors via
+  - CLI '--schedule <mins>'
+  - Config file 'monitors.schedule' field`;
+  } else if (opts.schedule && !ALLOWED_SCHEDULES.includes(opts.schedule)) {
+    reason = `Set default schedule(${
+      opts.schedule
+    }) to one of the allowed values - ${ALLOWED_SCHEDULES.join(',')}`;
+  }
+
+  if (!reason) return;
+
+  throw error(`${INVALID}
+
+${reason}
+
+${INSTALLATION_HELP}`);
+}
+
+async function overrideSettings(oldValue: string, newValue: string) {
+  const cwd = process.cwd();
+  const configPath = await findSyntheticsConfig(cwd, cwd);
+  if (!configPath) {
+    throw warn(`Unable to find synthetics config file: ${configPath}`);
+  }
+  const config = await readFile(configPath, 'utf-8');
+  const updatedConfig = config.replace(
+    `id: '${oldValue}'`,
+    `id: '${newValue}'`
+  );
+  await writeFile(configPath, updatedConfig, 'utf-8');
+}
+
+export async function catchIncorrectSettings(
+  settings: ProjectSettings,
+  options: PushOptions
+) {
+  let override = !settings.id;
+  if (settings.id && settings.id !== options.id) {
+    // Add an extra line to make it easier to read the prompt
+    write('');
+    ({ override } = await prompt<{ override: boolean }>({
+      type: 'confirm',
+      name: 'override',
+      skip() {
+        if (options.yes) {
+          this.initial = process.env.TEST_OVERRIDE ?? true;
+          return true;
+        }
+        return false;
+      },
+      message: `Monitors were pushed under the '${settings.id}' project. Are you sure you want to push them under the new '${options.id}' (note that this will duplicate the monitors, the old ones being orphaned)`,
+      initial: false,
+    }));
+    if (!override) {
+      throw warn('Push command Aborted');
+    }
+  }
+  if (override) {
+    await overrideSettings(settings.id, options.id);
+  }
+}
+
+export async function pushLegacy(
+  monitors: Monitor[],
+  options: PushOptions,
+  version: number
+) {
+  const noLightWeightSupport = semver.satisfies(version, '<8.5.0');
+  if (
+    noLightWeightSupport &&
+    monitors.some(monitor => monitor.type !== 'browser')
+  ) {
+    throw error(
+      `Aborted: Lightweight monitors are not supported in ${version}. Please upgrade to 8.5.0 or above.`
+    );
+  }
+
+  let schemas: MonitorSchema[] = [];
+  if (monitors.length > 0) {
+    progress(`bundling ${monitors.length} monitors`);
+    schemas = await buildMonitorSchema(monitors, false);
+    const chunks = getChunks(schemas, 10);
+    for (const chunk of chunks) {
+      await liveProgress(
+        createMonitorsLegacy({ schemas: chunk, keepStale: true, options }),
+        `creating or updating ${chunk.length} monitors`
+      );
+    }
+  } else {
+    await promptConfirmDeleteAll(options);
+  }
+  await liveProgress(
+    createMonitorsLegacy({ schemas, keepStale: false, options }),
+    `deleting all stale monitors`
+  );
+
+  done(`Pushed: ${grey(getMonitorManagementURL(options.url))}`);
+}
+
+// prints warning if any of the monitors has throttling settings enabled during push
+export function warnIfThrottled(monitors: Monitor[]) {
+  const throttled = monitors.some(monitor => monitor.config.throttling != null);
+  if (throttled) {
+    warn(THROTTLING_WARNING_MSG);
+  }
+}