@elastic/synthetics 1.0.0-beta.3 → 1.0.0-beta.32

package/src/plugins/plugin-manager.ts

@@ -23,7 +23,7 @@
  *
  */
 
-import { PluginOutput } from '../common_types';
+import { PluginOutput, Driver } from '../common_types';
 import {
   BrowserConsole,
   NetworkManager,
@@ -31,7 +31,6 @@ import {
   Tracing,
   TraceOptions,
 } from './';
-import { Driver } from '../core/gatherer';
 import { Step } from '../dsl';
 
 type PluginType = 'network' | 'trace' | 'performance' | 'browserconsole';
@@ -39,59 +38,78 @@ type Plugin = NetworkManager | Tracing | PerformanceManager | BrowserConsole;
 type PluginOptions = TraceOptions;
 
 export class PluginManager {
-  protected plugins = new Map<string, Plugin>();
-
+  protected plugins = new Map<PluginType, Plugin>();
+  public PLUGIN_TYPES: Array<PluginType> = [
+    'network',
+    'trace',
+    'performance',
+    'browserconsole',
+  ];
   constructor(private driver: Driver) {}
 
-  async start(type: PluginType, options?: PluginOptions) {
+  register(type: PluginType, options: PluginOptions) {
     let instance: Plugin;
     switch (type) {
       case 'network':
-        instance = new NetworkManager();
-        await instance.start(this.driver.client);
+        instance = new NetworkManager(this.driver);
         break;
       case 'trace':
-        instance = new Tracing(options);
-        await instance.start(this.driver.client);
+        instance = new Tracing(this.driver, options);
         break;
       case 'performance':
-        instance = new PerformanceManager(this.driver.client);
-        instance.start();
+        instance = new PerformanceManager(this.driver);
         break;
       case 'browserconsole':
-        instance = new BrowserConsole(this.driver.page);
-        instance.start();
+        instance = new BrowserConsole(this.driver);
         break;
     }
+    instance && this.plugins.set(type, instance);
+    return instance;
+  }
+
+  registerAll(options: PluginOptions) {
+    for (const type of this.PLUGIN_TYPES) {
+      this.register(type, options);
+    }
+  }
+
+  unregisterAll() {
+    for (const type of this.PLUGIN_TYPES) {
+      this.plugins.delete(type);
+    }
+  }
+
+  async stop(type: PluginType) {
+    const instance = this.plugins.get(type);
+    if (instance) {
+      return await instance.stop();
+    }
+    return {};
+  }
 
-    this.plugins.set(instance.constructor.name, instance);
+  async start(type: PluginType) {
+    const instance = this.plugins.get(type);
+    instance && (await instance.start());
     return instance;
   }
 
-  get<T extends Plugin>(Type: new (...args: any[]) => T): T {
-    return this.plugins.get(Type.name) as T;
+  get(type: PluginType) {
+    return this.plugins.get(type);
   }
 
   onStep(step: Step) {
-    this.get(BrowserConsole) && (this.get(BrowserConsole)._currentStep = step);
-    this.get(NetworkManager) && (this.get(NetworkManager)._currentStep = step);
+    (this.get('browserconsole') as BrowserConsole)._currentStep = step;
+    (this.get('network') as NetworkManager)._currentStep = step;
   }
 
-  async output(): Promise<PluginOutput> {
+  async output() {
     const data: PluginOutput = {};
-    try {
-      for (const [, plugin] of this.plugins) {
-        if (plugin instanceof NetworkManager) {
-          data.networkinfo = plugin.stop();
-        } else if (plugin instanceof Tracing) {
-          const result = await plugin.stop(this.driver.client);
-          Object.assign(data, { ...result });
-        } else if (plugin instanceof BrowserConsole) {
-          data.browserconsole = plugin.stop();
-        }
+    for (const [, plugin] of this.plugins) {
+      if (plugin instanceof NetworkManager) {
+        data.networkinfo = await plugin.stop();
+      } else if (plugin instanceof BrowserConsole) {
+        data.browserconsole = plugin.stop();
       }
-    } catch (e) {
-      console.error('Error capturing data', e.message);
     }
     return data;
   }

package/src/plugins/tracing.ts

@@ -23,8 +23,7 @@
  *
  */
 
-import { CDPSession } from 'playwright-chromium';
-import { PluginOutput } from '../common_types';
+import { Driver, PluginOutput } from '../common_types';
 import { Filmstrips } from '../sdk/trace-metrics';
 import { TraceEvent, TraceProcessor } from '../sdk/trace-processor';
 
@@ -38,9 +37,9 @@ export type TraceOptions = {
  * https://chromedevtools.github.io/devtools-protocol/tot/Tracing/
  */
 export class Tracing {
-  constructor(public options: TraceOptions) {}
+  constructor(private driver: Driver, private options: TraceOptions) {}
 
-  async start(client: CDPSession) {
+  async start() {
     const includedCategories = [
       // exclude all default categories
       '-*',
@@ -63,21 +62,21 @@ export class Tracing {
         'disabled-by-default-devtools.timeline'
       );
     }
-    await client.send('Tracing.start', {
+    await this.driver.client.send('Tracing.start', {
       /**
        * Using `ReportEvents` makes gathering trace events
        * much faster as opposed to using `ReturnAsStream` mode
        */
       transferMode: 'ReportEvents',
-      traceConfig: {
-        includedCategories,
-      },
+      categories: includedCategories.join(','),
+      options: 'sampling-frequency=10000', // 1000 is default
     });
   }
 
-  async stop(client: CDPSession) {
+  async stop() {
     const events = [];
     const collectListener = payload => events.push(...payload.value);
+    const { client } = this.driver;
     client.on('Tracing.dataCollected', collectListener);
 
     const [traceEvents] = await Promise.all([

package/src/push/bundler.ts

@@ -0,0 +1,100 @@
+/**
+ * MIT License
+ *
+ * Copyright (c) 2020-present, Elastic NV
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ *
+ */
+
+import path from 'path';
+import { unlink, readFile } from 'fs/promises';
+import { createWriteStream } from 'fs';
+import * as esbuild from 'esbuild';
+import archiver from 'archiver';
+import { commonOptions, MultiAssetPlugin, PluginData } from './plugin';
+
+function relativeToCwd(entry: string) {
+  return path.relative(process.cwd(), entry);
+}
+
+export class Bundler {
+  moduleMap = new Map<string, string>();
+  constructor() {}
+
+  async prepare(absPath: string) {
+    const addToMap = (data: PluginData) => {
+      this.moduleMap.set(data.path, data.contents);
+    };
+
+    const options: esbuild.BuildOptions = {
+      ...commonOptions(),
+      ...{
+        entryPoints: {
+          [absPath]: absPath,
+        },
+        plugins: [MultiAssetPlugin(addToMap)],
+      },
+    };
+    const result = await esbuild.build(options);
+    if (result.errors.length > 0) {
+      throw result.errors;
+    }
+    this.moduleMap.set(absPath, result.outputFiles[0].text);
+  }
+
+  async zip(outputPath: string) {
+    return new Promise((fulfill, reject) => {
+      const output = createWriteStream(outputPath);
+      const archive = archiver('zip', {
+        zlib: { level: 9 },
+      });
+      archive.on('error', reject);
+      output.on('close', fulfill);
+      archive.pipe(output);
+      for (const [path, content] of this.moduleMap.entries()) {
+        const relativePath = relativeToCwd(path);
+        // Date is fixed to Unix epoch so the file metadata is
+        // not modified everytime when files are bundled
+        archive.append(content, {
+          name: relativePath,
+          date: new Date('1970-01-01'),
+        });
+      }
+      archive.finalize();
+    });
+  }
+
+  async build(entry: string, output: string) {
+    await this.prepare(entry);
+    await this.zip(output);
+    const data = await this.encode(output);
+    await this.cleanup(output);
+    return data;
+  }
+
+  async encode(outputPath: string) {
+    return await readFile(outputPath, 'base64');
+  }
+
+  async cleanup(outputPath: string) {
+    this.moduleMap = new Map<string, string>();
+    await unlink(outputPath);
+  }
+}

package/src/push/index.ts

@@ -0,0 +1,202 @@
+/**
+ * MIT License
+ *
+ * Copyright (c) 2020-present, Elastic NV
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ *
+ */
+
+import { join } from 'path';
+import { existsSync } from 'fs';
+import { readFile, writeFile } from 'fs/promises';
+import { prompt } from 'enquirer';
+import { bold, yellow } from 'kleur/colors';
+import {
+  ok,
+  formatAPIError,
+  formatFailedMonitors,
+  formatNotFoundError,
+  formatStaleMonitors,
+} from './request';
+import { buildMonitorSchema, createMonitors } from './monitor';
+import { ProjectSettings } from '../generator';
+import { Monitor } from '../dsl/monitor';
+import {
+  progress,
+  write,
+  error,
+  SYNTHETICS_PATH,
+  aborted,
+  indent,
+  safeNDJSONParse,
+  done,
+} from '../helpers';
+import type { PushOptions } from '../common_types';
+
+export async function push(monitors: Monitor[], options: PushOptions) {
+  if (monitors.length === 0) {
+    throw 'No Monitors found';
+  }
+
+  const duplicates = trackDuplicates(monitors);
+  if (duplicates.size > 0) {
+    throw error(formatDuplicateError(duplicates));
+  }
+
+  progress(`preparing all monitors`);
+  const schemas = await buildMonitorSchema(monitors);
+  try {
+    progress(`creating all monitors`);
+    const { body, statusCode } = await createMonitors(schemas, options);
+    if (statusCode === 404) {
+      throw formatNotFoundError(await body.text());
+    }
+    if (!ok(statusCode)) {
+      const { error, message } = await body.json();
+      throw formatAPIError(statusCode, error, message);
+    }
+    body.setEncoding('utf-8');
+    for await (const data of body) {
+      // Its kind of hacky for now where Kibana streams the response by
+      // writing the data as NDJSON events (data can be interleaved), we
+      // distinguish the final data by checking if the event was a progress vs complete event
+      const chunks = safeNDJSONParse(data);
+      for (const chunk of chunks) {
+        if (typeof chunk === 'string') {
+          progress(chunk);
+          continue;
+        }
+        const { failedMonitors, failedStaleMonitors } = chunk;
+        if (failedMonitors && failedMonitors.length > 0) {
+          throw formatFailedMonitors(failedMonitors);
+        }
+        if (failedStaleMonitors.length > 0) {
+          write(yellow(formatStaleMonitors(failedStaleMonitors)));
+        }
+      }
+    }
+    done('Pushed');
+  } catch (e) {
+    error(e);
+  }
+}
+
+function trackDuplicates(monitors: Monitor[]) {
+  const monitorMap: Map<string, Monitor> = new Map();
+  const duplicates: Set<Monitor> = new Set();
+  for (const monitor of monitors) {
+    const id = monitor.config.id;
+    if (monitorMap.has(id)) {
+      duplicates.add(monitorMap.get(id));
+      duplicates.add(monitor);
+    }
+    monitorMap.set(id, monitor);
+  }
+  return duplicates;
+}
+
+export function formatDuplicateError(monitors: Set<Monitor>) {
+  let outer = bold(`Aborted: Duplicate monitors found\n`);
+
+  let inner = '';
+  for (const monitor of monitors) {
+    const { config, source } = monitor;
+    inner += `* ${config.id} - ${source.file}:${source.line}:${source.column}\n`;
+  }
+  outer += indent(inner);
+  return outer;
+}
+
+const PROJECT_SETTINGS_PATH = join(SYNTHETICS_PATH, 'project.json');
+const INSTALLATION_HELP = `Run 'npx @elastic/synthetics init' to create project with default settings.`;
+
+export async function loadSettings() {
+  if (!existsSync(PROJECT_SETTINGS_PATH)) {
+    throw error(`Aborted. Synthetics project not set up.
+
+${INSTALLATION_HELP}`);
+  }
+
+  const contents = await readFile(PROJECT_SETTINGS_PATH, 'utf-8');
+  return JSON.parse(contents) as ProjectSettings;
+}
+
+export function validateSettings(opts: PushOptions) {
+  const INVALID = 'Aborted. Invalid synthetics project settings.';
+
+  let reason = '';
+  if (!opts.project) {
+    reason = 'Set project id via CLI as `--project <id>`';
+  } else if (!opts.locations && !opts.privateLocations) {
+    reason = `Set default location for all monitors via
+  - CLI as '--locations <values...> or --privateLocations <values...>'
+  - Synthetics config file 'monitors.locations' | 'monitors.privateLocations' field`;
+  } else if (!opts.schedule) {
+    reason = `Set default schedule in minutes for all monitors via
+  - CLI as '--schedule <mins>'
+  - Synthetics config file 'monitors.schedule' field`;
+  }
+
+  if (!reason) return;
+
+  throw error(`${INVALID}
+
+${reason}
+
+${INSTALLATION_HELP}`);
+}
+
+async function overrideSettings(project: string) {
+  const settings = JSON.parse(await readFile(PROJECT_SETTINGS_PATH, 'utf-8'));
+  settings.project = project;
+
+  await writeFile(
+    PROJECT_SETTINGS_PATH,
+    JSON.stringify(settings, null, 2),
+    'utf-8'
+  );
+}
+
+export async function catchIncorrectSettings(
+  settings: ProjectSettings,
+  options: PushOptions
+) {
+  let override = !settings.project;
+  if (settings.project && settings.project !== options.project) {
+    if (typeof process.env.TEST_OVERRIDE != null) {
+      override = Boolean(process.env.TEST_OVERRIDE);
+    } else {
+      // Add an extra line to make it easier to read the prompt
+      write('');
+      ({ override } = await prompt<{ override: boolean }>({
+        type: 'confirm',
+        name: 'override',
+        message: `Monitors were pushed under the '${settings.project}' project. Are you sure you want to push them under the new '${options.project}' (note that this will duplicate the monitors, the old ones being orphaned)`,
+        initial: false,
+      }));
+    }
+    if (!override) {
+      throw aborted('Push command Aborted');
+    }
+  }
+  if (override) {
+    await overrideSettings(options.project);
+  }
+}

package/src/push/monitor.ts

@@ -0,0 +1,96 @@
+/**
+ * MIT License
+ *
+ * Copyright (c) 2020-present, Elastic NV
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ *
+ */
+
+import { mkdir, rm } from 'fs/promises';
+import { join } from 'path';
+import { Bundler } from './bundler';
+import { sendRequest } from './request';
+import { removeTrailingSlash, SYNTHETICS_PATH } from '../helpers';
+import { LocationsMap } from '../locations/public-locations';
+import { Monitor, MonitorConfig } from '../dsl/monitor';
+import { PushOptions } from '../common_types';
+
+export type MonitorSchema = Omit<MonitorConfig, 'locations'> & {
+  content: string;
+  locations: string[];
+  filter: Monitor['filter'];
+};
+
+type APISchema = {
+  project: string;
+  keep_stale: boolean;
+  monitors: MonitorSchema[];
+};
+
+function translateLocation(locations?: MonitorConfig['locations']) {
+  if (!locations) return [];
+  return locations.map(loc => LocationsMap[loc] || loc).filter(Boolean);
+}
+
+export async function buildMonitorSchema(monitors: Monitor[]) {
+  /**
+   * Set up the bundle artifacts path which can be used to
+   * create the bundles required for uploading journeys
+   */
+  const bundlePath = join(SYNTHETICS_PATH, 'bundles');
+  await mkdir(bundlePath, { recursive: true });
+  const bundler = new Bundler();
+  const schemas: MonitorSchema[] = [];
+
+  for (const monitor of monitors) {
+    const { source, config, filter } = monitor;
+    const outPath = join(bundlePath, config.name + '.zip');
+    const content = await bundler.build(source.file, outPath);
+    schemas.push({
+      ...config,
+      content,
+      locations: translateLocation(config.locations),
+      filter: filter,
+    });
+  }
+
+  await rm(bundlePath, { recursive: true });
+  return schemas;
+}
+
+export async function createMonitors(
+  monitors: MonitorSchema[],
+  options: PushOptions
+) {
+  const schema: APISchema = {
+    project: options.project,
+    keep_stale: false,
+    monitors,
+  };
+
+  return await sendRequest({
+    url:
+      removeTrailingSlash(options.url) +
+      `/s/${options.space}/api/synthetics/service/project/monitors`,
+    method: 'PUT',
+    auth: options.auth,
+    body: JSON.stringify(schema),
+  });
+}

package/src/push/plugin.ts

@@ -0,0 +1,119 @@
+/**
+ * MIT License
+ *
+ * Copyright (c) 2020-present, Elastic NV
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ *
+ */
+
+import path from 'path';
+import fs from 'fs/promises';
+import * as esbuild from 'esbuild';
+
+export function commonOptions(): esbuild.BuildOptions {
+  return {
+    bundle: true,
+    external: ['node_modules', '@elastic/synthetics'],
+    minify: false,
+    platform: 'node',
+    logLevel: 'silent',
+    format: 'esm',
+    write: false,
+    outExtension: {
+      '.js': '.js',
+    },
+  };
+}
+
+export type PluginData = {
+  path: string;
+  contents: string;
+};
+export type PluginCallback = (data: PluginData) => void;
+
+export function MultiAssetPlugin(callback: PluginCallback): esbuild.Plugin {
+  // Check that the path isn't in an external package by making sure it's at a standard
+  // local filesystem location
+  const isBare = (str: string) => {
+    // Note that we use `isAbsolute` to handle UNC/windows style paths like C:\path\to\thing
+    // This is not necessary for relative directories since `.\file` is not supported as an import
+    // nor is `~/path/to/file`.
+    if (path.isAbsolute(str) || str.startsWith('./') || str.startsWith('../')) {
+      return true;
+    }
+    return false;
+  };
+
+  return {
+    name: 'esbuild-multiasset-plugin',
+    setup(build) {
+      build.onResolve({ filter: /.*?/ }, async args => {
+        // External and other packages need be marked external to
+        // be removed from the bundle
+        if (
+          build.initialOptions.external?.includes(args.path) ||
+          !isBare(args.path)
+        ) {
+          return {
+            external: true,
+          };
+        }
+
+        if (args.kind === 'entry-point') {
+          return {
+            path: args.path,
+            namespace: 'asset',
+          };
+        }
+
+        // If the modules are resolved locally, then
+        // use the imported path to get full path
+        const entryPath =
+          path.join(path.dirname(args.importer), args.path) +
+          path.extname(args.importer);
+        // Spin off another build to copy over the imported modules without bundling
+        const result = await esbuild.build({
+          ...commonOptions(),
+          entryPoints: {
+            [entryPath]: entryPath,
+          },
+          bundle: false,
+          external: [],
+        });
+
+        callback({
+          path: entryPath,
+          contents: result.outputFiles[0].text,
+        });
+
+        return {
+          errors: result.errors,
+          external: true,
+        };
+      });
+
+      build.onLoad({ filter: /.*?/, namespace: 'asset' }, async args => {
+        const contents = await fs.readFile(args.path, 'utf-8');
+        callback({ path: args.path, contents });
+        return { contents };
+      });
+    },
+  };
+}