@elastic/synthetics 1.0.0-beta.31 → 1.0.0-beta.32

package/src/cli.ts

@@ -29,19 +29,37 @@ import { program, Option } from 'commander';
 import { cwd } from 'process';
 import { CliArgs, PushOptions } from './common_types';
 import { reporters } from './reporters';
-import { normalizeOptions, parseThrottling } from './options';
+import {
+  normalizeOptions,
+  parseThrottling,
+  getCommonCommandOpts,
+} from './options';
 import { loadTestFiles } from './loader';
 import { run } from './';
 import { runner } from './core';
 import { SyntheticsLocations } from './dsl/monitor';
-import { push } from './push';
+import {
+  push,
+  loadSettings,
+  validateSettings,
+  catchIncorrectSettings,
+} from './push';
+import {
+  formatLocations,
+  getLocations,
+  renderLocations,
+  LocationCmdOptions,
+} from './locations';
 import { resolve } from 'path';
 import { Generator } from './generator';
 import { error } from './helpers';
+import { LocationsMap } from './locations/public-locations';
 
 /* eslint-disable-next-line @typescript-eslint/no-var-requires */
 const { name, version } = require('../package.json');
 
+const { params, pattern, tags, match, playwrightOpts } = getCommonCommandOpts();
+
 program
   .name(`npx ${name}`)
   .usage('[options] [dir] [files] file')
@@ -49,20 +67,15 @@ program
     '-c, --config <path>',
     'configuration path (default: synthetics.config.js)'
   )
-  .option(
-    '-p, --params <jsonstring>',
-    'JSON object that gets injected to all journeys',
-    JSON.parse
-  )
+  .addOption(pattern)
+  .addOption(tags)
+  .addOption(match)
+  .addOption(params)
   .addOption(
     new Option('--reporter <value>', `output repoter format`).choices(
       Object.keys(reporters)
     )
   )
-  .option(
-    '--pattern <pattern>',
-    'RegExp file patterns to search inside directory'
-  )
   .option('--inline', 'Run inline journeys from heartbeat')
   .option('-r, --require <modules...>', 'module(s) to preload')
   .option('--no-headless', 'run browser in headful mode')
@@ -81,14 +94,6 @@ program
     '--dry-run',
     "don't actually execute anything, report only registered journeys"
   )
-  .option(
-    '--tags <name...>',
-    'run only journeys with a tag that matches the glob'
-  )
-  .option(
-    '--match <name>',
-    'run only journeys with a name or tag that matches the glob'
-  )
   .option(
     '--outfd <fd>',
     'specify a file descriptor for logs. Default is stdout',
@@ -117,11 +122,7 @@ program
     {}
   )
   .option('--no-throttling', 'Turns off default network throttling.')
-  .option(
-    '--playwright-options <jsonstring>',
-    'JSON object to pass in custom Playwright options for the agent. Options passed will be merged with Playwright options defined in your synthetics.config.js file. Options defined via --playwright-options take precedence.',
-    JSON.parse
-  )
+  .addOption(playwrightOpts)
   .version(version)
   .description('Run synthetic tests')
   .action(async (cliArgs: CliArgs) => {
@@ -149,19 +150,11 @@ program
 program
   .command('push')
   .description(
-    'Push journeys in the current directory to create monitors within the Kibana monitor management UI'
+    'Push all journeys in the current directory to create monitors within the Kibana monitor management UI'
   )
-  .option(
-    '--pattern <pattern>',
-    'RegExp file patterns to push inside current directory'
-  )
-  .option(
-    '--tags <name...>',
-    'push only journeys with a tag that matches the glob'
-  )
-  .option(
-    '--match <name>',
-    'push only journeys with a name or tag that matches the glob'
+  .requiredOption(
+    '--auth <auth>',
+    'API key used for Kibana authentication(https://www.elastic.co/guide/en/kibana/master/api-keys.html).'
   )
   .option(
     '--schedule <time-in-minutes>',
@@ -174,35 +167,37 @@ program
       'default list of locations from which your monitors will run.'
     ).choices(SyntheticsLocations)
   )
-  .requiredOption(
-    '--project <project-id>',
-    'id that will be used for logically grouping monitors'
+  .option(
+    '--private-locations <locations...>',
+    'default list of private locations from which your monitors will run.'
   )
-  .requiredOption('--url <url>', 'kibana URL to upload the monitors')
-  .requiredOption(
-    '--auth <auth>',
-    'API key used for Kibana authentication(https://www.elastic.co/guide/en/kibana/master/api-keys.html).'
+  .option('--url <url>', 'Kibana URL to upload the monitors')
+  .option(
+    '--project <id>',
+    'id that will be used for logically grouping monitors'
   )
   .option(
     '--space <space>',
-    'the target Kibana spaces for the pushed monitors — spaces help you organise pushed monitors.',
-    'default'
+    'the target Kibana spaces for the pushed monitors — spaces help you organise pushed monitors.'
   )
+  .addOption(pattern)
+  .addOption(tags)
+  .addOption(match)
+  .addOption(params)
+  .addOption(playwrightOpts)
   .action(async (cmdOpts: PushOptions) => {
     try {
+      const settings = await loadSettings();
       await loadTestFiles({ inline: false }, [cwd()]);
-      const options = normalizeOptions({ ...program.opts(), ...cmdOpts });
-      if (!options.schedule) {
-        throw error(`Set default schedule in minutes for all monitors via '--schedule <time-in-minutes>' OR
-  configure via Synthetics config file under 'monitors.schedule' field.`);
-      }
-
-      if (!options.locations) {
-        throw error(`Set default location for all monitors via CLI as '--locations <locations...>' OR
-  configure via Synthetics config file under 'monitors.locations' field.`);
-      }
+      const options = normalizeOptions({
+        ...program.opts(),
+        ...settings,
+        ...cmdOpts,
+      }) as PushOptions;
+      validateSettings(options);
+      await catchIncorrectSettings(settings, options);
       const monitors = runner.buildMonitors(options);
-      await push(monitors, cmdOpts);
+      await push(monitors, options);
     } catch (e) {
       e && console.error(e);
       process.exit(1);
@@ -211,9 +206,9 @@ program
 
 // Init command
 program
-  .command('init <project-dir>')
-  .description('Initalize Elastic synthetics project')
-  .action(async (dir: string) => {
+  .command('init [dir]')
+  .description('Initialize Elastic synthetics project')
+  .action(async (dir = '') => {
     try {
       const generator = await new Generator(resolve(process.cwd(), dir));
       await generator.setup();
@@ -223,4 +218,29 @@ program
     }
   });
 
+// Locations command
+program
+  .command('locations')
+  .description(
+    `List all locations to run the synthetics monitors. Pass optional '--url' and '--auth' to list private locations.`
+  )
+  .option('--url <url>', 'Kibana URL to fetch all public and private locations')
+  .option(
+    '--auth <auth>',
+    'API key used for Kibana authentication(https://www.elastic.co/guide/en/kibana/master/api-keys.html).'
+  )
+  .action(async (cmdOpts: LocationCmdOptions) => {
+    try {
+      let locations = Object.keys(LocationsMap);
+      if (cmdOpts.auth && cmdOpts.url) {
+        const allLocations = await getLocations(cmdOpts);
+        locations = formatLocations(allLocations);
+      }
+      renderLocations(locations);
+    } catch (e) {
+      e && error(e);
+      process.exit(1);
+    }
+  });
+
 program.parse(process.argv);

package/src/common_types.ts

@@ -212,6 +212,7 @@ type BaseArgs = {
   throttling?: ThrottlingOptions;
   schedule?: MonitorConfig['schedule'];
   locations?: MonitorConfig['locations'];
+  privateLocations?: MonitorConfig['privateLocations'];
 };
 
 export type CliArgs = BaseArgs & {
@@ -247,6 +248,7 @@ export type PushOptions = {
   space: string;
   schedule?: MonitorConfig['schedule'];
   locations?: MonitorConfig['locations'];
+  privateLocations?: MonitorConfig['privateLocations'];
 };
 
 export type PlaywrightOptions = LaunchOptions & BrowserContextOptions;

package/src/core/runner.ts

@@ -24,7 +24,7 @@
  */
 
 import { join } from 'path';
-import { rm } from 'fs/promises';
+import { mkdir, rm, writeFile } from 'fs/promises';
 import { Journey } from '../dsl/journey';
 import { Step } from '../dsl/step';
 import { reporters, Reporter } from '../reporters';
@@ -34,8 +34,6 @@ import {
   getTimestamp,
   runParallel,
   generateUniqueId,
-  mkdirAsync,
-  writeFileAsync,
 } from '../helpers';
 import {
   HooksCallback,
@@ -87,7 +85,7 @@ export default class Runner {
      * For each journey we create the screenshots folder for
      * caching all screenshots and clear them at end of each journey
      */
-    await mkdirAsync(this.screenshotPath, { recursive: true });
+    await mkdir(this.screenshotPath, { recursive: true });
     return {
       start,
       params: options.params,
@@ -116,7 +114,7 @@ export default class Runner {
         timestamp: getTimestamp(),
         data: buffer.toString('base64'),
       };
-      await writeFileAsync(
+      await writeFile(
         join(Runner.screenshotPath, fileName),
         JSON.stringify(screenshot)
       );
@@ -381,7 +379,7 @@ export default class Runner {
     /**
      * Set up the directory for caching screenshots
      */
-    await mkdirAsync(CACHE_PATH, { recursive: true });
+    await mkdir(CACHE_PATH, { recursive: true });
   }
 
   buildMonitors(options: RunOptions) {
@@ -393,6 +391,7 @@ export default class Runner {
       throttling: options.throttling,
       schedule: options.schedule,
       locations: options.locations,
+      privateLocations: options.privateLocations,
       params: options.params,
       playwrightOptions: options.playwrightOptions,
     });

package/src/dsl/monitor.ts

@@ -31,7 +31,7 @@ import {
   Params,
   PlaywrightOptions,
 } from '../common_types';
-import { LocationsMap } from './locations';
+import { LocationsMap } from '../locations/public-locations';
 
 export type SyntheticsLocationsType = keyof typeof LocationsMap;
 export const SyntheticsLocations = Object.keys(
@@ -45,6 +45,7 @@ export type MonitorConfig = {
   schedule?: number;
   enabled?: boolean;
   locations?: SyntheticsLocationsType[];
+  privateLocations?: string[];
   throttling?: ThrottlingOptions;
   screenshot?: ScreenshotOptions;
   params?: Params;

package/src/generator/index.ts

@@ -27,34 +27,56 @@ import { existsSync } from 'fs';
 import { mkdir, readFile, writeFile } from 'fs/promises';
 import { bold, cyan, yellow } from 'kleur/colors';
 import { join, relative, dirname, basename } from 'path';
-import { prompt } from 'enquirer';
-import { SyntheticsLocations } from '../dsl/monitor';
-import { progress, write as stdWrite } from '../helpers';
-import { getPackageManager, replaceTemplates, runCommand } from './utils';
+// @ts-ignore-next-line: has no exported member 'Input'
+import { prompt, Input } from 'enquirer';
+import { progress, removeTrailingSlash, write as stdWrite } from '../helpers';
+import {
+  getPackageManager,
+  replaceTemplates,
+  runCommand,
+  cloudIDToKibanaURL,
+} from './utils';
+import { formatLocations, getLocations, groupLocations } from '../locations';
 
-// Templates that are required for setting up new
-// synthetics project
+// Templates that are required for setting up new synthetics project
 const templateDir = join(__dirname, '..', '..', 'templates');
 
-type PromptOptions = {
-  locations: string;
+export type ProjectSettings = {
+  url: string;
+  space: string;
+  project: string;
+};
+
+type PromptOptions = ProjectSettings & {
+  locations: Array<string>;
+  privateLocations: Array<string>;
   schedule: number;
 };
 
 // exported for testing
-export const regularFiles = [
-  '.github/workflows/run-synthetics.yml',
-  'README.md',
+export const REGULAR_FILES_PATH = [
   'journeys/example.journey.ts',
-  'journeys/advanced-example.journey.ts',
   'journeys/advanced-example-helpers.ts',
+  'journeys/advanced-example.journey.ts',
+  '.github/workflows/run-synthetics.yml',
+  'README.md',
 ];
+export const SETTINGS_PATH = '.synthetics/project.json';
+export const CONFIG_PATH = 'synthetics.config.ts';
+
+// Files to be overriden by default if the project is initialized multiple times
+const DEFAULT_OVERRIDES = [SETTINGS_PATH, CONFIG_PATH];
 
 export class Generator {
   pkgManager = 'npm';
   constructor(public projectDir: string) {}
 
   async directory() {
+    progress(
+      `Initializing Synthetics project in '${
+        relative(process.cwd(), this.projectDir) || '.'
+      }'`
+    );
     if (!existsSync(this.projectDir)) {
       await mkdir(this.projectDir);
     }
@@ -64,12 +86,58 @@ export class Generator {
     if (process.env.TEST_QUESTIONS) {
       return JSON.parse(process.env.TEST_QUESTIONS);
     }
-    const question = [
+
+    const { onCloud } = await prompt<{ onCloud: string }>({
+      type: 'confirm',
+      name: 'onCloud',
+      initial: 'y',
+      message: 'Do you use Elastic Cloud',
+    });
+    const url = await new Input({
+      header: onCloud
+        ? yellow(
+            'Get cloud.id from your deployment https://www.elastic.co/guide/en/cloud/current/ec-cloud-id.html'
+          )
+        : '',
+      message: onCloud
+        ? 'What is your cloud.id'
+        : 'What is the url of your Kibana instance',
+      name: 'url',
+      required: true,
+      result(value) {
+        return onCloud ? cloudIDToKibanaURL(value) : value;
+      },
+    }).run();
+
+    const auth = await new Input({
+      name: 'auth',
+      header: yellow(
+        `Generate API key from Kibana ${removeTrailingSlash(
+          url
+        )}/app/uptime/manage-monitors/all`
+      ),
+      required: true,
+      message: 'What is your API key',
+    }).run();
+
+    const allLocations = await getLocations({ url, auth });
+    const locChoices = formatLocations(allLocations);
+    if (locChoices.length === 0) {
+      // TODO - Add link to docs
+      throw 'Follow the docs to set up your first private locations - <link>';
+    }
+
+    const monitorQues = [
       {
         type: 'select',
         name: 'locations',
-        message: 'Select the default location where you want to run monitors.',
-        choices: SyntheticsLocations,
+        hint: '(Use <space> to select, <return> to submit)',
+        message: 'Select the locations where you want to run monitors',
+        choices: locChoices,
+        multiple: true,
+        validate(value) {
+          return value.length === 0 ? `Select at least one option.` : true;
+        },
       },
       {
         type: 'numeral',
@@ -77,26 +145,50 @@ export class Generator {
         message: 'Set default schedule in minutes for all monitors',
         initial: 10,
       },
+      {
+        type: 'input',
+        name: 'project',
+        message: 'Choose project id to logically group monitors',
+        initial: basename(this.projectDir),
+      },
+      {
+        type: 'input',
+        name: 'space',
+        message: 'Choose the target Kibana space',
+        initial: 'default',
+      },
     ];
-    return await prompt<PromptOptions>(question);
+
+    // Split and group private and public locations from the answered list.
+    const answers = await prompt<PromptOptions>(monitorQues);
+    const { locations, privateLocations } = groupLocations(answers.locations);
+    return { ...answers, url, locations, privateLocations };
   }
 
   async files(answers: PromptOptions) {
     const fileMap = new Map<string, string>();
+
+    // Setup project.json
+    const projectJSON = {
+      url: answers.url,
+      space: answers.space,
+      project: answers.project,
+    };
+    fileMap.set(SETTINGS_PATH, JSON.stringify(projectJSON, null, 2) + '\n');
+
     // Setup Synthetics config file
-    const configFile = 'synthetics.config.ts';
     fileMap.set(
-      configFile,
+      CONFIG_PATH,
       replaceTemplates(
-        await readFile(join(templateDir, configFile), 'utf-8'),
+        await readFile(join(templateDir, CONFIG_PATH), 'utf-8'),
         answers
       )
     );
 
     // Setup non-templated files
     Promise.all(
-      regularFiles.map(async fn => {
-        fileMap.set(fn, await readFile(join(templateDir, fn), 'utf-8'));
+      REGULAR_FILES_PATH.map(async file => {
+        fileMap.set(file, await readFile(join(templateDir, file), 'utf-8'));
       })
     ).catch(e => {
       throw e;
@@ -110,20 +202,31 @@ export class Generator {
 
   async createFile(relativePath: string, content: string, override = false) {
     const absolutePath = join(this.projectDir, relativePath);
-    if (override || !existsSync(absolutePath)) {
-      progress(`Writing ${relative(process.cwd(), absolutePath)}.`);
-      await mkdir(dirname(absolutePath), { recursive: true });
-      await writeFile(absolutePath, content, 'utf-8');
+
+    if (
+      !override &&
+      !DEFAULT_OVERRIDES.includes(relativePath) &&
+      existsSync(absolutePath)
+    ) {
+      const { override } = await prompt<{ override: boolean }>({
+        type: 'confirm',
+        name: 'override',
+        message: `File ${relativePath} already exists. Override it?`,
+        initial: false,
+      });
+      if (!override) return;
     }
+
+    progress(`Writing ${relative(process.cwd(), absolutePath)}.`);
+    await mkdir(dirname(absolutePath), { recursive: true });
+    await writeFile(absolutePath, content, 'utf-8');
   }
 
   async package() {
     this.pkgManager = await getPackageManager(this.projectDir);
     const commands = new Map<string, string>();
     commands.set(
-      `Initializing Synthetics project using ${
-        this.pkgManager == 'yarn' ? 'Yarn' : 'NPM'
-      }`,
+      `Setting up project using ${this.pkgManager == 'yarn' ? 'Yarn' : 'NPM'}`,
       this.pkgManager == 'yarn' ? 'yarn init -y' : 'npm init -y'
     );
 
@@ -131,8 +234,8 @@ export class Generator {
     commands.set(
       `Installing @elastic/synthetics library`,
       this.pkgManager == 'yarn'
-        ? `yarn add -dev ${pkgName}`
-        : `npm i -d ${pkgName}`
+        ? `yarn add -dev ${pkgName} --silent`
+        : `npm i -d ${pkgName} --quiet`
     );
 
     // Execute commands
@@ -158,8 +261,7 @@ export class Generator {
     pkgJSON.scripts.test = 'npx @elastic/synthetics journeys';
 
     // Add push command
-    const project = basename(this.projectDir);
-    pkgJSON.scripts.push = `npx @elastic/synthetics push journeys --project ${project} --url http://localhost:5601 --auth apiKey`;
+    pkgJSON.scripts.push = `npx @elastic/synthetics push --auth $SYNTHETICS_API_KEY`;
 
     await this.createFile(
       filename,
@@ -189,10 +291,12 @@ All set, you can run below commands inside: ${this.projectDir}:
 
   Run synthetic tests: ${cyan(runCommand(this.pkgManager, 'test'))}
 
-  Push monitors to Kibana: ${cyan(runCommand(this.pkgManager, 'push'))}
+  Push monitors to Kibana: ${cyan(
+    'SYNTHETICS_API_KEY=<value> ' + runCommand(this.pkgManager, 'push')
+  )}
 
   ${yellow(
-    'Make sure to update the Kibana url and api keys before pushing monitors to Kibana.'
+    'Make sure to configure the SYNTHETICS_API_KEY before pushing monitors to Kibana.'
   )}
 
 Visit https://www.elastic.co/guide/en/observability/current/synthetic-run-tests.html to learn more.

package/src/generator/utils.ts

@@ -40,10 +40,15 @@ export function runCommand(pkgManager: string, command: string) {
   return `npm run ${command}`;
 }
 
-export function replaceTemplates(input: string, literals: Record<string, any>) {
-  for (const key in literals) {
-    const finalValue = literals[key];
+export function replaceTemplates(input: string, values: Record<string, any>) {
+  for (const key in values) {
+    const finalValue = values[key];
     input = input.replace(new RegExp(`'{{` + key + `}}'`, 'g'), () => {
+      if (Array.isArray(finalValue)) {
+        return String(finalValue)
+          .split(',')
+          .map(f => `'${f}'`);
+      }
       if (typeof finalValue == 'number') {
         return Number(finalValue);
       }
@@ -52,3 +57,24 @@ export function replaceTemplates(input: string, literals: Record<string, any>) {
   }
   return input;
 }
+
+export function cloudIDToKibanaURL(id: string) {
+  const ID_SEPARATOR = ':';
+  const VALUE_SEPARATOR = '$';
+  // Ignore values before the separator
+  const index = id.lastIndexOf(ID_SEPARATOR);
+  if (index >= 0) {
+    id = id.substring(index + 1);
+  }
+
+  // Decode the base64 string
+  const decoded = Buffer.from(id, 'base64').toString();
+  const words = decoded.split(VALUE_SEPARATOR);
+
+  // construct the kibana and ES urls from decoded value
+  const [host, port = '443'] = words[0].split(ID_SEPARATOR);
+  const [kibanaId, kibanaPort = port] = words[2].split(ID_SEPARATOR);
+  const kibanaURL = `https://${kibanaId}.${host}:${kibanaPort}`;
+
+  return kibanaURL;
+}