@elastic/synthetics 1.0.0-beta.30 → 1.0.0-beta.33

package/src/generator/index.ts

@@ -27,34 +27,56 @@ import { existsSync } from 'fs';
 import { mkdir, readFile, writeFile } from 'fs/promises';
 import { bold, cyan, yellow } from 'kleur/colors';
 import { join, relative, dirname, basename } from 'path';
-import { prompt } from 'enquirer';
-import { SyntheticsLocations } from '../dsl/monitor';
-import { progress, write as stdWrite } from '../helpers';
-import { getPackageManager, replaceTemplates, runCommand } from './utils';
+// @ts-ignore-next-line: has no exported member 'Input'
+import { prompt, Input } from 'enquirer';
+import { progress, removeTrailingSlash, write as stdWrite } from '../helpers';
+import {
+  getPackageManager,
+  replaceTemplates,
+  runCommand,
+  cloudIDToKibanaURL,
+} from './utils';
+import { formatLocations, getLocations, groupLocations } from '../locations';
 
-// Templates that are required for setting up new
-// synthetics project
+// Templates that are required for setting up new synthetics project
 const templateDir = join(__dirname, '..', '..', 'templates');
 
-type PromptOptions = {
-  locations: string;
+export type ProjectSettings = {
+  url: string;
+  space: string;
+  project: string;
+};
+
+type PromptOptions = ProjectSettings & {
+  locations: Array<string>;
+  privateLocations: Array<string>;
   schedule: number;
 };
 
 // exported for testing
-export const regularFiles = [
-  '.github/workflows/run-synthetics.yml',
-  'README.md',
+export const REGULAR_FILES_PATH = [
   'journeys/example.journey.ts',
-  'journeys/advanced-example.journey.ts',
   'journeys/advanced-example-helpers.ts',
+  'journeys/advanced-example.journey.ts',
+  '.github/workflows/run-synthetics.yml',
+  'README.md',
 ];
+export const SETTINGS_PATH = '.synthetics/project.json';
+export const CONFIG_PATH = 'synthetics.config.ts';
+
+// Files to be overriden by default if the project is initialized multiple times
+const DEFAULT_OVERRIDES = [SETTINGS_PATH, CONFIG_PATH];
 
 export class Generator {
   pkgManager = 'npm';
   constructor(public projectDir: string) {}
 
   async directory() {
+    progress(
+      `Initializing Synthetics project in '${
+        relative(process.cwd(), this.projectDir) || '.'
+      }'`
+    );
     if (!existsSync(this.projectDir)) {
       await mkdir(this.projectDir);
     }
@@ -64,12 +86,58 @@ export class Generator {
     if (process.env.TEST_QUESTIONS) {
       return JSON.parse(process.env.TEST_QUESTIONS);
     }
-    const question = [
+
+    const { onCloud } = await prompt<{ onCloud: string }>({
+      type: 'confirm',
+      name: 'onCloud',
+      initial: 'y',
+      message: 'Do you use Elastic Cloud',
+    });
+    const url = await new Input({
+      header: onCloud
+        ? yellow(
+            'Get cloud.id from your deployment https://www.elastic.co/guide/en/cloud/current/ec-cloud-id.html'
+          )
+        : '',
+      message: onCloud
+        ? 'What is your cloud.id'
+        : 'What is the url of your Kibana instance',
+      name: 'url',
+      required: true,
+      result(value) {
+        return onCloud ? cloudIDToKibanaURL(value) : value;
+      },
+    }).run();
+
+    const auth = await new Input({
+      name: 'auth',
+      header: yellow(
+        `Generate API key from Kibana ${removeTrailingSlash(
+          url
+        )}/app/uptime/manage-monitors/all`
+      ),
+      required: true,
+      message: 'What is your API key',
+    }).run();
+
+    const allLocations = await getLocations({ url, auth });
+    const locChoices = formatLocations(allLocations);
+    if (locChoices.length === 0) {
+      // TODO - Add link to docs
+      throw 'Follow the docs to set up your first private locations - <link>';
+    }
+
+    const monitorQues = [
       {
         type: 'select',
         name: 'locations',
-        message: 'Select the default location where you want to run monitors.',
-        choices: SyntheticsLocations,
+        hint: '(Use <space> to select, <return> to submit)',
+        message: 'Select the locations where you want to run monitors',
+        choices: locChoices,
+        multiple: true,
+        validate(value) {
+          return value.length === 0 ? `Select at least one option.` : true;
+        },
       },
       {
         type: 'numeral',
@@ -77,26 +145,50 @@ export class Generator {
         message: 'Set default schedule in minutes for all monitors',
         initial: 10,
       },
+      {
+        type: 'input',
+        name: 'project',
+        message: 'Choose project id to logically group monitors',
+        initial: basename(this.projectDir),
+      },
+      {
+        type: 'input',
+        name: 'space',
+        message: 'Choose the target Kibana space',
+        initial: 'default',
+      },
     ];
-    return await prompt<PromptOptions>(question);
+
+    // Split and group private and public locations from the answered list.
+    const answers = await prompt<PromptOptions>(monitorQues);
+    const { locations, privateLocations } = groupLocations(answers.locations);
+    return { ...answers, url, locations, privateLocations };
   }
 
   async files(answers: PromptOptions) {
     const fileMap = new Map<string, string>();
+
+    // Setup project.json
+    const projectJSON = {
+      url: answers.url,
+      space: answers.space,
+      project: answers.project,
+    };
+    fileMap.set(SETTINGS_PATH, JSON.stringify(projectJSON, null, 2) + '\n');
+
     // Setup Synthetics config file
-    const configFile = 'synthetics.config.ts';
     fileMap.set(
-      configFile,
+      CONFIG_PATH,
       replaceTemplates(
-        await readFile(join(templateDir, configFile), 'utf-8'),
+        await readFile(join(templateDir, CONFIG_PATH), 'utf-8'),
         answers
       )
     );
 
     // Setup non-templated files
     Promise.all(
-      regularFiles.map(async fn => {
-        fileMap.set(fn, await readFile(join(templateDir, fn), 'utf-8'));
+      REGULAR_FILES_PATH.map(async file => {
+        fileMap.set(file, await readFile(join(templateDir, file), 'utf-8'));
       })
     ).catch(e => {
       throw e;
@@ -110,20 +202,31 @@ export class Generator {
 
   async createFile(relativePath: string, content: string, override = false) {
     const absolutePath = join(this.projectDir, relativePath);
-    if (override || !existsSync(absolutePath)) {
-      progress(`Writing ${relative(process.cwd(), absolutePath)}.`);
-      await mkdir(dirname(absolutePath), { recursive: true });
-      await writeFile(absolutePath, content, 'utf-8');
+
+    if (
+      !override &&
+      !DEFAULT_OVERRIDES.includes(relativePath) &&
+      existsSync(absolutePath)
+    ) {
+      const { override } = await prompt<{ override: boolean }>({
+        type: 'confirm',
+        name: 'override',
+        message: `File ${relativePath} already exists. Override it?`,
+        initial: false,
+      });
+      if (!override) return;
     }
+
+    progress(`Writing ${relative(process.cwd(), absolutePath)}.`);
+    await mkdir(dirname(absolutePath), { recursive: true });
+    await writeFile(absolutePath, content, 'utf-8');
   }
 
   async package() {
     this.pkgManager = await getPackageManager(this.projectDir);
     const commands = new Map<string, string>();
     commands.set(
-      `Initializing Synthetics project using ${
-        this.pkgManager == 'yarn' ? 'Yarn' : 'NPM'
-      }`,
+      `Setting up project using ${this.pkgManager == 'yarn' ? 'Yarn' : 'NPM'}`,
       this.pkgManager == 'yarn' ? 'yarn init -y' : 'npm init -y'
     );
 
@@ -131,8 +234,8 @@ export class Generator {
     commands.set(
       `Installing @elastic/synthetics library`,
       this.pkgManager == 'yarn'
-        ? `yarn add -dev ${pkgName}`
-        : `npm i -d ${pkgName}`
+        ? `yarn add -dev ${pkgName} --silent`
+        : `npm i -d ${pkgName} --quiet`
     );
 
     // Execute commands
@@ -158,8 +261,7 @@ export class Generator {
     pkgJSON.scripts.test = 'npx @elastic/synthetics journeys';
 
     // Add push command
-    const project = basename(this.projectDir);
-    pkgJSON.scripts.push = `npx @elastic/synthetics push journeys --project ${project} --url http://localhost:5601 --auth apiKey`;
+    pkgJSON.scripts.push = `npx @elastic/synthetics push --auth $SYNTHETICS_API_KEY`;
 
     await this.createFile(
       filename,
@@ -189,10 +291,12 @@ All set, you can run below commands inside: ${this.projectDir}:
 
   Run synthetic tests: ${cyan(runCommand(this.pkgManager, 'test'))}
 
-  Push monitors to Kibana: ${cyan(runCommand(this.pkgManager, 'push'))}
+  Push monitors to Kibana: ${cyan(
+    'SYNTHETICS_API_KEY=<value> ' + runCommand(this.pkgManager, 'push')
+  )}
 
   ${yellow(
-    'Make sure to update the Kibana url and api keys before pushing monitors to Kibana.'
+    'Make sure to configure the SYNTHETICS_API_KEY before pushing monitors to Kibana.'
   )}
 
 Visit https://www.elastic.co/guide/en/observability/current/synthetic-run-tests.html to learn more.

package/src/generator/utils.ts

@@ -40,10 +40,15 @@ export function runCommand(pkgManager: string, command: string) {
   return `npm run ${command}`;
 }
 
-export function replaceTemplates(input: string, literals: Record<string, any>) {
-  for (const key in literals) {
-    const finalValue = literals[key];
+export function replaceTemplates(input: string, values: Record<string, any>) {
+  for (const key in values) {
+    const finalValue = values[key];
     input = input.replace(new RegExp(`'{{` + key + `}}'`, 'g'), () => {
+      if (Array.isArray(finalValue)) {
+        return String(finalValue)
+          .split(',')
+          .map(f => `'${f}'`);
+      }
       if (typeof finalValue == 'number') {
         return Number(finalValue);
       }
@@ -52,3 +57,24 @@ export function replaceTemplates(input: string, literals: Record<string, any>) {
   }
   return input;
 }
+
+export function cloudIDToKibanaURL(id: string) {
+  const ID_SEPARATOR = ':';
+  const VALUE_SEPARATOR = '$';
+  // Ignore values before the separator
+  const index = id.lastIndexOf(ID_SEPARATOR);
+  if (index >= 0) {
+    id = id.substring(index + 1);
+  }
+
+  // Decode the base64 string
+  const decoded = Buffer.from(id, 'base64').toString();
+  const words = decoded.split(VALUE_SEPARATOR);
+
+  // construct the kibana and ES urls from decoded value
+  const [host, port = '443'] = words[0].split(ID_SEPARATOR);
+  const [kibanaId, kibanaPort = port] = words[2].split(ID_SEPARATOR);
+  const kibanaURL = `https://${kibanaId}.${host}:${kibanaPort}`;
+
+  return kibanaURL;
+}

package/src/helpers.ts

@@ -23,11 +23,11 @@
  *
  */
 
-import { red, green, yellow, cyan, bold } from 'kleur/colors';
+import { red, green, yellow, cyan, bold, grey } from 'kleur/colors';
 import os from 'os';
 import { resolve, join, dirname } from 'path';
 import fs from 'fs';
-import { promisify } from 'util';
+import { lstat, readdir } from 'fs/promises';
 import { performance } from 'perf_hooks';
 import sourceMapSupport from '@cspotcode/source-map-support';
 import {
@@ -38,13 +38,6 @@ import {
   ThrottlingOptions,
 } from './common_types';
 
-const lstatAsync = promisify(fs.lstat);
-const readdirAsync = promisify(fs.readdir);
-
-export const readFileAsync = promisify(fs.readFile);
-export const writeFileAsync = promisify(fs.writeFile);
-export const mkdirAsync = promisify(fs.mkdir);
-
 const SEPARATOR = '\n';
 
 export function noop() {}
@@ -177,11 +170,11 @@ export async function totalist(
   pre = ''
 ) {
   dir = resolve('.', dir);
-  await readdirAsync(dir).then(arr => {
+  await readdir(dir).then(arr => {
     return Promise.all(
       arr.map(str => {
         const abs = join(dir, str);
-        return lstatAsync(abs).then(stats =>
+        return lstat(abs).then(stats =>
           stats.isDirectory()
             ? totalist(abs, callback, join(pre, str))
             : callback(join(pre, str), abs)
@@ -254,12 +247,18 @@ export function formatError(error: Error) {
 }
 
 const cwd = process.cwd();
+/**
+ * All the settings that are related to the Synthetics is stored
+ * under this directory
+ * Examples: Screenshots, Project setup
+ */
+export const SYNTHETICS_PATH = join(cwd, '.synthetics');
 /**
  * Synthetics cache path that is based on the process id to make sure
  * each process does not modify the caching layer used by other process
  * once we move to executing journeys in parallel
  */
-export const CACHE_PATH = join(cwd, '.synthetics', process.pid.toString());
+export const CACHE_PATH = join(SYNTHETICS_PATH, process.pid.toString());
 
 export function getDurationInUs(duration: number) {
   return Math.trunc(duration * 1e6);
@@ -322,6 +321,23 @@ export function wrapFnWithLocation<A extends unknown[], R>(
   };
 }
 
+// Safely parse ND JSON (Newline delimitted JSON) chunks
+export function safeNDJSONParse(chunks: string[]) {
+  // chunks may not be at proper newline boundaries, so we make sure everything is split
+  // on proper newlines
+  chunks = Array.isArray(chunks) ? chunks : [chunks];
+  const lines = chunks.join('\n').split(/\r?\n/);
+  return lines
+    .filter(l => l.match(/\S/)) // remove blank lines
+    .map(line => {
+      try {
+        return JSON.parse(line);
+      } catch (e) {
+        throw `Error ${e} could not parse data '${line}'`;
+      }
+    });
+}
+
 // Console helpers
 export function write(message: string) {
   process.stderr.write(message + '\n');
@@ -331,6 +347,10 @@ export function progress(message: string) {
   write(cyan(bold(`${symbols.progress} ${message}`)));
 }
 
+export function apiProgress(message: string) {
+  write(grey(`> ${message}`));
+}
+
 export function error(message: string) {
   write(red(message));
 }
@@ -338,3 +358,11 @@ export function error(message: string) {
 export function done(message: string) {
   write(bold(green(`${symbols['succeeded']} ${message}`)));
 }
+
+export function aborted(message: string) {
+  write(bold(yellow(`${symbols['warning']} ${message}`)));
+}
+
+export function removeTrailingSlash(url: string) {
+  return url.replace(/\/+$/, '');
+}

package/src/index.ts

@@ -52,6 +52,7 @@ export type {
   ChromiumBrowser,
   ChromiumBrowserContext,
   CDPSession,
+  APIRequestContext,
 } from 'playwright-chromium';
 
 /**

package/src/loader.ts

@@ -93,10 +93,19 @@ const loadInlineScript = source => {
     'browser',
     'params',
     'expect',
+    'request',
     source
   );
-  journey('inline', ({ page, context, browser, params }) => {
-    scriptFn.apply(null, [step, page, context, browser, params, expect]);
+  journey('inline', ({ page, context, browser, params, request }) => {
+    scriptFn.apply(null, [
+      step,
+      page,
+      context,
+      browser,
+      params,
+      expect,
+      request,
+    ]);
   });
 };
 

package/src/locations/index.ts

@@ -0,0 +1,108 @@
+/**
+ * MIT License
+ *
+ * Copyright (c) 2020-present, Elastic NV
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ *
+ */
+
+import {
+  formatAPIError,
+  formatNotFoundError,
+  sendRequest,
+  ok,
+} from '../push/request';
+import { indent, removeTrailingSlash, write } from '../helpers';
+
+export type LocationCmdOptions = {
+  auth: string;
+  url: string;
+};
+
+type LocationMetadata = {
+  id: string;
+  label: string;
+  isServiceManaged: boolean;
+};
+
+export type LocationAPIResponse = {
+  locations: Array<LocationMetadata>;
+};
+
+const PRIVATE_KEYWORD = '(private)';
+
+export async function getLocations(options: LocationCmdOptions) {
+  const url =
+    removeTrailingSlash(options.url) + '/internal/uptime/service/locations';
+  const { body, statusCode } = await sendRequest({
+    url,
+    method: 'GET',
+    auth: options.auth,
+  });
+  if (statusCode === 404) {
+    throw formatNotFoundError(await body.text());
+  }
+  if (!ok(statusCode)) {
+    const { error, message } = await body.json();
+    throw formatAPIError(statusCode, error, message);
+  }
+  return ((await body.json()) as LocationAPIResponse).locations;
+}
+
+export function formatLocations(locations: Array<LocationMetadata>) {
+  const formatted: Array<string> = [];
+  for (const location of locations) {
+    let name = location.label;
+    if (location.isServiceManaged) {
+      [, name] = name.includes('-') ? name.split('-') : [, name];
+      name = name.toLowerCase().trim().split(' ').join('_');
+    } else {
+      name = `${name}${PRIVATE_KEYWORD}`;
+    }
+    formatted.push(name);
+  }
+  return formatted;
+}
+
+export function groupLocations(locations: Array<string>) {
+  const grouped = { locations: [], privateLocations: [] };
+  for (const loc of locations) {
+    if (loc.includes(PRIVATE_KEYWORD)) {
+      grouped.privateLocations.push(loc.replace(PRIVATE_KEYWORD, ''));
+    } else {
+      grouped.locations.push(loc);
+    }
+  }
+  return grouped;
+}
+
+export function renderLocations(locations: Array<string>) {
+  let outer = 'Available locations: \n';
+  let inner = '';
+  for (const location of locations) {
+    inner += `* ${location}\n`;
+  }
+  outer += indent(inner);
+  outer += `\nSet default location for monitors via
+  - Synthetics config file 'monitors.locations' | 'monitors.privateLocations' field
+  - Monitor API 'monitor.use({ locations: ["japan"], privateLocations: ["custom-location"] }')`;
+
+  write(outer);
+}

package/src/options.ts

@@ -24,6 +24,7 @@
  */
 
 import merge from 'deepmerge';
+import { createOption } from 'commander';
 import { readConfig } from './config';
 import { getNetworkConditions, DEFAULT_THROTTLING_OPTIONS } from './helpers';
 import type { CliArgs, RunOptions, ThrottlingOptions } from './common_types';
@@ -136,6 +137,8 @@ export function normalizeOptions(cliArgs: CliArgs): RunOptions {
 
   options.schedule = cliArgs.schedule ?? monitor?.schedule;
   options.locations = cliArgs.locations ?? monitor?.locations;
+  options.privateLocations =
+    cliArgs.privateLocations ?? monitor?.privateLocations;
 
   return options;
 }
@@ -171,3 +174,38 @@ export function parseThrottling(value: string, prev?: string) {
   }
   return JSON.parse(value || prev);
 }
+
+export function getCommonCommandOpts() {
+  const params = createOption(
+    '-p, --params <jsonstring>',
+    'JSON object that gets injected to all journeys'
+  );
+  params.argParser(JSON.parse);
+
+  const playwrightOpts = createOption(
+    '--playwright-options <jsonstring>',
+    'JSON object to pass in custom Playwright options for the agent. Options passed will be merged with Playwright options defined in your synthetics.config.js file.'
+  );
+  playwrightOpts.argParser(JSON.parse);
+
+  const pattern = createOption(
+    '--pattern <pattern>',
+    'RegExp file patterns to search inside directory'
+  );
+  const tags = createOption(
+    '--tags <name...>',
+    'run only journeys with a tag that matches the glob'
+  );
+  const match = createOption(
+    '--match <name>',
+    'run only journeys with a name or tag that matches the glob'
+  );
+
+  return {
+    params,
+    playwrightOpts,
+    pattern,
+    tags,
+    match,
+  };
+}

package/src/push/bundler.ts

@@ -65,12 +65,17 @@ export class Bundler {
       const archive = archiver('zip', {
         zlib: { level: 9 },
       });
-      archive.on('error', err => reject(err));
+      archive.on('error', reject);
       output.on('close', fulfill);
       archive.pipe(output);
       for (const [path, content] of this.moduleMap.entries()) {
         const relativePath = relativeToCwd(path);
-        archive.append(content, { name: relativePath });
+        // Date is fixed to Unix epoch so the file metadata is
+        // not modified everytime when files are bundled
+        archive.append(content, {
+          name: relativePath,
+          date: new Date('1970-01-01'),
+        });
       }
       archive.finalize();
     });