@elastic/synthetics 1.0.0-beta.31 → 1.0.0-beta.32

package/src/helpers.ts

@@ -27,7 +27,7 @@ import { red, green, yellow, cyan, bold } from 'kleur/colors';
 import os from 'os';
 import { resolve, join, dirname } from 'path';
 import fs from 'fs';
-import { promisify } from 'util';
+import { lstat, readdir } from 'fs/promises';
 import { performance } from 'perf_hooks';
 import sourceMapSupport from '@cspotcode/source-map-support';
 import {
@@ -38,13 +38,6 @@ import {
   ThrottlingOptions,
 } from './common_types';
 
-const lstatAsync = promisify(fs.lstat);
-const readdirAsync = promisify(fs.readdir);
-
-export const readFileAsync = promisify(fs.readFile);
-export const writeFileAsync = promisify(fs.writeFile);
-export const mkdirAsync = promisify(fs.mkdir);
-
 const SEPARATOR = '\n';
 
 export function noop() {}
@@ -177,11 +170,11 @@ export async function totalist(
   pre = ''
 ) {
   dir = resolve('.', dir);
-  await readdirAsync(dir).then(arr => {
+  await readdir(dir).then(arr => {
     return Promise.all(
       arr.map(str => {
         const abs = join(dir, str);
-        return lstatAsync(abs).then(stats =>
+        return lstat(abs).then(stats =>
           stats.isDirectory()
             ? totalist(abs, callback, join(pre, str))
             : callback(join(pre, str), abs)
@@ -254,12 +247,18 @@ export function formatError(error: Error) {
 }
 
 const cwd = process.cwd();
+/**
+ * All the settings that are related to the Synthetics is stored
+ * under this directory
+ * Examples: Screenshots, Project setup
+ */
+export const SYNTHETICS_PATH = join(cwd, '.synthetics');
 /**
  * Synthetics cache path that is based on the process id to make sure
  * each process does not modify the caching layer used by other process
  * once we move to executing journeys in parallel
  */
-export const CACHE_PATH = join(cwd, '.synthetics', process.pid.toString());
+export const CACHE_PATH = join(SYNTHETICS_PATH, process.pid.toString());
 
 export function getDurationInUs(duration: number) {
   return Math.trunc(duration * 1e6);
@@ -322,6 +321,23 @@ export function wrapFnWithLocation<A extends unknown[], R>(
   };
 }
 
+// Safely parse ND JSON (Newline delimitted JSON) chunks
+export function safeNDJSONParse(chunks: string[]) {
+  // chunks may not be at proper newline boundaries, so we make sure everything is split
+  // on proper newlines
+  chunks = Array.isArray(chunks) ? chunks : [chunks];
+  const lines = chunks.join('\n').split(/\r?\n/);
+  return lines
+    .filter(l => l.match(/\S/)) // remove blank lines
+    .map(line => {
+      try {
+        return JSON.parse(line);
+      } catch (e) {
+        throw `Error ${e} could not parse data '${line}'`;
+      }
+    });
+}
+
 // Console helpers
 export function write(message: string) {
   process.stderr.write(message + '\n');
@@ -338,3 +354,11 @@ export function error(message: string) {
 export function done(message: string) {
   write(bold(green(`${symbols['succeeded']} ${message}`)));
 }
+
+export function aborted(message: string) {
+  write(bold(yellow(`${symbols['warning']} ${message}`)));
+}
+
+export function removeTrailingSlash(url: string) {
+  return url.replace(/\/+$/, '');
+}

package/src/locations/index.ts

@@ -0,0 +1,108 @@
+/**
+ * MIT License
+ *
+ * Copyright (c) 2020-present, Elastic NV
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ *
+ */
+
+import {
+  formatAPIError,
+  formatNotFoundError,
+  sendRequest,
+  ok,
+} from '../push/request';
+import { indent, removeTrailingSlash, write } from '../helpers';
+
+export type LocationCmdOptions = {
+  auth: string;
+  url: string;
+};
+
+type LocationMetadata = {
+  id: string;
+  label: string;
+  isServiceManaged: boolean;
+};
+
+export type LocationAPIResponse = {
+  locations: Array<LocationMetadata>;
+};
+
+const PRIVATE_KEYWORD = '(private)';
+
+export async function getLocations(options: LocationCmdOptions) {
+  const url =
+    removeTrailingSlash(options.url) + '/internal/uptime/service/locations';
+  const { body, statusCode } = await sendRequest({
+    url,
+    method: 'GET',
+    auth: options.auth,
+  });
+  if (statusCode === 404) {
+    throw formatNotFoundError(await body.text());
+  }
+  if (!ok(statusCode)) {
+    const { error, message } = await body.json();
+    throw formatAPIError(statusCode, error, message);
+  }
+  return ((await body.json()) as LocationAPIResponse).locations;
+}
+
+export function formatLocations(locations: Array<LocationMetadata>) {
+  const formatted: Array<string> = [];
+  for (const location of locations) {
+    let name = location.label;
+    if (location.isServiceManaged) {
+      [, name] = name.includes('-') ? name.split('-') : [, name];
+      name = name.toLowerCase().trim().split(' ').join('_');
+    } else {
+      name = `${name}${PRIVATE_KEYWORD}`;
+    }
+    formatted.push(name);
+  }
+  return formatted;
+}
+
+export function groupLocations(locations: Array<string>) {
+  const grouped = { locations: [], privateLocations: [] };
+  for (const loc of locations) {
+    if (loc.includes(PRIVATE_KEYWORD)) {
+      grouped.privateLocations.push(loc.replace(PRIVATE_KEYWORD, ''));
+    } else {
+      grouped.locations.push(loc);
+    }
+  }
+  return grouped;
+}
+
+export function renderLocations(locations: Array<string>) {
+  let outer = 'Available locations: \n';
+  let inner = '';
+  for (const location of locations) {
+    inner += `* ${location}\n`;
+  }
+  outer += indent(inner);
+  outer += `\nSet default location for monitors via
+  - Synthetics config file 'monitors.locations' | 'monitors.privateLocations' field
+  - Monitor API 'monitor.use({ locations: ["japan"], privateLocations: ["custom-location"] }')`;
+
+  write(outer);
+}

package/src/options.ts

@@ -24,6 +24,7 @@
  */
 
 import merge from 'deepmerge';
+import { createOption } from 'commander';
 import { readConfig } from './config';
 import { getNetworkConditions, DEFAULT_THROTTLING_OPTIONS } from './helpers';
 import type { CliArgs, RunOptions, ThrottlingOptions } from './common_types';
@@ -136,6 +137,8 @@ export function normalizeOptions(cliArgs: CliArgs): RunOptions {
 
   options.schedule = cliArgs.schedule ?? monitor?.schedule;
   options.locations = cliArgs.locations ?? monitor?.locations;
+  options.privateLocations =
+    cliArgs.privateLocations ?? monitor?.privateLocations;
 
   return options;
 }
@@ -171,3 +174,38 @@ export function parseThrottling(value: string, prev?: string) {
   }
   return JSON.parse(value || prev);
 }
+
+export function getCommonCommandOpts() {
+  const params = createOption(
+    '-p, --params <jsonstring>',
+    'JSON object that gets injected to all journeys'
+  );
+  params.argParser(JSON.parse);
+
+  const playwrightOpts = createOption(
+    '--playwright-options <jsonstring>',
+    'JSON object to pass in custom Playwright options for the agent. Options passed will be merged with Playwright options defined in your synthetics.config.js file.'
+  );
+  playwrightOpts.argParser(JSON.parse);
+
+  const pattern = createOption(
+    '--pattern <pattern>',
+    'RegExp file patterns to search inside directory'
+  );
+  const tags = createOption(
+    '--tags <name...>',
+    'run only journeys with a tag that matches the glob'
+  );
+  const match = createOption(
+    '--match <name>',
+    'run only journeys with a name or tag that matches the glob'
+  );
+
+  return {
+    params,
+    playwrightOpts,
+    pattern,
+    tags,
+    match,
+  };
+}

package/src/push/bundler.ts

@@ -65,12 +65,17 @@ export class Bundler {
       const archive = archiver('zip', {
         zlib: { level: 9 },
       });
-      archive.on('error', err => reject(err));
+      archive.on('error', reject);
       output.on('close', fulfill);
       archive.pipe(output);
       for (const [path, content] of this.moduleMap.entries()) {
         const relativePath = relativeToCwd(path);
-        archive.append(content, { name: relativePath });
+        // Date is fixed to Unix epoch so the file metadata is
+        // not modified everytime when files are bundled
+        archive.append(content, {
+          name: relativePath,
+          date: new Date('1970-01-01'),
+        });
       }
       archive.finalize();
     });

package/src/push/index.ts

@@ -23,19 +23,43 @@
  *
  */
 
+import { join } from 'path';
+import { existsSync } from 'fs';
+import { readFile, writeFile } from 'fs/promises';
+import { prompt } from 'enquirer';
+import { bold, yellow } from 'kleur/colors';
 import {
-  createMonitors,
   ok,
   formatAPIError,
   formatFailedMonitors,
   formatNotFoundError,
+  formatStaleMonitors,
 } from './request';
+import { buildMonitorSchema, createMonitors } from './monitor';
+import { ProjectSettings } from '../generator';
 import { Monitor } from '../dsl/monitor';
-import { PushOptions } from '../common_types';
-import { buildMonitorSchema } from './monitor';
-import { progress, error, done } from '../helpers';
+import {
+  progress,
+  write,
+  error,
+  SYNTHETICS_PATH,
+  aborted,
+  indent,
+  safeNDJSONParse,
+  done,
+} from '../helpers';
+import type { PushOptions } from '../common_types';
 
 export async function push(monitors: Monitor[], options: PushOptions) {
+  if (monitors.length === 0) {
+    throw 'No Monitors found';
+  }
+
+  const duplicates = trackDuplicates(monitors);
+  if (duplicates.size > 0) {
+    throw error(formatDuplicateError(duplicates));
+  }
+
   progress(`preparing all monitors`);
   const schemas = await buildMonitorSchema(monitors);
   try {
@@ -48,12 +72,131 @@ export async function push(monitors: Monitor[], options: PushOptions) {
       const { error, message } = await body.json();
       throw formatAPIError(statusCode, error, message);
     }
-    const { failedMonitors } = await body.json();
-    if (failedMonitors.length > 0) {
-      throw formatFailedMonitors(failedMonitors);
+    body.setEncoding('utf-8');
+    for await (const data of body) {
+      // Its kind of hacky for now where Kibana streams the response by
+      // writing the data as NDJSON events (data can be interleaved), we
+      // distinguish the final data by checking if the event was a progress vs complete event
+      const chunks = safeNDJSONParse(data);
+      for (const chunk of chunks) {
+        if (typeof chunk === 'string') {
+          progress(chunk);
+          continue;
+        }
+        const { failedMonitors, failedStaleMonitors } = chunk;
+        if (failedMonitors && failedMonitors.length > 0) {
+          throw formatFailedMonitors(failedMonitors);
+        }
+        if (failedStaleMonitors.length > 0) {
+          write(yellow(formatStaleMonitors(failedStaleMonitors)));
+        }
+      }
     }
     done('Pushed');
   } catch (e) {
     error(e);
   }
 }
+
+function trackDuplicates(monitors: Monitor[]) {
+  const monitorMap: Map<string, Monitor> = new Map();
+  const duplicates: Set<Monitor> = new Set();
+  for (const monitor of monitors) {
+    const id = monitor.config.id;
+    if (monitorMap.has(id)) {
+      duplicates.add(monitorMap.get(id));
+      duplicates.add(monitor);
+    }
+    monitorMap.set(id, monitor);
+  }
+  return duplicates;
+}
+
+export function formatDuplicateError(monitors: Set<Monitor>) {
+  let outer = bold(`Aborted: Duplicate monitors found\n`);
+
+  let inner = '';
+  for (const monitor of monitors) {
+    const { config, source } = monitor;
+    inner += `* ${config.id} - ${source.file}:${source.line}:${source.column}\n`;
+  }
+  outer += indent(inner);
+  return outer;
+}
+
+const PROJECT_SETTINGS_PATH = join(SYNTHETICS_PATH, 'project.json');
+const INSTALLATION_HELP = `Run 'npx @elastic/synthetics init' to create project with default settings.`;
+
+export async function loadSettings() {
+  if (!existsSync(PROJECT_SETTINGS_PATH)) {
+    throw error(`Aborted. Synthetics project not set up.
+
+${INSTALLATION_HELP}`);
+  }
+
+  const contents = await readFile(PROJECT_SETTINGS_PATH, 'utf-8');
+  return JSON.parse(contents) as ProjectSettings;
+}
+
+export function validateSettings(opts: PushOptions) {
+  const INVALID = 'Aborted. Invalid synthetics project settings.';
+
+  let reason = '';
+  if (!opts.project) {
+    reason = 'Set project id via CLI as `--project <id>`';
+  } else if (!opts.locations && !opts.privateLocations) {
+    reason = `Set default location for all monitors via
+  - CLI as '--locations <values...> or --privateLocations <values...>'
+  - Synthetics config file 'monitors.locations' | 'monitors.privateLocations' field`;
+  } else if (!opts.schedule) {
+    reason = `Set default schedule in minutes for all monitors via
+  - CLI as '--schedule <mins>'
+  - Synthetics config file 'monitors.schedule' field`;
+  }
+
+  if (!reason) return;
+
+  throw error(`${INVALID}
+
+${reason}
+
+${INSTALLATION_HELP}`);
+}
+
+async function overrideSettings(project: string) {
+  const settings = JSON.parse(await readFile(PROJECT_SETTINGS_PATH, 'utf-8'));
+  settings.project = project;
+
+  await writeFile(
+    PROJECT_SETTINGS_PATH,
+    JSON.stringify(settings, null, 2),
+    'utf-8'
+  );
+}
+
+export async function catchIncorrectSettings(
+  settings: ProjectSettings,
+  options: PushOptions
+) {
+  let override = !settings.project;
+  if (settings.project && settings.project !== options.project) {
+    if (typeof process.env.TEST_OVERRIDE != null) {
+      override = Boolean(process.env.TEST_OVERRIDE);
+    } else {
+      // Add an extra line to make it easier to read the prompt
+      write('');
+      ({ override } = await prompt<{ override: boolean }>({
+        type: 'confirm',
+        name: 'override',
+        message: `Monitors were pushed under the '${settings.project}' project. Are you sure you want to push them under the new '${options.project}' (note that this will duplicate the monitors, the old ones being orphaned)`,
+        initial: false,
+      }));
+    }
+    if (!override) {
+      throw aborted('Push command Aborted');
+    }
+  }
+  if (override) {
+    await overrideSettings(options.project);
+  }
+}

package/src/push/monitor.ts

@@ -23,12 +23,14 @@
  *
  */
 
-import { mkdir } from 'fs/promises';
+import { mkdir, rm } from 'fs/promises';
 import { join } from 'path';
 import { Bundler } from './bundler';
-import { CACHE_PATH } from '../helpers';
-import { LocationsMap } from '../dsl/locations';
+import { sendRequest } from './request';
+import { removeTrailingSlash, SYNTHETICS_PATH } from '../helpers';
+import { LocationsMap } from '../locations/public-locations';
 import { Monitor, MonitorConfig } from '../dsl/monitor';
+import { PushOptions } from '../common_types';
 
 export type MonitorSchema = Omit<MonitorConfig, 'locations'> & {
   content: string;
@@ -36,19 +38,23 @@ export type MonitorSchema = Omit<MonitorConfig, 'locations'> & {
   filter: Monitor['filter'];
 };
 
-function translateLocation(locations: MonitorConfig['locations']) {
-  return locations.map(loc => LocationsMap[loc]).filter(Boolean);
+type APISchema = {
+  project: string;
+  keep_stale: boolean;
+  monitors: MonitorSchema[];
+};
+
+function translateLocation(locations?: MonitorConfig['locations']) {
+  if (!locations) return [];
+  return locations.map(loc => LocationsMap[loc] || loc).filter(Boolean);
 }
 
 export async function buildMonitorSchema(monitors: Monitor[]) {
-  if (monitors.length == 0) {
-    throw new Error('No Monitors found');
-  }
   /**
    * Set up the bundle artifacts path which can be used to
    * create the bundles required for uploading journeys
    */
-  const bundlePath = join(CACHE_PATH, 'bundles');
+  const bundlePath = join(SYNTHETICS_PATH, 'bundles');
   await mkdir(bundlePath, { recursive: true });
   const bundler = new Bundler();
   const schemas: MonitorSchema[] = [];
@@ -64,5 +70,27 @@ export async function buildMonitorSchema(monitors: Monitor[]) {
       filter: filter,
     });
   }
+
+  await rm(bundlePath, { recursive: true });
   return schemas;
 }
+
+export async function createMonitors(
+  monitors: MonitorSchema[],
+  options: PushOptions
+) {
+  const schema: APISchema = {
+    project: options.project,
+    keep_stale: false,
+    monitors,
+  };
+
+  return await sendRequest({
+    url:
+      removeTrailingSlash(options.url) +
+      `/s/${options.space}/api/synthetics/service/project/monitors`,
+    method: 'PUT',
+    auth: options.auth,
+    body: JSON.stringify(schema),
+  });
+}

package/src/push/request.ts

@@ -24,45 +24,23 @@
  */
 
 import { bold } from 'kleur/colors';
-import { request } from 'undici';
-import { PushOptions } from '../common_types';
+import { Dispatcher, request } from 'undici';
 import { indent, symbols } from '../helpers';
-import { MonitorSchema } from './monitor';
 
 /* eslint-disable @typescript-eslint/no-var-requires */
 const { version } = require('../../package.json');
 
-export type APISchema = {
-  project: string;
-  keep_stale: boolean;
-  monitors: MonitorSchema[];
+export type APIRequestOptions = {
+  url: string;
+  method: Dispatcher.HttpMethod;
+  auth: string;
+  body?: string;
 };
 
-function getAPIUrl(options: PushOptions) {
-  // remove trialing / as well
-  return (
-    options.url.replace(/\/+$/, '') +
-    `/s/${options.space}/api/synthetics/service/project/monitors`
-  );
-}
-
-export function ok(statusCode: number) {
-  return statusCode >= 200 && statusCode <= 299;
-}
-
-export async function createMonitors(
-  monitors: MonitorSchema[],
-  options: PushOptions
-) {
-  const schema: APISchema = {
-    project: options.project,
-    keep_stale: false,
-    monitors,
-  };
-
-  return await request(getAPIUrl(options), {
-    method: 'PUT',
-    body: JSON.stringify(schema),
+export async function sendRequest(options: APIRequestOptions) {
+  return await request(options.url, {
+    method: options.method,
+    body: options.body,
     headers: {
       authorization: `ApiKey ${options.auth}`,
       'content-type': 'application/json',
@@ -72,6 +50,10 @@ export async function createMonitors(
   });
 }
 
+export function ok(statusCode: number) {
+  return statusCode >= 200 && statusCode <= 299;
+}
+
 export type APIMonitorError = {
   id?: string;
   reason: string;
@@ -98,8 +80,8 @@ export function formatAPIError(
   return outer;
 }
 
-export function formatFailedMonitors(errors: APIMonitorError[]) {
-  let outer = bold(`${symbols['failed']} Error\n`);
+function formatMonitorError(errors: APIMonitorError[]) {
+  let outer = '';
   for (const error of errors) {
     const monitorId = error.id ? `: monitor(${error.id})` : '';
     let inner = bold(`> ${error.reason}${monitorId}\n`);
@@ -109,3 +91,13 @@ export function formatFailedMonitors(errors: APIMonitorError[]) {
   }
   return outer;
 }
+
+export function formatFailedMonitors(errors: APIMonitorError[]) {
+  const heading = bold(`${symbols['failed']} Error\n`);
+  return heading + formatMonitorError(errors);
+}
+
+export function formatStaleMonitors(errors: APIMonitorError[]) {
+  const heading = bold(`${symbols['warning']} Warnings\n`);
+  return heading + formatMonitorError(errors);
+}

package/templates/journeys/advanced-example-helpers.ts

@@ -12,7 +12,7 @@ export const loadAppStep = (page: Page, url: string) => {
 
 export const addTaskStep = (page: Page, task: string) => {
   step(`add task ${task}`, async () => {
-    const input = await page.$('input.new-todo');
+    const input = await page.locator('input.new-todo');
     await input.type(task);
     await input.press('Enter');
   });
@@ -21,12 +21,13 @@ export const addTaskStep = (page: Page, task: string) => {
 const todosSelector = 'ul.todo-list li.todo';
 
 export const findTask = async (page: Page, task: string) => {
-  return await page.waitForSelector(`${todosSelector} >> text="${task}"`);
+  return await page.locator(`${todosSelector} >> text="${task}"`);
 };
 
-export const assertTaskListSizeStep = async (page: Page, size: number) => {
-  step(`check that task list has exactly ${size} elements`, async () => {
-    expect((await page.$$(todosSelector)).length).toBe(size);
+export const assertTaskListCountStep = async (page: Page, count: number) => {
+  step(`check that task list has exactly ${count} elements`, async () => {
+    const tasks = await page.locator(todosSelector);
+    expect(await tasks.count()).toBe(count);
   });
 };
 
@@ -40,14 +41,13 @@ export const destroyTaskStep = async (page: Page, task: string) => {
   step(`destroy task '${task}'`, async () => {
     const label = await findTask(page, task);
     // xpath indexes arrays starting at 1!!! Easy to forget!
-    const li = await label.$('xpath=ancestor::li[1]');
-    const destroyButton = await li.$('button');
-
+    const li = await label.locator('xpath=ancestor::li[1]');
+    const destroyButton = await li.locator('button');
     // The destroy button is not visible until hovered. Setup a click test which
     // will wait up to 30s for the button to be visible.
     const clickFuture = destroyButton.click();
     // now hover, making the destroy button clickable
-    li.hover();
+    await li.hover();
     // now we are done
     await clickFuture;
   });