@elastic/synthetics 1.0.0-beta.30 → 1.0.0-beta.33

package/src/push/index.ts

@@ -23,24 +23,71 @@
  *
  */
 
+import { join } from 'path';
+import { existsSync } from 'fs';
+import { readFile, writeFile } from 'fs/promises';
+import { prompt } from 'enquirer';
+import { bold, yellow } from 'kleur/colors';
 import {
-  createMonitors,
   ok,
   formatAPIError,
   formatFailedMonitors,
   formatNotFoundError,
+  formatStaleMonitors,
 } from './request';
+import { buildMonitorSchema, createMonitors, MonitorSchema } from './monitor';
+import { ProjectSettings } from '../generator';
 import { Monitor } from '../dsl/monitor';
-import { PushOptions } from '../common_types';
-import { buildMonitorSchema } from './monitor';
-import { progress, error, done } from '../helpers';
+import {
+  progress,
+  apiProgress,
+  write,
+  error,
+  SYNTHETICS_PATH,
+  aborted,
+  indent,
+  safeNDJSONParse,
+  done,
+} from '../helpers';
+import type { PushOptions } from '../common_types';
 
 export async function push(monitors: Monitor[], options: PushOptions) {
+  if (monitors.length === 0) {
+    throw 'No Monitors found';
+  }
+  const duplicates = trackDuplicates(monitors);
+  if (duplicates.size > 0) {
+    throw error(formatDuplicateError(duplicates));
+  }
+
   progress(`preparing all monitors`);
   const schemas = await buildMonitorSchema(monitors);
+
+  progress(`creating all monitors`);
+  for (const schema of schemas) {
+    await pushMonitors({ schemas: [schema], keepStale: true, options });
+  }
+
+  progress(`deleting all stale monitors`);
+  await pushMonitors({ schemas, keepStale: false, options });
+  done('Pushed');
+}
+
+export async function pushMonitors({
+  schemas,
+  keepStale,
+  options,
+}: {
+  schemas: MonitorSchema[];
+  keepStale: boolean;
+  options: PushOptions;
+}) {
   try {
-    progress(`creating all monitors`);
-    const { body, statusCode } = await createMonitors(schemas, options);
+    const { body, statusCode } = await createMonitors(
+      schemas,
+      options,
+      keepStale
+    );
     if (statusCode === 404) {
       throw formatNotFoundError(await body.text());
     }
@@ -48,12 +95,132 @@ export async function push(monitors: Monitor[], options: PushOptions) {
       const { error, message } = await body.json();
       throw formatAPIError(statusCode, error, message);
     }
-    const { failedMonitors } = await body.json();
-    if (failedMonitors.length > 0) {
-      throw formatFailedMonitors(failedMonitors);
+    body.setEncoding('utf-8');
+    for await (const data of body) {
+      // Its kind of hacky for now where Kibana streams the response by
+      // writing the data as NDJSON events (data can be interleaved), we
+      // distinguish the final data by checking if the event was a progress vs complete event
+      const chunks = safeNDJSONParse(data);
+      for (const chunk of chunks) {
+        if (typeof chunk === 'string') {
+          // TODO: add progress back for all states once we get the fix
+          // on kibana side
+          keepStale && apiProgress(chunk);
+          continue;
+        }
+        const { failedMonitors, failedStaleMonitors } = chunk;
+        if (failedMonitors && failedMonitors.length > 0) {
+          throw formatFailedMonitors(failedMonitors);
+        }
+        if (failedStaleMonitors.length > 0) {
+          write(yellow(formatStaleMonitors(failedStaleMonitors)));
+        }
+      }
     }
-    done('Pushed');
   } catch (e) {
     error(e);
   }
 }
+
+function trackDuplicates(monitors: Monitor[]) {
+  const monitorMap: Map<string, Monitor> = new Map();
+  const duplicates: Set<Monitor> = new Set();
+  for (const monitor of monitors) {
+    const id = monitor.config.id;
+    if (monitorMap.has(id)) {
+      duplicates.add(monitorMap.get(id));
+      duplicates.add(monitor);
+    }
+    monitorMap.set(id, monitor);
+  }
+  return duplicates;
+}
+
+export function formatDuplicateError(monitors: Set<Monitor>) {
+  let outer = bold(`Aborted: Duplicate monitors found\n`);
+
+  let inner = '';
+  for (const monitor of monitors) {
+    const { config, source } = monitor;
+    inner += `* ${config.id} - ${source.file}:${source.line}:${source.column}\n`;
+  }
+  outer += indent(inner);
+  return outer;
+}
+
+const PROJECT_SETTINGS_PATH = join(SYNTHETICS_PATH, 'project.json');
+const INSTALLATION_HELP = `Run 'npx @elastic/synthetics init' to create project with default settings.`;
+
+export async function loadSettings() {
+  if (!existsSync(PROJECT_SETTINGS_PATH)) {
+    throw error(`Aborted. Synthetics project not set up.
+
+${INSTALLATION_HELP}`);
+  }
+
+  const contents = await readFile(PROJECT_SETTINGS_PATH, 'utf-8');
+  return JSON.parse(contents) as ProjectSettings;
+}
+
+export function validateSettings(opts: PushOptions) {
+  const INVALID = 'Aborted. Invalid synthetics project settings.';
+
+  let reason = '';
+  if (!opts.project) {
+    reason = 'Set project id via CLI as `--project <id>`';
+  } else if (!opts.locations && !opts.privateLocations) {
+    reason = `Set default location for all monitors via
+  - CLI as '--locations <values...> or --privateLocations <values...>'
+  - Synthetics config file 'monitors.locations' | 'monitors.privateLocations' field`;
+  } else if (!opts.schedule) {
+    reason = `Set default schedule in minutes for all monitors via
+  - CLI as '--schedule <mins>'
+  - Synthetics config file 'monitors.schedule' field`;
+  }
+
+  if (!reason) return;
+
+  throw error(`${INVALID}
+
+${reason}
+
+${INSTALLATION_HELP}`);
+}
+
+async function overrideSettings(project: string) {
+  const settings = JSON.parse(await readFile(PROJECT_SETTINGS_PATH, 'utf-8'));
+  settings.project = project;
+
+  await writeFile(
+    PROJECT_SETTINGS_PATH,
+    JSON.stringify(settings, null, 2),
+    'utf-8'
+  );
+}
+
+export async function catchIncorrectSettings(
+  settings: ProjectSettings,
+  options: PushOptions
+) {
+  let override = !settings.project;
+  if (settings.project && settings.project !== options.project) {
+    if (typeof process.env.TEST_OVERRIDE != null) {
+      override = Boolean(process.env.TEST_OVERRIDE);
+    } else {
+      // Add an extra line to make it easier to read the prompt
+      write('');
+      ({ override } = await prompt<{ override: boolean }>({
+        type: 'confirm',
+        name: 'override',
+        message: `Monitors were pushed under the '${settings.project}' project. Are you sure you want to push them under the new '${options.project}' (note that this will duplicate the monitors, the old ones being orphaned)`,
+        initial: false,
+      }));
+    }
+    if (!override) {
+      throw aborted('Push command Aborted');
+    }
+  }
+  if (override) {
+    await overrideSettings(options.project);
+  }
+}

package/src/push/monitor.ts

@@ -23,12 +23,14 @@
  *
  */
 
-import { mkdir } from 'fs/promises';
+import { mkdir, rm } from 'fs/promises';
 import { join } from 'path';
 import { Bundler } from './bundler';
-import { CACHE_PATH } from '../helpers';
-import { LocationsMap } from '../dsl/locations';
+import { sendRequest } from './request';
+import { removeTrailingSlash, SYNTHETICS_PATH } from '../helpers';
+import { LocationsMap } from '../locations/public-locations';
 import { Monitor, MonitorConfig } from '../dsl/monitor';
+import { PushOptions } from '../common_types';
 
 export type MonitorSchema = Omit<MonitorConfig, 'locations'> & {
   content: string;
@@ -36,19 +38,23 @@ export type MonitorSchema = Omit<MonitorConfig, 'locations'> & {
   filter: Monitor['filter'];
 };
 
-function translateLocation(locations: MonitorConfig['locations']) {
-  return locations.map(loc => LocationsMap[loc]).filter(Boolean);
+export type APISchema = {
+  project: string;
+  keep_stale: boolean;
+  monitors: MonitorSchema[];
+};
+
+function translateLocation(locations?: MonitorConfig['locations']) {
+  if (!locations) return [];
+  return locations.map(loc => LocationsMap[loc] || loc).filter(Boolean);
 }
 
 export async function buildMonitorSchema(monitors: Monitor[]) {
-  if (monitors.length == 0) {
-    throw new Error('No Monitors found');
-  }
   /**
    * Set up the bundle artifacts path which can be used to
    * create the bundles required for uploading journeys
    */
-  const bundlePath = join(CACHE_PATH, 'bundles');
+  const bundlePath = join(SYNTHETICS_PATH, 'bundles');
   await mkdir(bundlePath, { recursive: true });
   const bundler = new Bundler();
   const schemas: MonitorSchema[] = [];
@@ -64,5 +70,28 @@ export async function buildMonitorSchema(monitors: Monitor[]) {
       filter: filter,
     });
   }
+
+  await rm(bundlePath, { recursive: true });
   return schemas;
 }
+
+export async function createMonitors(
+  monitors: MonitorSchema[],
+  options: PushOptions,
+  keepStale: boolean
+) {
+  const schema: APISchema = {
+    project: options.project,
+    keep_stale: keepStale,
+    monitors,
+  };
+
+  return await sendRequest({
+    url:
+      removeTrailingSlash(options.url) +
+      `/s/${options.space}/api/synthetics/service/project/monitors`,
+    method: 'PUT',
+    auth: options.auth,
+    body: JSON.stringify(schema),
+  });
+}

package/src/push/plugin.ts

@@ -49,8 +49,13 @@ export type PluginData = {
 export type PluginCallback = (data: PluginData) => void;
 
 export function MultiAssetPlugin(callback: PluginCallback): esbuild.Plugin {
+  // Check that the path isn't in an external package by making sure it's at a standard
+  // local filesystem location
   const isBare = (str: string) => {
-    if (str.startsWith('/') || str.startsWith('./') || str.startsWith('../')) {
+    // Note that we use `isAbsolute` to handle UNC/windows style paths like C:\path\to\thing
+    // This is not necessary for relative directories since `.\file` is not supported as an import
+    // nor is `~/path/to/file`.
+    if (path.isAbsolute(str) || str.startsWith('./') || str.startsWith('../')) {
       return true;
     }
     return false;

package/src/push/request.ts

@@ -24,54 +24,37 @@
  */
 
 import { bold } from 'kleur/colors';
-import { request } from 'undici';
-import { PushOptions } from '../common_types';
+import { Dispatcher, request } from 'undici';
 import { indent, symbols } from '../helpers';
-import { MonitorSchema } from './monitor';
 
 /* eslint-disable @typescript-eslint/no-var-requires */
 const { version } = require('../../package.json');
 
-export type APISchema = {
-  project: string;
-  keep_stale: boolean;
-  monitors: MonitorSchema[];
+export type APIRequestOptions = {
+  url: string;
+  method: Dispatcher.HttpMethod;
+  auth: string;
+  body?: string;
 };
 
-function getAPIUrl(options: PushOptions) {
-  // remove trialing / as well
-  return (
-    options.url.replace(/\/+$/, '') +
-    `/s/${options.space}/api/synthetics/service/project/monitors`
-  );
-}
-
-export function ok(statusCode: number) {
-  return statusCode >= 200 && statusCode <= 299;
-}
-
-export async function createMonitors(
-  monitors: MonitorSchema[],
-  options: PushOptions
-) {
-  const schema: APISchema = {
-    project: options.project,
-    keep_stale: false,
-    monitors,
-  };
-
-  return await request(getAPIUrl(options), {
-    method: 'PUT',
-    body: JSON.stringify(schema),
+export async function sendRequest(options: APIRequestOptions) {
+  return await request(options.url, {
+    method: options.method,
+    body: options.body,
     headers: {
       authorization: `ApiKey ${options.auth}`,
       'content-type': 'application/json',
       'user-agent': `Elastic/Synthetics ${version}`,
       'kbn-xsrf': 'true',
     },
+    headersTimeout: 60 * 1000
   });
 }
 
+export function ok(statusCode: number) {
+  return statusCode >= 200 && statusCode <= 299;
+}
+
 export type APIMonitorError = {
   id?: string;
   reason: string;
@@ -98,8 +81,8 @@ export function formatAPIError(
   return outer;
 }
 
-export function formatFailedMonitors(errors: APIMonitorError[]) {
-  let outer = bold(`${symbols['failed']} Error\n`);
+function formatMonitorError(errors: APIMonitorError[]) {
+  let outer = '';
   for (const error of errors) {
     const monitorId = error.id ? `: monitor(${error.id})` : '';
     let inner = bold(`> ${error.reason}${monitorId}\n`);
@@ -109,3 +92,13 @@ export function formatFailedMonitors(errors: APIMonitorError[]) {
   }
   return outer;
 }
+
+export function formatFailedMonitors(errors: APIMonitorError[]) {
+  const heading = bold(`${symbols['failed']} Error\n`);
+  return heading + formatMonitorError(errors);
+}
+
+export function formatStaleMonitors(errors: APIMonitorError[]) {
+  const heading = bold(`${symbols['warning']} Warnings\n`);
+  return heading + formatMonitorError(errors);
+}

package/templates/journeys/advanced-example-helpers.ts

@@ -12,7 +12,7 @@ export const loadAppStep = (page: Page, url: string) => {
 
 export const addTaskStep = (page: Page, task: string) => {
   step(`add task ${task}`, async () => {
-    const input = await page.$('input.new-todo');
+    const input = await page.locator('input.new-todo');
     await input.type(task);
     await input.press('Enter');
   });
@@ -21,12 +21,13 @@ export const addTaskStep = (page: Page, task: string) => {
 const todosSelector = 'ul.todo-list li.todo';
 
 export const findTask = async (page: Page, task: string) => {
-  return await page.waitForSelector(`${todosSelector} >> text="${task}"`);
+  return await page.locator(`${todosSelector} >> text="${task}"`);
 };
 
-export const assertTaskListSizeStep = async (page: Page, size: number) => {
-  step(`check that task list has exactly ${size} elements`, async () => {
-    expect((await page.$$(todosSelector)).length).toBe(size);
+export const assertTaskListCountStep = async (page: Page, count: number) => {
+  step(`check that task list has exactly ${count} elements`, async () => {
+    const tasks = await page.locator(todosSelector);
+    expect(await tasks.count()).toBe(count);
   });
 };
 
@@ -40,14 +41,13 @@ export const destroyTaskStep = async (page: Page, task: string) => {
   step(`destroy task '${task}'`, async () => {
     const label = await findTask(page, task);
     // xpath indexes arrays starting at 1!!! Easy to forget!
-    const li = await label.$('xpath=ancestor::li[1]');
-    const destroyButton = await li.$('button');
-
+    const li = await label.locator('xpath=ancestor::li[1]');
+    const destroyButton = await li.locator('button');
     // The destroy button is not visible until hovered. Setup a click test which
     // will wait up to 30s for the button to be visible.
     const clickFuture = destroyButton.click();
     // now hover, making the destroy button clickable
-    li.hover();
+    await li.hover();
     // now we are done
     await clickFuture;
   });

package/templates/journeys/advanced-example.journey.ts

@@ -2,7 +2,7 @@ import { journey } from '@elastic/synthetics';
 import {
   loadAppStep,
   addTaskStep,
-  assertTaskListSizeStep,
+  assertTaskListCountStep,
   checkForTaskStep,
   destroyTaskStep,
 } from './advanced-example-helpers';
@@ -17,10 +17,10 @@ journey('addition and completion of single task', ({ page, params }) => {
 
   loadAppStep(page, params.url);
   addTaskStep(page, testText);
-  assertTaskListSizeStep(page, 1);
+  assertTaskListCountStep(page, 1);
   checkForTaskStep(page, testText);
   destroyTaskStep(page, testText);
-  assertTaskListSizeStep(page, 0);
+  assertTaskListCountStep(page, 0);
 });
 
 journey('adding and removing multiple tasks', ({ page, params }) => {
@@ -31,13 +31,13 @@ journey('adding and removing multiple tasks', ({ page, params }) => {
     addTaskStep(page, t);
   });
 
-  assertTaskListSizeStep(page, 3);
+  assertTaskListCountStep(page, 3);
 
   // remove the middle task and check that it worked
   destroyTaskStep(page, testTasks[1]);
-  assertTaskListSizeStep(page, 2);
+  assertTaskListCountStep(page, 2);
 
   // add a new task and check it exists
   addTaskStep(page, 'Task 4');
-  assertTaskListSizeStep(page, 3);
+  assertTaskListCountStep(page, 3);
 });

package/templates/journeys/example.journey.ts

@@ -12,7 +12,7 @@ journey('My Example Journey', ({ page, params }) => {
   });
 
   step('assert title', async () => {
-    const header = await page.$('h1');
+    const header = await page.locator('h1');
     expect(await header.textContent()).toBe('todos');
   });
 });

package/templates/synthetics.config.ts

@@ -13,7 +13,8 @@ export default env => {
      */
     monitor: {
       schedule: '{{schedule}}',
-      locations: ["'{{locations}}'"],
+      locations: ['{{locations}}'],
+      privateLocations: ['{{privateLocations}}'],
     },
   };
   if (env !== 'development') {

package/dist/dsl/locations.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"locations.d.ts","sourceRoot":"","sources":["../../src/dsl/locations.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;GAuBG;AAGH,eAAO,MAAM,YAAY;;;;;;;;;;;CAWxB,CAAC"}

package/dist/dsl/locations.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"locations.js","sourceRoot":"","sources":["../../src/dsl/locations.ts"],"names":[],"mappings":";AAAA;;;;;;;;;;;;;;;;;;;;;;;GAuBG;;;AAEH,sDAAsD;AACzC,QAAA,YAAY,GAAG;IAC1B,KAAK,EAAE,mBAAmB;IAC1B,KAAK,EAAE,eAAe;IACtB,SAAS,EAAE,mBAAmB;IAC9B,cAAc,EAAE,wBAAwB;IACxC,cAAc,EAAE,gBAAgB;IAChC,OAAO,EAAE,gBAAgB;IACzB,WAAW,EAAE,2BAA2B;IACxC,MAAM,EAAE,sBAAsB;IAC9B,OAAO,EAAE,YAAY;IACrB,OAAO,EAAE,YAAY;CACtB,CAAC"}