@elastic/synthetics 1.0.0-beta.39 → 1.0.0-beta.40

package/src/push/index.ts

@@ -22,118 +22,107 @@
  * THE SOFTWARE.
  *
  */
-
+import semver from 'semver';
 import { readFile, writeFile } from 'fs/promises';
 import { prompt } from 'enquirer';
 import { bold, grey } from 'kleur/colors';
 import {
-  ok,
-  formatAPIError,
-  formatFailedMonitors,
-  formatNotFoundError,
-  formatStaleMonitors,
-} from './request';
-import { buildMonitorSchema, createMonitors, MonitorSchema } from './monitor';
+  getLocalMonitors,
+  buildMonitorSchema,
+  diffMonitors as diffMonitorHashIDs,
+  MonitorSchema,
+} from './monitor';
 import { ALLOWED_SCHEDULES, Monitor } from '../dsl/monitor';
 import {
   progress,
-  apiProgress,
+  liveProgress,
   write,
   error,
   warn,
   indent,
-  safeNDJSONParse,
   done,
   getMonitorManagementURL,
-  getArrayChunks,
 } from '../helpers';
 import type { PushOptions, ProjectSettings } from '../common_types';
 import { findSyntheticsConfig, readConfig } from '../config';
+import {
+  bulkDeleteMonitors,
+  bulkGetMonitors,
+  bulkPutMonitors,
+  getVersion,
+  createMonitorsLegacy,
+  CHUNK_SIZE,
+} from './kibana_api';
+import { getChunks, logDiff } from './utils';
 
 export async function push(monitors: Monitor[], options: PushOptions) {
-  let schemas: MonitorSchema[] = [];
-  if (monitors.length > 0) {
-    const duplicates = trackDuplicates(monitors);
-    if (duplicates.size > 0) {
-      throw error(formatDuplicateError(duplicates));
-    }
+  const duplicates = trackDuplicates(monitors);
+  if (duplicates.size > 0) {
+    throw error(formatDuplicateError(duplicates));
+  }
+  progress(`Pushing monitors for project: ${options.id}`);
 
-    progress(`preparing all monitors`);
-    schemas = await buildMonitorSchema(monitors);
+  const stackVersion = await getVersion(options);
+  const isV2 = semver.satisfies(stackVersion, '>=8.6.0');
+  if (!isV2) {
+    return await pushLegacy(monitors, options, stackVersion);
+  }
+
+  const local = getLocalMonitors(monitors);
+  const { monitors: remote } = await bulkGetMonitors(options);
+  const { newIDs, changedIDs, removedIDs, unchangedIDs } = diffMonitorHashIDs(
+    local,
+    remote
+  );
+  logDiff(newIDs, changedIDs, removedIDs, unchangedIDs);
 
-    progress(`creating all monitors`);
-    const chunks = getArrayChunks(schemas, 10);
+  const updatedMonitors = new Set<string>([...changedIDs, ...newIDs]);
+  if (updatedMonitors.size > 0) {
+    const toBundle = monitors.filter(m => updatedMonitors.has(m.config.id));
+    progress(`bundling ${toBundle.length} monitors`);
+    const schemas = await buildMonitorSchema(toBundle, true);
+    const chunks = getChunks(schemas, CHUNK_SIZE);
     for (const chunk of chunks) {
-      await pushMonitors({ schemas: chunk, keepStale: true, options });
+      await liveProgress(
+        bulkPutMonitors(options, chunk),
+        `creating or updating ${chunk.length} monitors`
+      );
     }
-  } else {
-    write('');
-    const { deleteAll } = await prompt<{ deleteAll: boolean }>({
-      type: 'confirm',
-      skip() {
-        if (options.yes) {
-          this.initial = process.env.TEST_OVERRIDE ?? true;
-          return true;
-        }
-        return false;
-      },
-      name: 'deleteAll',
-      message: `Pushing without any monitors will delete all monitors associated with the project.\n Do you want to continue?`,
-      initial: false,
-    });
-    if (!deleteAll) {
-      throw warn('Push command Aborted');
+  }
+
+  if (removedIDs.size > 0) {
+    if (updatedMonitors.size === 0 && unchangedIDs.size === 0) {
+      await promptConfirmDeleteAll(options);
+    }
+    const chunks = getChunks(Array.from(removedIDs), CHUNK_SIZE);
+    for (const chunk of chunks) {
+      await liveProgress(
+        bulkDeleteMonitors(options, chunk),
+        `deleting ${chunk.length} monitors`
+      );
     }
   }
-  progress(`deleting all stale monitors`);
-  await pushMonitors({ schemas, keepStale: false, options });
 
   done(`Pushed: ${grey(getMonitorManagementURL(options.url))}`);
 }
 
-export async function pushMonitors({
-  schemas,
-  keepStale,
-  options,
-}: {
-  schemas: MonitorSchema[];
-  keepStale: boolean;
-  options: PushOptions;
-}) {
-  const { body, statusCode } = await createMonitors(
-    schemas,
-    options,
-    keepStale
-  );
-  if (statusCode === 404) {
-    throw formatNotFoundError(await body.text());
-  }
-  if (!ok(statusCode)) {
-    const { error, message } = await body.json();
-    throw formatAPIError(statusCode, error, message);
-  }
-  const allchunks = [];
-  for await (const data of body) {
-    allchunks.push(Buffer.from(data));
-  }
-  const chunks = safeNDJSONParse(Buffer.concat(allchunks).toString('utf-8'));
-  // Its kind of hacky for now where Kibana streams the response by
-  // writing the data as NDJSON events (data can be interleaved), we
-  // distinguish the final data by checking if the event was a progress vs complete event
-  for (const chunk of chunks) {
-    if (typeof chunk === 'string') {
-      // TODO: add progress back for all states once we get the fix
-      // on kibana side
-      keepStale && apiProgress(chunk);
-      continue;
-    }
-    const { failedMonitors, failedStaleMonitors } = chunk;
-    if (failedMonitors && failedMonitors.length > 0) {
-      throw formatFailedMonitors(failedMonitors);
-    }
-    if (failedStaleMonitors.length > 0) {
-      throw formatStaleMonitors(failedStaleMonitors);
-    }
+async function promptConfirmDeleteAll(options: PushOptions) {
+  write('');
+  const { deleteAll } = await prompt<{ deleteAll: boolean }>({
+    type: 'confirm',
+    skip() {
+      if (options.yes) {
+        this.initial = process.env.TEST_OVERRIDE ?? true;
+        return true;
+      }
+      return false;
+    },
+    name: 'deleteAll',
+    message: `Pushing without any monitors will delete all monitors associated with the project.\n Do you want to continue?`,
+    initial: false,
+  });
+  if (!deleteAll) {
+    throw warn('Push command Aborted');
   }
 }
 
@@ -174,7 +163,7 @@ export async function loadSettings() {
     }
     return config.project || ({} as ProjectSettings);
   } catch (e) {
-    throw error(`Aborted (missing synthetics config file), Project not set up corrrectly.
+    throw error(`Aborted (missing synthetics config file), Project not set up correctly.
 
 ${INSTALLATION_HELP}`);
   }
@@ -254,3 +243,40 @@ export async function catchIncorrectSettings(
     await overrideSettings(settings.id, options.id);
   }
 }
+
+export async function pushLegacy(
+  monitors: Monitor[],
+  options: PushOptions,
+  version: number
+) {
+  const noLightWeightSupport = semver.satisfies(version, '<8.5.0');
+  if (
+    noLightWeightSupport &&
+    monitors.some(monitor => monitor.type !== 'browser')
+  ) {
+    throw error(
+      `Aborted: Lightweight monitors are not supported in ${version}. Please upgrade to 8.5.0 or above.`
+    );
+  }
+
+  let schemas: MonitorSchema[] = [];
+  if (monitors.length > 0) {
+    progress(`bundling ${monitors.length} monitors`);
+    schemas = await buildMonitorSchema(monitors, false);
+    const chunks = getChunks(schemas, 10);
+    for (const chunk of chunks) {
+      await liveProgress(
+        createMonitorsLegacy({ schemas: chunk, keepStale: true, options }),
+        `creating or updating ${chunk.length} monitors`
+      );
+    }
+  } else {
+    await promptConfirmDeleteAll(options);
+  }
+  await liveProgress(
+    createMonitorsLegacy({ schemas, keepStale: false, options }),
+    `deleting all stale monitors`
+  );
+
+  done(`Pushed: ${grey(getMonitorManagementURL(options.url))}`);
+}

package/src/push/kibana_api.ts

@@ -0,0 +1,191 @@
+/**
+ * MIT License
+ *
+ * Copyright (c) 2020-present, Elastic NV
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ *
+ */
+
+import { PushOptions } from '../common_types';
+import { safeNDJSONParse } from '../helpers';
+import { MonitorHashID, MonitorSchema } from './monitor';
+import {
+  formatFailedMonitors,
+  formatStaleMonitors,
+  handleError,
+  sendReqAndHandleError,
+  sendRequest,
+  APIMonitorError,
+} from './request';
+import { generateURL } from './utils';
+
+// Default chunk size for bulk put / delete
+export const CHUNK_SIZE = 100;
+
+export type PutResponse = {
+  createdMonitors: string[];
+  updatedMonitors: string[];
+  failedMonitors: APIMonitorError[];
+};
+
+export async function bulkPutMonitors(
+  options: PushOptions,
+  schemas: MonitorSchema[]
+) {
+  const resp = await sendReqAndHandleError<PutResponse>({
+    url: generateURL(options, 'bulk_update') + '/_bulk_update',
+    method: 'PUT',
+    auth: options.auth,
+    body: JSON.stringify({ monitors: schemas }),
+  });
+  const { failedMonitors } = resp;
+  if (failedMonitors && failedMonitors.length > 0) {
+    throw formatFailedMonitors(failedMonitors);
+  }
+  return resp;
+}
+
+export type GetResponse = {
+  total: number;
+  monitors: MonitorHashID[];
+  after_key?: string;
+};
+
+export async function bulkGetMonitors(
+  options: PushOptions
+): Promise<GetResponse> {
+  const allMonitors: MonitorHashID[] = [];
+
+  const resp = await fetchMonitors(options);
+  allMonitors.push(...resp.monitors);
+  let afterKey = resp.afterKey;
+  const total = resp.total;
+
+  while (allMonitors.length < total) {
+    const resp = await fetchMonitors(options, afterKey);
+    allMonitors.push(...resp.monitors);
+    afterKey = resp.afterKey;
+  }
+
+  return {
+    total,
+    monitors: allMonitors,
+  };
+}
+
+const fetchMonitors = async (options: PushOptions, afterKey?: string) => {
+  let url = generateURL(options, 'bulk_get');
+  if (afterKey) {
+    url += `?search_after=${afterKey}`;
+  }
+  const resp = await sendReqAndHandleError<GetResponse>({
+    url,
+    method: 'GET',
+    auth: options.auth,
+  });
+  return {
+    afterKey: resp.after_key,
+    total: resp.total,
+    monitors: resp.monitors,
+  };
+};
+
+export type DeleteResponse = {
+  deleted_monitors: string[];
+};
+
+export async function bulkDeleteMonitors(
+  options: PushOptions,
+  monitorIDs: string[]
+) {
+  return await sendReqAndHandleError<DeleteResponse>({
+    url: generateURL(options, 'bulk_delete') + '/_bulk_delete',
+    method: 'DELETE',
+    auth: options.auth,
+    body: JSON.stringify({ monitors: monitorIDs }),
+  });
+}
+
+type StatusResponse = {
+  version: {
+    number: number;
+  };
+};
+
+export async function getVersion(options: PushOptions) {
+  const data = await sendReqAndHandleError<StatusResponse>({
+    url: generateURL(options, 'status'),
+    method: 'GET',
+    auth: options.auth,
+  });
+  return data.version.number;
+}
+
+export type LegacyAPISchema = {
+  project: string;
+  keep_stale: boolean;
+  monitors: MonitorSchema[];
+};
+
+export async function createMonitorsLegacy({
+  schemas,
+  keepStale,
+  options,
+}: {
+  schemas: MonitorSchema[];
+  keepStale: boolean;
+  options: PushOptions;
+}) {
+  const schema: LegacyAPISchema = {
+    project: options.id,
+    keep_stale: keepStale,
+    monitors: schemas,
+  };
+  const url = generateURL(options, 'legacy');
+  const { body, statusCode } = await sendRequest({
+    url,
+    method: 'PUT',
+    auth: options.auth,
+    body: JSON.stringify(schema),
+  });
+
+  const resBody = await handleError(statusCode, url, body);
+  const allchunks = [];
+  for await (const data of resBody) {
+    allchunks.push(Buffer.from(data));
+  }
+  const chunks = safeNDJSONParse(Buffer.concat(allchunks).toString('utf-8'));
+  // Its kind of hacky for now where Kibana streams the response by
+  // writing the data as NDJSON events (data can be interleaved), we
+  // distinguish the final data by checking if the event was a progress vs complete event
+  for (const chunk of chunks) {
+    if (typeof chunk === 'string') {
+      // Ignore the progress from Kibana as we chunk the requests
+      continue;
+    }
+    const { failedMonitors, failedStaleMonitors } = chunk;
+    if (failedMonitors && failedMonitors.length > 0) {
+      throw formatFailedMonitors(failedMonitors);
+    }
+    if (failedStaleMonitors.length > 0) {
+      throw formatStaleMonitors(failedStaleMonitors);
+    }
+  }
+}

package/src/push/monitor.ts

@@ -28,14 +28,7 @@ import { join } from 'path';
 import { LineCounter, parseDocument, YAMLSeq, YAMLMap } from 'yaml';
 import { bold, red } from 'kleur/colors';
 import { Bundler } from './bundler';
-import { sendRequest } from './request';
-import {
-  removeTrailingSlash,
-  SYNTHETICS_PATH,
-  totalist,
-  indent,
-  warn,
-} from '../helpers';
+import { SYNTHETICS_PATH, totalist, indent, warn } from '../helpers';
 import { LocationsMap } from '../locations/public-locations';
 import { ALLOWED_SCHEDULES, Monitor, MonitorConfig } from '../dsl/monitor';
 import { PushOptions } from '../common_types';
@@ -44,12 +37,14 @@ export type MonitorSchema = Omit<MonitorConfig, 'locations'> & {
   locations: string[];
   content?: string;
   filter?: Monitor['filter'];
+  hash?: string;
 };
 
-export type APISchema = {
-  project: string;
-  keep_stale: boolean;
-  monitors: MonitorSchema[];
+// Abbreviated monitor info, as often returned by the API,
+// just the journey ID and hash
+export type MonitorHashID = {
+  journey_id?: string;
+  hash?: string;
 };
 
 function translateLocation(locations?: MonitorConfig['locations']) {
@@ -57,7 +52,67 @@ function translateLocation(locations?: MonitorConfig['locations']) {
   return locations.map(loc => LocationsMap[loc] || loc).filter(Boolean);
 }
 
-export async function buildMonitorSchema(monitors: Monitor[]) {
+class RemoteDiffResult {
+  // The set of monitor IDs that have been added
+  newIDs = new Set<string>();
+  // Monitor IDs that are different locally than remotely
+  changedIDs = new Set<string>();
+  // Monitor IDs that are no longer present locally
+  removedIDs = new Set<string>();
+  // Monitor IDs that are identical on the remote server
+  unchangedIDs = new Set<string>();
+}
+
+export function diffMonitors(
+  local: MonitorHashID[],
+  remote: MonitorHashID[]
+): RemoteDiffResult {
+  const result = new RemoteDiffResult();
+  const localMonitorsIDToHash = new Map<string, string>();
+  for (const hashID of local) {
+    localMonitorsIDToHash.set(hashID.journey_id, hashID.hash);
+  }
+  const remoteMonitorsIDToHash = new Map<string, string>();
+  for (const hashID of remote) {
+    remoteMonitorsIDToHash.set(hashID.journey_id, hashID.hash);
+  }
+
+  // Compare local to remote
+  for (const [localID, localHash] of localMonitorsIDToHash) {
+    // Hash is reset to '' when a monitor is edited on the UI
+    if (!remoteMonitorsIDToHash.has(localID)) {
+      result.newIDs.add(localID);
+    } else {
+      const remoteHash = remoteMonitorsIDToHash.get(localID);
+      if (remoteHash != localHash) {
+        result.changedIDs.add(localID);
+      } else if (remoteHash === localHash) {
+        result.unchangedIDs.add(localID);
+      }
+    }
+    // We no longer need to process this ID, removing it here
+    // reduces the numbers considered in the next phase
+    remoteMonitorsIDToHash.delete(localID);
+  }
+
+  for (const [id] of remoteMonitorsIDToHash) {
+    result.removedIDs.add(id);
+  }
+  return result;
+}
+
+export function getLocalMonitors(monitors: Monitor[]) {
+  const data: MonitorHashID[] = [];
+  for (const monitor of monitors) {
+    data.push({
+      journey_id: monitor.config.id,
+      hash: monitor.hash(),
+    });
+  }
+  return data;
+}
+
+export async function buildMonitorSchema(monitors: Monitor[], isV2: boolean) {
   /**
    * Set up the bundle artifacts path which can be used to
    * create the bundles required for uploading journeys
@@ -69,10 +124,13 @@ export async function buildMonitorSchema(monitors: Monitor[]) {
 
   for (const monitor of monitors) {
     const { source, config, filter, type } = monitor;
-    const schema = {
+    const schema: MonitorSchema = {
       ...config,
       locations: translateLocation(config.locations),
     };
+    if (isV2) {
+      schema.hash = monitor.hash();
+    }
     if (type === 'browser') {
       const outPath = join(bundlePath, config.name + '.zip');
       const content = await bundler.build(source.file, outPath);
@@ -216,24 +274,3 @@ export function nearestSchedule(schedule: number) {
   }
   return ALLOWED_SCHEDULES[end];
 }
-
-export async function createMonitors(
-  monitors: MonitorSchema[],
-  options: PushOptions,
-  keepStale: boolean
-) {
-  const schema: APISchema = {
-    project: options.id,
-    keep_stale: keepStale,
-    monitors,
-  };
-
-  return await sendRequest({
-    url:
-      removeTrailingSlash(options.url) +
-      `/s/${options.space}/api/synthetics/service/project/monitors`,
-    method: 'PUT',
-    auth: options.auth,
-    body: JSON.stringify(schema),
-  });
-}