@elastic/synthetics 1.0.0-beta.34 → 1.0.0-beta.36

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@elastic/synthetics",
-  "version": "1.0.0-beta.34",
+  "version": "1.0.0-beta.36",
   "description": "Elastic synthetic monitoring agent",
   "main": "dist/index.js",
   "types": "dist/index.d.ts",
@@ -42,39 +42,40 @@
   "dependencies": {
     "@cspotcode/source-map-support": "^0.8.1",
     "@esbuild-plugins/node-resolve": "^0.1.4",
-    "archiver": "^5.3.0",
-    "commander": "^9.0.0",
+    "archiver": "^5.3.1",
+    "commander": "^9.4.0",
     "deepmerge": "^4.2.2",
     "enquirer": "^2.3.6",
-    "esbuild": "^0.14.27",
+    "esbuild": "^0.14.54",
     "expect": "^28.1.3",
     "http-proxy": "^1.18.1",
-    "kleur": "^4.1.4",
-    "micromatch": "^4.0.4",
-    "playwright-chromium": "=1.20.1",
-    "playwright-core": "=1.20.1",
+    "kleur": "^4.1.5",
+    "micromatch": "^4.0.5",
+    "playwright-chromium": "=1.26.0",
+    "playwright-core": "=1.26.0",
     "sharp": "^0.31.0",
-    "snakecase-keys": "^3.2.1",
-    "sonic-boom": "^2.6.0",
-    "ts-node": "^10.9.0",
-    "typescript": "^4.7.4",
-    "undici": "^5.10.0"
+    "snakecase-keys": "^4.0.0",
+    "sonic-boom": "^3.2.0",
+    "ts-node": "^10.9.1",
+    "typescript": "^4.8.3",
+    "undici": "^5.10.0",
+    "yaml": "^2.1.1"
   },
   "devDependencies": {
-    "@types/jest": "^28.1.3",
-    "@types/micromatch": "^4.0.1",
-    "@types/node": "^16.11.44",
-    "@types/sharp": "^0.28.2",
-    "@typescript-eslint/eslint-plugin": "^5.0.0",
-    "@typescript-eslint/parser": "^5.0.0",
-    "eslint": "^8.0.1",
+    "@types/jest": "^28.1.8",
+    "@types/micromatch": "^4.0.2",
+    "@types/node": "^16.11.59",
+    "@types/sharp": "^0.28.6",
+    "@typescript-eslint/eslint-plugin": "^5.38.0",
+    "@typescript-eslint/parser": "^5.38.0",
+    "eslint": "^8.23.1",
     "husky": "^4.3.6",
     "is-positive": "3.1.0",
     "jest": "^28.1.3",
     "lint-staged": "^10.5.3",
-    "prettier": "^2.4.1",
+    "prettier": "^2.7.1",
     "rimraf": "^3.0.2",
-    "ts-jest": "^28.0.5",
+    "ts-jest": "^28.0.8",
     "unzipper": "^0.10.11"
   },
   "engines": {

package/src/cli.ts

@@ -54,6 +54,7 @@ import { resolve } from 'path';
 import { Generator } from './generator';
 import { error } from './helpers';
 import { LocationsMap } from './locations/public-locations';
+import { createLightweightMonitors } from './push/monitor';
 
 /* eslint-disable-next-line @typescript-eslint/no-var-requires */
 const { name, version } = require('../package.json');
@@ -184,6 +185,7 @@ program
     '--space <space>',
     'the target Kibana spaces for the pushed monitors — spaces help you organise pushed monitors.'
   )
+  .option('-y, --yes', 'skip all questions and run non-interactively')
   .addOption(pattern)
   .addOption(tags)
   .addOption(match)
@@ -191,7 +193,8 @@ program
   .addOption(playwrightOpts)
   .action(async (cmdOpts: PushOptions) => {
     try {
-      await loadTestFiles({ inline: false }, [cwd()]);
+      const workDir = cwd();
+      await loadTestFiles({ inline: false, ...program.opts() }, [workDir]);
       const settings = await loadSettings();
       const options = normalizeOptions({
         ...program.opts(),
@@ -201,6 +204,7 @@ program
       validateSettings(options);
       await catchIncorrectSettings(settings, options);
       const monitors = runner.buildMonitors(options);
+      monitors.push(...(await createLightweightMonitors(workDir, options)));
       await push(monitors, options);
     } catch (e) {
       e && console.error(e);

package/src/common_types.ts

@@ -241,11 +241,12 @@ export type RunOptions = BaseArgs & {
   reporter?: BuiltInReporterName | ReporterInstance;
 };
 
-export type PushOptions = ProjectSettings & {
+export type PushOptions = Partial<ProjectSettings> & {
   auth: string;
   schedule?: MonitorConfig['schedule'];
   locations?: MonitorConfig['locations'];
   privateLocations?: MonitorConfig['privateLocations'];
+  yes?: boolean;
 };
 
 export type ProjectSettings = {

package/src/dsl/journey.ts

@@ -92,6 +92,7 @@ export class Journey {
     this.monitor = new Monitor({
       name: this.name,
       id: this.id,
+      type: 'browser',
       tags: this.tags ?? [],
       ...config,
     });

package/src/dsl/monitor.ts

@@ -41,6 +41,7 @@ export const SyntheticsLocations = Object.keys(
 export type MonitorConfig = {
   id?: string;
   name?: string;
+  type?: string;
   tags?: string[];
   schedule?: number;
   enabled?: boolean;
@@ -73,6 +74,10 @@ export class Monitor {
     });
   }
 
+  get type() {
+    return this.config.type;
+  }
+
   setSource(source: Location) {
     this.source = source;
   }

package/src/formatter/javascript.ts

@@ -26,7 +26,7 @@
 import {
   JavaScriptLanguageGenerator,
   JavaScriptFormatter,
-} from 'playwright-core/lib/server/supplements/recorder/javascript';
+} from 'playwright-core/lib/server/recorder/javascript';
 
 export type Step = {
   actions: ActionInContext[];

package/src/helpers.ts

@@ -359,7 +359,7 @@ export function done(message: string) {
   write(bold(green(`${symbols['succeeded']} ${message}`)));
 }
 
-export function aborted(message: string) {
+export function warn(message: string) {
   write(bold(yellow(`${symbols['warning']} ${message}`)));
 }
 

package/src/plugins/network.ts

@@ -283,12 +283,12 @@ export class NetworkManager {
         networkEntry.request.body = {
           bytes: sizes.requestBodySize,
         };
-        networkEntry.response.bytes =
-          sizes.responseHeadersSize + sizes.responseBodySize;
+        const transferSize = sizes.responseHeadersSize + sizes.responseBodySize;
+        networkEntry.transferSize = transferSize;
+        networkEntry.response.bytes = transferSize;
         networkEntry.response.body = {
           bytes: sizes.responseBodySize,
         };
-        networkEntry.transferSize = sizes.responseBodySize;
       })
     );
   }

package/src/push/index.ts

@@ -25,7 +25,7 @@
 
 import { readFile, writeFile } from 'fs/promises';
 import { prompt } from 'enquirer';
-import { bold, yellow } from 'kleur/colors';
+import { bold } from 'kleur/colors';
 import {
   ok,
   formatAPIError,
@@ -40,7 +40,7 @@ import {
   apiProgress,
   write,
   error,
-  aborted,
+  warn,
   indent,
   safeNDJSONParse,
   done,
@@ -49,24 +49,42 @@ import type { PushOptions, ProjectSettings } from '../common_types';
 import { findSyntheticsConfig, readConfig } from '../config';
 
 export async function push(monitors: Monitor[], options: PushOptions) {
-  if (monitors.length === 0) {
-    throw 'No Monitors found';
-  }
-  const duplicates = trackDuplicates(monitors);
-  if (duplicates.size > 0) {
-    throw error(formatDuplicateError(duplicates));
-  }
+  let schemas: MonitorSchema[] = [];
+  if (monitors.length > 0) {
+    const duplicates = trackDuplicates(monitors);
+    if (duplicates.size > 0) {
+      throw error(formatDuplicateError(duplicates));
+    }
 
-  progress(`preparing all monitors`);
-  const schemas = await buildMonitorSchema(monitors);
+    progress(`preparing all monitors`);
+    schemas = await buildMonitorSchema(monitors);
 
-  progress(`creating all monitors`);
-  for (const schema of schemas) {
-    await pushMonitors({ schemas: [schema], keepStale: true, options });
+    progress(`creating all monitors`);
+    for (const schema of schemas) {
+      await pushMonitors({ schemas: [schema], keepStale: true, options });
+    }
+  } else {
+    write('');
+    const { deleteAll } = await prompt<{ deleteAll: boolean }>({
+      type: 'confirm',
+      skip() {
+        if (options.yes) {
+          this.initial = process.env.TEST_OVERRIDE ?? true;
+          return true;
+        }
+        return false;
+      },
+      name: 'deleteAll',
+      message: `Pushing without any monitors will delete all monitors associated with the project.\n Do you want to continue?`,
+      initial: false,
+    });
+    if (!deleteAll) {
+      throw warn('Push command Aborted');
+    }
   }
-
   progress(`deleting all stale monitors`);
   await pushMonitors({ schemas, keepStale: false, options });
+
   done('Pushed');
 }
 
@@ -79,43 +97,39 @@ export async function pushMonitors({
   keepStale: boolean;
   options: PushOptions;
 }) {
-  try {
-    const { body, statusCode } = await createMonitors(
-      schemas,
-      options,
-      keepStale
-    );
-    if (statusCode === 404) {
-      throw formatNotFoundError(await body.text());
-    }
-    if (!ok(statusCode)) {
-      const { error, message } = await body.json();
-      throw formatAPIError(statusCode, error, message);
-    }
-    body.setEncoding('utf-8');
-    for await (const data of body) {
-      // Its kind of hacky for now where Kibana streams the response by
-      // writing the data as NDJSON events (data can be interleaved), we
-      // distinguish the final data by checking if the event was a progress vs complete event
-      const chunks = safeNDJSONParse(data);
-      for (const chunk of chunks) {
-        if (typeof chunk === 'string') {
-          // TODO: add progress back for all states once we get the fix
-          // on kibana side
-          keepStale && apiProgress(chunk);
-          continue;
-        }
-        const { failedMonitors, failedStaleMonitors } = chunk;
-        if (failedMonitors && failedMonitors.length > 0) {
-          throw formatFailedMonitors(failedMonitors);
-        }
-        if (failedStaleMonitors.length > 0) {
-          write(yellow(formatStaleMonitors(failedStaleMonitors)));
-        }
+  const { body, statusCode } = await createMonitors(
+    schemas,
+    options,
+    keepStale
+  );
+  if (statusCode === 404) {
+    throw formatNotFoundError(await body.text());
+  }
+  if (!ok(statusCode)) {
+    const { error, message } = await body.json();
+    throw formatAPIError(statusCode, error, message);
+  }
+  body.setEncoding('utf-8');
+  for await (const data of body) {
+    // Its kind of hacky for now where Kibana streams the response by
+    // writing the data as NDJSON events (data can be interleaved), we
+    // distinguish the final data by checking if the event was a progress vs complete event
+    const chunks = safeNDJSONParse(data);
+    for (const chunk of chunks) {
+      if (typeof chunk === 'string') {
+        // TODO: add progress back for all states once we get the fix
+        // on kibana side
+        keepStale && apiProgress(chunk);
+        continue;
+      }
+      const { failedMonitors, failedStaleMonitors } = chunk;
+      if (failedMonitors && failedMonitors.length > 0) {
+        throw formatFailedMonitors(failedMonitors);
+      }
+      if (failedStaleMonitors.length > 0) {
+        throw formatStaleMonitors(failedStaleMonitors);
       }
     }
-  } catch (e) {
-    error(e);
   }
 }
 
@@ -149,7 +163,7 @@ const INSTALLATION_HELP = `Run 'npx @elastic/synthetics init' to create project
 
 export async function loadSettings() {
   try {
-    const config = await readConfig('asd');
+    const config = await readConfig(process.env['NODE_ENV'] || 'development');
     // Missing config file, fake throw to capture as missing file
     if (Object.keys(config).length === 0) {
       throw '';
@@ -193,7 +207,7 @@ async function overrideSettings(oldValue: string, newValue: string) {
   const cwd = process.cwd();
   const configPath = await findSyntheticsConfig(cwd, cwd);
   if (!configPath) {
-    throw aborted(`Unable to find synthetics config file: ${configPath}`);
+    throw warn(`Unable to find synthetics config file: ${configPath}`);
   }
   const config = await readFile(configPath, 'utf-8');
   const updatedConfig = config.replace(
@@ -209,20 +223,23 @@ export async function catchIncorrectSettings(
 ) {
   let override = !settings.id;
   if (settings.id && settings.id !== options.id) {
-    if (process.env.TEST_OVERRIDE != null) {
-      override = Boolean(process.env.TEST_OVERRIDE);
-    } else {
-      // Add an extra line to make it easier to read the prompt
-      write('');
-      ({ override } = await prompt<{ override: boolean }>({
-        type: 'confirm',
-        name: 'override',
-        message: `Monitors were pushed under the '${settings.id}' project. Are you sure you want to push them under the new '${options.id}' (note that this will duplicate the monitors, the old ones being orphaned)`,
-        initial: false,
-      }));
-    }
+    // Add an extra line to make it easier to read the prompt
+    write('');
+    ({ override } = await prompt<{ override: boolean }>({
+      type: 'confirm',
+      name: 'override',
+      skip() {
+        if (options.yes) {
+          this.initial = process.env.TEST_OVERRIDE ?? true;
+          return true;
+        }
+        return false;
+      },
+      message: `Monitors were pushed under the '${settings.id}' project. Are you sure you want to push them under the new '${options.id}' (note that this will duplicate the monitors, the old ones being orphaned)`,
+      initial: false,
+    }));
     if (!override) {
-      throw aborted('Push command Aborted');
+      throw warn('Push command Aborted');
     }
   }
   if (override) {

package/src/push/monitor.ts

@@ -23,19 +23,27 @@
  *
  */
 
-import { mkdir, rm } from 'fs/promises';
+import { mkdir, rm, readFile } from 'fs/promises';
 import { join } from 'path';
+import { LineCounter, parseDocument, YAMLSeq, YAMLMap } from 'yaml';
+import { bold, red } from 'kleur/colors';
 import { Bundler } from './bundler';
 import { sendRequest } from './request';
-import { removeTrailingSlash, SYNTHETICS_PATH } from '../helpers';
+import {
+  removeTrailingSlash,
+  SYNTHETICS_PATH,
+  totalist,
+  indent,
+  warn,
+} from '../helpers';
 import { LocationsMap } from '../locations/public-locations';
 import { Monitor, MonitorConfig } from '../dsl/monitor';
 import { PushOptions } from '../common_types';
 
 export type MonitorSchema = Omit<MonitorConfig, 'locations'> & {
-  content: string;
   locations: string[];
-  filter: Monitor['filter'];
+  content?: string;
+  filter?: Monitor['filter'];
 };
 
 export type APISchema = {
@@ -60,21 +68,134 @@ export async function buildMonitorSchema(monitors: Monitor[]) {
   const schemas: MonitorSchema[] = [];
 
   for (const monitor of monitors) {
-    const { source, config, filter } = monitor;
-    const outPath = join(bundlePath, config.name + '.zip');
-    const content = await bundler.build(source.file, outPath);
-    schemas.push({
+    const { source, config, filter, type } = monitor;
+    const schema = {
       ...config,
-      content,
       locations: translateLocation(config.locations),
-      filter: filter,
-    });
+    };
+    if (type === 'browser') {
+      const outPath = join(bundlePath, config.name + '.zip');
+      const content = await bundler.build(source.file, outPath);
+      Object.assign(schema, { content, filter });
+    }
+    schemas.push(schema);
   }
 
   await rm(bundlePath, { recursive: true });
   return schemas;
 }
 
+export async function createLightweightMonitors(
+  workDir: string,
+  options: PushOptions
+) {
+  const lwFiles = new Set<string>();
+  await totalist(workDir, (rel, abs) => {
+    if (/.(yml|yaml)$/.test(rel)) {
+      lwFiles.add(abs);
+    }
+  });
+  // Warn users about schedule that are less than 60 seconds
+  if (lwFiles.size > 0) {
+    warn(
+      'lightweight monitors schedule < 1 minute resolution are not supported, will get reset to their nearest minute interval.'
+    );
+  }
+
+  const monitors: Monitor[] = [];
+  for (const file of lwFiles.values()) {
+    const content = await readFile(file, 'utf-8');
+    const lineCounter = new LineCounter();
+    const parsedDoc = parseDocument(content, {
+      lineCounter,
+      keepSourceTokens: true,
+    });
+    // Skip other yml files that are not relevant
+    const monitorSeq = parsedDoc.get('heartbeat.monitors') as YAMLSeq<YAMLMap>;
+    if (!monitorSeq) {
+      continue;
+    }
+
+    for (const monNode of monitorSeq.items) {
+      // Skip browser monitors and disabled monitors from pushing
+      if (
+        monNode.get('type') === 'browser' ||
+        monNode.get('enabled') === false
+      ) {
+        continue;
+      }
+      const config = monNode.toJSON();
+      const { line, col } = lineCounter.linePos(monNode.srcToken.offset);
+      try {
+        const mon = buildMonitorFromYaml(config, options);
+        mon.setSource({ file, line, column: col });
+        monitors.push(mon);
+      } catch (e) {
+        let outer = bold(`Aborted: ${e}\n`);
+        outer += indent(
+          `* ${config.id || config.name} - ${file}:${line}:${col}\n`
+        );
+        throw red(outer);
+      }
+    }
+  }
+  return monitors;
+}
+
+const REQUIRED_MONITOR_FIELDS = ['id', 'name'];
+export function buildMonitorFromYaml(
+  config: MonitorConfig,
+  options: PushOptions
+) {
+  // Validate required fields
+  for (const field of REQUIRED_MONITOR_FIELDS) {
+    if (!config[field]) {
+      throw `Monitor ${field} is required`;
+    }
+  }
+  const schedule = config.schedule && parseSchedule(String(config.schedule));
+  const privateLocations =
+    config['private_locations'] || options.privateLocations;
+  delete config['private_locations'];
+
+  return new Monitor({
+    locations: options.locations,
+    ...config,
+    privateLocations,
+    schedule: schedule || options.schedule,
+  });
+}
+
+export function parseSchedule(schedule: string) {
+  const EVERY_SYNTAX = '@every';
+  if (!(schedule + '').startsWith(EVERY_SYNTAX)) {
+    throw `Monitor schedule format(${schedule}) not supported: use '@every' syntax instead`;
+  }
+
+  const duration = schedule.substring(EVERY_SYNTAX.length + 1);
+  // split between non-digit (\D) and a digit (\d)
+  const durations = duration.split(/(?<=\D)(?=\d)/g);
+  let minutes = 0;
+  for (const dur of durations) {
+    // split between a digit and non-digit
+    const [value, format] = dur.split(/(?<=\d)(?=\D)/g);
+    // Calculate based on the duration symbol
+    const scheduleValue = parseInt(value, 10);
+    switch (format) {
+      case 's':
+        minutes += Math.ceil(scheduleValue / 60);
+        break;
+      case 'm':
+        minutes += scheduleValue;
+        break;
+      case 'h':
+        minutes += scheduleValue * 60;
+        break;
+    }
+  }
+  return minutes;
+}
+
 export async function createMonitors(
   monitors: MonitorSchema[],
   options: PushOptions,

package/src/push/plugin.ts

@@ -27,8 +27,12 @@ import { isAbsolute, dirname, extname, join } from 'path';
 import fs from 'fs/promises';
 import * as esbuild from 'esbuild';
 
-const SOURCE_DIR = join(__dirname, '..', '..');
-const SOURCE_NODE_MODULES = join(SOURCE_DIR, 'node_modules');
+// ROOT directory of the Package - /
+const ROOT_DIR = join(__dirname, '..', '..');
+// Source of the package - /src, /dist, etc.
+const SOURCE_DIR = join(__dirname, '..');
+// Node modules directory of the package - /node_modules
+const SOURCE_NODE_MODULES = join(ROOT_DIR, 'node_modules');
 
 export function commonOptions(): esbuild.BuildOptions {
   return {

package/src/push/request.ts

@@ -23,7 +23,7 @@
  *
  */
 
-import { bold } from 'kleur/colors';
+import { bold, red, yellow } from 'kleur/colors';
 import { Dispatcher, request } from 'undici';
 import { indent, symbols } from '../helpers';
 
@@ -47,7 +47,7 @@ export async function sendRequest(options: APIRequestOptions) {
       'user-agent': `Elastic/Synthetics ${version}`,
       'kbn-xsrf': 'true',
     },
-    headersTimeout: 60 * 1000
+    headersTimeout: 60 * 1000,
   });
 }
 
@@ -62,8 +62,10 @@ export type APIMonitorError = {
 };
 
 export function formatNotFoundError(message: string) {
-  return bold(
-    `${symbols['failed']} Please check your kibana url and try again - 404:${message}`
+  return red(
+    bold(
+      `${symbols['failed']} Please check your kibana url and try again - 404:${message}`
+    )
   );
 }
 
@@ -78,7 +80,7 @@ export function formatAPIError(
   );
   inner += indent(message, '    ');
   outer += indent(inner);
-  return outer;
+  return red(outer);
 }
 
 function formatMonitorError(errors: APIMonitorError[]) {
@@ -95,10 +97,10 @@ function formatMonitorError(errors: APIMonitorError[]) {
 
 export function formatFailedMonitors(errors: APIMonitorError[]) {
   const heading = bold(`${symbols['failed']} Error\n`);
-  return heading + formatMonitorError(errors);
+  return red(heading + formatMonitorError(errors));
 }
 
 export function formatStaleMonitors(errors: APIMonitorError[]) {
   const heading = bold(`${symbols['warning']} Warnings\n`);
-  return heading + formatMonitorError(errors);
+  return yellow(heading + formatMonitorError(errors));
 }