@elastic/synthetics 1.17.2 → 1.18.1

package/src/push/bundler.ts

@@ -24,25 +24,19 @@
  */
 
 import path from 'path';
-import { stat, unlink, readFile } from 'fs/promises';
+import { unlink, readFile } from 'fs/promises';
 import { createWriteStream } from 'fs';
 import * as esbuild from 'esbuild';
 import archiver from 'archiver';
 import { commonOptions } from '../core/transform';
 import { SyntheticsBundlePlugin } from './plugin';
 
-// 1500KB Max Gzipped limit for bundled code to be pushed as Kibana project monitors.
-const SIZE_LIMIT_KB = 1500;
-
 function relativeToCwd(entry: string) {
   return path.relative(process.cwd(), entry);
 }
 
 export class Bundler {
-  moduleMap = new Map<string, string>();
-  constructor() {}
-
-  async prepare(absPath: string) {
+  async bundle(absPath: string) {
     const options: esbuild.BuildOptions = {
       ...commonOptions(),
       ...{
@@ -59,56 +53,42 @@ export class Bundler {
     if (result.errors.length > 0) {
       throw result.errors;
     }
-    this.moduleMap.set(absPath, result.outputFiles[0].text);
+    return result.outputFiles[0].text;
   }
 
-  async zip(outputPath: string) {
+  async zip(source: string, code: string, dest: string) {
     return new Promise((fulfill, reject) => {
-      const output = createWriteStream(outputPath);
+      const output = createWriteStream(dest);
       const archive = archiver('zip', {
         zlib: { level: 9 },
       });
       archive.on('error', reject);
       output.on('close', fulfill);
       archive.pipe(output);
-      for (const [path, content] of this.moduleMap.entries()) {
-        const relativePath = relativeToCwd(path);
-        // Date is fixed to Unix epoch so the file metadata is
-        // not modified everytime when files are bundled
-        archive.append(content, {
-          name: relativePath,
-          date: new Date('1970-01-01'),
-        });
-      }
+      const relativePath = relativeToCwd(source);
+      // Date is fixed to Unix epoch so the file metadata is
+      // not modified everytime when files are bundled
+      archive.append(code, {
+        name: relativePath,
+        date: new Date('1970-01-01'),
+      });
       archive.finalize();
     });
   }
 
   async build(entry: string, output: string) {
-    await this.prepare(entry);
-    await this.zip(output);
-    const data = await this.encode(output);
-    await this.checkSize(output);
+    const code = await this.bundle(entry);
+    await this.zip(entry, code, output);
+    const content = await this.encode(output);
     await this.cleanup(output);
-    return data;
+    return content;
   }
 
   async encode(outputPath: string) {
     return await readFile(outputPath, 'base64');
   }
 
-  async checkSize(outputPath: string) {
-    const { size } = await stat(outputPath);
-    const sizeKb = size / 1024;
-    if (sizeKb > SIZE_LIMIT_KB) {
-      throw new Error(
-        `Bundled monitor code exceeds the recommended ${SIZE_LIMIT_KB}KB limit. Please check your dependencies and try again.`
-      );
-    }
-  }
-
   async cleanup(outputPath: string) {
-    this.moduleMap = new Map<string, string>();
     await unlink(outputPath);
   }
 }

package/src/push/index.ts

@@ -24,7 +24,7 @@
  */
 import { readFile, writeFile } from 'fs/promises';
 import { prompt } from 'enquirer';
-import { bold, grey } from 'kleur/colors';
+import { bold, underline } from 'kleur/colors';
 import {
   getLocalMonitors,
   buildMonitorSchema,
@@ -50,15 +50,20 @@ import {
   bulkGetMonitors,
   bulkPutMonitors,
   createMonitorsLegacy,
-  CHUNK_SIZE,
+  BATCH_SIZE,
+  MAX_PAYLOAD_SIZE_KB,
 } from './kibana_api';
 import {
-  getChunks,
+  getBatches,
+  getSizedBatches,
   isBulkAPISupported,
   isLightweightMonitorSupported,
   logDiff,
+  logGroups,
+  printBytes,
 } from './utils';
-import { log } from '../core/logger';
+import { runLocal } from './run-local';
+import { inDebugMode } from '../core/globals';
 
 export async function push(monitors: Monitor[], options: PushOptions) {
   if (parseInt(process.env.CHUNK_SIZE) > 250) {
@@ -84,7 +89,7 @@ export async function push(monitors: Monitor[], options: PushOptions) {
   const { monitors: remote } = await bulkGetMonitors(options);
 
   progress(`preparing ${monitors.length} monitors`);
-  const schemas = await buildMonitorSchema(monitors, true);
+  const { schemas, sizes } = await buildMonitorSchema(monitors, true);
   const local = getLocalMonitors(schemas);
 
   const { newIDs, changedIDs, removedIDs, unchangedIDs } = diffMonitorHashIDs(
@@ -92,21 +97,47 @@ export async function push(monitors: Monitor[], options: PushOptions) {
     remote
   );
   logDiff(newIDs, changedIDs, removedIDs, unchangedIDs);
+  if (inDebugMode()) {
+    logGroups(sizes, newIDs, changedIDs, removedIDs, unchangedIDs);
+    // show bundle size for the whole project
+    let totalSize = 0;
+    for (const value of sizes.values()) {
+      totalSize += value;
+    }
+    progress(
+      `total size of the ${sizes.size} monitors payload is ` +
+        printBytes(totalSize)
+    );
+  }
+
+  if (options.dryRun) {
+    progress('Running browser monitors in dry run mode');
+    await runLocal(schemas);
+    progress('Dry run completed');
+    return;
+  }
 
   const updatedMonitors = new Set<string>([...changedIDs, ...newIDs]);
   if (updatedMonitors.size > 0) {
     const updatedMonSchemas = schemas.filter(s => updatedMonitors.has(s.id));
-    const chunks = getChunks(updatedMonSchemas, CHUNK_SIZE);
-    for (const chunk of chunks) {
+    const batches = getSizedBatches(
+      updatedMonSchemas,
+      sizes,
+      MAX_PAYLOAD_SIZE_KB,
+      BATCH_SIZE
+    );
+    if (batches.length > 1) {
+      progress(`Monitors will be pushed as ${batches.length} batches.`);
+    }
+    for (const batch of batches) {
       await liveProgress(
-        bulkPutMonitors(options, chunk),
-        `creating or updating ${chunk.length} monitors`
+        bulkPutMonitors(options, batch),
+        `creating or updating ${batch.length} monitors`
       );
     }
   }
 
   if (removedIDs.size > 0) {
-    log(`deleting monitor ids: ${Array.from(removedIDs.keys()).join(', ')}`);
     if (updatedMonitors.size === 0 && unchangedIDs.size === 0) {
       await confirmDelete(
         `Pushing without any monitors will delete all monitors associated with the project.\n Do you want to continue?`,
@@ -118,16 +149,15 @@ export async function push(monitors: Monitor[], options: PushOptions) {
         options.yes
       );
     }
-    const chunks = getChunks(Array.from(removedIDs), CHUNK_SIZE);
-    for (const chunk of chunks) {
+    const batches = getBatches(Array.from(removedIDs), BATCH_SIZE);
+    for (const batch of batches) {
       await liveProgress(
-        bulkDeleteMonitors(options, chunk),
-        `deleting ${chunk.length} monitors`
+        bulkDeleteMonitors(options, batch),
+        `deleting ${batch.length} monitors`
       );
     }
   }
-
-  done(`Pushed: ${grey(getMonitorManagementURL(options.url))}`);
+  done(`Pushed: ${underline(getMonitorManagementURL(options.url))} `);
 }
 
 async function confirmDelete(message: string, skip: boolean) {
@@ -223,8 +253,9 @@ export function validateSettings(opts: PushOptions) {
   - CLI '--schedule <mins>'
   - Config file 'monitors.schedule' field`;
   } else if (opts.schedule && !ALLOWED_SCHEDULES.includes(opts.schedule)) {
-    reason = `Set default schedule(${opts.schedule
-      }) to one of the allowed values - ${ALLOWED_SCHEDULES.join(',')}`;
+    reason = `Set default schedule(${
+      opts.schedule
+    }) to one of the allowed values - ${ALLOWED_SCHEDULES.join(',')}`;
   } else if (
     (opts.locations ?? []).length > 0 &&
     (opts?.playwrightOptions?.clientCertificates ?? []).filter(cert => {
@@ -299,14 +330,18 @@ export async function pushLegacy(monitors: Monitor[], options: PushOptions) {
   }
 
   let schemas: MonitorSchema[] = [];
+  let sizes: Map<string, number> = new Map();
   if (monitors.length > 0) {
     progress(`preparing ${monitors.length} monitors`);
-    schemas = await buildMonitorSchema(monitors, false);
-    const chunks = getChunks(schemas, 10);
-    for (const chunk of chunks) {
+    ({ schemas, sizes } = await buildMonitorSchema(monitors, false));
+    const batches = getSizedBatches(schemas, sizes, MAX_PAYLOAD_SIZE_KB, 10);
+    if (batches.length > 1) {
+      progress(`Monitors will be pushed as ${batches.length} batches.`);
+    }
+    for (const batch of batches) {
       await liveProgress(
-        createMonitorsLegacy({ schemas: chunk, keepStale: true, options }),
-        `creating or updating ${chunk.length} monitors`
+        createMonitorsLegacy({ schemas: batch, keepStale: true, options }),
+        `creating or updating ${batch.length} monitors`
       );
     }
   } else {
@@ -320,7 +355,7 @@ export async function pushLegacy(monitors: Monitor[], options: PushOptions) {
     `deleting all stale monitors`
   );
 
-  done(`Pushed: ${grey(getMonitorManagementURL(options.url))}`);
+  done(`Pushed: ${underline(getMonitorManagementURL(options.url))}`);
 }
 
 // prints warning if any of the monitors has throttling settings enabled during push

package/src/push/kibana_api.ts

@@ -36,8 +36,9 @@ import {
 } from './request';
 import { generateURL } from './utils';
 
-// Default chunk size for bulk put / delete
-export const CHUNK_SIZE = parseInt(process.env.CHUNK_SIZE) || 100;
+// Default batch size for bulk put / delete
+export const BATCH_SIZE = parseInt(process.env.CHUNK_SIZE) || 250;
+export const MAX_PAYLOAD_SIZE_KB = 50 * 1000; // 50 MB in KB
 
 export type PutResponse = {
   createdMonitors: string[];
@@ -49,12 +50,15 @@ export async function bulkPutMonitors(
   options: PushOptions,
   schemas: MonitorSchema[]
 ) {
+  const url = generateURL(options, 'bulk_update') + '/_bulk_update';
+
   const resp = await sendReqAndHandleError<PutResponse>({
-    url: generateURL(options, 'bulk_update') + '/_bulk_update',
+    url,
     method: 'PUT',
     auth: options.auth,
     body: JSON.stringify({ monitors: schemas }),
   });
+
   const { failedMonitors } = resp;
   if (failedMonitors && failedMonitors.length > 0) {
     throw formatFailedMonitors(failedMonitors);

package/src/push/monitor.ts

@@ -42,12 +42,15 @@ import { isParamOptionSupported, normalizeMonitorName } from './utils';
 
 // Allowed extensions for lightweight monitor files
 const ALLOWED_LW_EXTENSIONS = ['.yml', '.yaml'];
+// 1500kB Max Gzipped limit for bundled monitor code to be pushed as Kibana project monitors.
+const SIZE_LIMIT_KB = 1500;
 
 export type MonitorSchema = Omit<MonitorConfig, 'locations'> & {
   locations: string[];
   content?: string;
   filter?: Monitor['filter'];
   hash?: string;
+  size?: number;
 };
 
 // Abbreviated monitor info, as often returned by the API,
@@ -131,6 +134,7 @@ export async function buildMonitorSchema(monitors: Monitor[], isV2: boolean) {
   await mkdir(bundlePath, { recursive: true });
   const bundler = new Bundler();
   const schemas: MonitorSchema[] = [];
+  const sizes: Map<string, number> = new Map();
 
   for (const monitor of monitors) {
     const { source, config, filter, type } = monitor;
@@ -148,6 +152,17 @@ export async function buildMonitorSchema(monitors: Monitor[], isV2: boolean) {
       monitor.setContent(content);
       Object.assign(schema, { content, filter });
     }
+    const size = monitor.size();
+    const sizeKB = Math.round(size / 1000);
+    if (sizeKB > SIZE_LIMIT_KB) {
+      let outer = bold(
+        `Aborted: Bundled code ${sizeKB}kB exceeds the recommended ${SIZE_LIMIT_KB}kB limit. Please check the dependencies imported.\n`
+      );
+      const inner = `* ${config.id} - ${source.file}:${source.line}:${source.column}\n`;
+      outer += indent(inner);
+      throw red(outer);
+    }
+    sizes.set(config.id, size);
     /**
      * Generate hash only after the bundled content is created
      * to capture code changes in imported files
@@ -159,7 +174,7 @@ export async function buildMonitorSchema(monitors: Monitor[], isV2: boolean) {
   }
 
   await rm(bundlePath, { recursive: true });
-  return schemas;
+  return { schemas, sizes };
 }
 
 export async function createLightweightMonitors(
@@ -225,9 +240,7 @@ export async function createLightweightMonitors(
       const monitor = mergedConfig[i];
       // Skip browser monitors from the YML files
       if (monitor['type'] === 'browser') {
-        warn(
-          `Browser monitors from ${file} are skipped.`
-        );
+        warn(`Browser monitors from ${file} are skipped.`);
         continue;
       }
       const { line, col } = lineCounter.linePos(offsets[i]);
@@ -288,7 +301,7 @@ export function buildMonitorFromYaml(
   });
 
   /**
-   * Params support is only available for lighweight monitors
+   * Params support is only available for lightweight monitors
    * post 8.7.2 stack
    */
   if (isParamOptionSupported(options.kibanaVersion)) {

package/src/push/request.ts

@@ -58,8 +58,14 @@ export async function sendReqAndHandleError<T>(
   options: APIRequestOptions
 ): Promise<T> {
   const { statusCode, body } = await sendRequest(options);
+
   return (
-    await handleError(statusCode, options.url, body)
+    await handleError(
+      statusCode,
+      options.url,
+      body,
+      statusCode === 413 ? `${options.body?.length} bytes sent` : ''
+    )
   ).json() as Promise<T>;
 }
 
@@ -74,7 +80,8 @@ type APIError = {
 export async function handleError(
   statusCode: number,
   url: string,
-  body: Dispatcher.ResponseData['body']
+  body: Dispatcher.ResponseData['body'],
+  extraMessage?: string
 ): Promise<Dispatcher.ResponseData['body']> {
   if (statusCode === 404) {
     throw formatNotFoundError(url, await body.text());
@@ -84,9 +91,19 @@ export async function handleError(
       const resp = await body.text();
       parsed = JSON.parse(resp) as APIError;
     } catch (e) {
-      throw formatAPIError(statusCode, 'unexpected error', e.message);
+      throw formatAPIError(
+        statusCode,
+        'unexpected error',
+        e.message,
+        extraMessage
+      );
     }
-    throw formatAPIError(statusCode, parsed.error, parsed.message);
+    throw formatAPIError(
+      statusCode,
+      parsed.error,
+      parsed.message,
+      extraMessage
+    );
   }
 
   return body;
@@ -111,16 +128,18 @@ export function formatNotFoundError(url: string, message: string) {
 }
 
 export function formatAPIError(
-  statuCode: number,
+  statusCode: number,
   error: string,
-  message: string
+  message: string,
+  extraMessage = ''
 ) {
   let outer = bold(`${symbols['failed']} Error\n`);
   let inner = bold(
-    `${symbols['failed']} monitor creation failed - ${statuCode}:${error}\n`
+    `${symbols['failed']} monitor creation failed - ${statusCode}:${error}\n`
   );
   inner += indent(message, '    ');
   outer += indent(inner);
+  outer += extraMessage ? indent(extraMessage, '    ') : '';
   return red(outer);
 }
 

package/src/push/run-local.ts

@@ -0,0 +1,155 @@
+/**
+ * MIT License
+ *
+ * Copyright (c) 2020-present, Elastic NV
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ *
+ */
+
+import { execFileSync, spawn } from 'child_process';
+import { rm, writeFile } from 'fs/promises';
+import { createReadStream } from 'fs';
+import { tmpdir } from 'os';
+import { Extract } from 'unzip-stream';
+import { red } from 'kleur/colors';
+import { join } from 'path';
+import { pathToFileURL } from 'url';
+import { MonitorSchema } from './monitor';
+
+async function unzipFile(zipPath, destination) {
+  return new Promise<void>((resolve, reject) => {
+    createReadStream(zipPath)
+      .pipe(Extract({ path: destination }))
+      .on('close', resolve)
+      .on('error', err =>
+        reject(new Error(`failed to extract zip ${zipPath} : ${err.message}`))
+      );
+  });
+}
+
+async function runNpmInstall(directory) {
+  return new Promise<void>((resolve, reject) => {
+    const flags = [
+      '--no-audit', // Prevent audit checks
+      '--no-update-notifier', // Prevent update checks
+      '--no-fund', // No need for package funding messages here
+      '--package-lock=false', // no need to write package lock here
+      '--progress=false', // no need to display progress
+    ];
+
+    const npmInstall = spawn('npm', ['install', ...flags], {
+      cwd: directory,
+      stdio: 'ignore',
+    });
+    npmInstall.on('close', code => {
+      if (code === 0) {
+        resolve();
+      } else {
+        reject(new Error(`npm install failed with exit code ${code}`));
+      }
+    });
+    npmInstall.on('error', err =>
+      reject(new Error(`failed to setup: ${err.message}`))
+    );
+  });
+}
+
+async function runTest(directory, schema: MonitorSchema) {
+  return new Promise<void>((resolve, reject) => {
+    const runTest = spawn(
+      'npx',
+      [
+        '@elastic/synthetics',
+        '.',
+        '--playwright-options',
+        JSON.stringify(schema.playwrightOptions),
+        '--params',
+        JSON.stringify(schema.params),
+      ],
+      {
+        cwd: directory,
+        stdio: 'inherit',
+      }
+    );
+
+    runTest.on('close', resolve);
+    runTest.on('error', err => {
+      reject(
+        new Error(`Failed to execute @elastic/synthetics : ${err.message}`)
+      );
+    });
+  });
+}
+
+async function writePkgJSON(dir: string, synthPath: string) {
+  const packageJsonContent = {
+    name: 'project-journey',
+    private: 'true',
+    dependencies: {
+      '@elastic/synthetics': pathToFileURL(synthPath),
+    },
+  };
+  await writeFile(
+    join(dir, 'package.json'),
+    JSON.stringify(packageJsonContent, null, 2),
+    'utf-8'
+  );
+}
+
+async function extract(
+  schema: MonitorSchema,
+  zipPath: string,
+  unzipPath: string
+) {
+  if (schema.type !== 'browser') {
+    return;
+  }
+  const content = schema.content;
+  await writeFile(zipPath, content, 'base64');
+  await unzipFile(zipPath, unzipPath);
+}
+
+export async function runLocal(schemas: MonitorSchema[]) {
+  // lookup installed bin path of a node module
+  const resolvedPath = execFileSync('which', ['elastic-synthetics'], {
+    encoding: 'utf8',
+  }).trim();
+  const synthPath = resolvedPath.replace(
+    join('bin', 'elastic-synthetics'),
+    join('lib', 'node_modules', '@elastic/synthetics')
+  );
+  const rand = Date.now();
+  const zipPath = join(tmpdir(), `synthetics-zip-${rand}.zip`);
+  const unzipPath = join(tmpdir(), `synthetics-unzip-${rand}`);
+  try {
+    for (const schema of schemas) {
+      await extract(schema, zipPath, unzipPath);
+    }
+    await writePkgJSON(unzipPath, synthPath);
+    await runNpmInstall(unzipPath);
+    // TODO: figure out a way to collect all params and Playwright options
+    await runTest(unzipPath, schemas[0]);
+  } catch (e) {
+    throw red(`Aborted: ${e.message}`);
+  } finally {
+    await rm(zipPath, { recursive: true, force: true });
+    await rm(unzipPath, { recursive: true, force: true });
+  }
+}