@elastic/synthetics 1.0.0-beta.39 → 1.0.0-beta.40

package/src/common_types.ts

@@ -209,7 +209,7 @@ type BaseArgs = {
   ignoreHttpsErrors?: boolean;
   playwrightOptions?: PlaywrightOptions;
   quietExitCode?: boolean;
-  throttling?: ThrottlingOptions;
+  throttling?: MonitorConfig['throttling'];
   schedule?: MonitorConfig['schedule'];
   locations?: MonitorConfig['locations'];
   privateLocations?: MonitorConfig['privateLocations'];
@@ -226,7 +226,6 @@ export type CliArgs = BaseArgs & {
   richEvents?: boolean;
   capability?: Array<string>;
   ignoreHttpsErrors?: boolean;
-  throttling?: boolean | ThrottlingOptions;
 };
 
 export type RunOptions = BaseArgs & {

package/src/core/runner.ts

@@ -397,15 +397,12 @@ export default class Runner {
     });
 
     const { match, tags } = options;
-
     const monitors: Monitor[] = [];
     for (const journey of this.journeys) {
       if (!journey.isMatch(match, tags)) {
         continue;
       }
-
       this.#currentJourney = journey;
-
       /**
        * Execute dummy callback to get all monitor specific
        * configurations for the current journey

package/src/core/transform.ts

@@ -0,0 +1,160 @@
+/**
+ * MIT License
+ *
+ * Copyright (c) 2020-present, Elastic NV
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ *
+ */
+
+import { addHook } from 'pirates';
+import {
+  transformSync,
+  Loader,
+  CommonOptions,
+  TransformOptions,
+} from 'esbuild';
+import path from 'path';
+import sourceMapSupport from 'source-map-support';
+
+// Cache that holds the sourcemap content for each file
+const sourceMaps: Map<string, string> = new Map();
+
+// Register the source-map-support library to resolve the sourcemaps
+// for the files that are transpiled by esbuild
+/**
+ * Register the source-map-support library to resolve the sourcemaps
+ * for the files that are transpiled by esbuild and cache the sourcemaps
+ * for each file.
+ *
+ * Caching the sourcemap file is important, as it is used by the Synthetics agent
+ * to resolve the stack traces of journey, monitor and step functions.
+ */
+sourceMapSupport.install({
+  environment: 'node',
+  handleUncaughtExceptions: false,
+  retrieveSourceMap(source) {
+    if (!sourceMaps.has(source)) return null;
+    return {
+      map: JSON.parse(sourceMaps.get(source)),
+      url: source,
+    };
+  },
+});
+
+/**
+ * Default list of files and corresponding loaders we support
+ * while pushing project based monitors
+ */
+const LOADERS: Record<string, Loader> = {
+  '.ts': 'ts',
+  '.js': 'js',
+  '.mjs': 'js',
+  '.cjs': 'js',
+};
+
+const getLoader = (filename: string) => {
+  const ext = path.extname(filename);
+  return LOADERS[ext] || 'default';
+};
+
+export function commonOptions(): CommonOptions {
+  /**
+   * We are not minifying the code as we want it to be in sync with previous
+   * transformation phase and introduce it later.
+   */
+  return {
+    minify: false,
+    minifyIdentifiers: false,
+    minifySyntax: false,
+    minifyWhitespace: false,
+    sourcemap: 'both',
+    sourcesContent: false,
+    platform: 'node',
+    logLevel: 'silent',
+    format: 'cjs',
+    target: `node${process.version.slice(1)}`,
+  };
+}
+
+/**
+ * Transform the given code using esbuild and save the corresponding
+ * map file in memory to be retrived later.
+ */
+export function transform(
+  code: string,
+  filename: string,
+  options: TransformOptions = {}
+) {
+  const result = transformSync(code, {
+    ...commonOptions(),
+    sourcefile: filename,
+    loader: getLoader(filename),
+    /**
+     * Add this only for the transformation phase, using it on
+     * bundling phase would disable tree shaking and uncessary bloat
+     *
+     * Ensures backwards compatability with tsc's implicit strict behaviour
+     */
+    tsconfigRaw: {
+      compilerOptions: {
+        alwaysStrict: true,
+      },
+    },
+    ...options,
+  });
+
+  const warnings = result.warnings;
+  if (warnings && warnings.length > 0) {
+    for (const warning of warnings) {
+      console.log(warning.location);
+      console.log(warning.text);
+    }
+  }
+
+  /**
+   * Cache the sourcemap contents in memory, so we can look it up
+   * later when we try to resolve the sourcemap for a given file.
+   *
+   * Every time the synthetics agent is started, we register the source-map-support
+   * library to resolve the sourcemaps for the files that are transpiled by esbuild.
+   */
+  if (result.map) {
+    sourceMaps.set(filename, result.map);
+  }
+  return result;
+}
+
+/**
+ * Install the pirates hook to transform the code on the fly
+ * for all of the imported files.
+ */
+export function installTransform() {
+  const revertPirates = addHook(
+    (source: string, filename: string) => {
+      const { code } = transform(source, filename);
+      return code;
+    },
+    { exts: ['.ts', '.js', '.mjs', '.cjs'] }
+  );
+
+  return () => {
+    revertPirates();
+  };
+}

package/src/dsl/journey.ts

@@ -97,6 +97,7 @@ export class Journey {
       ...config,
     });
     this.monitor.setSource(this.location);
+    this.monitor.setContent(this.callback.toString());
     this.monitor.setFilter({ match: this.name });
   }
 

package/src/dsl/monitor.ts

@@ -23,6 +23,7 @@
  *
  */
 
+import { createHash } from 'crypto';
 import merge from 'deepmerge';
 import { bold, red } from 'kleur/colors';
 import {
@@ -50,7 +51,7 @@ export type MonitorConfig = {
   enabled?: boolean;
   locations?: SyntheticsLocationsType[];
   privateLocations?: string[];
-  throttling?: ThrottlingOptions;
+  throttling?: boolean | ThrottlingOptions;
   screenshot?: ScreenshotOptions;
   params?: Params;
   playwrightOptions?: PlaywrightOptions;
@@ -62,6 +63,7 @@ type MonitorFilter = {
 };
 
 export class Monitor {
+  content?: string;
   source?: Location;
   filter: MonitorFilter;
   constructor(public config: MonitorConfig = {}) {}
@@ -84,6 +86,14 @@ export class Monitor {
   setSource(source: Location) {
     this.source = source;
   }
+
+  /**
+   * The underlying journey code of the monitor
+   */
+  setContent(content = '') {
+    this.content = content;
+  }
+
   /**
    * If journey files are colocated within the same file during
    * push command, when we invoke synthetics from HB we rely on
@@ -94,6 +104,22 @@ export class Monitor {
     this.filter = filter;
   }
 
+  /**
+   * Hash is used to identify if the monitor has changed since the last time
+   * it was pushed to Kibana. Change is based on three factors:
+   * - Monitor configuration
+   * - Code changes
+   * - File path changes
+   */
+  hash(): string {
+    const hash = createHash('sha256');
+    return hash
+      .update(JSON.stringify(this.config))
+      .update(this.content || '')
+      .update(this.source?.file || '')
+      .digest('base64');
+  }
+
   validate() {
     const schedule = this.config.schedule;
     if (ALLOWED_SCHEDULES.includes(schedule)) {

package/src/generator/index.ts

@@ -64,9 +64,6 @@ export const REGULAR_FILES_PATH = [
 ];
 export const CONFIG_PATH = 'synthetics.config.ts';
 
-// Files to be overriden by default if the project is initialized multiple times
-const DEFAULT_OVERRIDES = [CONFIG_PATH];
-
 export class Generator {
   pkgManager = 'npm';
   constructor(public projectDir: string) {}
@@ -197,11 +194,7 @@ export class Generator {
   async createFile(relativePath: string, content: string, override = false) {
     const absolutePath = join(this.projectDir, relativePath);
 
-    if (
-      !override &&
-      !DEFAULT_OVERRIDES.includes(relativePath) &&
-      existsSync(absolutePath)
-    ) {
+    if (!override && existsSync(absolutePath)) {
       const { override } = await prompt<{ override: boolean }>({
         type: 'confirm',
         name: 'override',
@@ -252,10 +245,13 @@ export class Generator {
       pkgJSON.scripts = {};
     }
     // Add test command
-    pkgJSON.scripts.test = 'npx @elastic/synthetics journeys';
-
+    if (!pkgJSON.scripts.test) {
+      pkgJSON.scripts.test = 'npx @elastic/synthetics journeys';
+    }
     // Add push command
-    pkgJSON.scripts.push = 'npx @elastic/synthetics push';
+    if (!pkgJSON.scripts.push) {
+      pkgJSON.scripts.push = 'npx @elastic/synthetics push';
+    }
 
     await this.createFile(
       filename,
@@ -274,7 +270,9 @@ export class Generator {
     if (!gitIgnore.includes('node_modules')) {
       gitIgnore += 'node_modules/\n';
     }
-    gitIgnore += '.synthetics/\n';
+    if (!gitIgnore.includes('.synthetics')) {
+      gitIgnore += '.synthetics/\n';
+    }
     await writeFile(gitIgnorePath, gitIgnore, 'utf-8');
   }
 

package/src/helpers.ts

@@ -29,7 +29,7 @@ import { resolve, join, dirname } from 'path';
 import fs from 'fs';
 import { lstat, readdir } from 'fs/promises';
 import { performance } from 'perf_hooks';
-import sourceMapSupport from '@cspotcode/source-map-support';
+import sourceMapSupport from 'source-map-support';
 import {
   HooksArgs,
   HooksCallback,
@@ -233,10 +233,21 @@ export function rewriteErrorStack(stack: string, indexes: [number, number]) {
   return stack;
 }
 
-export function formatError(error: Error) {
-  if (!(error instanceof Error)) {
+// formatError prefers receiving proper Errors, but since at runtime
+// non Error exceptions can be thrown, it tolerates though. The
+// redundant type Error | any expresses that.
+export function formatError(error: Error | any) {
+  if (error === undefined || error === null) {
     return;
   }
+
+  if (!(error instanceof Error)) {
+    return {
+      message: `Error "${error}" received, with type "${typeof error}". (Do not throw exceptions without using \`new Error("my message")\`)`,
+      name: '',
+      stack: '',
+    };
+  }
   const { name, message, stack } = error;
   const indexes = findPWLogsIndexes(message);
   return {
@@ -300,8 +311,8 @@ export function wrapFnWithLocation<A extends unknown[], R>(
     const _prepareStackTrace = Error.prepareStackTrace;
     Error.prepareStackTrace = (_, stackFrames) => {
       // Deafult CallSite would not map to the original transpiled source
-      // from ts-node properly, So we wrap it with the library that knows
-      // how to retrive those source-map for the transpiled code
+      // correctly, So we use source-map-support to map the CallSite to the
+      // original source from our cached source map
       const frame: NodeJS.CallSite = sourceMapSupport.wrapCallSite(
         stackFrames[1]
       );
@@ -339,16 +350,27 @@ export function safeNDJSONParse(data: string | string[]) {
 }
 
 // Console helpers
-export function write(message: string) {
-  process.stderr.write(message + '\n');
+export function write(message: string, live?: boolean) {
+  process.stderr.write(message + (live ? '\r' : '\n'));
 }
 
 export function progress(message: string) {
   write(cyan(bold(`${symbols.progress} ${message}`)));
 }
 
-export function apiProgress(message: string) {
-  write(grey(`> ${message}`));
+export async function liveProgress(promise: Promise<any>, message: string) {
+  const start = now();
+  const interval = setInterval(() => {
+    apiProgress(`${message} (${Math.trunc(now() - start)}ms)`, true);
+  }, 500);
+  promise.finally(() => clearInterval(interval));
+  const result = await promise;
+  apiProgress(`${message} (${Math.trunc(now() - start)}ms)`);
+  return result;
+}
+
+export function apiProgress(message: string, live = false) {
+  write(grey(`> ${message}`), live);
 }
 
 export function error(message: string) {
@@ -370,11 +392,3 @@ export function removeTrailingSlash(url: string) {
 export function getMonitorManagementURL(url) {
   return removeTrailingSlash(url) + '/app/uptime/manage-monitors/all';
 }
-
-export function getArrayChunks<T>(arr: T[], chunkSize: number): T[][] {
-  const chunks = [];
-  for (let i = 0; i < arr.length; i += chunkSize) {
-    chunks.push(arr.slice(i, i + chunkSize));
-  }
-  return chunks;
-}

package/src/loader.ts

@@ -35,29 +35,25 @@ import {
   totalist,
   findPkgJsonByTraversing,
 } from './helpers';
+import { installTransform } from './core/transform';
 
 const resolvedCwd = cwd();
 
-export async function loadTestFiles(options: CliArgs, args: string[]) {
-  /**
-   * Transform `.ts` files out of the box by invoking
-   * the `ts-node` via `transpile-only` mode that compiles
-   * TS files without doing any extensive type checks.
-   *
-   * We must register `ts-node` _before_ loading inline
-   * scripts too because otherwise we will not be able to
-   * require `.ts` configuration files.
-   */
-  /* eslint-disable-next-line @typescript-eslint/no-var-requires */
-  require('ts-node').register({
-    transpileOnly: true,
-    compilerOptions: {
-      esModuleInterop: true,
-      allowJs: true,
-      target: 'es2020',
-    },
-  });
+/**
+ * Perform global setup process required for running the test suites
+ * and also for bundling the monitors. The process includes
+ * - Transpiling the TS/JS test files
+ * - Loading these files for running test suites
+ */
+export async function globalSetup(options: CliArgs, args: string[]) {
+  const revert = installTransform();
+  await loadTestFiles(options, args);
+  return () => {
+    revert();
+  };
+}
 
+export async function loadTestFiles(options: CliArgs, args: string[]) {
   /**
    * Preload modules before running the tests
    */

package/src/locations/index.ts

@@ -23,13 +23,9 @@
  *
  */
 
-import {
-  formatAPIError,
-  formatNotFoundError,
-  sendRequest,
-  ok,
-} from '../push/request';
-import { indent, removeTrailingSlash, write } from '../helpers';
+import { sendReqAndHandleError } from '../push/request';
+import { indent, write } from '../helpers';
+import { generateURL } from '../push/utils';
 
 export type LocationCmdOptions = {
   auth: string;
@@ -49,21 +45,12 @@ export type LocationAPIResponse = {
 const PRIVATE_KEYWORD = '(private)';
 
 export async function getLocations(options: LocationCmdOptions) {
-  const url =
-    removeTrailingSlash(options.url) + '/internal/uptime/service/locations';
-  const { body, statusCode } = await sendRequest({
-    url,
+  const resp = await sendReqAndHandleError<LocationAPIResponse>({
+    url: generateURL(options, 'location'),
     method: 'GET',
     auth: options.auth,
   });
-  if (statusCode === 404) {
-    throw formatNotFoundError(await body.text());
-  }
-  if (!ok(statusCode)) {
-    const { error, message } = await body.json();
-    throw formatAPIError(statusCode, error, message);
-  }
-  return ((await body.json()) as LocationAPIResponse).locations;
+  return resp.locations;
 }
 
 export function formatLocations(locations: Array<LocationMetadata>) {

package/src/options.ts

@@ -123,16 +123,24 @@ export function normalizeOptions(cliArgs: CliArgs): RunOptions {
   if (cliArgs.throttling) {
     const throttleConfig = merge.all([
       DEFAULT_THROTTLING_OPTIONS,
-      monitor?.throttling || {},
-      cliArgs.throttling as ThrottlingOptions,
+      toObject(monitor?.throttling),
+      toObject(cliArgs.throttling),
     ]);
-    options.throttling = throttleConfig;
-    options.networkConditions = getNetworkConditions(throttleConfig);
+    if (monitor?.throttling !== false) {
+      options.throttling = throttleConfig;
+      options.networkConditions = getNetworkConditions(throttleConfig);
+    } else {
+      /**
+       * If the throttling is disabled via Project monitor config, use it a source
+       * of truth and disable it for pushing those monitors.
+       */
+      options.throttling = false;
+    }
   } else {
     /**
      * Do not apply throttling when `--no-throttling` flag is passed
      */
-    options.throttling = {};
+    options.throttling = false;
   }
 
   options.schedule = cliArgs.schedule ?? monitor?.schedule;
@@ -143,6 +151,14 @@ export function normalizeOptions(cliArgs: CliArgs): RunOptions {
   return options;
 }
 
+function toObject(value: boolean | Record<string, any>): Record<string, any> {
+  const defaulVal = {};
+  if (typeof value === 'boolean') {
+    return defaulVal;
+  }
+  return value || defaulVal;
+}
+
 /**
  * Parses the throttling CLI settings and also
  * adapts to the format to keep the backwards compatability

package/src/push/bundler.ts

@@ -27,9 +27,9 @@ import path from 'path';
 import { stat, unlink, readFile } from 'fs/promises';
 import { createWriteStream } from 'fs';
 import * as esbuild from 'esbuild';
-import NodeResolve from '@esbuild-plugins/node-resolve';
 import archiver from 'archiver';
-import { commonOptions, MultiAssetPlugin, PluginData } from './plugin';
+import { commonOptions } from '../core/transform';
+import { SyntheticsBundlePlugin } from './plugin';
 
 const SIZE_LIMIT_KB = 800;
 
@@ -42,22 +42,15 @@ export class Bundler {
   constructor() {}
 
   async prepare(absPath: string) {
-    const addToMap = (data: PluginData) => {
-      this.moduleMap.set(data.path, data.contents);
-    };
-
     const options: esbuild.BuildOptions = {
       ...commonOptions(),
       ...{
-        entryPoints: {
-          [absPath]: absPath,
-        },
-        plugins: [
-          MultiAssetPlugin(addToMap),
-          NodeResolve({
-            extensions: ['.ts', '.js'],
-          }),
-        ],
+        entryPoints: [absPath],
+        bundle: true,
+        write: false,
+        sourcemap: 'inline',
+        external: ['@elastic/synthetics'],
+        plugins: [SyntheticsBundlePlugin()],
       },
     };
     const result = await esbuild.build(options);