@elastic/synthetics 1.0.0-beta.32 → 1.0.0-beta.34

package/src/cli.ts

@@ -72,7 +72,7 @@ program
   .addOption(match)
   .addOption(params)
   .addOption(
-    new Option('--reporter <value>', `output repoter format`).choices(
+    new Option('--reporter <value>', `output reporter format`).choices(
       Object.keys(reporters)
     )
   )
@@ -152,9 +152,13 @@ program
   .description(
     'Push all journeys in the current directory to create monitors within the Kibana monitor management UI'
   )
-  .requiredOption(
-    '--auth <auth>',
-    'API key used for Kibana authentication(https://www.elastic.co/guide/en/kibana/master/api-keys.html).'
+  .addOption(
+    new Option(
+      '--auth <auth>',
+      'API key used for Kibana authentication(https://www.elastic.co/guide/en/kibana/master/api-keys.html).'
+    )
+      .env('SYNTHETICS_API_KEY')
+      .makeOptionMandatory(true)
   )
   .option(
     '--schedule <time-in-minutes>',
@@ -173,8 +177,8 @@ program
   )
   .option('--url <url>', 'Kibana URL to upload the monitors')
   .option(
-    '--project <id>',
-    'id that will be used for logically grouping monitors'
+    '--id <id>',
+    'project id that will be used for logically grouping monitors'
   )
   .option(
     '--space <space>',
@@ -187,8 +191,8 @@ program
   .addOption(playwrightOpts)
   .action(async (cmdOpts: PushOptions) => {
     try {
-      const settings = await loadSettings();
       await loadTestFiles({ inline: false }, [cwd()]);
+      const settings = await loadSettings();
       const options = normalizeOptions({
         ...program.opts(),
         ...settings,

package/src/common_types.ts

@@ -241,21 +241,25 @@ export type RunOptions = BaseArgs & {
   reporter?: BuiltInReporterName | ReporterInstance;
 };
 
-export type PushOptions = {
+export type PushOptions = ProjectSettings & {
   auth: string;
-  url: string;
-  project: string;
-  space: string;
   schedule?: MonitorConfig['schedule'];
   locations?: MonitorConfig['locations'];
   privateLocations?: MonitorConfig['privateLocations'];
 };
 
+export type ProjectSettings = {
+  id: string;
+  url: string;
+  space: string;
+};
+
 export type PlaywrightOptions = LaunchOptions & BrowserContextOptions;
 export type SyntheticsConfig = {
   params?: Params;
   playwrightOptions?: PlaywrightOptions;
   monitor?: MonitorConfig;
+  project?: ProjectSettings;
 };
 
 /** Runner Payload types */

package/src/config.ts

@@ -80,7 +80,7 @@ function getConfigFile(ext: string) {
   return 'synthetics.config' + ext;
 }
 
-function findSyntheticsConfig(resolvePath, cwd) {
+export function findSyntheticsConfig(resolvePath, cwd) {
   const configPath = ['.js', '.ts']
     .map(ext => resolve(resolvePath, getConfigFile(ext)))
     .find(isFile);

package/src/generator/index.ts

@@ -37,16 +37,11 @@ import {
   cloudIDToKibanaURL,
 } from './utils';
 import { formatLocations, getLocations, groupLocations } from '../locations';
+import type { ProjectSettings } from '../common_types';
 
 // Templates that are required for setting up new synthetics project
 const templateDir = join(__dirname, '..', '..', 'templates');
 
-export type ProjectSettings = {
-  url: string;
-  space: string;
-  project: string;
-};
-
 type PromptOptions = ProjectSettings & {
   locations: Array<string>;
   privateLocations: Array<string>;
@@ -61,11 +56,10 @@ export const REGULAR_FILES_PATH = [
   '.github/workflows/run-synthetics.yml',
   'README.md',
 ];
-export const SETTINGS_PATH = '.synthetics/project.json';
 export const CONFIG_PATH = 'synthetics.config.ts';
 
 // Files to be overriden by default if the project is initialized multiple times
-const DEFAULT_OVERRIDES = [SETTINGS_PATH, CONFIG_PATH];
+const DEFAULT_OVERRIDES = [CONFIG_PATH];
 
 export class Generator {
   pkgManager = 'npm';
@@ -123,8 +117,7 @@ export class Generator {
     const allLocations = await getLocations({ url, auth });
     const locChoices = formatLocations(allLocations);
     if (locChoices.length === 0) {
-      // TODO - Add link to docs
-      throw 'Follow the docs to set up your first private locations - <link>';
+      throw 'Follow the docs to set up your first private locations - https://www.elastic.co/guide/en/observability/current/uptime-set-up-choose-agent.html#private-locations';
     }
 
     const monitorQues = [
@@ -147,7 +140,7 @@ export class Generator {
       },
       {
         type: 'input',
-        name: 'project',
+        name: 'id',
         message: 'Choose project id to logically group monitors',
         initial: basename(this.projectDir),
       },
@@ -168,14 +161,6 @@ export class Generator {
   async files(answers: PromptOptions) {
     const fileMap = new Map<string, string>();
 
-    // Setup project.json
-    const projectJSON = {
-      url: answers.url,
-      space: answers.space,
-      project: answers.project,
-    };
-    fileMap.set(SETTINGS_PATH, JSON.stringify(projectJSON, null, 2) + '\n');
-
     // Setup Synthetics config file
     fileMap.set(
       CONFIG_PATH,
@@ -261,7 +246,7 @@ export class Generator {
     pkgJSON.scripts.test = 'npx @elastic/synthetics journeys';
 
     // Add push command
-    pkgJSON.scripts.push = `npx @elastic/synthetics push --auth $SYNTHETICS_API_KEY`;
+    pkgJSON.scripts.push = 'npx @elastic/synthetics push';
 
     await this.createFile(
       filename,

package/src/generator/utils.ts

@@ -52,6 +52,9 @@ export function replaceTemplates(input: string, values: Record<string, any>) {
       if (typeof finalValue == 'number') {
         return Number(finalValue);
       }
+      if (typeof finalValue == 'string') {
+        return `'${finalValue}'`;
+      }
       return finalValue;
     });
   }

package/src/helpers.ts

@@ -23,7 +23,7 @@
  *
  */
 
-import { red, green, yellow, cyan, bold } from 'kleur/colors';
+import { red, green, yellow, cyan, bold, grey } from 'kleur/colors';
 import os from 'os';
 import { resolve, join, dirname } from 'path';
 import fs from 'fs';
@@ -347,6 +347,10 @@ export function progress(message: string) {
   write(cyan(bold(`${symbols.progress} ${message}`)));
 }
 
+export function apiProgress(message: string) {
+  write(grey(`> ${message}`));
+}
+
 export function error(message: string) {
   write(red(message));
 }

package/src/push/bundler.ts

@@ -24,12 +24,15 @@
  */
 
 import path from 'path';
-import { unlink, readFile } from 'fs/promises';
+import { stat, unlink, readFile } from 'fs/promises';
 import { createWriteStream } from 'fs';
 import * as esbuild from 'esbuild';
+import NodeResolve from '@esbuild-plugins/node-resolve';
 import archiver from 'archiver';
 import { commonOptions, MultiAssetPlugin, PluginData } from './plugin';
 
+const SIZE_LIMIT_KB = 800;
+
 function relativeToCwd(entry: string) {
   return path.relative(process.cwd(), entry);
 }
@@ -49,7 +52,12 @@ export class Bundler {
         entryPoints: {
           [absPath]: absPath,
         },
-        plugins: [MultiAssetPlugin(addToMap)],
+        plugins: [
+          MultiAssetPlugin(addToMap),
+          NodeResolve({
+            extensions: ['.ts', '.js'],
+          }),
+        ],
       },
     };
     const result = await esbuild.build(options);
@@ -85,6 +93,7 @@ export class Bundler {
     await this.prepare(entry);
     await this.zip(output);
     const data = await this.encode(output);
+    await this.checkSize(output);
     await this.cleanup(output);
     return data;
   }
@@ -93,6 +102,16 @@ export class Bundler {
     return await readFile(outputPath, 'base64');
   }
 
+  async checkSize(outputPath: string) {
+    const { size } = await stat(outputPath);
+    const sizeKb = size / 1024;
+    if (sizeKb > SIZE_LIMIT_KB) {
+      throw new Error(
+        `You have monitors whose size exceeds the ${SIZE_LIMIT_KB}KB limit.`
+      );
+    }
+  }
+
   async cleanup(outputPath: string) {
     this.moduleMap = new Map<string, string>();
     await unlink(outputPath);

package/src/push/index.ts

@@ -23,8 +23,6 @@
  *
  */
 
-import { join } from 'path';
-import { existsSync } from 'fs';
 import { readFile, writeFile } from 'fs/promises';
 import { prompt } from 'enquirer';
 import { bold, yellow } from 'kleur/colors';
@@ -35,26 +33,25 @@ import {
   formatNotFoundError,
   formatStaleMonitors,
 } from './request';
-import { buildMonitorSchema, createMonitors } from './monitor';
-import { ProjectSettings } from '../generator';
+import { buildMonitorSchema, createMonitors, MonitorSchema } from './monitor';
 import { Monitor } from '../dsl/monitor';
 import {
   progress,
+  apiProgress,
   write,
   error,
-  SYNTHETICS_PATH,
   aborted,
   indent,
   safeNDJSONParse,
   done,
 } from '../helpers';
-import type { PushOptions } from '../common_types';
+import type { PushOptions, ProjectSettings } from '../common_types';
+import { findSyntheticsConfig, readConfig } from '../config';
 
 export async function push(monitors: Monitor[], options: PushOptions) {
   if (monitors.length === 0) {
     throw 'No Monitors found';
   }
-
   const duplicates = trackDuplicates(monitors);
   if (duplicates.size > 0) {
     throw error(formatDuplicateError(duplicates));
@@ -62,9 +59,32 @@ export async function push(monitors: Monitor[], options: PushOptions) {
 
   progress(`preparing all monitors`);
   const schemas = await buildMonitorSchema(monitors);
+
+  progress(`creating all monitors`);
+  for (const schema of schemas) {
+    await pushMonitors({ schemas: [schema], keepStale: true, options });
+  }
+
+  progress(`deleting all stale monitors`);
+  await pushMonitors({ schemas, keepStale: false, options });
+  done('Pushed');
+}
+
+export async function pushMonitors({
+  schemas,
+  keepStale,
+  options,
+}: {
+  schemas: MonitorSchema[];
+  keepStale: boolean;
+  options: PushOptions;
+}) {
   try {
-    progress(`creating all monitors`);
-    const { body, statusCode } = await createMonitors(schemas, options);
+    const { body, statusCode } = await createMonitors(
+      schemas,
+      options,
+      keepStale
+    );
     if (statusCode === 404) {
       throw formatNotFoundError(await body.text());
     }
@@ -80,7 +100,9 @@ export async function push(monitors: Monitor[], options: PushOptions) {
       const chunks = safeNDJSONParse(data);
       for (const chunk of chunks) {
         if (typeof chunk === 'string') {
-          progress(chunk);
+          // TODO: add progress back for all states once we get the fix
+          // on kibana side
+          keepStale && apiProgress(chunk);
           continue;
         }
         const { failedMonitors, failedStaleMonitors } = chunk;
@@ -92,7 +114,6 @@ export async function push(monitors: Monitor[], options: PushOptions) {
         }
       }
     }
-    done('Pushed');
   } catch (e) {
     error(e);
   }
@@ -124,34 +145,39 @@ export function formatDuplicateError(monitors: Set<Monitor>) {
   return outer;
 }
 
-const PROJECT_SETTINGS_PATH = join(SYNTHETICS_PATH, 'project.json');
 const INSTALLATION_HELP = `Run 'npx @elastic/synthetics init' to create project with default settings.`;
 
 export async function loadSettings() {
-  if (!existsSync(PROJECT_SETTINGS_PATH)) {
-    throw error(`Aborted. Synthetics project not set up.
+  try {
+    const config = await readConfig('asd');
+    // Missing config file, fake throw to capture as missing file
+    if (Object.keys(config).length === 0) {
+      throw '';
+    }
+    return config.project || ({} as ProjectSettings);
+  } catch (e) {
+    throw error(`Aborted (missing synthetics config file), Project not set up corrrectly.
 
 ${INSTALLATION_HELP}`);
   }
-
-  const contents = await readFile(PROJECT_SETTINGS_PATH, 'utf-8');
-  return JSON.parse(contents) as ProjectSettings;
 }
 
 export function validateSettings(opts: PushOptions) {
   const INVALID = 'Aborted. Invalid synthetics project settings.';
 
   let reason = '';
-  if (!opts.project) {
-    reason = 'Set project id via CLI as `--project <id>`';
+  if (!opts.id) {
+    reason = `Set project id via
+  - CLI '--id <id>'
+  - Config file 'project.id' field`;
   } else if (!opts.locations && !opts.privateLocations) {
     reason = `Set default location for all monitors via
-  - CLI as '--locations <values...> or --privateLocations <values...>'
-  - Synthetics config file 'monitors.locations' | 'monitors.privateLocations' field`;
+  - CLI '--locations <values...> or --privateLocations <values...>'
+  - Config file 'monitors.locations' | 'monitors.privateLocations' field`;
   } else if (!opts.schedule) {
     reason = `Set default schedule in minutes for all monitors via
-  - CLI as '--schedule <mins>'
-  - Synthetics config file 'monitors.schedule' field`;
+  - CLI '--schedule <mins>'
+  - Config file 'monitors.schedule' field`;
   }
 
   if (!reason) return;
@@ -163,24 +189,27 @@ ${reason}
 ${INSTALLATION_HELP}`);
 }
 
-async function overrideSettings(project: string) {
-  const settings = JSON.parse(await readFile(PROJECT_SETTINGS_PATH, 'utf-8'));
-  settings.project = project;
-
-  await writeFile(
-    PROJECT_SETTINGS_PATH,
-    JSON.stringify(settings, null, 2),
-    'utf-8'
+async function overrideSettings(oldValue: string, newValue: string) {
+  const cwd = process.cwd();
+  const configPath = await findSyntheticsConfig(cwd, cwd);
+  if (!configPath) {
+    throw aborted(`Unable to find synthetics config file: ${configPath}`);
+  }
+  const config = await readFile(configPath, 'utf-8');
+  const updatedConfig = config.replace(
+    `id: '${oldValue}'`,
+    `id: '${newValue}'`
   );
+  await writeFile(configPath, updatedConfig, 'utf-8');
 }
 
 export async function catchIncorrectSettings(
   settings: ProjectSettings,
   options: PushOptions
 ) {
-  let override = !settings.project;
-  if (settings.project && settings.project !== options.project) {
-    if (typeof process.env.TEST_OVERRIDE != null) {
+  let override = !settings.id;
+  if (settings.id && settings.id !== options.id) {
+    if (process.env.TEST_OVERRIDE != null) {
       override = Boolean(process.env.TEST_OVERRIDE);
     } else {
       // Add an extra line to make it easier to read the prompt
@@ -188,7 +217,7 @@ export async function catchIncorrectSettings(
       ({ override } = await prompt<{ override: boolean }>({
         type: 'confirm',
         name: 'override',
-        message: `Monitors were pushed under the '${settings.project}' project. Are you sure you want to push them under the new '${options.project}' (note that this will duplicate the monitors, the old ones being orphaned)`,
+        message: `Monitors were pushed under the '${settings.id}' project. Are you sure you want to push them under the new '${options.id}' (note that this will duplicate the monitors, the old ones being orphaned)`,
         initial: false,
       }));
     }
@@ -197,6 +226,6 @@ export async function catchIncorrectSettings(
     }
   }
   if (override) {
-    await overrideSettings(options.project);
+    await overrideSettings(settings.id, options.id);
   }
 }

package/src/push/monitor.ts

@@ -38,7 +38,7 @@ export type MonitorSchema = Omit<MonitorConfig, 'locations'> & {
   filter: Monitor['filter'];
 };
 
-type APISchema = {
+export type APISchema = {
   project: string;
   keep_stale: boolean;
   monitors: MonitorSchema[];
@@ -77,11 +77,12 @@ export async function buildMonitorSchema(monitors: Monitor[]) {
 
 export async function createMonitors(
   monitors: MonitorSchema[],
-  options: PushOptions
+  options: PushOptions,
+  keepStale: boolean
 ) {
   const schema: APISchema = {
-    project: options.project,
-    keep_stale: false,
+    project: options.id,
+    keep_stale: keepStale,
     monitors,
   };
 

package/src/push/plugin.ts

@@ -23,14 +23,17 @@
  *
  */
 
-import path from 'path';
+import { isAbsolute, dirname, extname, join } from 'path';
 import fs from 'fs/promises';
 import * as esbuild from 'esbuild';
 
+const SOURCE_DIR = join(__dirname, '..', '..');
+const SOURCE_NODE_MODULES = join(SOURCE_DIR, 'node_modules');
+
 export function commonOptions(): esbuild.BuildOptions {
   return {
     bundle: true,
-    external: ['node_modules', '@elastic/synthetics'],
+    external: ['@elastic/synthetics'],
     minify: false,
     platform: 'node',
     logLevel: 'silent',
@@ -55,27 +58,49 @@ export function MultiAssetPlugin(callback: PluginCallback): esbuild.Plugin {
     // Note that we use `isAbsolute` to handle UNC/windows style paths like C:\path\to\thing
     // This is not necessary for relative directories since `.\file` is not supported as an import
     // nor is `~/path/to/file`.
-    if (path.isAbsolute(str) || str.startsWith('./') || str.startsWith('../')) {
+    if (isAbsolute(str) || str.startsWith('./') || str.startsWith('../')) {
       return true;
     }
     return false;
   };
 
+  // If we're importing the @elastic/synthetics package
+  // directly from source instead of using the fully
+  // qualified name, we must skip it too. That's just
+  // so it doesn't get bundled on tests or when we locally
+  // refer to the package itself.
+  const isLocalSynthetics = (entryPath: string) => {
+    return entryPath.startsWith(SOURCE_DIR);
+  };
+
+  // When importing the local synthetics module directly
+  // it may import its own local dependencies, so we must
+  // make sure those will be resolved using Node's resolution
+  // algorithm, as they're still "node_modules" that we must bundle
+  const isLocalSyntheticsModule = (str: string) => {
+    return str.startsWith(SOURCE_NODE_MODULES);
+  };
+
   return {
     name: 'esbuild-multiasset-plugin',
     setup(build) {
       build.onResolve({ filter: /.*?/ }, async args => {
         // External and other packages need be marked external to
         // be removed from the bundle
-        if (
-          build.initialOptions.external?.includes(args.path) ||
-          !isBare(args.path)
-        ) {
+        if (build.initialOptions.external?.includes(args.path)) {
           return {
             external: true,
           };
         }
 
+        if (
+          !isBare(args.path) ||
+          args.importer.includes('/node_modules/') ||
+          isLocalSyntheticsModule(args.importer)
+        ) {
+          return;
+        }
+
         if (args.kind === 'entry-point') {
           return {
             path: args.path,
@@ -86,8 +111,12 @@ export function MultiAssetPlugin(callback: PluginCallback): esbuild.Plugin {
         // If the modules are resolved locally, then
         // use the imported path to get full path
         const entryPath =
-          path.join(path.dirname(args.importer), args.path) +
-          path.extname(args.importer);
+          join(dirname(args.importer), args.path) + extname(args.importer);
+
+        if (isLocalSynthetics(entryPath)) {
+          return { external: true };
+        }
+
         // Spin off another build to copy over the imported modules without bundling
         const result = await esbuild.build({
           ...commonOptions(),

package/src/push/request.ts

@@ -47,6 +47,7 @@ export async function sendRequest(options: APIRequestOptions) {
       'user-agent': `Elastic/Synthetics ${version}`,
       'kbn-xsrf': 'true',
     },
+    headersTimeout: 60 * 1000
   });
 }
 

package/templates/synthetics.config.ts

@@ -16,6 +16,14 @@ export default env => {
       locations: ['{{locations}}'],
       privateLocations: ['{{privateLocations}}'],
     },
+    /**
+     * Project monitors settings
+     */
+    project: {
+      id: '{{id}}',
+      url: '{{url}}',
+      space: '{{space}}',
+    },
   };
   if (env !== 'development') {
     /**