@depup/artillery 2.0.30-depup.0

package/lib/launch-platform.js

@@ -0,0 +1,439 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+const { SSMS } = require('@artilleryio/int-core').ssms;
+const { loadPlugins, loadPluginsConfig } = require('./load-plugins');
+
+const EventEmitter = require('eventemitter3');
+const debug = require('debug')('core');
+
+const p = require('node:util').promisify;
+const _ = require('lodash');
+
+const PlatformLocal = require('./platform/local');
+const PlatformLambda = require('./platform/aws-lambda');
+const PlatformAzureACI = require('./platform/az/aci');
+
+async function createLauncher(script, payload, opts, launcherOpts) {
+  launcherOpts = launcherOpts || {
+    platform: 'local',
+    mode: 'distribute'
+  };
+  let l;
+  try {
+    l = new Launcher(script, payload, opts, launcherOpts);
+  } catch (err) {
+    console.log(err);
+    return null;
+  }
+
+  return l;
+}
+class Launcher {
+  constructor(script, payload, opts, launcherOpts) {
+    this.script = script;
+    this.payload = payload;
+    this.opts = opts;
+
+    this.exitedWorkersCount = 0;
+    this.workerMessageBuffer = [];
+
+    this.metricsByPeriod = {}; // individual intermediates by worker
+    this.finalReportsByWorker = {};
+
+    this.events = new EventEmitter();
+
+    this.pluginEvents = new EventEmitter();
+    this.pluginEventsLegacy = new EventEmitter();
+
+    this.launcherOpts = launcherOpts;
+
+    this.periodsReportedFor = [];
+
+    if (launcherOpts.platform === 'local') {
+      this.platform = new PlatformLocal(script, payload, opts, launcherOpts);
+    } else if (launcherOpts.platform === 'aws:lambda') {
+      this.platform = new PlatformLambda(script, payload, opts, launcherOpts);
+    } else if (launcherOpts.platform === 'az:aci') {
+      this.platform = new PlatformAzureACI(script, payload, opts, launcherOpts);
+    } else {
+      throw new Error(`Unknown platform: ${launcherOpts.platform}`);
+    }
+
+    this.phaseStartedEventsSeen = {};
+    this.phaseCompletedEventsSeen = {};
+
+    this.eventsByWorker = {};
+  }
+
+  async initWorkerEvents(workerEvents) {
+    workerEvents.on('workerError', (_workerId, message) => {
+      const { id, error, level, aggregatable, logs } = message;
+
+      if (level !== 'warn') {
+        this.exitedWorkersCount++;
+      }
+
+      if (aggregatable) {
+        this.workerMessageBuffer.push(message);
+      } else {
+        global.artillery.log(`[${id}]: ${error.message}`);
+        if (logs) {
+          global.artillery.log(logs);
+        }
+      }
+
+      this.events.emit('workerError', message);
+    });
+
+    workerEvents.on('phaseStarted', (_workerId, message) => {
+      // Note - we send only the first event for a phase, not all of them
+      if (
+        typeof this.phaseStartedEventsSeen[message.phase.index] === 'undefined'
+      ) {
+        this.phaseStartedEventsSeen[message.phase.index] = Date.now();
+        const fullPhase = {
+          //get back original phase without any splitting for workers
+          ...this.script.config.phases[message.phase.index],
+          index: message.phase.index,
+          id: message.phase.id,
+          startTime: this.phaseStartedEventsSeen[message.phase.index]
+        };
+
+        this.events.emit('phaseStarted', fullPhase);
+        this.pluginEvents.emit('phaseStarted', fullPhase);
+        this.pluginEventsLegacy.emit('phaseStarted', fullPhase);
+
+        global.artillery.globalEvents.emit('phaseStarted', fullPhase);
+      }
+    });
+
+    workerEvents.on('phaseCompleted', (_workerId, message) => {
+      if (
+        typeof this.phaseCompletedEventsSeen[message.phase.index] ===
+        'undefined'
+      ) {
+        this.phaseCompletedEventsSeen[message.phase.index] = Date.now();
+        const fullPhase = {
+          //get back original phase without any splitting for workers
+          ...this.script.config.phases[message.phase.index],
+          id: message.phase.id,
+          index: message.phase.index,
+          startTime: this.phaseStartedEventsSeen[message.phase.index],
+          endTime: message.phase.endTime
+        };
+
+        this.events.emit('phaseCompleted', fullPhase);
+        this.pluginEvents.emit('phaseCompleted', fullPhase);
+        this.pluginEventsLegacy.emit('phaseCompleted', fullPhase);
+        global.artillery.globalEvents.emit('phaseCompleted', fullPhase);
+      }
+    });
+
+    // We are not going to receive stats events from workers
+    // which have zero arrivals for a phase. (This can only happen
+    // in "distribute" mode.)
+    workerEvents.on('stats', (_workerId, message) => {
+      const workerStats = SSMS.deserializeMetrics(message.stats);
+      const period = workerStats.period;
+      if (typeof this.metricsByPeriod[period] === 'undefined') {
+        this.metricsByPeriod[period] = [];
+      }
+      // TODO: might want the full message here, with worker ID etc
+      this.metricsByPeriod[period].push(workerStats);
+    });
+
+    workerEvents.on('done', async (workerId, message) => {
+      this.exitedWorkersCount++;
+      this.finalReportsByWorker[workerId] = SSMS.deserializeMetrics(
+        message.report
+      );
+    });
+
+    workerEvents.on('log', async (_workerId, message) => {
+      artillery.globalEvents.emit('log', ...message.args);
+    });
+
+    workerEvents.on('setSuggestedExitCode', (_workerId, message) => {
+      artillery.suggestedExitCode = message.code;
+    });
+  }
+
+  async initPlugins() {
+    const plugins = await loadPlugins(
+      this.script.config.plugins,
+      this.script,
+      this.opts
+    );
+
+    //
+    // init plugins
+    //
+    for (const [name, result] of Object.entries(plugins)) {
+      if (result.isLoaded) {
+        if (result.version === 3) {
+          // TODO: load the plugin, subscribe to events
+          // global.artillery.plugins[name] = result.plugin;
+        } else {
+          //           global.artillery.log(`WARNING: Legacy plugin detected: ${name}
+          // See https://artillery.io/docs/resources/core/v2.html for more details.`,
+          //                                'warn');
+
+          // NOTE:
+          // We are giving v1 and v2 plugins a throw-away script
+          // object because we only care about the plugin setting
+          // up event handlers here. The plugins will be loaded
+          // properly in individual workers where they will have the
+          // opportunity to attach custom code, modify the script
+          // object etc.
+          // If we let a plugin access to the actual script object,
+          // and it happens to attach code to it (with a custom
+          // processor function for example) - spawning a worker
+          // will fail.
+          const dummyScript = JSON.parse(JSON.stringify(this.script));
+          dummyScript.config = {
+            ...dummyScript.config,
+            // Load additional plugins configuration from the environment
+            plugins: loadPluginsConfig(this.script.config.plugins)
+          };
+
+          if (result.version === 1) {
+            result.plugin = new result.PluginExport(
+              dummyScript.config,
+              this.pluginEventsLegacy
+            );
+            global.artillery.plugins.push(result);
+          } else if (result.version === 2) {
+            if (result.PluginExport.LEGACY_METRICS_FORMAT === false) {
+              result.plugin = new result.PluginExport.Plugin(
+                dummyScript,
+                this.pluginEvents,
+                this.opts
+              );
+            } else {
+              result.plugin = new result.PluginExport.Plugin(
+                dummyScript,
+                this.pluginEventsLegacy,
+                this.opts
+              );
+            }
+            global.artillery.plugins.push(result);
+          } else {
+            // TODO: print warning
+          }
+        }
+      } else {
+        global.artillery.log(`WARNING: Could not load plugin: ${name}`, 'warn');
+        global.artillery.log(result.msg, 'warn');
+        // global.artillery.log(result.error, 'warn');
+      }
+    }
+  }
+
+  async handleAllWorkersFinished() {
+    const allWorkersDone =
+      this.exitedWorkersCount === this.platform.getDesiredWorkerCount();
+    if (allWorkersDone) {
+      clearInterval(this.i1);
+      clearInterval(this.i2);
+
+      // Flush messages from workers
+      await this.flushWorkerMessages(0);
+      await this.flushIntermediateMetrics(true);
+
+      const pds = Object.keys(this.finalReportsByWorker).map(
+        (k) => this.finalReportsByWorker[k]
+      );
+
+      const statsByPeriod = Object.values(SSMS.mergeBuckets(pds));
+      const stats = SSMS.pack(statsByPeriod);
+
+      stats.summaries = {};
+      for (const [name, value] of Object.entries(stats.histograms || {})) {
+        const summary = SSMS.summarizeHistogram(value);
+        stats.summaries[name] = summary;
+      }
+
+      clearInterval(this.workerExitWatcher);
+
+      // Relay event to workers
+      this.pluginEvents.emit('done', stats);
+
+      global.artillery.globalEvents.emit('done', stats);
+      this.pluginEventsLegacy.emit('done', SSMS.legacyReport(stats));
+
+      this.events.emit('done', stats);
+    }
+  }
+
+  async flushWorkerMessages(maxAge = 9000) {
+    // Collect messages older than maxAge msec and group by log message:
+    const now = Date.now();
+    const okToPrint = this.workerMessageBuffer.filter(
+      (m) => now - m.ts > maxAge
+    );
+    this.workerMessageBuffer = this.workerMessageBuffer.filter(
+      (m) => now - m.ts <= maxAge
+    );
+
+    const readyMessages = okToPrint.reduce((acc, message) => {
+      const { error } = message;
+      // TODO: Take event type and level into account
+      if (typeof acc[error.message] === 'undefined') {
+        acc[error.message] = [];
+      }
+      acc[error.message].push(message);
+      return acc;
+    }, {});
+
+    for (const [_logMessage, messageObjects] of Object.entries(readyMessages)) {
+      if (messageObjects[0].error) {
+        global.artillery.log(
+          `[${messageObjects[0].id}] ${messageObjects[0].error.message}`,
+          messageObjects[0].level
+        );
+      } else {
+        // Expect a msg property:
+        global.artillery.log(
+          `[${messageObjects[0].id}] ${messageObjects[0].msg}`,
+          messageObjects[0].level
+        );
+      }
+    }
+  }
+
+  async flushIntermediateMetrics(flushAll = false) {
+    if (Object.keys(this.metricsByPeriod).length === 0) {
+      debug('No metrics received yet');
+      return;
+    }
+
+    // We always look at the earliest period available so that reports come in chronological order
+    const unreportedPeriods = Object.keys(this.metricsByPeriod)
+      .filter((x) => this.periodsReportedFor.indexOf(x) === -1)
+      .sort();
+
+    const earliestPeriodAvailable = unreportedPeriods[0];
+
+    // TODO: better name. One above is earliestNotAlreadyReported
+    const earliest = Object.keys(this.metricsByPeriod).sort()[0];
+    if (this.periodsReportedFor.indexOf(earliest) > -1) {
+      global.artillery.log(
+        'Warning: multiple batches of metrics for period',
+        earliest,
+        new Date(Number(earliest))
+      );
+
+      delete this.metricsByPeriod[earliest]; // FIXME: need to merge them in for the final report
+    }
+
+    // Dynamically adjust the duration we're willing to wait for. This matters on SQS where messages are received
+    // in batches of 10 and more workers => need to wait longer.
+    const MAX_WAIT_FOR_PERIOD_MS =
+      (Math.ceil(this.platform.getDesiredWorkerCount() / 10) * 3 + 30) * 1000;
+
+    debug({
+      now: Date.now(),
+      count: this.platform.getDesiredWorkerCount(),
+      earliestPeriodAvailable,
+      earliest,
+      MAX_WAIT_FOR_PERIOD_MS,
+      numReports: this.metricsByPeriod[earliestPeriodAvailable]?.length,
+      periodsReportedFor: this.periodsReportedFor,
+      metricsByPeriod: Object.keys(this.metricsByPeriod)
+    });
+
+    const allWorkersReportedForPeriod =
+      this.metricsByPeriod[earliestPeriodAvailable]?.length ===
+      this.platform.getDesiredWorkerCount();
+    const waitedLongEnough =
+      Date.now() - Number(earliestPeriodAvailable) > MAX_WAIT_FOR_PERIOD_MS;
+
+    if (flushAll) {
+      for (const period of unreportedPeriods) {
+        this.emitIntermediatesForPeriod(period);
+      }
+    } else if (
+      typeof earliestPeriodAvailable !== 'undefined' &&
+      (allWorkersReportedForPeriod || waitedLongEnough)
+    ) {
+      this.emitIntermediatesForPeriod(earliestPeriodAvailable);
+      // TODO: autoscaling. Handle workers that drop off or join, and update count
+    } else {
+      debug('Waiting for more workerStats before emitting stats event');
+    }
+  }
+
+  emitIntermediatesForPeriod(period) {
+    debug(
+      'Report @',
+      new Date(Number(period)),
+      'made up of items:',
+      this.metricsByPeriod[String(period)].length
+    );
+
+    // TODO: Track how many workers provided metrics in the metrics report
+    // summarize histograms for console reporter:
+    const merged = SSMS.mergeBuckets(this.metricsByPeriod[String(period)]);
+    const stats = merged[String(period)];
+
+    stats.summaries = {};
+    for (const [name, value] of Object.entries(stats.histograms || {})) {
+      const summary = SSMS.summarizeHistogram(value);
+      stats.summaries[name] = summary;
+    }
+
+    delete this.metricsByPeriod[String(period)];
+
+    this.periodsReportedFor.push(period);
+    this.pluginEvents.emit('stats', stats);
+    global.artillery.globalEvents.emit('stats', stats);
+    this.pluginEventsLegacy.emit('stats', SSMS.legacyReport(stats));
+
+    this.events.emit('stats', stats);
+  }
+
+  async run() {
+    await this.initPlugins();
+
+    this.i1 = setInterval(async () => {
+      await this.flushWorkerMessages();
+    }, 1 * 1000).unref();
+
+    this.i2 = setInterval(async () => {
+      this.flushIntermediateMetrics();
+    }, 2 * 1000).unref();
+
+    this.workerExitWatcher = setInterval(async () => {
+      await this.handleAllWorkersFinished();
+    }, 2 * 1000);
+
+    await this.initWorkerEvents(this.platform.events);
+    await this.platform.startJob();
+    debug('workers running');
+  }
+
+  async shutdown() {
+    await this.platform.shutdown();
+
+    // TODO: flush worker messages, and intermediate stats
+
+    // Unload plugins
+    // TODO: v3 plugins
+    if (global.artillery?.plugins) {
+      for (const o of global.artillery.plugins) {
+        if (o.plugin.cleanup) {
+          try {
+            await p(o.plugin.cleanup.bind(o.plugin))();
+            debug('plugin unloaded:', o.name);
+          } catch (cleanupErr) {
+            global.artillery.log(cleanupErr, 'error');
+          }
+        }
+      }
+    }
+  }
+}
+
+module.exports = createLauncher;

package/lib/load-plugins.js

@@ -0,0 +1,113 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+const debug = require('debug')('core');
+const path = require('node:path');
+
+// Additional paths to load plugins can be set via ARTILLERY_PLUGIN_PATH
+// Additional plugin config mafy be set via ARTILLERY_PLUGINS (as JSON)
+// Version may be: v1, v2, v3 or any
+function loadPluginsConfig(pluginSpecs) {
+  let additionalPlugins = {};
+
+  if (process.env.ARTILLERY_PLUGINS) {
+    try {
+      additionalPlugins = JSON.parse(process.env.ARTILLERY_PLUGINS);
+    } catch (ignoreErr) {
+      debug(ignoreErr);
+    }
+  }
+
+  return Object.assign({}, pluginSpecs, additionalPlugins);
+}
+
+async function loadPlugins(pluginSpecs, testScript) {
+  let requirePaths = [''];
+
+  // requirePaths = requirePaths.concat(pro.getPluginPath());
+
+  if (process.env.ARTILLERY_PLUGIN_PATH) {
+    requirePaths = requirePaths.concat(
+      process.env.ARTILLERY_PLUGIN_PATH.split(':')
+    );
+  }
+
+  pluginSpecs = loadPluginsConfig(pluginSpecs);
+
+  const results = {};
+  for (const [name, config] of Object.entries(pluginSpecs)) {
+    const result = await loadPlugin(name, config, requirePaths, testScript);
+    results[name] = result;
+  }
+
+  return results;
+}
+
+async function loadPlugin(name, config, requirePaths, testScript) {
+  // TODO: Take scope in directly - don't need the full script
+  const pluginConfigScope = config.scope || testScript.config.pluginsScope;
+  const pluginPrefix = pluginConfigScope
+    ? pluginConfigScope
+    : 'artillery-plugin-';
+  const requireString = pluginPrefix + name;
+  let PluginExport, pluginErr, loadedFrom, version;
+
+  for (const p of requirePaths) {
+    debug('Looking for plugin in:', p);
+    try {
+      loadedFrom = path.join(p, requireString);
+      PluginExport = require(loadedFrom);
+      if (typeof PluginExport === 'function') {
+        version = 1;
+      } else if (
+        typeof PluginExport === 'object' &&
+        typeof PluginExport.Plugin === 'function'
+      ) {
+        version = 2;
+      } // TODO: Add v3
+    } catch (err) {
+      debug(err);
+      pluginErr = err;
+    }
+
+    if (typeof PluginExport !== 'undefined') {
+      break;
+    }
+  }
+
+  if (!PluginExport) {
+    let msg;
+
+    if (!pluginErr) {
+      msg = `WARNING: Could not initialize plugin: ${name}`;
+    } else {
+      if (pluginErr.code === 'MODULE_NOT_FOUND') {
+        msg = `WARNING: Plugin ${name} specified but module ${requireString} could not be found (${pluginErr.code})`;
+      } else {
+        msg = `WARNING: Could not initialize plugin: ${name} (${pluginErr.message})`;
+      }
+    }
+
+    return {
+      name,
+      isLoaded: false,
+      isInitialized: false,
+      msg: msg,
+      error: pluginErr
+    };
+  } else {
+    debug('Plugin %s loaded from %s', name, requireString);
+    return {
+      name,
+      isLoaded: true,
+      isInitialized: false,
+
+      PluginExport,
+      loadedFrom,
+      version
+    };
+  }
+}
+
+module.exports = { loadPlugins, loadPlugin, loadPluginsConfig };

package/lib/platform/aws/aws-cloudwatch.js

@@ -0,0 +1,106 @@
+const {
+  CloudWatchLogsClient,
+  PutRetentionPolicyCommand
+} = require('@aws-sdk/client-cloudwatch-logs');
+const debug = require('debug')('artillery:aws-cloudwatch');
+
+const allowedRetentionDays = [
+  1, 3, 5, 7, 14, 30, 60, 90, 120, 150, 180, 365, 400, 545, 731, 1096, 1827,
+  2192, 2557, 2922, 3288, 3653
+];
+
+async function _putCloudwatchRetentionPolicy(
+  logGroupName,
+  retentionInDays,
+  region
+) {
+  const cloudwatchlogs = new CloudWatchLogsClient({
+    apiVersion: '2014-11-06',
+    region
+  });
+  const putRetentionPolicyParams = {
+    logGroupName,
+    retentionInDays
+  };
+
+  return cloudwatchlogs.send(
+    new PutRetentionPolicyCommand(putRetentionPolicyParams)
+  );
+}
+
+function setCloudwatchRetention(
+  logGroupName,
+  retentionInDays,
+  region,
+  options = { maxRetries: 5, waitPerRetry: 1000 }
+) {
+  if (!allowedRetentionDays.includes(retentionInDays)) {
+    console.log(
+      `WARNING: Skipping setting CloudWatch retention, as invalid value specified: ${retentionInDays}. Allowed values are: ${allowedRetentionDays.join(
+        ', '
+      )}`
+    );
+    return;
+  }
+
+  const interval = setInterval(
+    async (opts) => {
+      debug(
+        `Trying to set CloudWatch Log group ${logGroupName} retention policy to ${retentionInDays} days`
+      );
+      opts.incr = (opts.incr || 0) + 1;
+
+      try {
+        const _res = await _putCloudwatchRetentionPolicy(
+          logGroupName,
+          retentionInDays,
+          region
+        );
+        debug(
+          `Successfully set CloudWatch Logs retention policy to ${retentionInDays} days`
+        );
+        clearInterval(interval);
+      } catch (error) {
+        const resumeTestMessage =
+          'The test will continue without setting the retention policy.';
+        if (error?.code === 'AccessDeniedException') {
+          console.log(`\n${error.message}`);
+          console.log(
+            '\nWARNING: Missing logs:PutRetentionPolicy permission to set CloudWatch retention policy. Please ensure the IAM role has the necessary permissions:\nhttps://docs.art/fargate#iam-permissions'
+          );
+          console.log(`${resumeTestMessage}\n`);
+          clearInterval(interval);
+          return;
+        }
+
+        if (error?.code !== 'ResourceNotFoundException') {
+          console.log(`\n${error.message}`);
+          console.log(
+            '\nWARNING: Unexpected error setting CloudWatch retention policy\n'
+          );
+          console.log(`${resumeTestMessage}\n`);
+          clearInterval(interval);
+          return;
+        }
+
+        if (opts.incr >= opts.maxRetries) {
+          console.log(`\n${error.message}`);
+          console.log(
+            `\nWARNING: Cannot find log group ${logGroupName}\nMax retries exceeded setting CloudWatch retention policy:`
+          );
+          console.log(`${resumeTestMessage}\n`);
+          clearInterval(interval);
+          return;
+        }
+      }
+    },
+    options.waitPerRetry,
+    options
+  );
+
+  return interval;
+}
+
+module.exports = {
+  setCloudwatchRetention
+};