@atom8n/n8n-benchmark 2.0.0

package/scripts/run-for-n8n-setup.mjs

@@ -0,0 +1,175 @@
+#!/usr/bin/env zx
+/**
+ * This script runs the benchmarks for the given n8n setup.
+ */
+// @ts-check
+import path from 'path';
+import { $, argv, fs } from 'zx';
+import { DockerComposeClient } from './clients/docker-compose-client.mjs';
+import { flagsObjectToCliArgs } from './utils/flags.mjs';
+import { EOL } from 'os';
+
+const paths = {
+	n8nSetupsDir: path.join(__dirname, 'n8n-setups'),
+	mockApiDataPath: path.join(__dirname, 'mock-api'),
+};
+
+const N8N_ENCRYPTION_KEY = 'very-secret-encryption-key';
+
+/**
+ * Discovers runner services in the docker-compose setup, where the name matches exactly `runners` or ends with `_runners`
+ */
+async function discoverRunnerServices(dockerComposeClient) {
+	const result = await dockerComposeClient.$('config', '--services');
+
+	return result.stdout
+		.trim()
+		.split(EOL)
+		.filter((service) => service === 'runners' || service.endsWith('_runners'));
+}
+
+async function main() {
+	const [n8nSetupToUse] = argv._;
+	validateN8nSetup(n8nSetupToUse);
+
+	const composeFilePath = path.join(paths.n8nSetupsDir, n8nSetupToUse);
+	const setupScriptPath = path.join(paths.n8nSetupsDir, n8nSetupToUse, 'setup.mjs');
+	const n8nTag = argv.n8nDockerTag || process.env.N8N_DOCKER_TAG || 'latest';
+	const benchmarkTag = argv.benchmarkDockerTag || process.env.BENCHMARK_DOCKER_TAG || 'latest';
+	const k6ApiToken = argv.k6ApiToken || process.env.K6_API_TOKEN || undefined;
+	const resultWebhookUrl =
+		argv.resultWebhookUrl || process.env.BENCHMARK_RESULT_WEBHOOK_URL || undefined;
+	const resultWebhookAuthHeader =
+		argv.resultWebhookAuthHeader || process.env.BENCHMARK_RESULT_WEBHOOK_AUTH_HEADER || undefined;
+	const baseRunDir = argv.runDir || process.env.RUN_DIR || '/n8n';
+	const n8nLicenseCert = argv.n8nLicenseCert || process.env.N8N_LICENSE_CERT || undefined;
+	const n8nLicenseActivationKey = process.env.N8N_LICENSE_ACTIVATION_KEY || undefined;
+	const n8nLicenseTenantId = argv.n8nLicenseTenantId || process.env.N8N_LICENSE_TENANT_ID || '1';
+	const envTag = argv.env || 'local';
+	const vus = argv.vus;
+	const duration = argv.duration;
+	const scenarioFilter = argv.scenarioFilter;
+
+	const hasN8nLicense = !!n8nLicenseCert || !!n8nLicenseActivationKey;
+	if (n8nSetupToUse === 'scaling-multi-main' && !hasN8nLicense) {
+		console.error(
+			'n8n license is required to run the multi-main scaling setup. Please provide N8N_LICENSE_CERT or N8N_LICENSE_ACTIVATION_KEY (and N8N_LICENSE_TENANT_ID if needed)',
+		);
+		process.exit(1);
+	}
+
+	if (!fs.existsSync(baseRunDir)) {
+		console.error(
+			`The run directory "${baseRunDir}" does not exist. Please specify a valid directory using --runDir`,
+		);
+		process.exit(1);
+	}
+
+	const runDir = path.join(baseRunDir, n8nSetupToUse);
+	fs.emptyDirSync(runDir);
+
+	const dockerComposeClient = new DockerComposeClient({
+		$: $({
+			cwd: composeFilePath,
+			verbose: true,
+			env: {
+				PATH: process.env.PATH,
+				N8N_VERSION: n8nTag,
+				N8N_LICENSE_CERT: n8nLicenseCert,
+				N8N_LICENSE_ACTIVATION_KEY: n8nLicenseActivationKey,
+				N8N_LICENSE_TENANT_ID: n8nLicenseTenantId,
+				N8N_ENCRYPTION_KEY,
+				BENCHMARK_VERSION: benchmarkTag,
+				K6_API_TOKEN: k6ApiToken,
+				BENCHMARK_RESULT_WEBHOOK_URL: resultWebhookUrl,
+				BENCHMARK_RESULT_WEBHOOK_AUTH_HEADER: resultWebhookAuthHeader,
+				RUN_DIR: runDir,
+				MOCK_API_DATA_PATH: paths.mockApiDataPath,
+			},
+		}),
+	});
+
+	// Run the setup script if it exists
+	if (fs.existsSync(setupScriptPath)) {
+		const setupScript = await import(setupScriptPath);
+		await setupScript.setup({ runDir });
+	}
+
+	try {
+		const runnerServices = await discoverRunnerServices(dockerComposeClient);
+		await dockerComposeClient.$('up', '-d', '--remove-orphans', 'n8n', ...runnerServices);
+
+		const tags = Object.entries({
+			Env: envTag,
+			N8nVersion: n8nTag,
+			N8nSetup: n8nSetupToUse,
+		})
+			.map(([key, value]) => `${key}=${value}`)
+			.join(',');
+
+		const cliArgs = flagsObjectToCliArgs({
+			scenarioNamePrefix: n8nSetupToUse,
+			scenarioFilter,
+			vus,
+			duration,
+			tags,
+		});
+
+		await dockerComposeClient.$('run', 'benchmark', 'run', ...cliArgs);
+	} catch (error) {
+		console.error('An error occurred while running the benchmarks:');
+		console.error(error.message);
+		console.error('');
+		await printContainerStatus(dockerComposeClient);
+		throw error;
+	} finally {
+		await dumpLogs(dockerComposeClient);
+		await dockerComposeClient.$('down');
+	}
+}
+
+async function printContainerStatus(dockerComposeClient) {
+	console.error('Container statuses:');
+	await dockerComposeClient.$('ps', '-a');
+}
+
+async function dumpLogs(dockerComposeClient) {
+	console.info('Container logs:');
+	await dockerComposeClient.$('logs');
+}
+
+function printUsage() {
+	const availableSetups = getAllN8nSetups();
+	console.log('Usage: zx runForN8nSetup.mjs --runDir /path/for/n8n/data <n8n setup to use>');
+	console.log(`   eg: zx runForN8nSetup.mjs --runDir /path/for/n8n/data ${availableSetups[0]}`);
+	console.log('');
+	console.log('Flags:');
+	console.log(
+		'  --runDir <path>             Directory to share with the n8n container for storing data. Default is /n8n',
+	);
+	console.log('  --n8nDockerTag <tag>        Docker tag for n8n image. Default is latest');
+	console.log(
+		'  --benchmarkDockerTag <tag>  Docker tag for benchmark cli image. Default is latest',
+	);
+	console.log('  --k6ApiToken <token>        K6 API token to upload the results');
+	console.log('');
+	console.log('Available setups:');
+	console.log(availableSetups.join(', '));
+}
+
+/**
+ * @returns {string[]}
+ */
+function getAllN8nSetups() {
+	return fs.readdirSync(paths.n8nSetupsDir);
+}
+
+function validateN8nSetup(givenSetup) {
+	const availableSetups = getAllN8nSetups();
+	if (!availableSetups.includes(givenSetup)) {
+		printUsage();
+		process.exit(1);
+	}
+}
+
+main();

package/scripts/run-in-cloud.mjs

@@ -0,0 +1,167 @@
+#!/usr/bin/env zx
+/**
+ * Script to run benchmarks on the cloud benchmark environment.
+ * This script will:
+ * 	1. Provision a benchmark environment using Terraform.
+ * 	2. Run the benchmarks on the VM.
+ * 	3. Destroy the cloud environment.
+ *
+ * NOTE: Must be run in the root of the package.
+ */
+// @ts-check
+import { sleep, which, $, tmpdir } from 'zx';
+import path from 'path';
+import { SshClient } from './clients/ssh-client.mjs';
+import { TerraformClient } from './clients/terraform-client.mjs';
+import { flagsObjectToCliArgs } from './utils/flags.mjs';
+
+/**
+ * @typedef {Object} BenchmarkEnv
+ * @property {string} vmName
+ * @property {string} ip
+ * @property {string} sshUsername
+ * @property {string} sshPrivateKeyPath
+ */
+
+/**
+ * @typedef {Object} Config
+ * @property {boolean} isVerbose
+ * @property {string[]} n8nSetupsToUse
+ * @property {string} n8nTag
+ * @property {string} benchmarkTag
+ * @property {string} [k6ApiToken]
+ * @property {string} [resultWebhookUrl]
+ * @property {string} [resultWebhookAuthHeader]
+ * @property {string} [n8nLicenseCert]
+ * @property {string} [vus]
+ * @property {string} [duration]
+ * @property {string} [scenarioFilter]
+ *
+ * @param {Config} config
+ */
+export async function runInCloud(config) {
+	await ensureDependencies();
+
+	const terraformClient = new TerraformClient({
+		isVerbose: config.isVerbose,
+	});
+
+	const benchmarkEnv = await terraformClient.getTerraformOutputs();
+
+	await runBenchmarksOnVm(config, benchmarkEnv);
+}
+
+async function ensureDependencies() {
+	await which('terraform');
+	await which('az');
+}
+
+/**
+ * @param {Config} config
+ * @param {BenchmarkEnv} benchmarkEnv
+ */
+async function runBenchmarksOnVm(config, benchmarkEnv) {
+	console.log(`Setting up the environment...`);
+
+	const sshClient = new SshClient({
+		ip: benchmarkEnv.ip,
+		username: benchmarkEnv.sshUsername,
+		privateKeyPath: benchmarkEnv.sshPrivateKeyPath,
+		verbose: config.isVerbose,
+	});
+
+	await ensureVmIsReachable(sshClient);
+
+	const scriptsDir = await transferScriptsToVm(sshClient, config);
+
+	// Bootstrap the environment with dependencies
+	console.log('Running bootstrap script...');
+	const bootstrapScriptPath = path.join(scriptsDir, 'bootstrap.sh');
+	await sshClient.ssh(`chmod a+x ${bootstrapScriptPath} && ${bootstrapScriptPath}`);
+
+	// Give some time for the VM to be ready
+	await sleep(1000);
+
+	const failures = [];
+
+	for (const n8nSetup of config.n8nSetupsToUse) {
+		try {
+			await runBenchmarkForN8nSetup({
+				config,
+				sshClient,
+				scriptsDir,
+				n8nSetup,
+			});
+		} catch (error) {
+			console.error(`Benchmark failed for ${n8nSetup}:`, error.message);
+			failures.push(n8nSetup);
+		}
+	}
+
+	if (failures.length > 0) {
+		throw new Error(`Benchmarks failed for setups: ${failures.join(', ')}`);
+	}
+}
+
+/**
+ * @param {{ config: Config; sshClient: any; scriptsDir: string; n8nSetup: string; }} opts
+ */
+async function runBenchmarkForN8nSetup({ config, sshClient, scriptsDir, n8nSetup }) {
+	console.log(`Running benchmarks for ${n8nSetup}...`);
+	const runScriptPath = path.join(scriptsDir, 'run-for-n8n-setup.mjs');
+
+	const cliArgs = flagsObjectToCliArgs({
+		n8nDockerTag: config.n8nTag,
+		benchmarkDockerTag: config.benchmarkTag,
+		k6ApiToken: config.k6ApiToken,
+		resultWebhookUrl: config.resultWebhookUrl,
+		resultWebhookAuthHeader: config.resultWebhookAuthHeader,
+		n8nLicenseCert: config.n8nLicenseCert,
+		scenarioFilter: config.scenarioFilter,
+		vus: config.vus,
+		duration: config.duration,
+		env: 'cloud',
+	});
+
+	const flagsString = cliArgs.join(' ');
+
+	await sshClient.ssh(`npx zx ${runScriptPath} ${flagsString} ${n8nSetup}`, {
+		// Test run should always log its output
+		verbose: true,
+	});
+}
+
+async function ensureVmIsReachable(sshClient) {
+	try {
+		await sshClient.ssh('echo "VM is reachable"');
+	} catch (error) {
+		console.error(`VM is not reachable: ${error.message}`);
+		console.error(
+			`Did you provision the cloud environment first with 'pnpm provision-cloud-env'? You can also run the benchmarks locally with 'pnpm run benchmark-locally'.`,
+		);
+		process.exit(1);
+	}
+}
+
+/**
+ * @returns Path where the scripts are located on the VM
+ */
+async function transferScriptsToVm(sshClient, config) {
+	const cwd = process.cwd();
+	const scriptsDir = path.resolve(cwd, './scripts');
+	const tarFilename = 'scripts.tar.gz';
+	const scriptsTarPath = path.join(tmpdir('n8n-benchmark'), tarFilename);
+
+	const $$ = $({ verbose: config.isVerbose });
+
+	// Compress the scripts folder
+	await $$`tar -czf ${scriptsTarPath} ${scriptsDir} -C ${cwd} ./scripts`;
+
+	// Transfer the scripts to the VM
+	await sshClient.scp(scriptsTarPath, `~/${tarFilename}`);
+
+	// Extract the scripts on the VM
+	await sshClient.ssh(`tar -xzf ~/${tarFilename}`);
+
+	return '~/scripts';
+}

package/scripts/run-locally.mjs

@@ -0,0 +1,73 @@
+#!/usr/bin/env zx
+/**
+ * Script to run benchmarks on the cloud benchmark environment.
+ * This script will:
+ * 	1. Provision a benchmark environment using Terraform.
+ * 	2. Run the benchmarks on the VM.
+ * 	3. Destroy the cloud environment.
+ *
+ * NOTE: Must be run in the root of the package.
+ */
+// @ts-check
+import { $ } from 'zx';
+import path from 'path';
+import { flagsObjectToCliArgs } from './utils/flags.mjs';
+
+/**
+ * @typedef {Object} BenchmarkEnv
+ * @property {string} vmName
+ */
+
+const paths = {
+	scriptsDir: path.join(path.resolve('scripts')),
+};
+
+/**
+ * @typedef {Object} Config
+ * @property {boolean} isVerbose
+ * @property {string[]} n8nSetupsToUse
+ * @property {string} n8nTag
+ * @property {string} benchmarkTag
+ * @property {string} [runDir]
+ * @property {string} [k6ApiToken]
+ * @property {string} [resultWebhookUrl]
+ * @property {string} [resultWebhookAuthHeader]
+ * @property {string} [n8nLicenseCert]
+ * @property {string} [vus]
+ * @property {string} [duration]
+ * @property {string} [scenarioFilter]
+ *
+ * @param {Config} config
+ */
+export async function runLocally(config) {
+	const runScriptPath = path.join(paths.scriptsDir, 'run-for-n8n-setup.mjs');
+
+	const cliArgs = flagsObjectToCliArgs({
+		n8nDockerTag: config.n8nTag,
+		benchmarkDockerTag: config.benchmarkTag,
+		runDir: config.runDir,
+		vus: config.vus,
+		duration: config.duration,
+		scenarioFilter: config.scenarioFilter,
+		env: 'local',
+	});
+
+	try {
+		for (const n8nSetup of config.n8nSetupsToUse) {
+			console.log(`Running benchmarks for n8n setup: ${n8nSetup}`);
+
+			await $({
+				env: {
+					...process.env,
+					K6_API_TOKEN: config.k6ApiToken,
+					BENCHMARK_RESULT_WEBHOOK_URL: config.resultWebhookUrl,
+					BENCHMARK_RESULT_WEBHOOK_AUTH_HEADER: config.resultWebhookAuthHeader,
+					N8N_LICENSE_CERT: config.n8nLicenseCert,
+				},
+			})`npx ${runScriptPath} ${cliArgs} ${n8nSetup}`;
+		}
+	} catch (error) {
+		console.error('An error occurred while running the benchmarks:');
+		console.error(error);
+	}
+}

package/scripts/run.mjs

@@ -0,0 +1,192 @@
+#!/usr/bin/env zx
+/**
+ * Script to run benchmarks either on the cloud benchmark environment or locally.
+ * The cloud environment needs to be provisioned using Terraform before running the benchmarks.
+ *
+ * NOTE: Must be run in the root of the package.
+ */
+// @ts-check
+import fs from 'fs';
+import minimist from 'minimist';
+import path from 'path';
+import { runInCloud } from './run-in-cloud.mjs';
+import { runLocally } from './run-locally.mjs';
+
+const paths = {
+	n8nSetupsDir: path.join(path.resolve('scripts'), 'n8n-setups'),
+};
+
+async function main() {
+	const config = await parseAndValidateConfig();
+
+	const n8nSetupsToUse =
+		config.n8nSetupToUse === 'all' ? readAvailableN8nSetups() : [config.n8nSetupToUse];
+
+	console.log('Using n8n tag', config.n8nTag);
+	console.log('Using benchmark cli tag', config.benchmarkTag);
+	console.log('Using environment', config.env);
+	console.log('Using n8n setups', n8nSetupsToUse.join(', '));
+	console.log('');
+
+	if (config.env === 'cloud') {
+		await runInCloud({
+			benchmarkTag: config.benchmarkTag,
+			isVerbose: config.isVerbose,
+			k6ApiToken: config.k6ApiToken,
+			resultWebhookUrl: config.resultWebhookUrl,
+			resultWebhookAuthHeader: config.resultWebhookAuthHeader,
+			n8nLicenseCert: config.n8nLicenseCert,
+			n8nTag: config.n8nTag,
+			n8nSetupsToUse,
+			vus: config.vus,
+			duration: config.duration,
+			scenarioFilter: config.scenarioFilter,
+		});
+	} else if (config.env === 'local') {
+		await runLocally({
+			benchmarkTag: config.benchmarkTag,
+			isVerbose: config.isVerbose,
+			k6ApiToken: config.k6ApiToken,
+			resultWebhookUrl: config.resultWebhookUrl,
+			resultWebhookAuthHeader: config.resultWebhookAuthHeader,
+			n8nLicenseCert: config.n8nLicenseCert,
+			n8nTag: config.n8nTag,
+			runDir: config.runDir,
+			n8nSetupsToUse,
+			vus: config.vus,
+			duration: config.duration,
+			scenarioFilter: config.scenarioFilter,
+		});
+	} else {
+		console.error('Invalid env:', config.env);
+		printUsage();
+		process.exit(1);
+	}
+}
+
+function readAvailableN8nSetups() {
+	const setups = fs.readdirSync(paths.n8nSetupsDir);
+
+	return setups;
+}
+
+/**
+ * @typedef {Object} Config
+ * @property {boolean} isVerbose
+ * @property {'cloud' | 'local'} env
+ * @property {string} n8nSetupToUse
+ * @property {string} n8nTag
+ * @property {string} benchmarkTag
+ * @property {string} [k6ApiToken]
+ * @property {string} [resultWebhookUrl]
+ * @property {string} [resultWebhookAuthHeader]
+ * @property {string} [n8nLicenseCert]
+ * @property {string} [runDir]
+ * @property {string} [vus]
+ * @property {string} [duration]
+ * @property {string} [scenarioFilter]
+ *
+ * @returns {Promise<Config>}
+ */
+async function parseAndValidateConfig() {
+	const args = minimist(process.argv.slice(3), {
+		boolean: ['debug', 'help'],
+	});
+
+	if (args.help) {
+		printUsage();
+		process.exit(0);
+	}
+
+	const n8nSetupToUse = await getAndValidateN8nSetup(args);
+	const isVerbose = args.debug || false;
+	const n8nTag = args.n8nTag || process.env.N8N_DOCKER_TAG || 'latest';
+	const benchmarkTag = args.benchmarkTag || process.env.BENCHMARK_DOCKER_TAG || 'latest';
+	const k6ApiToken = args.k6ApiToken || process.env.K6_API_TOKEN || undefined;
+	const resultWebhookUrl =
+		args.resultWebhookUrl || process.env.BENCHMARK_RESULT_WEBHOOK_URL || undefined;
+	const resultWebhookAuthHeader =
+		args.resultWebhookAuthHeader || process.env.BENCHMARK_RESULT_WEBHOOK_AUTH_HEADER || undefined;
+	const n8nLicenseCert = args.n8nLicenseCert || process.env.N8N_LICENSE_CERT || undefined;
+	const runDir = args.runDir || undefined;
+	const env = args.env || 'local';
+	const vus = args.vus;
+	const duration = args.duration;
+	const scenarioFilter = args.scenarioFilter;
+
+	if (!env) {
+		printUsage();
+		process.exit(1);
+	}
+
+	return {
+		isVerbose,
+		env,
+		n8nSetupToUse,
+		n8nTag,
+		benchmarkTag,
+		k6ApiToken,
+		resultWebhookUrl,
+		resultWebhookAuthHeader,
+		n8nLicenseCert,
+		runDir,
+		vus,
+		duration,
+		scenarioFilter,
+	};
+}
+
+/**
+ * @param {minimist.ParsedArgs} args
+ */
+async function getAndValidateN8nSetup(args) {
+	// Last parameter is the n8n setup to use
+	const n8nSetupToUse = args._[args._.length - 1];
+	if (!n8nSetupToUse || n8nSetupToUse === 'all') {
+		return 'all';
+	}
+
+	const availableSetups = readAvailableN8nSetups();
+
+	if (!availableSetups.includes(n8nSetupToUse)) {
+		printUsage();
+		process.exit(1);
+	}
+
+	return n8nSetupToUse;
+}
+
+function printUsage() {
+	const availableSetups = readAvailableN8nSetups();
+
+	console.log(`Usage: zx scripts/${path.basename(__filename)} [n8n setup name]`);
+	console.log(`   eg: zx scripts/${path.basename(__filename)}`);
+	console.log('');
+	console.log('Options:');
+	console.log(
+		`  [n8n setup name]     Against which n8n setup to run the benchmarks. One of: ${['all', ...availableSetups].join(', ')}. Default is all`,
+	);
+	console.log(
+		'  --env                Env where to run the benchmarks. Either cloud or local. Default is local.',
+	);
+	console.log('  --debug              Enable verbose output');
+	console.log('  --n8nTag             Docker tag for n8n image. Default is latest');
+	console.log('  --benchmarkTag       Docker tag for benchmark cli image. Default is latest');
+	console.log('  --scenarioFilter     Filter scenarios by name (case-insensitive)');
+	console.log('  --vus                How many concurrent requests to make');
+	console.log('  --duration           Test duration, e.g. 1m or 30s');
+	console.log(
+		'  --k6ApiToken         API token for k6 cloud. Default is read from K6_API_TOKEN env var. If omitted, k6 cloud will not be used',
+	);
+	console.log(
+		'  --runDir         Directory to share with the n8n container for storing data. Needed only for local runs.',
+	);
+	console.log('');
+}
+
+main().catch((error) => {
+	console.error('An error occurred while running the benchmarks:');
+	console.error(error);
+
+	process.exit(1);
+});

package/scripts/utils/flags.mjs

@@ -0,0 +1,20 @@
+// @ts-check
+
+/**
+ * Converts an object of flags to an array of CLI arguments.
+ *
+ * @param {Record<string, string | undefined>} flags
+ *
+ * @returns {string[]}
+ */
+export function flagsObjectToCliArgs(flags) {
+	return Object.entries(flags)
+		.filter(([, value]) => value !== undefined)
+		.map(([key, value]) => {
+			if (typeof value === 'string' && value.includes(' ')) {
+				return `--${key}="${value}"`;
+			} else {
+				return `--${key}=${value}`;
+			}
+		});
+}

package/src/commands/list.ts

@@ -0,0 +1,26 @@
+import { Command } from '@oclif/core';
+
+import { testScenariosPath } from '@/config/common-flags';
+import { ScenarioLoader } from '@/scenario/scenario-loader';
+
+export default class ListCommand extends Command {
+	static description = 'List all available scenarios';
+
+	static flags = {
+		testScenariosPath,
+	};
+
+	async run() {
+		const { flags } = await this.parse(ListCommand);
+		const scenarioLoader = new ScenarioLoader();
+
+		const allScenarios = scenarioLoader.loadAll(flags.testScenariosPath);
+
+		console.log('Available test scenarios:');
+		console.log('');
+
+		for (const scenario of allScenarios) {
+			console.log('\t', scenario.name, ':', scenario.description);
+		}
+	}
+}