@atom8n/n8n-benchmark 2.0.0

package/src/test-execution/app-metrics-poller.ts

@@ -0,0 +1,81 @@
+/**
+ * Polls the /metrics endpoint from an n8n instance to collect application metrics
+ * during benchmark test runs.
+ *
+ * A Poller can be started and stopped once. After it's stopped, it cannot be restarted.
+ * Instead, a new poller instance should be created.
+ */
+export class AppMetricsPoller {
+	private intervalId: NodeJS.Timeout | undefined = undefined;
+	private metricsData: string[] = [];
+	private isRunning = false;
+	private isStopped = false;
+
+	constructor(
+		private readonly metricsUrl: string,
+		private readonly pollIntervalMs: number = 5000,
+	) {}
+
+	/**
+	 * Starts polling the metrics endpoint
+	 */
+	start() {
+		if (this.isRunning) {
+			throw new Error('Metrics poller is already running');
+		}
+		if (this.isStopped) {
+			throw new Error('Metrics poller has been stopped and cannot be restarted');
+		}
+
+		this.isRunning = true;
+		this.metricsData = [];
+
+		// Immediately poll once to get initial metrics
+		void this.pollMetrics();
+
+		// Set up interval polling
+		this.intervalId = setInterval(() => {
+			void this.pollMetrics();
+		}, this.pollIntervalMs);
+	}
+
+	/**
+	 * Stops polling the metrics endpoint
+	 */
+	stop() {
+		if (this.intervalId) {
+			clearInterval(this.intervalId);
+			this.intervalId = undefined;
+		}
+		this.isRunning = false;
+		this.isStopped = true;
+	}
+
+	/**
+	 * Gets all collected metrics data
+	 */
+	getMetricsData(): string[] {
+		return this.metricsData;
+	}
+
+	/**
+	 * Polls the metrics endpoint once
+	 */
+	private async pollMetrics() {
+		try {
+			const response = await fetch(this.metricsUrl);
+
+			if (!response.ok) {
+				console.warn(`Failed to poll metrics: ${response.status} ${response.statusText}`);
+				return;
+			}
+
+			const metricsText = await response.text();
+			this.metricsData.push(metricsText);
+		} catch (error) {
+			console.warn(
+				`Error polling metrics: ${error instanceof Error ? error.message : String(error)}`,
+			);
+		}
+	}
+}

package/src/test-execution/k6-executor.ts

@@ -0,0 +1,192 @@
+import fs from 'fs';
+import assert from 'node:assert/strict';
+import path from 'path';
+import { $, which, tmpfile } from 'zx';
+
+import { AppMetricsPoller } from '@/test-execution/app-metrics-poller';
+import { buildTestReport, type K6Tag } from '@/test-execution/test-report';
+import type { Scenario } from '@/types/scenario';
+export type { K6Tag };
+
+export type K6ExecutorOpts = {
+	k6ExecutablePath: string;
+	/** How many concurrent requests to make */
+	vus: number;
+	/** Test duration, e.g. 1m or 30s */
+	duration: string;
+	k6Out?: string;
+	k6ApiToken?: string;
+	n8nApiBaseUrl: string;
+	tags?: K6Tag[];
+	resultsWebhook?: {
+		url: string;
+		authHeader: string;
+	};
+	/** Configuration for polling app metrics during test runs */
+	appMetricsPolling?: {
+		enabled: boolean;
+		/** Poll interval in milliseconds. Defaults to 5000ms (5 seconds) */
+		intervalMs?: number;
+	};
+};
+
+export type K6RunOpts = {
+	/** Name of the scenario run. Used e.g. when the run is reported to k6 cloud */
+	scenarioRunName: string;
+};
+
+/**
+ * Flag for the k6 CLI.
+ * @example ['--duration', '1m']
+ * @example ['--quiet']
+ */
+type K6CliFlag = [string | number] | [string, string | number];
+
+/**
+ * Executes test scenarios using k6
+ */
+export class K6Executor {
+	/**
+	 * This script is dynamically injected into the k6 test script to generate
+	 * a summary report of the test execution.
+	 */
+	private readonly handleSummaryScript = `
+import { textSummary } from 'https://jslib.k6.io/k6-summary/0.0.2/index.js';
+export function handleSummary(data) {
+	return {
+		stdout: textSummary(data),
+		'{{scenarioName}}.summary.json': JSON.stringify(data),
+	};
+}
+`;
+
+	constructor(private readonly opts: K6ExecutorOpts) {}
+
+	async executeTestScenario(scenario: Scenario, { scenarioRunName }: K6RunOpts) {
+		const augmentedTestScriptPath = this.augmentSummaryScript(scenario, scenarioRunName);
+		const runDirPath = path.dirname(augmentedTestScriptPath);
+
+		const flags: K6CliFlag[] = [
+			['--quiet'],
+			['--duration', this.opts.duration],
+			['--vus', this.opts.vus],
+		];
+
+		if (this.opts.k6Out) {
+			flags.push(['--out', this.opts.k6Out]);
+		} else if (!this.opts.resultsWebhook && this.opts.k6ApiToken) {
+			flags.push(['--out', 'cloud']);
+		}
+
+		const flattedFlags = flags.flat(2);
+
+		const k6ExecutablePath = await this.resolveK6ExecutablePath();
+
+		// Start app metrics polling if enabled
+		let metricsPoller: AppMetricsPoller | undefined;
+		if (this.opts.appMetricsPolling?.enabled) {
+			const metricsUrl = `${this.opts.n8nApiBaseUrl}/metrics`;
+			const intervalMs = this.opts.appMetricsPolling.intervalMs ?? 5000;
+			metricsPoller = new AppMetricsPoller(metricsUrl, intervalMs);
+			metricsPoller.start();
+			console.log(`Started polling app metrics from ${metricsUrl} every ${intervalMs}ms`);
+		}
+
+		try {
+			await $({
+				cwd: runDirPath,
+				env: {
+					API_BASE_URL: this.opts.n8nApiBaseUrl,
+					DATA_TABLE_ID: scenario.dataTableId,
+					K6_CLOUD_TOKEN: this.opts.k6ApiToken,
+				},
+				stdio: 'inherit',
+			})`${k6ExecutablePath} run ${flattedFlags} ${augmentedTestScriptPath}`;
+		} finally {
+			// Stop metrics polling regardless of test outcome
+			if (metricsPoller) {
+				metricsPoller.stop();
+				console.log('Stopped polling app metrics');
+			}
+		}
+
+		console.log('\n');
+
+		if (this.opts.resultsWebhook) {
+			const endOfTestSummary = this.loadEndOfTestSummary(runDirPath, scenarioRunName);
+			const appMetricsData = metricsPoller?.getMetricsData();
+
+			const testReport = buildTestReport(
+				scenario,
+				endOfTestSummary,
+				[
+					...(this.opts.tags ?? []),
+					{ name: 'Vus', value: this.opts.vus.toString() },
+					{ name: 'Duration', value: this.opts.duration.toString() },
+				],
+				appMetricsData,
+			);
+
+			await this.sendTestReport(testReport);
+		}
+	}
+
+	async sendTestReport(testReport: unknown) {
+		assert(this.opts.resultsWebhook);
+
+		const response = await fetch(this.opts.resultsWebhook.url, {
+			method: 'POST',
+			body: JSON.stringify(testReport),
+			headers: {
+				Authorization: this.opts.resultsWebhook.authHeader,
+				'Content-Type': 'application/json',
+			},
+		});
+
+		if (!response.ok) {
+			console.warn(`Failed to send test summary: ${response.status} ${await response.text()}`);
+		}
+	}
+
+	/**
+	 * Augments the test script with a summary script
+	 *
+	 * @returns Absolute path to the augmented test script
+	 */
+	private augmentSummaryScript(scenario: Scenario, scenarioRunName: string) {
+		const fullTestScriptPath = path.join(scenario.scenarioDirPath, scenario.scriptPath);
+		const testScript = fs.readFileSync(fullTestScriptPath, 'utf8');
+		const summaryScript = this.handleSummaryScript.replace('{{scenarioName}}', scenarioRunName);
+
+		const augmentedTestScript = `${testScript}\n\n${summaryScript}`;
+
+		const tempFilePath = tmpfile(`${scenarioRunName}.js`, augmentedTestScript);
+
+		return tempFilePath;
+	}
+
+	private loadEndOfTestSummary(dir: string, scenarioRunName: string): K6EndOfTestSummary {
+		const summaryReportPath = path.join(dir, `${scenarioRunName}.summary.json`);
+		const summaryReport = fs.readFileSync(summaryReportPath, 'utf8');
+
+		try {
+			return JSON.parse(summaryReport) as K6EndOfTestSummary;
+		} catch (error) {
+			throw new Error(`Failed to parse the summary report at ${summaryReportPath}`);
+		}
+	}
+
+	/**
+	 * @returns Resolved path to the k6 executable
+	 */
+	private async resolveK6ExecutablePath(): Promise<string> {
+		const k6ExecutablePath = await which(this.opts.k6ExecutablePath, { nothrow: true });
+		if (!k6ExecutablePath) {
+			throw new Error(
+				'Could not find k6 executable based on your `PATH`. Please ensure k6 is available in your system and add it to your `PATH` or specify the path to the k6 executable using the `K6_PATH` environment variable.',
+			);
+		}
+
+		return k6ExecutablePath;
+	}
+}

package/src/test-execution/k6-summary.ts

@@ -0,0 +1,255 @@
+/**
+Example JSON:
+
+{
+	"options": {
+		"summaryTrendStats": ["avg", "min", "med", "max", "p(90)", "p(95)"],
+		"summaryTimeUnit": "",
+		"noColor": false
+	},
+	"state": { "isStdOutTTY": false, "isStdErrTTY": false, "testRunDurationMs": 23.374 },
+	"metrics": {
+		"http_req_tls_handshaking": {
+			"type": "trend",
+			"contains": "time",
+			"values": { "avg": 0, "min": 0, "med": 0, "max": 0, "p(90)": 0, "p(95)": 0 }
+		},
+		"checks": {
+			"type": "rate",
+			"contains": "default",
+			"values": { "rate": 1, "passes": 1, "fails": 0 }
+		},
+		"http_req_sending": {
+			"type": "trend",
+			"contains": "time",
+			"values": {
+				"p(90)": 0.512,
+				"p(95)": 0.512,
+				"avg": 0.512,
+				"min": 0.512,
+				"med": 0.512,
+				"max": 0.512
+			}
+		},
+		"http_reqs": {
+			"contains": "default",
+			"values": { "count": 1, "rate": 42.78257893385813 },
+			"type": "counter"
+		},
+		"http_req_blocked": {
+			"contains": "time",
+			"values": {
+				"avg": 1.496,
+				"min": 1.496,
+				"med": 1.496,
+				"max": 1.496,
+				"p(90)": 1.496,
+				"p(95)": 1.496
+			},
+			"type": "trend"
+		},
+		"data_received": {
+			"type": "counter",
+			"contains": "data",
+			"values": { "count": 269, "rate": 11508.513733207838 }
+		},
+		"iterations": {
+			"type": "counter",
+			"contains": "default",
+			"values": { "count": 1, "rate": 42.78257893385813 }
+		},
+		"http_req_waiting": {
+			"type": "trend",
+			"contains": "time",
+			"values": {
+				"p(95)": 18.443,
+				"avg": 18.443,
+				"min": 18.443,
+				"med": 18.443,
+				"max": 18.443,
+				"p(90)": 18.443
+			}
+		},
+		"http_req_receiving": {
+			"type": "trend",
+			"contains": "time",
+			"values": {
+				"avg": 0.186,
+				"min": 0.186,
+				"med": 0.186,
+				"max": 0.186,
+				"p(90)": 0.186,
+				"p(95)": 0.186
+			}
+		},
+		"http_req_duration{expected_response:true}": {
+			"type": "trend",
+			"contains": "time",
+			"values": {
+				"max": 19.141,
+				"p(90)": 19.141,
+				"p(95)": 19.141,
+				"avg": 19.141,
+				"min": 19.141,
+				"med": 19.141
+			}
+		},
+		"iteration_duration": {
+			"type": "trend",
+			"contains": "time",
+			"values": {
+				"avg": 22.577833,
+				"min": 22.577833,
+				"med": 22.577833,
+				"max": 22.577833,
+				"p(90)": 22.577833,
+				"p(95)": 22.577833
+			}
+		},
+		"http_req_connecting": {
+			"type": "trend",
+			"contains": "time",
+			"values": {
+				"avg": 0.673,
+				"min": 0.673,
+				"med": 0.673,
+				"max": 0.673,
+				"p(90)": 0.673,
+				"p(95)": 0.673
+			}
+		},
+		"http_req_failed": {
+			"type": "rate",
+			"contains": "default",
+			"values": { "rate": 0, "passes": 0, "fails": 1 }
+		},
+		"http_req_duration": {
+			"type": "trend",
+			"contains": "time",
+			"values": {
+				"p(90)": 19.141,
+				"p(95)": 19.141,
+				"avg": 19.141,
+				"min": 19.141,
+				"med": 19.141,
+				"max": 19.141
+			}
+		},
+		"data_sent": {
+			"type": "counter",
+			"contains": "data",
+			"values": { "count": 102, "rate": 4363.82305125353 }
+		}
+	},
+	"root_group": {
+		"name": "",
+		"path": "",
+		"id": "d41d8cd98f00b204e9800998ecf8427e",
+		"groups": [],
+		"checks": [
+			{
+				"name": "is status 200",
+				"path": "::is status 200",
+				"id": "548d37ca5f33793206f7832e7cea54fb",
+				"passes": 1,
+				"fails": 0
+			}
+		]
+	}
+}
+ */
+
+type TrendStat = 'avg' | 'min' | 'med' | 'max' | 'p(90)' | 'p(95)';
+type MetricType = 'trend' | 'rate' | 'counter';
+type MetricContains = 'time' | 'default' | 'data';
+
+interface TrendValues {
+	avg: number;
+	min: number;
+	med: number;
+	max: number;
+	'p(90)': number;
+	'p(95)': number;
+}
+
+interface RateValues {
+	rate: number;
+	passes: number;
+	fails: number;
+}
+
+interface CounterValues {
+	count: number;
+	rate: number;
+}
+
+interface K6TrendMetric {
+	type: 'trend';
+	contains: 'time';
+	values: TrendValues;
+}
+
+interface RateMetric {
+	type: 'rate';
+	contains: 'default';
+	values: RateValues;
+}
+
+interface K6CounterMetric {
+	type: 'counter';
+	contains: MetricContains;
+	values: CounterValues;
+}
+
+interface Options {
+	summaryTrendStats: TrendStat[];
+	summaryTimeUnit: string;
+	noColor: boolean;
+}
+
+interface State {
+	isStdOutTTY: boolean;
+	isStdErrTTY: boolean;
+	testRunDurationMs: number;
+}
+
+interface Metrics {
+	http_req_tls_handshaking: K6TrendMetric;
+	checks: RateMetric;
+	http_req_sending: K6TrendMetric;
+	http_reqs: K6CounterMetric;
+	http_req_blocked: K6TrendMetric;
+	data_received: K6CounterMetric;
+	iterations: K6CounterMetric;
+	http_req_waiting: K6TrendMetric;
+	http_req_receiving: K6TrendMetric;
+	'http_req_duration{expected_response:true}': K6TrendMetric;
+	iteration_duration: K6TrendMetric;
+	http_req_connecting: K6TrendMetric;
+	http_req_failed: RateMetric;
+	http_req_duration: K6TrendMetric;
+	data_sent: K6CounterMetric;
+}
+
+interface K6Check {
+	name: string;
+	path: string;
+	id: string;
+	passes: number;
+	fails: number;
+}
+
+interface RootGroup {
+	name: string;
+	path: string;
+	id: string;
+	groups: unknown[];
+	checks: K6Check[];
+}
+
+interface K6EndOfTestSummary {
+	options: Options;
+	state: State;
+	metrics: Metrics;
+	root_group: RootGroup;
+}

package/src/test-execution/prometheus-metrics-parser.ts

@@ -0,0 +1,63 @@
+/**
+ * Parses Prometheus metrics text format and extracts time series data
+ */
+export class PrometheusMetricsParser {
+	/**
+	 * Extracts all values for a specific metric from collected metrics data
+	 */
+	static extractMetricValues(metricsData: string[], metricName: string): number[] {
+		const values: number[] = [];
+
+		for (const metricsText of metricsData) {
+			const lines = metricsText.split('\n');
+
+			for (const line of lines) {
+				// Skip comments and empty lines
+				if (line.startsWith('#') || line.trim() === '') {
+					continue;
+				}
+
+				// Parse metric line: metric_name{labels} value
+				// For example: n8n_nodejs_heap_size_total_bytes 149159936
+				// For simplicity, we'll just check if the line starts with the metric name
+				if (line.startsWith(metricName)) {
+					const parts = line.split(/\s+/);
+					if (parts.length >= 2) {
+						const value = parseFloat(parts[parts.length - 1]);
+						if (!isNaN(value)) {
+							values.push(value);
+						}
+					}
+				}
+			}
+		}
+
+		return values;
+	}
+
+	/**
+	 * Calculates statistics for a metric from collected data
+	 */
+	static calculateMetricStats(
+		metricsData: string[],
+		metricName: string,
+	): { max: number; avg: number; min: number; count: number } | null {
+		const values = this.extractMetricValues(metricsData, metricName);
+
+		if (values.length === 0) {
+			return null;
+		}
+
+		const max = Math.max(...values);
+		const min = Math.min(...values);
+		const sum = values.reduce((a, b) => a + b, 0);
+		const avg = sum / values.length;
+
+		return {
+			max,
+			min,
+			avg,
+			count: values.length,
+		};
+	}
+}