@atom8n/n8n-benchmark 2.0.0

package/src/commands/run.ts

@@ -0,0 +1,140 @@
+import { Command, Flags } from '@oclif/core';
+
+import { testScenariosPath } from '@/config/common-flags';
+import { N8nApiClient } from '@/n8n-api-client/n8n-api-client';
+import { ScenarioDataFileLoader } from '@/scenario/scenario-data-loader';
+import { ScenarioLoader } from '@/scenario/scenario-loader';
+import type { K6Tag } from '@/test-execution/k6-executor';
+import { K6Executor } from '@/test-execution/k6-executor';
+import { ScenarioRunner } from '@/test-execution/scenario-runner';
+
+export default class RunCommand extends Command {
+	static description = 'Run all (default) or specified test scenarios';
+
+	static flags = {
+		testScenariosPath,
+		scenarioFilter: Flags.string({
+			char: 'f',
+			description: 'Filter scenarios by name',
+		}),
+		scenarioNamePrefix: Flags.string({
+			description: 'Prefix for the scenario name',
+			default: 'Unnamed',
+		}),
+		n8nBaseUrl: Flags.string({
+			description: 'The base URL for the n8n instance',
+			default: 'http://localhost:5678',
+			env: 'N8N_BASE_URL',
+		}),
+		n8nUserEmail: Flags.string({
+			description: 'The email address of the n8n user',
+			default: 'benchmark-user@n8n.io',
+			env: 'N8N_USER_EMAIL',
+		}),
+		k6ExecutablePath: Flags.string({
+			doc: 'The path to the k6 binary',
+			default: 'k6',
+			env: 'K6_PATH',
+		}),
+		k6ApiToken: Flags.string({
+			doc: 'The API token for k6 cloud',
+			default: undefined,
+			env: 'K6_API_TOKEN',
+		}),
+		out: Flags.string({
+			description: 'The --out flag for k6',
+			default: undefined,
+			env: 'K6_OUT',
+		}),
+		resultWebhookUrl: Flags.string({
+			doc: 'The URL where the benchmark results should be sent to',
+			default: undefined,
+			env: 'BENCHMARK_RESULT_WEBHOOK_URL',
+		}),
+		resultWebhookAuthHeader: Flags.string({
+			doc: 'The Authorization header value for the benchmark results webhook',
+			default: undefined,
+			env: 'BENCHMARK_RESULT_WEBHOOK_AUTH_HEADER',
+		}),
+		n8nUserPassword: Flags.string({
+			description: 'The password of the n8n user',
+			default: 'VerySecret!123',
+			env: 'N8N_USER_PASSWORD',
+		}),
+		tags: Flags.string({
+			char: 't',
+			description: 'Tags to attach to the run. Comma separated list of key=value pairs',
+		}),
+		vus: Flags.integer({
+			description: 'Number of concurrent requests to make',
+			default: 5,
+		}),
+		duration: Flags.string({
+			description: 'Duration of the test with a unit, e.g. 1m',
+			default: '1m',
+		}),
+		collectAppMetrics: Flags.boolean({
+			description: 'Collect app metrics from the /metrics endpoint during test runs',
+			default: false,
+			env: 'COLLECT_APP_METRICS',
+		}),
+		appMetricsPollInterval: Flags.integer({
+			description: 'App metrics polling interval in milliseconds',
+			default: 5000,
+			env: 'APP_METRICS_POLL_INTERVAL',
+		}),
+	};
+
+	async run() {
+		const { flags } = await this.parse(RunCommand);
+		const tags = await this.parseTags();
+		const scenarioLoader = new ScenarioLoader();
+
+		const scenarioRunner = new ScenarioRunner(
+			new N8nApiClient(flags.n8nBaseUrl),
+			new ScenarioDataFileLoader(),
+			new K6Executor({
+				duration: flags.duration,
+				vus: flags.vus,
+				k6Out: flags.out,
+				k6ExecutablePath: flags.k6ExecutablePath,
+				k6ApiToken: flags.k6ApiToken,
+				n8nApiBaseUrl: flags.n8nBaseUrl,
+				tags,
+				resultsWebhook: flags.resultWebhookUrl
+					? {
+							url: flags.resultWebhookUrl,
+							authHeader: flags.resultWebhookAuthHeader,
+						}
+					: undefined,
+				appMetricsPolling: flags.collectAppMetrics
+					? {
+							enabled: true,
+							intervalMs: flags.appMetricsPollInterval,
+						}
+					: undefined,
+			}),
+			{
+				email: flags.n8nUserEmail,
+				password: flags.n8nUserPassword,
+			},
+			flags.scenarioNamePrefix,
+		);
+
+		const allScenarios = scenarioLoader.loadAll(flags.testScenariosPath, flags.scenarioFilter);
+
+		await scenarioRunner.runManyScenarios(allScenarios);
+	}
+
+	private async parseTags(): Promise<K6Tag[]> {
+		const { flags } = await this.parse(RunCommand);
+		if (!flags.tags) {
+			return [];
+		}
+
+		return flags.tags.split(',').map((tag) => {
+			const [name, value] = tag.split('=');
+			return { name, value };
+		});
+	}
+}

package/src/config/common-flags.ts

@@ -0,0 +1,6 @@
+import { Flags } from '@oclif/core';
+
+export const testScenariosPath = Flags.string({
+	description: 'The path to the scenarios',
+	default: 'scenarios',
+});

package/src/n8n-api-client/authenticated-n8n-api-client.ts

@@ -0,0 +1,88 @@
+import type { AxiosRequestConfig } from 'axios';
+
+import { N8nApiClient } from './n8n-api-client';
+
+export class AuthenticatedN8nApiClient extends N8nApiClient {
+	constructor(
+		apiBaseUrl: string,
+		private readonly authCookie: string,
+	) {
+		super(apiBaseUrl);
+	}
+
+	static async createUsingUsernameAndPassword(
+		apiClient: N8nApiClient,
+		loginDetails: {
+			email: string;
+			password: string;
+		},
+	): Promise<AuthenticatedN8nApiClient> {
+		const response = await apiClient.restApiRequest('/login', {
+			method: 'POST',
+			data: {
+				emailOrLdapLoginId: loginDetails.email,
+				password: loginDetails.password,
+			},
+		});
+
+		if (response.data === 'n8n is starting up. Please wait') {
+			await apiClient.delay(1000);
+			return await this.createUsingUsernameAndPassword(apiClient, loginDetails);
+		}
+
+		const cookieHeader = response.headers['set-cookie'];
+		const authCookie = Array.isArray(cookieHeader) ? cookieHeader.join('; ') : cookieHeader;
+		if (!authCookie) {
+			throw new Error(
+				'Did not receive authentication cookie even tho login succeeded: ' +
+					JSON.stringify(
+						{
+							status: response.status,
+							headers: response.headers,
+							data: response.data,
+						},
+						null,
+						2,
+					),
+			);
+		}
+
+		return new AuthenticatedN8nApiClient(apiClient.apiBaseUrl, authCookie);
+	}
+
+	async get<T>(endpoint: string) {
+		return await this.authenticatedRequest<T>(endpoint, {
+			method: 'GET',
+		});
+	}
+
+	async post<T>(endpoint: string, data: unknown) {
+		return await this.authenticatedRequest<T>(endpoint, {
+			method: 'POST',
+			data,
+		});
+	}
+
+	async patch<T>(endpoint: string, data: unknown) {
+		return await this.authenticatedRequest<T>(endpoint, {
+			method: 'PATCH',
+			data,
+		});
+	}
+
+	async delete<T>(endpoint: string) {
+		return await this.authenticatedRequest<T>(endpoint, {
+			method: 'DELETE',
+		});
+	}
+
+	protected async authenticatedRequest<T>(endpoint: string, init: Omit<AxiosRequestConfig, 'url'>) {
+		return await this.restApiRequest<T>(endpoint, {
+			...init,
+			headers: {
+				...init.headers,
+				cookie: this.authCookie,
+			},
+		});
+	}
+}

package/src/n8n-api-client/credentials-api-client.ts

@@ -0,0 +1,28 @@
+import type { Credential } from '@/n8n-api-client/n8n-api-client.types';
+
+import type { AuthenticatedN8nApiClient } from './authenticated-n8n-api-client';
+
+export class CredentialApiClient {
+	constructor(private readonly apiClient: AuthenticatedN8nApiClient) {}
+
+	async getAllCredentials(): Promise<Credential[]> {
+		const response = await this.apiClient.get<{ count: number; data: Credential[] }>(
+			'/credentials',
+		);
+
+		return response.data.data;
+	}
+
+	async createCredential(credential: Credential): Promise<Credential> {
+		const response = await this.apiClient.post<{ data: Credential }>('/credentials', {
+			...credential,
+			id: undefined,
+		});
+
+		return response.data.data;
+	}
+
+	async deleteCredential(credentialId: Credential['id']): Promise<void> {
+		await this.apiClient.delete(`/credentials/${credentialId}`);
+	}
+}

package/src/n8n-api-client/data-table-api-client.ts

@@ -0,0 +1,30 @@
+import type { DataTable } from '@/n8n-api-client/n8n-api-client.types';
+
+import type { AuthenticatedN8nApiClient } from './authenticated-n8n-api-client';
+
+export class DataTableApiClient {
+	constructor(private readonly apiClient: AuthenticatedN8nApiClient) {}
+
+	async getAllDataTables(): Promise<DataTable[]> {
+		const response = await this.apiClient.get<{ data: { count: number; data: DataTable[] } }>(
+			'/data-tables-global',
+		);
+
+		return response.data.data.data;
+	}
+
+	async deleteDataTable(projectId: string, dataTableId: DataTable['id']): Promise<void> {
+		await this.apiClient.delete(`/projects/${projectId}/data-tables/${dataTableId}`);
+	}
+
+	async createDataTable(projectId: string, dataTable: DataTable): Promise<DataTable> {
+		const response = await this.apiClient.post<{ data: DataTable }>(
+			`/projects/${projectId}/data-tables`,
+			{
+				...dataTable,
+			},
+		);
+
+		return response.data.data;
+	}
+}

package/src/n8n-api-client/n8n-api-client.ts

@@ -0,0 +1,85 @@
+import type { AxiosError, AxiosRequestConfig } from 'axios';
+import axios from 'axios';
+
+export class N8nApiClient {
+	constructor(readonly apiBaseUrl: string) {}
+
+	async waitForInstanceToBecomeOnline(): Promise<void> {
+		const HEALTH_ENDPOINT = 'healthz';
+		const START_TIME = Date.now();
+		const INTERVAL_MS = 1000;
+		const TIMEOUT_MS = 60_000;
+
+		while (Date.now() - START_TIME < TIMEOUT_MS) {
+			try {
+				const response = await axios.request<{ status: 'ok' }>({
+					url: `${this.apiBaseUrl}/${HEALTH_ENDPOINT}`,
+					method: 'GET',
+				});
+
+				if (response.status === 200 && response.data.status === 'ok') {
+					return;
+				}
+			} catch {}
+
+			console.log(`n8n instance not online yet, retrying in ${INTERVAL_MS / 1000} seconds...`);
+			await this.delay(INTERVAL_MS);
+		}
+
+		throw new Error(`n8n instance did not come online within ${TIMEOUT_MS / 1000} seconds`);
+	}
+
+	async setupOwnerIfNeeded(loginDetails: { email: string; password: string }) {
+		const response = await this.restApiRequest<{ message: string }>('/owner/setup', {
+			method: 'POST',
+			data: {
+				email: loginDetails.email,
+				password: loginDetails.password,
+				firstName: 'Test',
+				lastName: 'User',
+			},
+			// Don't throw on non-2xx responses
+			validateStatus: () => true,
+		});
+
+		const responsePayload = response.data;
+
+		if (response.status === 200) {
+			console.log('Owner setup successful');
+		} else if (response.status === 400) {
+			if (responsePayload.message === 'Instance owner already setup')
+				console.log('Owner already set up');
+		} else if (response.status === 404) {
+			// The n8n instance setup owner endpoint not be available yet even tho
+			// the health endpoint returns ok. In this case we simply retry.
+			console.log('Owner setup endpoint not available yet, retrying in 1s...');
+			await this.delay(1000);
+			await this.setupOwnerIfNeeded(loginDetails);
+		} else {
+			throw new Error(
+				`Owner setup failed with status ${response.status}: ${responsePayload.message}`,
+			);
+		}
+	}
+
+	async restApiRequest<T>(endpoint: string, init: Omit<AxiosRequestConfig, 'url'>) {
+		try {
+			return await axios.request<T>({
+				...init,
+				url: this.getRestEndpointUrl(endpoint),
+			});
+		} catch (e) {
+			const error = e as AxiosError;
+			console.error(`[ERROR] Request failed ${init.method} ${endpoint}`, error?.response?.data);
+			throw error;
+		}
+	}
+
+	async delay(ms: number): Promise<void> {
+		return await new Promise((resolve) => setTimeout(resolve, ms));
+	}
+
+	protected getRestEndpointUrl(endpoint: string) {
+		return `${this.apiBaseUrl}/rest${endpoint}`;
+	}
+}

package/src/n8n-api-client/n8n-api-client.types.ts

@@ -0,0 +1,27 @@
+/**
+ * n8n workflow. This is a simplified version of the actual workflow object.
+ */
+export type Workflow = {
+	id: string;
+	name: string;
+	versionId: string;
+	tags?: string[];
+};
+
+export type Credential = {
+	id: string;
+	name: string;
+	type: string;
+};
+
+export type DataTableColumn = {
+	name: string;
+	type: 'string' | 'number' | 'boolean' | 'date';
+};
+
+export type DataTable = {
+	id?: string;
+	projectId?: number;
+	name: string;
+	columns: DataTableColumn[];
+};

package/src/n8n-api-client/project-api-client.ts

@@ -0,0 +1,11 @@
+import type { AuthenticatedN8nApiClient } from './authenticated-n8n-api-client';
+
+export class ProjectApiClient {
+	constructor(private readonly apiClient: AuthenticatedN8nApiClient) {}
+
+	async getPersonalProject(): Promise<string> {
+		const response = await this.apiClient.get<{ data: { id: string } }>('/projects/personal');
+
+		return response.data.data.id;
+	}
+}

package/src/n8n-api-client/workflows-api-client.ts

@@ -0,0 +1,38 @@
+import type { Workflow } from '@/n8n-api-client/n8n-api-client.types';
+
+import type { AuthenticatedN8nApiClient } from './authenticated-n8n-api-client';
+
+export class WorkflowApiClient {
+	constructor(private readonly apiClient: AuthenticatedN8nApiClient) {}
+
+	async getAllWorkflows(): Promise<Workflow[]> {
+		const response = await this.apiClient.get<{ count: number; data: Workflow[] }>('/workflows');
+
+		return response.data.data;
+	}
+
+	async createWorkflow(workflow: unknown): Promise<Workflow> {
+		const response = await this.apiClient.post<{ data: Workflow }>('/workflows', workflow);
+
+		return response.data.data;
+	}
+
+	async activateWorkflow(workflow: Workflow): Promise<Workflow> {
+		const response = await this.apiClient.post<{ data: Workflow }>(
+			`/workflows/${workflow.id}/activate`,
+			{
+				versionId: workflow.versionId,
+			},
+		);
+
+		return response.data.data;
+	}
+
+	async archiveWorkflow(workflowId: Workflow['id']): Promise<void> {
+		await this.apiClient.post(`/workflows/${workflowId}/archive`, {});
+	}
+
+	async deleteWorkflow(workflowId: Workflow['id']): Promise<void> {
+		await this.apiClient.delete(`/workflows/${workflowId}`);
+	}
+}

package/src/scenario/scenario-data-loader.ts

@@ -0,0 +1,75 @@
+import * as fs from 'node:fs';
+import * as path from 'node:path';
+
+import type { Workflow, Credential, DataTable } from '@/n8n-api-client/n8n-api-client.types';
+import type { Scenario } from '@/types/scenario';
+
+export type LoadableScenarioData = {
+	workflows: Workflow[];
+	credentials: Credential[];
+	dataTable: DataTable | null;
+};
+
+/**
+ * Loads scenario data files from FS
+ */
+export class ScenarioDataFileLoader {
+	async loadDataForScenario(scenario: Scenario): Promise<LoadableScenarioData> {
+		const workflows = await Promise.all(
+			scenario.scenarioData.workflowFiles?.map((workflowFilePath) =>
+				this.loadSingleWorkflowFromFile(path.join(scenario.scenarioDirPath, workflowFilePath)),
+			) ?? [],
+		);
+
+		const credentials = await Promise.all(
+			scenario.scenarioData.credentialFiles?.map((credentialFilePath) =>
+				this.loadSingleCredentialFromFile(path.join(scenario.scenarioDirPath, credentialFilePath)),
+			) ?? [],
+		);
+
+		const dataTable = scenario.scenarioData.dataTableFile
+			? this.loadSingleDataTableFromFile(
+					path.join(scenario.scenarioDirPath, scenario.scenarioData.dataTableFile),
+				)
+			: null;
+
+		return {
+			workflows,
+			credentials,
+			dataTable,
+		};
+	}
+
+	private loadSingleCredentialFromFile(credentialFilePath: string): Credential {
+		const fileContent = fs.readFileSync(credentialFilePath, 'utf8');
+
+		try {
+			return JSON.parse(fileContent) as Credential;
+		} catch (error) {
+			const e = error as Error;
+			throw new Error(`Failed to parse credential file ${credentialFilePath}: ${e.message}`);
+		}
+	}
+
+	private loadSingleWorkflowFromFile(workflowFilePath: string): Workflow {
+		const fileContent = fs.readFileSync(workflowFilePath, 'utf8');
+
+		try {
+			return JSON.parse(fileContent) as Workflow;
+		} catch (error) {
+			const e = error as Error;
+			throw new Error(`Failed to parse workflow file ${workflowFilePath}: ${e.message}`);
+		}
+	}
+
+	private loadSingleDataTableFromFile(dataTableFilePath: string): DataTable {
+		const fileContent = fs.readFileSync(dataTableFilePath, 'utf8');
+
+		try {
+			return JSON.parse(fileContent) as DataTable;
+		} catch (error) {
+			const e = error as Error;
+			throw new Error(`Failed to parse data table file ${dataTableFilePath}: ${e.message}`);
+		}
+	}
+}

package/src/scenario/scenario-loader.ts

@@ -0,0 +1,90 @@
+import { createHash } from 'node:crypto';
+import * as fs from 'node:fs';
+import * as path from 'path';
+
+import type { Scenario, ScenarioManifest } from '@/types/scenario';
+
+export class ScenarioLoader {
+	/**
+	 * Loads all scenarios from the given path
+	 */
+	loadAll(pathToScenarios: string, filter?: string): Scenario[] {
+		pathToScenarios = path.resolve(pathToScenarios);
+		const scenarioFolders = fs
+			.readdirSync(pathToScenarios, { withFileTypes: true })
+			.filter((dirent) => dirent.isDirectory())
+			.map((dirent) => dirent.name);
+
+		const scenarios: Scenario[] = [];
+
+		for (const folder of scenarioFolders) {
+			if (filter && folder.toLowerCase() !== filter.toLowerCase()) {
+				continue;
+			}
+			const scenarioPath = path.join(pathToScenarios, folder);
+			const manifestFileName = `${folder}.manifest.json`;
+			const scenarioManifestPath = path.join(pathToScenarios, folder, manifestFileName);
+			if (!fs.existsSync(scenarioManifestPath)) {
+				console.warn(`Scenario at ${scenarioPath} is missing the ${manifestFileName} file`);
+				continue;
+			}
+
+			// Load the scenario manifest file
+			const [scenario, validationErrors] =
+				this.loadAndValidateScenarioManifest(scenarioManifestPath);
+			if (validationErrors) {
+				console.warn(
+					`Scenario at ${scenarioPath} has the following validation errors: ${validationErrors.join(', ')}`,
+				);
+				continue;
+			}
+
+			scenarios.push({
+				...scenario,
+				id: this.formScenarioId(scenarioPath),
+				scenarioDirPath: scenarioPath,
+			});
+		}
+
+		return scenarios;
+	}
+
+	private loadAndValidateScenarioManifest(
+		scenarioManifestPath: string,
+	): [ScenarioManifest, null] | [null, string[]] {
+		const [scenario, error] = this.loadScenarioManifest(scenarioManifestPath);
+		if (!scenario) {
+			return [null, [error]];
+		}
+
+		const validationErrors: string[] = [];
+
+		if (!scenario.name) {
+			validationErrors.push(`Scenario at ${scenarioManifestPath} is missing a name`);
+		}
+		if (!scenario.description) {
+			validationErrors.push(`Scenario at ${scenarioManifestPath} is missing a description`);
+		}
+
+		return validationErrors.length === 0 ? [scenario, null] : [null, validationErrors];
+	}
+
+	private loadScenarioManifest(
+		scenarioManifestPath: string,
+	): [ScenarioManifest, null] | [null, string] {
+		try {
+			const scenario = JSON.parse(
+				fs.readFileSync(scenarioManifestPath, 'utf8'),
+			) as ScenarioManifest;
+
+			return [scenario, null];
+		} catch (error) {
+			const message = error instanceof Error ? error.message : JSON.stringify(error);
+			return [null, `Failed to parse manifest ${scenarioManifestPath}: ${message}`];
+		}
+	}
+
+	private formScenarioId(scenarioPath: string): string {
+		return createHash('sha256').update(scenarioPath).digest('hex');
+	}
+}