@atom8n/n8n-benchmark 2.0.0

package/src/test-execution/scenario-data-importer.ts

@@ -0,0 +1,165 @@
+import type { AuthenticatedN8nApiClient } from '@/n8n-api-client/authenticated-n8n-api-client';
+import { CredentialApiClient } from '@/n8n-api-client/credentials-api-client';
+import { DataTableApiClient } from '@/n8n-api-client/data-table-api-client';
+import type { Workflow, Credential, DataTable } from '@/n8n-api-client/n8n-api-client.types';
+import { ProjectApiClient } from '@/n8n-api-client/project-api-client';
+import { WorkflowApiClient } from '@/n8n-api-client/workflows-api-client';
+import type { LoadableScenarioData } from '@/scenario/scenario-data-loader';
+
+/**
+ * Imports scenario data into an n8n instance
+ */
+export class ScenarioDataImporter {
+	private readonly workflowApiClient: WorkflowApiClient;
+	private readonly credentialApiClient: CredentialApiClient;
+	private readonly dataTableApiClient: DataTableApiClient;
+	private readonly projectApiClient: ProjectApiClient;
+
+	constructor(n8nApiClient: AuthenticatedN8nApiClient) {
+		this.workflowApiClient = new WorkflowApiClient(n8nApiClient);
+		this.credentialApiClient = new CredentialApiClient(n8nApiClient);
+		this.dataTableApiClient = new DataTableApiClient(n8nApiClient);
+		this.projectApiClient = new ProjectApiClient(n8nApiClient);
+	}
+
+	private replaceValuesInObject(obj: unknown, searchText: string, targetText: string) {
+		if (Array.isArray(obj)) {
+			obj.map((item) => this.replaceValuesInObject(item, searchText, targetText));
+		} else if (typeof obj === 'object' && obj !== null) {
+			for (const [key, value] of Object.entries(obj)) {
+				if (typeof value === 'string' && value === searchText) {
+					(obj as Record<string, unknown>)[key] = targetText;
+				} else {
+					this.replaceValuesInObject(value, searchText, targetText);
+				}
+			}
+		}
+	}
+
+	async importTestScenarioData(data: LoadableScenarioData) {
+		const existingWorkflows = await this.workflowApiClient.getAllWorkflows();
+		const existingCredentials = await this.credentialApiClient.getAllCredentials();
+		const existingDataTables = await this.dataTableApiClient.getAllDataTables();
+
+		for (const credential of data.credentials) {
+			const createdCredential = await this.importCredentials({ existingCredentials, credential });
+
+			// We need to update the id and name of the credential in the workflows
+			for (const workflow of data.workflows) {
+				this.replaceValuesInObject(workflow, credential.id, createdCredential.id);
+				this.replaceValuesInObject(workflow, credential.name, createdCredential.name);
+			}
+		}
+
+		for (const workflow of data.workflows) {
+			await this.importWorkflow({ existingWorkflows, workflow });
+		}
+
+		let dataTableId: string | undefined;
+		if (data.dataTable) {
+			dataTableId = await this.importDataTable({ existingDataTables, dataTable: data.dataTable });
+		}
+
+		return { dataTableId };
+	}
+
+	/**
+	 * Imports a single credential into n8n removing any existing credentials with the same name
+	 * @param opts
+	 * @returns
+	 */
+	private async importCredentials(opts: {
+		existingCredentials: Credential[];
+		credential: Credential;
+	}) {
+		const existingCredentials = this.findExistingCredentials(
+			opts.existingCredentials,
+			opts.credential,
+		);
+		if (existingCredentials.length > 0) {
+			for (const toDelete of existingCredentials) {
+				await this.credentialApiClient.deleteCredential(toDelete.id);
+			}
+		}
+
+		return await this.credentialApiClient.createCredential({
+			...opts.credential,
+			name: this.getBenchmarkCredentialName(opts.credential),
+		});
+	}
+
+	private async importDataTable(opts: { existingDataTables: DataTable[]; dataTable: DataTable }) {
+		const { existingDataTables, dataTable } = opts;
+
+		const projectId = await this.projectApiClient.getPersonalProject();
+
+		const existingTable = existingDataTables.find(
+			(dt) => dt.name === this.getBenchmarkDataTableName(dataTable),
+		);
+
+		if (existingTable) {
+			await this.dataTableApiClient.deleteDataTable(projectId, existingTable.id);
+		}
+
+		const { id } = await this.dataTableApiClient.createDataTable(projectId, {
+			...dataTable,
+			name: this.getBenchmarkDataTableName(dataTable),
+		});
+
+		return id;
+	}
+
+	/**
+	 * Imports a single workflow into n8n removing any existing workflows with the same name
+	 */
+	private async importWorkflow(opts: { existingWorkflows: Workflow[]; workflow: Workflow }) {
+		const existingWorkflows = this.findExistingWorkflows(opts.existingWorkflows, opts.workflow);
+		if (existingWorkflows.length > 0) {
+			for (const toDelete of existingWorkflows) {
+				await this.workflowApiClient.archiveWorkflow(toDelete.id);
+				await this.workflowApiClient.deleteWorkflow(toDelete.id);
+			}
+		}
+
+		const createdWorkflow = await this.workflowApiClient.createWorkflow({
+			...opts.workflow,
+			name: this.getBenchmarkWorkflowName(opts.workflow),
+		});
+
+		return await this.workflowApiClient.activateWorkflow(createdWorkflow);
+	}
+
+	private findExistingCredentials(
+		existingCredentials: Credential[],
+		credentialToImport: Credential,
+	): Credential[] {
+		const benchmarkCredentialName = this.getBenchmarkCredentialName(credentialToImport);
+
+		return existingCredentials.filter(
+			(existingCredential) => existingCredential.name === benchmarkCredentialName,
+		);
+	}
+
+	private findExistingWorkflows(
+		existingWorkflows: Workflow[],
+		workflowToImport: Workflow,
+	): Workflow[] {
+		const benchmarkWorkflowName = this.getBenchmarkWorkflowName(workflowToImport);
+
+		return existingWorkflows.filter(
+			(existingWorkflow) => existingWorkflow.name === benchmarkWorkflowName,
+		);
+	}
+
+	private getBenchmarkCredentialName(credential: Credential) {
+		return `[BENCHMARK] ${credential.name}`;
+	}
+
+	private getBenchmarkWorkflowName(workflow: Workflow) {
+		return `[BENCHMARK] ${workflow.name}`;
+	}
+
+	private getBenchmarkDataTableName(dataTable: DataTable) {
+		return `[BENCHMARK] ${dataTable.name}`;
+	}
+}

package/src/test-execution/scenario-runner.ts

@@ -0,0 +1,76 @@
+import { sleep } from 'zx';
+
+import { AuthenticatedN8nApiClient } from '@/n8n-api-client/authenticated-n8n-api-client';
+import type { N8nApiClient } from '@/n8n-api-client/n8n-api-client';
+import type { ScenarioDataFileLoader } from '@/scenario/scenario-data-loader';
+import { ScenarioDataImporter } from '@/test-execution/scenario-data-importer';
+import type { Scenario } from '@/types/scenario';
+
+import type { K6Executor } from './k6-executor';
+
+/**
+ * Runs scenarios
+ */
+export class ScenarioRunner {
+	constructor(
+		private readonly n8nClient: N8nApiClient,
+		private readonly dataLoader: ScenarioDataFileLoader,
+		private readonly k6Executor: K6Executor,
+		private readonly ownerConfig: {
+			email: string;
+			password: string;
+		},
+		private readonly scenarioPrefix: string,
+	) {}
+
+	async runManyScenarios(scenarios: Scenario[]) {
+		console.log(`Waiting for n8n ${this.n8nClient.apiBaseUrl} to become online`);
+		await this.n8nClient.waitForInstanceToBecomeOnline();
+
+		console.log('Setting up owner');
+		await this.n8nClient.setupOwnerIfNeeded(this.ownerConfig);
+
+		const authenticatedN8nClient = await AuthenticatedN8nApiClient.createUsingUsernameAndPassword(
+			this.n8nClient,
+			this.ownerConfig,
+		);
+		const testDataImporter = new ScenarioDataImporter(authenticatedN8nClient);
+
+		for (const scenario of scenarios) {
+			await this.runSingleTestScenario(testDataImporter, scenario);
+		}
+	}
+
+	private async runSingleTestScenario(testDataImporter: ScenarioDataImporter, scenario: Scenario) {
+		const scenarioRunName = this.formTestScenarioRunName(scenario);
+		console.log('Running scenario:', scenarioRunName);
+
+		console.log('Loading and importing data');
+		const testData = await this.dataLoader.loadDataForScenario(scenario);
+		const { dataTableId } = await testDataImporter.importTestScenarioData(testData);
+
+		// Wait for 1s before executing the scenario to ensure that the workflows are activated.
+		// In multi-main mode it can take some time before the workflow becomes active.
+		await sleep(1000);
+
+		console.log('Executing scenario script');
+		await this.k6Executor.executeTestScenario(
+			{
+				...scenario,
+				dataTableId,
+			},
+			{
+				scenarioRunName,
+			},
+		);
+	}
+
+	/**
+	 * Forms a name for the scenario by combining prefix and scenario name.
+	 * The benchmarks are ran against different n8n setups, so we use the
+	 * prefix to differentiate between them.
+	 */
+	private formTestScenarioRunName(scenario: Scenario) {
+		return `${this.scenarioPrefix}-${scenario.name}`;
+	}
+}

package/src/test-execution/test-report.ts

@@ -0,0 +1,152 @@
+import { nanoid } from 'nanoid';
+
+import { PrometheusMetricsParser } from '@/test-execution/prometheus-metrics-parser';
+import type { Scenario } from '@/types/scenario';
+
+export type K6Tag = {
+	name: string;
+	value: string;
+};
+
+export type Check = {
+	name: string;
+	passes: number;
+	fails: number;
+};
+
+export type CounterMetric = {
+	type: 'counter';
+	count: number;
+	rate: number;
+};
+
+export type TrendMetric = {
+	type: 'trend';
+	'p(95)': number;
+	avg: number;
+	min: number;
+	med: number;
+	max: number;
+	'p(90)': number;
+};
+
+export type AppMetricStats = {
+	max: number;
+	avg: number;
+	min: number;
+	count: number;
+};
+
+export type AppMetricsReport = {
+	heapSizeTotal?: AppMetricStats;
+	heapSizeUsed?: AppMetricStats;
+	externalMemory?: AppMetricStats;
+	eventLoopLag?: AppMetricStats;
+};
+
+export type TestReport = {
+	runId: string;
+	ts: string; // ISO8601
+	scenarioName: string;
+	tags: K6Tag[];
+	metrics: {
+		iterations: CounterMetric;
+		dataReceived: CounterMetric;
+		dataSent: CounterMetric;
+		httpRequests: CounterMetric;
+		httpRequestDuration: TrendMetric;
+		httpRequestSending: TrendMetric;
+		httpRequestReceiving: TrendMetric;
+		httpRequestWaiting: TrendMetric;
+	};
+	checks: Check[];
+	appMetrics?: AppMetricsReport;
+};
+
+function k6CheckToCheck(check: K6Check): Check {
+	return {
+		name: check.name,
+		passes: check.passes,
+		fails: check.fails,
+	};
+}
+
+function k6CounterToCounter(counter: K6CounterMetric): CounterMetric {
+	return {
+		type: 'counter',
+		count: counter.values.count,
+		rate: counter.values.rate,
+	};
+}
+
+function k6TrendToTrend(trend: K6TrendMetric): TrendMetric {
+	return {
+		type: 'trend',
+		'p(90)': trend.values['p(90)'],
+		avg: trend.values.avg,
+		min: trend.values.min,
+		med: trend.values.med,
+		max: trend.values.max,
+		'p(95)': trend.values['p(95)'],
+	};
+}
+
+/**
+ * Builds an app metrics report from collected Prometheus metrics data
+ */
+export function buildAppMetricsReport(metricsData: string[]): AppMetricsReport {
+	const heapSizeTotal = PrometheusMetricsParser.calculateMetricStats(
+		metricsData,
+		'n8n_nodejs_heap_size_total_bytes',
+	);
+	const heapSizeUsed = PrometheusMetricsParser.calculateMetricStats(
+		metricsData,
+		'n8n_nodejs_heap_size_used_bytes',
+	);
+	const externalMemory = PrometheusMetricsParser.calculateMetricStats(
+		metricsData,
+		'n8n_nodejs_external_memory_bytes',
+	);
+	const eventLoopLag = PrometheusMetricsParser.calculateMetricStats(
+		metricsData,
+		'n8n_nodejs_eventloop_lag_seconds',
+	);
+
+	return {
+		...(heapSizeTotal && { heapSizeTotal }),
+		...(heapSizeUsed && { heapSizeUsed }),
+		...(externalMemory && { externalMemory }),
+		...(eventLoopLag && { eventLoopLag }),
+	};
+}
+
+/**
+ * Converts the k6 test summary to a test report
+ */
+export function buildTestReport(
+	scenario: Scenario,
+	endOfTestSummary: K6EndOfTestSummary,
+	tags: K6Tag[],
+	appMetricsData?: string[],
+): TestReport {
+	const appMetrics = appMetricsData ? buildAppMetricsReport(appMetricsData) : undefined;
+
+	return {
+		runId: nanoid(),
+		ts: new Date().toISOString(),
+		scenarioName: scenario.name,
+		tags,
+		checks: endOfTestSummary.root_group.checks.map(k6CheckToCheck),
+		metrics: {
+			dataReceived: k6CounterToCounter(endOfTestSummary.metrics.data_received),
+			dataSent: k6CounterToCounter(endOfTestSummary.metrics.data_sent),
+			httpRequests: k6CounterToCounter(endOfTestSummary.metrics.http_reqs),
+			httpRequestDuration: k6TrendToTrend(endOfTestSummary.metrics.http_req_duration),
+			httpRequestSending: k6TrendToTrend(endOfTestSummary.metrics.http_req_sending),
+			httpRequestReceiving: k6TrendToTrend(endOfTestSummary.metrics.http_req_receiving),
+			httpRequestWaiting: k6TrendToTrend(endOfTestSummary.metrics.http_req_waiting),
+			iterations: k6CounterToCounter(endOfTestSummary.metrics.iterations),
+		},
+		...(appMetrics && { appMetrics }),
+	};
+}

package/src/types/scenario.ts

@@ -0,0 +1,33 @@
+export type ScenarioData = {
+	/** Relative paths to the workflow files */
+	workflowFiles?: string[];
+	/** Relative paths to the credential files */
+	credentialFiles?: string[];
+	/** Relative paths to the data table files */
+	dataTableFile?: string;
+};
+
+/**
+ * Configuration that defines the benchmark scenario
+ */
+export type ScenarioManifest = {
+	/** The name of the scenario */
+	name: string;
+	/** A longer description of the scenario */
+	description: string;
+	/** Relative path to the k6 script */
+	scriptPath: string;
+	/** Data to import before running the scenario */
+	scenarioData: ScenarioData;
+};
+
+/**
+ * Scenario with additional metadata
+ */
+export type Scenario = ScenarioManifest & {
+	id: string;
+	/** Path to the directory containing the scenario */
+	scenarioDirPath: string;
+	/** ID of the data table created for the scenario, if any */
+	dataTableId?: string;
+};

package/tsconfig.build.json

@@ -0,0 +1,9 @@
+{
+	"extends": ["./tsconfig.json", "@n8n/typescript-config/tsconfig.build.json"],
+	"compilerOptions": {
+		"rootDir": "src",
+		"outDir": "dist",
+		"tsBuildInfoFile": "dist/build.tsbuildinfo"
+	},
+	"include": ["src/**/*.ts"]
+}

package/tsconfig.json

@@ -0,0 +1,14 @@
+{
+	"extends": [
+		"@n8n/typescript-config/tsconfig.common.json",
+		"@n8n/typescript-config/tsconfig.backend.json"
+	],
+	"compilerOptions": {
+		"rootDir": ".",
+		"baseUrl": "src",
+		"paths": {
+			"@/*": ["./*"]
+		}
+	},
+	"include": ["src/**/*.ts"]
+}