@atom8n/n8n-benchmark 2.0.0

package/scenarios/multiple-webhooks/multiple-webhooks9.json

@@ -0,0 +1,25 @@
+{
+	"createdAt": "2024-08-06T12:19:51.268Z",
+	"updatedAt": "2024-08-06T12:20:45.000Z",
+	"name": "Multiple Webhook 9",
+	"active": true,
+	"nodes": [
+		{
+			"parameters": { "path": "multiple-webhook9", "options": {} },
+			"id": "34ac4500-9a29-4f4f-a604-134aa2cb2889",
+			"name": "Webhook",
+			"type": "n8n-nodes-base.webhook",
+			"typeVersion": 2,
+			"position": [760, 400],
+			"webhookId": "47fe25ac-1376-4ee7-b9de-3fff1d49281c"
+		}
+	],
+	"connections": {},
+	"settings": { "executionOrder": "v1" },
+	"staticData": null,
+	"meta": { "templateCredsSetupCompleted": true, "responseMode": "lastNode", "options": {} },
+	"pinData": {},
+	"versionId": "106d4d2c-49c0-45b8-8421-99cc0c8b1589",
+	"triggerCount": 1,
+	"tags": []
+}

package/scenarios/py-code-node/py-code-node.json

@@ -0,0 +1,98 @@
+{
+	"createdAt": "2024-08-06T12:19:51.268Z",
+	"updatedAt": "2024-08-06T12:20:45.000Z",
+	"name": "Python Code Node",
+	"active": true,
+	"nodes": [
+		{
+			"parameters": {
+				"respondWith": "allIncomingItems",
+				"options": {}
+			},
+			"type": "n8n-nodes-base.respondToWebhook",
+			"typeVersion": 1.1,
+			"position": [1280, 460],
+			"id": "0067e317-09b8-478a-8c50-e19b4c9e294c",
+			"name": "Respond to Webhook"
+		},
+		{
+			"parameters": {
+				"language": "pythonNative",
+				"mode": "runOnceForEachItem",
+				"pythonCode": "def pseudo_random(seed_str, max_val):\n    return hash(seed_str) % max_val\n\n# Add new field\n_item['json']['age'] = 10 + pseudo_random(str(_item['json']['email']), 30)\n\n# Mutate existing field\n_item['json']['password'] = '*' * len(_item['json']['password'])\n\n# Remove field\nif 'lastname' in _item['json']:\n    del _item['json']['lastname']\n\n# New object field\nemail_parts = _item['json']['email'].split('@')\n_item['json']['emailData'] = {\n    'user': email_parts[0],\n    'domain': email_parts[1]\n}\n\nreturn _item"
+			},
+			"type": "n8n-nodes-base.code",
+			"typeVersion": 2,
+			"position": [1040, 460],
+			"id": "56d751c0-0d30-43c3-89fa-bebf3a9d436f",
+			"name": "OnceForEachItemPythonCode"
+		},
+		{
+			"parameters": {
+				"httpMethod": "POST",
+				"path": "py-code-node-benchmark",
+				"responseMode": "responseNode",
+				"options": {}
+			},
+			"type": "n8n-nodes-base.webhook",
+			"typeVersion": 2,
+			"position": [580, 460],
+			"id": "417d749d-156c-4ffe-86ea-336f702dc5da",
+			"name": "Webhook",
+			"webhookId": "34ca1895-ccf4-4a4a-8bb8-a042f5edb567"
+		},
+		{
+			"parameters": {
+				"language": "pythonNative",
+				"pythonCode": "def pseudo_random_string(length):\n    characters = 'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789'\n    result = ''\n    seed = hash(str(length)) % 1000\n    for i in range(length):\n        index = (hash(str(seed + i)) % len(characters))\n        result += characters[index]\n        seed = (seed * 31 + i) % 1000\n    return result\n\ndef random_uid():\n    lengths = [8, 4, 4, 4, 8]\n    parts = [pseudo_random_string(length) for length in lengths]\n    return '-'.join(parts)\n\ndef random_email():\n    return f\"{pseudo_random_string(8)}@{pseudo_random_string(10)}.com\"\n\ndef random_person():\n    return {\n        'uid': random_uid(),\n        'email': random_email(),\n        'firstname': pseudo_random_string(5),\n        'lastname': pseudo_random_string(12),\n        'password': pseudo_random_string(10)\n    }\n\nreturn [{'json': random_person()} for _ in range(100)]"
+			},
+			"id": "c30db155-73ca-48b9-8860-c3fe7a0926fb",
+			"name": "Code",
+			"type": "n8n-nodes-base.code",
+			"typeVersion": 2,
+			"position": [820, 460]
+		}
+	],
+	"connections": {
+		"OnceForEachItemPythonCode": {
+			"main": [
+				[
+					{
+						"node": "Respond to Webhook",
+						"type": "main",
+						"index": 0
+					}
+				]
+			]
+		},
+		"Webhook": {
+			"main": [
+				[
+					{
+						"node": "Code",
+						"type": "main",
+						"index": 0
+					}
+				]
+			]
+		},
+		"Code": {
+			"main": [
+				[
+					{
+						"node": "OnceForEachItemPythonCode",
+						"type": "main",
+						"index": 0
+					}
+				]
+			]
+		}
+	},
+	"settings": { "executionOrder": "v1" },
+	"staticData": null,
+	"meta": { "templateCredsSetupCompleted": true, "responseMode": "lastNode", "options": {} },
+	"pinData": {},
+	"versionId": "840a38a1-ba37-433d-9f20-de73f5131a2b",
+	"triggerCount": 1,
+	"tags": []
+}

package/scenarios/py-code-node/py-code-node.manifest.json

@@ -0,0 +1,7 @@
+{
+	"$schema": "../scenario.schema.json",
+	"name": "CodeNodePython",
+	"description": "A Python Code Node that first generates 100 items and then runs once for each item and adds, modifies and removes properties. The data is returned with RespondToWebhook Node.",
+	"scenarioData": { "workflowFiles": ["py-code-node.json"] },
+	"scriptPath": "py-code-node.script.js"
+}

package/scenarios/py-code-node/py-code-node.script.js

@@ -0,0 +1,29 @@
+import http from 'k6/http';
+import { check } from 'k6';
+
+const apiBaseUrl = __ENV.API_BASE_URL;
+
+export default function () {
+	const res = http.post(`${apiBaseUrl}/webhook/py-code-node-benchmark`, {});
+
+	if (res.status !== 200) {
+		console.error(
+			`Invalid response. Received status ${res.status}. Body: ${JSON.stringify(res.body)}`,
+		);
+	}
+
+	check(res, {
+		'is status 200': (r) => r.status === 200,
+		'has items in response': (r) => {
+			if (r.status !== 200) return false;
+
+			try {
+				const body = JSON.parse(r.body);
+				return Array.isArray(body) ? body.length === 100 : false;
+			} catch (error) {
+				console.error('Error parsing response body: ', error);
+				return false;
+			}
+		},
+	});
+}

package/scenarios/scenario.schema.json

@@ -0,0 +1,51 @@
+{
+	"definitions": {
+		"ScenarioData": {
+			"type": "object",
+			"properties": {
+				"workflowFiles": {
+					"type": "array",
+					"items": {
+						"type": "string"
+					}
+				},
+				"credentialFiles": {
+					"type": "array",
+					"items": {
+						"type": "string"
+					}
+				},
+				"dataTableFile": {
+					"type": "string"
+				}
+			},
+			"required": [],
+			"additionalProperties": false
+		}
+	},
+	"type": "object",
+	"properties": {
+		"$schema": {
+			"type": "string",
+			"description": "The JSON schema to validate this file"
+		},
+		"name": {
+			"type": "string",
+			"description": "The name of the scenario"
+		},
+		"description": {
+			"type": "string",
+			"description": "A longer description of the scenario"
+		},
+		"scriptPath": {
+			"type": "string",
+			"description": "Relative path to the k6 test script"
+		},
+		"scenarioData": {
+			"$ref": "#/definitions/ScenarioData",
+			"description": "Data to import before running the scenario"
+		}
+	},
+	"required": ["name", "description", "scriptPath", "scenarioData"],
+	"additionalProperties": false
+}

package/scenarios/set-node-expressions/set-node-expressions.json

@@ -0,0 +1,91 @@
+{
+	"createdAt": "2024-09-03T11:30:26.333Z",
+	"updatedAt": "2024-09-03T11:42:52.000Z",
+	"name": "Set Node Expressions",
+	"active": false,
+	"nodes": [
+		{
+			"parameters": {
+				"httpMethod": "POST",
+				"path": "set-expressions-benchmark",
+				"responseMode": "responseNode",
+				"options": {}
+			},
+			"type": "n8n-nodes-base.webhook",
+			"typeVersion": 2,
+			"position": [40, 0],
+			"id": "5babc228-2b89-48cb-8337-28416e867874",
+			"name": "Webhook",
+			"webhookId": "f6f1750d-b734-496f-afe8-26e8e393ca87"
+		},
+		{
+			"parameters": { "respondWith": "allIncomingItems", "options": {} },
+			"type": "n8n-nodes-base.respondToWebhook",
+			"typeVersion": 1.1,
+			"position": [640, 0],
+			"id": "4146a3fb-403c-4cfc-9d38-8af4d16a8440",
+			"name": "Respond to Webhook"
+		},
+		{
+			"parameters": {
+				"assignments": {
+					"assignments": [
+						{
+							"id": "48c46098-f411-41f7-8f0a-1da372340a4e",
+							"name": "oneToOneCopy",
+							"value": "={{ $json.headers.host }}",
+							"type": "string"
+						},
+						{
+							"id": "5d90808b-1c1a-4065-ac51-6d61bd03e564",
+							"name": "={{ $json.headers['user-agent'].slice(0, 4) }}",
+							"value": "Set key with expression",
+							"type": "string"
+						},
+						{
+							"id": "8a74ac24-1f43-43ba-969d-87bfd2f401ce",
+							"name": "Multiple variables",
+							"value": "={{ $json.executionMode + ' ' + $json.webhookUrl }}",
+							"type": "string"
+						},
+						{
+							"id": "93eba201-79d9-4305-a246-f9c8ec50ebab",
+							"name": "Static value",
+							"value": 42,
+							"type": "number"
+						},
+						{
+							"id": "0470a712-c795-44ab-9dcc-05a3f67698bb",
+							"name": "Object",
+							"value": "={{ $json.headers }}",
+							"type": "object"
+						},
+						{
+							"id": "eb671167-da14-4b55-8eea-31ab7bedae10",
+							"name": "Array",
+							"value": "={{ Object.values($json.headers) }}",
+							"type": "array"
+						}
+					]
+				},
+				"options": {}
+			},
+			"type": "n8n-nodes-base.set",
+			"typeVersion": 3.4,
+			"position": [360, 0],
+			"id": "0cb5e82d-f61e-4d91-8fa9-365e382a4d75",
+			"name": "Edit Fields"
+		}
+	],
+	"connections": {
+		"Webhook": { "main": [[{ "node": "Edit Fields", "type": "main", "index": 0 }]] },
+		"Edit Fields": { "main": [[{ "node": "Respond to Webhook", "type": "main", "index": 0 }]] }
+	},
+	"settings": { "executionOrder": "v1" },
+	"staticData": null,
+	"meta": null,
+	"pinData": {},
+	"versionId": "04fd543e-3923-4092-8c2b-2b4262ccbb38",
+	"triggerCount": 0,
+	"tags": []
+}

package/scenarios/set-node-expressions/set-node-expressions.manifest.json

@@ -0,0 +1,7 @@
+{
+	"$schema": "../scenario.schema.json",
+	"name": "SetNodeExpressions",
+	"description": "Expressions in a Set node",
+	"scenarioData": { "workflowFiles": ["set-node-expressions.json"] },
+	"scriptPath": "set-node-expressions.script.js"
+}

package/scenarios/set-node-expressions/set-node-expressions.script.js

@@ -0,0 +1,18 @@
+import http from 'k6/http';
+import { check } from 'k6';
+
+const apiBaseUrl = __ENV.API_BASE_URL;
+
+export default function () {
+	const res = http.post(`${apiBaseUrl}/webhook/set-expressions-benchmark`, {});
+
+	if (res.status !== 200) {
+		console.error(
+			`Invalid response. Received status ${res.status}. Body: ${JSON.stringify(res.body)}`,
+		);
+	}
+
+	check(res, {
+		'is status 200': (r) => r.status === 200,
+	});
+}

package/scenarios/single-webhook/single-webhook.json

@@ -0,0 +1,25 @@
+{
+	"createdAt": "2024-08-06T12:19:51.268Z",
+	"updatedAt": "2024-08-06T12:20:45.000Z",
+	"name": "Single Webhook",
+	"active": true,
+	"nodes": [
+		{
+			"parameters": { "path": "single-webhook", "options": {} },
+			"id": "7587ab0e-cc15-424f-83c0-c887a0eb97fb",
+			"name": "Webhook",
+			"type": "n8n-nodes-base.webhook",
+			"typeVersion": 2,
+			"position": [760, 400],
+			"webhookId": "fa563fc2-c73f-4631-99a1-39c16f1f858f"
+		}
+	],
+	"connections": {},
+	"settings": { "executionOrder": "v1" },
+	"staticData": null,
+	"meta": { "templateCredsSetupCompleted": true, "responseMode": "lastNode", "options": {} },
+	"pinData": {},
+	"versionId": "840a38a1-ba37-433d-9f20-de73f5131a2b",
+	"triggerCount": 1,
+	"tags": []
+}

package/scenarios/single-webhook/single-webhook.manifest.json

@@ -0,0 +1,7 @@
+{
+	"$schema": "../scenario.schema.json",
+	"name": "SingleWebhook",
+	"description": "A single webhook trigger that responds with a 200 status code",
+	"scenarioData": { "workflowFiles": ["single-webhook.json"] },
+	"scriptPath": "single-webhook.script.js"
+}

package/scenarios/single-webhook/single-webhook.script.js

@@ -0,0 +1,18 @@
+import http from 'k6/http';
+import { check } from 'k6';
+
+const apiBaseUrl = __ENV.API_BASE_URL;
+
+export default function () {
+	const res = http.get(`${apiBaseUrl}/webhook/single-webhook`);
+
+	if (res.status !== 200) {
+		console.error(
+			`Invalid response. Received status ${res.status}. Body: ${JSON.stringify(res.body)}`,
+		);
+	}
+
+	check(res, {
+		'is status 200': (r) => r.status === 200,
+	});
+}

package/scripts/bootstrap.sh

@@ -0,0 +1,63 @@
+#!/bin/bash
+#
+# Script to initialize the benchmark environment on a VM
+#
+
+set -euo pipefail;
+
+CURRENT_USER=$(whoami)
+
+# Mount the data disk
+# First wait for the disk to become available
+WAIT_TIME=0
+MAX_WAIT_TIME=60
+
+while [ ! -e /dev/sdc ]; do
+    if [ $WAIT_TIME -ge $MAX_WAIT_TIME ]; then
+        echo "Error: /dev/sdc did not become available within $MAX_WAIT_TIME seconds."
+        exit 1
+    fi
+
+    echo "Waiting for /dev/sdc to be available... ($WAIT_TIME/$MAX_WAIT_TIME)"
+    sleep 1
+    WAIT_TIME=$((WAIT_TIME + 1))
+done
+
+# Then mount it
+if [ -d "/n8n" ]; then
+	echo "Data disk already mounted. Clearing it..."
+	sudo rm -rf /n8n/*
+	sudo rm -rf /n8n/.[!.]*
+else
+	sudo mkdir -p /n8n
+	sudo parted /dev/sdc --script mklabel gpt mkpart xfspart xfs 0% 100%
+	sudo mkfs.xfs /dev/sdc1
+	sudo partprobe /dev/sdc1
+	sudo mount /dev/sdc1 /n8n
+	sudo chown -R "$CURRENT_USER":"$CURRENT_USER" /n8n
+fi
+
+### Remove unneeded dependencies
+# TTY
+sudo systemctl disable getty@tty1.service
+sudo systemctl disable serial-getty@ttyS0.service
+# Snap
+sudo systemctl disable snapd.service
+# Unattended upgrades
+sudo systemctl disable unattended-upgrades.service
+# Cron
+sudo systemctl disable cron.service
+
+# Include nodejs v20 repository
+curl -fsSL https://deb.nodesource.com/setup_20.x -o nodesource_setup.sh
+sudo -E bash nodesource_setup.sh
+
+# Install docker, docker compose, nodejs
+sudo DEBIAN_FRONTEND=noninteractive apt-get update -yq
+sudo DEBIAN_FRONTEND=noninteractive apt-get install -yq docker.io docker-compose nodejs
+
+# Add the current user to the docker group
+sudo usermod -aG docker "$CURRENT_USER"
+
+# Install zx
+npm install zx

package/scripts/clients/docker-compose-client.mjs

@@ -0,0 +1,45 @@
+import { which } from 'zx';
+
+export class DockerComposeClient {
+	/**
+	 *
+	 * @param {{ $: Shell; verbose?: boolean }} opts
+	 */
+	constructor({ $ }) {
+		this.$$ = $;
+	}
+
+	async $(...args) {
+		await this.resolveExecutableIfNeeded();
+
+		if (this.isCompose) {
+			return await this.$$`docker-compose ${args}`;
+		} else {
+			return await this.$$`docker compose ${args}`;
+		}
+	}
+
+	async resolveExecutableIfNeeded() {
+		if (this.isResolved) {
+			return;
+		}
+
+		// The VM deployment doesn't have `docker compose` available,
+		// so try to resolve the `docker-compose` first
+		const compose = await which('docker-compose', { nothrow: true });
+		if (compose) {
+			this.isResolved = true;
+			this.isCompose = true;
+			return;
+		}
+
+		const docker = await which('docker', { nothrow: true });
+		if (docker) {
+			this.isResolved = true;
+			this.isCompose = false;
+			return;
+		}
+
+		throw new Error('Could not resolve docker-compose or docker');
+	}
+}

package/scripts/clients/ssh-client.mjs

@@ -0,0 +1,37 @@
+// @ts-check
+import { $ } from 'zx';
+
+export class SshClient {
+	/**
+	 *
+	 * @param {{ privateKeyPath: string; ip: string; username: string; verbose?: boolean }} param0
+	 */
+	constructor({ privateKeyPath, ip, username, verbose = false }) {
+		this.verbose = verbose;
+		this.privateKeyPath = privateKeyPath;
+		this.ip = ip;
+		this.username = username;
+
+		this.$$ = $({
+			verbose,
+		});
+	}
+
+	/**
+	 * @param {string} command
+	 * @param {{ verbose?: boolean }} [options]
+	 */
+	async ssh(command, options = {}) {
+		const $$ = options?.verbose ? $({ verbose: true }) : this.$$;
+
+		const target = `${this.username}@${this.ip}`;
+
+		await $$`ssh -i ${this.privateKeyPath} -o StrictHostKeyChecking=accept-new ${target} ${command}`;
+	}
+
+	async scp(source, destination) {
+		const target = `${this.username}@${this.ip}:${destination}`;
+		await this
+			.$$`scp -i ${this.privateKeyPath} -o StrictHostKeyChecking=accept-new ${source} ${target}`;
+	}
+}

package/scripts/clients/terraform-client.mjs

@@ -0,0 +1,71 @@
+// @ts-check
+
+import path from 'path';
+import { $, fs } from 'zx';
+
+const paths = {
+	infraCodeDir: path.resolve('infra'),
+	terraformStateFile: path.join(path.resolve('infra'), 'terraform.tfstate'),
+};
+
+export class TerraformClient {
+	constructor({ isVerbose = false }) {
+		this.isVerbose = isVerbose;
+		this.$$ = $({
+			cwd: paths.infraCodeDir,
+			verbose: isVerbose,
+		});
+	}
+
+	/**
+	 * Provisions the environment
+	 */
+	async provisionEnvironment() {
+		console.log('Provisioning cloud environment...');
+
+		await this.$$`terraform init`;
+		await this.$$`terraform apply -input=false -auto-approve`;
+	}
+
+	/**
+	 * @typedef {Object} BenchmarkEnv
+	 * @property {string} vmName
+	 * @property {string} ip
+	 * @property {string} sshUsername
+	 * @property {string} sshPrivateKeyPath
+	 *
+	 * @returns {Promise<BenchmarkEnv>}
+	 */
+	async getTerraformOutputs() {
+		const privateKeyName = await this.extractPrivateKey();
+
+		return {
+			ip: await this.getTerraformOutput('ip'),
+			sshUsername: await this.getTerraformOutput('ssh_username'),
+			sshPrivateKeyPath: path.join(paths.infraCodeDir, privateKeyName),
+			vmName: await this.getTerraformOutput('vm_name'),
+		};
+	}
+
+	hasTerraformState() {
+		return fs.existsSync(paths.terraformStateFile);
+	}
+
+	async destroyEnvironment() {
+		console.log('Destroying cloud environment...');
+
+		await this.$$`terraform destroy -input=false -auto-approve`;
+	}
+
+	async getTerraformOutput(key) {
+		const output = await this.$$`terraform output -raw ${key}`;
+		return output.stdout.trim();
+	}
+
+	async extractPrivateKey() {
+		await this.$$`terraform output -raw ssh_private_key > privatekey.pem`;
+		await this.$$`chmod 600 privatekey.pem`;
+
+		return 'privatekey.pem';
+	}
+}