@contember/engine-server 1.4.0-rc.8 → 2.0.0-alpha.8

package/esbuild.cjs

@@ -0,0 +1,42 @@
+const path = require('path')
+const esbuild = require('esbuild')
+const {nodeExternalsPlugin} = require('esbuild-node-externals');
+const glob = require('fast-glob')
+
+const root = path.dirname(path.dirname(__dirname))
+
+;(async () => {
+	const packageJsonFiles = await glob('packages/*/package.json')
+	const buildServer = entrypoint => esbuild.build({
+		entryPoints: [entrypoint],
+		bundle: true,
+		platform: 'node',
+		format: 'esm',
+		sourcemap: 'external',
+		outfile: './server/server.js',
+		plugins: [
+			nodeExternalsPlugin({
+				packagePath: packageJsonFiles,
+				allowList: it => it.startsWith('@contember/')
+			}),
+			{
+				name: 'resolve contember',
+				setup(build) {
+					build.onResolve({filter: /^@contember\/.+$/}, args => {
+						const pkg = args.path.match(/@contember\/(.+)/)[1]
+						return {path: path.join(root, 'packages', pkg, pkg.includes('/') ? 'index.ts' : 'src/index.ts')}
+					})
+				},
+			},
+		],
+	})
+
+	await buildServer('./packages/engine-server/src/start.ts')
+
+})()
+.catch(e => {
+	console.error(e)
+	process.exit(1)
+})
+
+

package/package.json

@@ -1,26 +1,40 @@
 {
   "name": "@contember/engine-server",
-  "version": "1.4.0-rc.8",
+  "version": "2.0.0-alpha.8",
   "license": "Apache-2.0",
-  "main": "dist/src/index.js",
-  "typings": "dist/src/index.d.ts",
-  "scripts": {
-    "test-e2e": "vitest --dir ./tests/e2e --no-threads"
+  "main": "./dist/production/index.js",
+  "typings": "./dist/types/index.d.ts",
+  "exports": {
+    ".": {
+      "import": {
+        "types": "./dist/types/index.d.ts",
+        "development": "./dist/development/index.js",
+        "production": "./dist/production/index.js",
+        "typescript": "./src/index.ts",
+        "default": "./dist/production/index.js"
+      },
+      "require": {
+        "types": "./dist/types/index.d.ts",
+        "development": "./dist/development/index.cjs",
+        "production": "./dist/production/index.cjs",
+        "typescript": "./src/index.ts",
+        "default": "./dist/production/index.cjs"
+      }
+    }
   },
   "dependencies": {
-    "@contember/engine-http": "1.4.0-rc.8",
-    "@contember/engine-plugins": "1.4.0-rc.8",
-    "@contember/engine-s3-plugin": "1.4.0-rc.8",
-    "@contember/engine-vimeo-plugin": "1.4.0-rc.8",
+    "@contember/engine-actions": "2.0.0-alpha.8",
+    "@contember/engine-http": "2.0.0-alpha.8",
+    "@contember/engine-plugins": "2.0.0-alpha.8",
+    "@contember/engine-s3-plugin": "2.0.0-alpha.8",
+    "@contember/engine-vimeo-plugin": "2.0.0-alpha.8",
     "graphql": "^16.8.1",
     "pg": "^8.11.3",
     "uuid": "^9.0.1"
   },
   "devDependencies": {
-    "@contember/engine-api-tester": "1.4.0-rc.8",
     "@types/koa-compress": "^4.0.6",
-    "@types/node": "^20.9.0",
-    "@types/supertest": "^2.0.16",
-    "supertest": "^6.3.3"
-  }
+    "@types/node": "^20.9.0"
+  },
+  "type": "module"
 }

package/src/cluster.ts

@@ -0,0 +1,95 @@
+import cluster, { Worker } from 'node:cluster'
+import { timeout } from './utils'
+
+export const getClusterProcessType = (isClusterMode: boolean): ProcessType =>
+	!isClusterMode ? 'singleNode' : cluster.isMaster ? 'clusterMaster' : 'clusterWorker'
+
+export type ProcessType =
+	| 'singleNode'
+	| 'clusterMaster'
+	| 'clusterWorker'
+
+const MSG_WORKER_STARTED = 'msg_worker_started'
+
+export const waitForWorker = (worker: Worker, timeoutMs: number) => {
+	let ok = false
+	return Promise.race([
+		new Promise(async resolve => {
+			const listener = (message: any) => {
+				if ('type' in message && message.type === MSG_WORKER_STARTED) {
+					ok = true
+					resolve(null)
+					worker.removeListener('message', listener)
+				}
+			}
+			await worker.on('message', listener)
+		}),
+		timeout(timeoutMs).then(() => {
+			if (!ok) {
+				throw new Error('Worker start timed out')
+			}
+			return true
+		}),
+	])
+}
+
+export const notifyWorkerStarted = () =>
+	process.send?.({
+		type: MSG_WORKER_STARTED,
+	})
+
+export class WorkerManager {
+
+	private isTerminating = false
+
+	private workers: Set<Worker> = new Set()
+
+	public async start({ workerCount, env }: { workerCount: number; env?: any }): Promise<void> {
+		if (this.isTerminating) {
+			throw new Error(`Worker manager is terminating`)
+		}
+		for (let i = 0; i < workerCount; i++) {
+			await this.startWorker(env)
+			if (this.isTerminating) {
+				return
+			}
+		}
+	}
+
+	private async startWorker(env?: any) {
+		const worker = cluster.fork(env)
+		worker.on('exit', async (code, signal) => {
+			this.workers.delete(worker)
+			if (!this.isTerminating) {
+				// eslint-disable-next-line no-console
+				console.log(`Worker ${worker.process.pid} died with signal ${signal}, restarting`)
+				await timeout(2000)
+				await this.startWorker(env)
+			}
+		})
+		this.workers.add(worker)
+		await waitForWorker(worker, 15000)
+	}
+
+	public async terminate(signal: NodeJS.Signals): Promise<void> {
+		if (this.isTerminating) {
+			throw new Error(`Worker manager is terminating`)
+		}
+		this.isTerminating = true
+		await Promise.allSettled(Array.from(this.workers).map(async it => {
+			if (!it) {
+				return
+			}
+			// eslint-disable-next-line no-console
+			console.log(`Terminating worker ${it.process.pid}`)
+			const disconnectPromise = new Promise(resolve => it.once('disconnect', resolve))
+			it.disconnect()
+			await disconnectPromise
+			const killPromise = new Promise(resolve => it.once('exit', resolve))
+			it.kill(signal)
+			await killPromise
+			// eslint-disable-next-line no-console
+			console.log(`Worker ${it.process.pid} terminated`)
+		}))
+	}
+}

package/src/loadPlugins.ts

@@ -1,7 +1,8 @@
+import ActionsPlugin from '@contember/engine-actions'
 import { Plugin } from '@contember/engine-plugins'
 import S3Plugin from '@contember/engine-s3-plugin'
 import VimeoPlugin from '@contember/engine-vimeo-plugin'
 
 export default function loadPlugins(): Promise<Plugin[]> {
-	return Promise.resolve([new S3Plugin(), new VimeoPlugin()])
+	return Promise.resolve([new S3Plugin(), new VimeoPlugin(), new ActionsPlugin()])
 }

package/src/start.ts

@@ -2,13 +2,21 @@
 
 import {
 	createContainer,
-	createDefaultLogger, createSentryLoggerHandler,
+	createDefaultLogger,
+	createSentryLoggerHandler,
+	getServerVersion,
+	isDebugMode,
 	listenOnProcessTermination,
+	printStartInfo,
+	resolveServerConfig,
 	serverConfigSchema,
 	TerminationJob,
-	getServerVersion, isDebugMode, printStartInfo, resolveServerConfig,
 } from '@contember/engine-http'
 import loadPlugins from './loadPlugins'
+import cluster from 'node:cluster'
+import * as os from 'node:os'
+import { timeout } from './utils'
+import { getClusterProcessType, notifyWorkerStarted, WorkerManager } from './cluster'
 
 const logger = createDefaultLogger()
 process.on('warning', message => {
@@ -16,38 +24,141 @@ process.on('warning', message => {
 })
 ;(async () => {
 	const isDebug = isDebugMode()
-	const terminationJobs: TerminationJob[] = []
-	listenOnProcessTermination(terminationJobs, logger)
 
-	const version = getServerVersion()
+	const version = await getServerVersion()
 	printStartInfo({ version, isDebug }, logger)
 	const plugins = await loadPlugins()
-	const { serverConfig, projectConfigResolver, tenantConfigResolver } = await resolveServerConfig({
-		plugins,
-		serverConfigSchema,
-	})
+	const { serverConfig, projectConfigResolver, tenantConfigResolver } = await resolveServerConfig({ plugins, serverConfigSchema })
 
-	const sentryTransport = createSentryLoggerHandler(serverConfig.logging.sentry?.dsn)
-	if (sentryTransport !== null) {
-		logger.addHandler(sentryTransport)
+	if (process.argv[2] === 'validate') {
+		process.exit(0)
 	}
 
+	const sentryhandler = createSentryLoggerHandler(serverConfig.logging?.sentry?.dsn)
+	if (sentryhandler !== null) {
+		logger.addHandler(sentryhandler)
+	}
+
+	const workerConfig = serverConfig.workerCount || 1
+
+	const workerCount = workerConfig === 'auto' ? os.cpus().length : Number(workerConfig)
+	const isClusterMode = serverConfig.workerCount !== undefined
+
+	const processType = getClusterProcessType(isClusterMode)
+
 	const container = createContainer({
 		debugMode: isDebug,
 		serverConfig,
 		projectConfigResolver,
 		tenantConfigResolver,
 		plugins,
+		processType,
 		version,
 		logger,
 	})
 
-	const initializedProjects: string[] = await container.initializer.initialize()
+	let initializedProjects: string[] = []
+	const terminationJobs: TerminationJob[] = []
+	listenOnProcessTermination(terminationJobs, logger)
+
+	if (cluster.isMaster) {
+		const monitoringPort = serverConfig.monitoringPort
+		const monitoringServer = container.monitoringKoa.listen(monitoringPort, () => {
+			logger.info(`Monitoring running on http://localhost:${monitoringPort}`)
+		})
+		terminationJobs.push(async () => {
+			await new Promise<any>(resolve => monitoringServer.close(resolve))
+			logger.info('Monitoring server terminated')
+		})
+		if (!serverConfig.projectGroup) {
+			initializedProjects = await container.initializer.initialize()
+		}
+	}
+
+	const port = serverConfig.port
+	const printStarted = () => {
+		if (serverConfig.projectGroup) {
+			logger.info('Contember Cloud running')
+		} else {
+			logger.info(`Contember API running on http://localhost:${port}`)
+		}
+		if (initializedProjects.length) {
+			logger.info(`Initialized projects: ${initializedProjects.join(', ')}`)
+		}
+	}
+
+	const startApplication = async () => {
+		const httpServerPromise = container.application.listen()
+		terminationJobs.push(async () => {
+			await (await httpServerPromise).close()
+			logger.info('API server terminated')
+		})
+		await httpServerPromise
+	}
+
+	const startWorker = async (workerName: string) => {
+		const worker = container.applicationWorkers.getWorker(workerName)
+		const runningWorker = worker.run({
+			logger,
+			onError: e => {
+				logger.crit(e)
+				process.exit(1)
+			},
+		})
+		terminationJobs.push(async () => {
+			(await runningWorker).end()
+		})
+		await runningWorker
+		logger.info(`Contember Worker ${workerName} started.`)
+
+	}
+
+	const applicationWorker = serverConfig.applicationWorker
+	if (applicationWorker && applicationWorker !== 'all') {
+		await startWorker(applicationWorker)
+		if (isClusterMode) {
+			notifyWorkerStarted()
+		}
+	} else if (isClusterMode) {
+		if (cluster.isMaster) {
+			logger.info(`Master ${process.pid} is running`)
+			const workerManager = new WorkerManager()
+			terminationJobs.push(async ({ signal }) => {
+				await workerManager.terminate(signal)
+				logger.info('Workers terminated')
+			})
+
+			await workerManager.start({ workerCount })
+
+			if (applicationWorker === 'all') {
+				for (const workerName of container.applicationWorkers.getWorkerNames()) {
+					await workerManager.start({
+						workerCount, env: {
+							CONTEMBER_APPLICATION_WORKER: workerName,
+						},
+					})
+				}
+			}
+			printStarted()
+		} else {
+			logger.info(`Starting worker ${process.pid}`)
+
+			// this line somehow ensures, that worker waits for termination of all jobs
+			process.on('disconnect', () => timeout(0))
+
+			await startApplication()
+			notifyWorkerStarted()
+		}
+	} else {
+		await startApplication()
+		if (applicationWorker === 'all') {
+			for (const workerName of container.applicationWorkers.getWorkerNames()) {
+				await startWorker(workerName)
+			}
+		}
+		printStarted()
+	}
 
-	const httpServerPromise = container.application.listen()
-	terminationJobs.push(async () => await (await httpServerPromise).close())
-	logger.info(`Contember API running on http://localhost:${(serverConfig.port)}`)
-	logger.info(initializedProjects.length ? `Initialized projects: ${initializedProjects.join(', ')}` : 'No project initialized')
 })().catch(e => {
 	logger.crit(e)
 	process.exit(1)

package/src/tsconfig.json

@@ -1,9 +1,10 @@
 {
 	"extends": "../tsconfig.settings.json",
 	"compilerOptions": {
-		"outDir": "../dist/src"
+		"outDir": "../dist/types"
 	},
 	"references": [
+		{ "path": "../../engine-actions/src" },
 		{ "path": "../../engine-http/src" },
 		{ "path": "../../engine-plugins/src" },
 		{ "path": "../../engine-s3-plugin/src" },

package/src/utils.ts

@@ -0,0 +1,4 @@
+export const timeout = (ms: number) =>
+	new Promise(resolve => {
+		setTimeout(resolve, ms)
+	})

package/tsconfig.json

@@ -3,6 +3,5 @@
 	"include": [],
 	"references": [
 		{ "path": "./src" },
-		{ "path": "./tests" }
 	]
 }

package/types/prom-client/index.d.ts

@@ -0,0 +1,17 @@
+import * as promClient from 'prom-client'
+
+declare module 'prom-client' {
+	export type Collector = () => void
+
+	interface Registry {
+		/**
+		 * Add metric collector, which is invoked on scrape
+		 */
+		registerCollector(collectorFn: Collector): void
+
+		/**
+		 * Get all registered collector functions
+		 */
+		collectors(): Collector[]
+	}
+}

package/dist/src/loadPlugins.js

@@ -1,12 +0,0 @@
-"use strict";
-var __importDefault = (this && this.__importDefault) || function (mod) {
-    return (mod && mod.__esModule) ? mod : { "default": mod };
-};
-Object.defineProperty(exports, "__esModule", { value: true });
-const engine_s3_plugin_1 = __importDefault(require("@contember/engine-s3-plugin"));
-const engine_vimeo_plugin_1 = __importDefault(require("@contember/engine-vimeo-plugin"));
-function loadPlugins() {
-    return Promise.resolve([new engine_s3_plugin_1.default(), new engine_vimeo_plugin_1.default()]);
-}
-exports.default = loadPlugins;
-//# sourceMappingURL=loadPlugins.js.map

package/dist/src/loadPlugins.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"loadPlugins.js","sourceRoot":"","sources":["../../src/loadPlugins.ts"],"names":[],"mappings":";;;;;AACA,mFAAkD;AAClD,yFAAwD;AAExD,SAAwB,WAAW;IAClC,OAAO,OAAO,CAAC,OAAO,CAAC,CAAC,IAAI,0BAAQ,EAAE,EAAE,IAAI,6BAAW,EAAE,CAAC,CAAC,CAAA;AAC5D,CAAC;AAFD,8BAEC"}

package/dist/src/start.js

@@ -1,47 +0,0 @@
-#!/usr/bin/env node
-"use strict";
-var __importDefault = (this && this.__importDefault) || function (mod) {
-    return (mod && mod.__esModule) ? mod : { "default": mod };
-};
-Object.defineProperty(exports, "__esModule", { value: true });
-const engine_http_1 = require("@contember/engine-http");
-const loadPlugins_1 = __importDefault(require("./loadPlugins"));
-const logger = (0, engine_http_1.createDefaultLogger)();
-process.on('warning', message => {
-    logger.warn(message);
-});
-(async () => {
-    var _a;
-    const isDebug = (0, engine_http_1.isDebugMode)();
-    const terminationJobs = [];
-    (0, engine_http_1.listenOnProcessTermination)(terminationJobs, logger);
-    const version = (0, engine_http_1.getServerVersion)();
-    (0, engine_http_1.printStartInfo)({ version, isDebug }, logger);
-    const plugins = await (0, loadPlugins_1.default)();
-    const { serverConfig, projectConfigResolver, tenantConfigResolver } = await (0, engine_http_1.resolveServerConfig)({
-        plugins,
-        serverConfigSchema: engine_http_1.serverConfigSchema,
-    });
-    const sentryTransport = (0, engine_http_1.createSentryLoggerHandler)((_a = serverConfig.logging.sentry) === null || _a === void 0 ? void 0 : _a.dsn);
-    if (sentryTransport !== null) {
-        logger.addHandler(sentryTransport);
-    }
-    const container = (0, engine_http_1.createContainer)({
-        debugMode: isDebug,
-        serverConfig,
-        projectConfigResolver,
-        tenantConfigResolver,
-        plugins,
-        version,
-        logger,
-    });
-    const initializedProjects = await container.initializer.initialize();
-    const httpServerPromise = container.application.listen();
-    terminationJobs.push(async () => await (await httpServerPromise).close());
-    logger.info(`Contember API running on http://localhost:${(serverConfig.port)}`);
-    logger.info(initializedProjects.length ? `Initialized projects: ${initializedProjects.join(', ')}` : 'No project initialized');
-})().catch(e => {
-    logger.crit(e);
-    process.exit(1);
-});
-//# sourceMappingURL=start.js.map

package/dist/src/start.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"start.js","sourceRoot":"","sources":["../../src/start.ts"],"names":[],"mappings":";;;;;;AAEA,wDAO+B;AAC/B,gEAAuC;AAEvC,MAAM,MAAM,GAAG,IAAA,iCAAmB,GAAE,CAAA;AACpC,OAAO,CAAC,EAAE,CAAC,SAAS,EAAE,OAAO,CAAC,EAAE;IAC/B,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAA;AACrB,CAAC,CAAC,CACD;AAAA,CAAC,KAAK,IAAI,EAAE;;IACZ,MAAM,OAAO,GAAG,IAAA,yBAAW,GAAE,CAAA;IAC7B,MAAM,eAAe,GAAqB,EAAE,CAAA;IAC5C,IAAA,wCAA0B,EAAC,eAAe,EAAE,MAAM,CAAC,CAAA;IAEnD,MAAM,OAAO,GAAG,IAAA,8BAAgB,GAAE,CAAA;IAClC,IAAA,4BAAc,EAAC,EAAE,OAAO,EAAE,OAAO,EAAE,EAAE,MAAM,CAAC,CAAA;IAC5C,MAAM,OAAO,GAAG,MAAM,IAAA,qBAAW,GAAE,CAAA;IACnC,MAAM,EAAE,YAAY,EAAE,qBAAqB,EAAE,oBAAoB,EAAE,GAAG,MAAM,IAAA,iCAAmB,EAAC;QAC/F,OAAO;QACP,kBAAkB,EAAlB,gCAAkB;KAClB,CAAC,CAAA;IAEF,MAAM,eAAe,GAAG,IAAA,uCAAyB,EAAC,MAAA,YAAY,CAAC,OAAO,CAAC,MAAM,0CAAE,GAAG,CAAC,CAAA;IACnF,IAAI,eAAe,KAAK,IAAI,EAAE;QAC7B,MAAM,CAAC,UAAU,CAAC,eAAe,CAAC,CAAA;KAClC;IAED,MAAM,SAAS,GAAG,IAAA,6BAAe,EAAC;QACjC,SAAS,EAAE,OAAO;QAClB,YAAY;QACZ,qBAAqB;QACrB,oBAAoB;QACpB,OAAO;QACP,OAAO;QACP,MAAM;KACN,CAAC,CAAA;IAEF,MAAM,mBAAmB,GAAa,MAAM,SAAS,CAAC,WAAW,CAAC,UAAU,EAAE,CAAA;IAE9E,MAAM,iBAAiB,GAAG,SAAS,CAAC,WAAW,CAAC,MAAM,EAAE,CAAA;IACxD,eAAe,CAAC,IAAI,CAAC,KAAK,IAAI,EAAE,CAAC,MAAM,CAAC,MAAM,iBAAiB,CAAC,CAAC,KAAK,EAAE,CAAC,CAAA;IACzE,MAAM,CAAC,IAAI,CAAC,6CAA6C,CAAC,YAAY,CAAC,IAAI,CAAC,EAAE,CAAC,CAAA;IAC/E,MAAM,CAAC,IAAI,CAAC,mBAAmB,CAAC,MAAM,CAAC,CAAC,CAAC,yBAAyB,mBAAmB,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC,wBAAwB,CAAC,CAAA;AAC/H,CAAC,CAAC,EAAE,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE;IACd,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;IACd,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;AAChB,CAAC,CAAC,CAAA"}