@contember/engine-actions 2.2.0-alpha.4 → 2.2.0-alpha.6

package/package.json

@@ -22,26 +22,26 @@
     }
   },
   "dependencies": {
-    "@contember/authorization": "2.2.0-alpha.4",
-    "@contember/database": "2.2.0-alpha.4",
-    "@contember/database-migrations": "2.2.0-alpha.4",
-    "@contember/engine-common": "2.2.0-alpha.4",
-    "@contember/engine-content-api": "2.2.0-alpha.4",
-    "@contember/engine-http": "2.2.0-alpha.4",
-    "@contember/engine-plugins": "2.2.0-alpha.4",
-    "@contember/engine-system-api": "2.2.0-alpha.4",
-    "@contember/engine-tenant-api": "2.2.0-alpha.4",
-    "@contember/graphql-utils": "2.2.0-alpha.4",
-    "@contember/logger": "2.2.0-alpha.4",
-    "@contember/schema": "2.2.0-alpha.4",
-    "@contember/schema-utils": "2.2.0-alpha.4",
-    "@contember/typesafe": "2.2.0-alpha.4",
+    "@contember/authorization": "2.2.0-alpha.6",
+    "@contember/database": "2.2.0-alpha.6",
+    "@contember/database-migrations": "2.2.0-alpha.6",
+    "@contember/engine-common": "2.2.0-alpha.6",
+    "@contember/engine-content-api": "2.2.0-alpha.6",
+    "@contember/engine-http": "2.2.0-alpha.6",
+    "@contember/engine-plugins": "2.2.0-alpha.6",
+    "@contember/engine-system-api": "2.2.0-alpha.6",
+    "@contember/engine-tenant-api": "2.2.0-alpha.6",
+    "@contember/graphql-utils": "2.2.0-alpha.6",
+    "@contember/logger": "2.2.0-alpha.6",
+    "@contember/schema": "2.2.0-alpha.6",
+    "@contember/schema-utils": "2.2.0-alpha.6",
+    "@contember/typesafe": "2.2.0-alpha.6",
     "@graphql-tools/schema": "^10.0.31",
     "graphql-tag": "^2.12.6",
     "prom-client": "^12.0.0"
   },
   "devDependencies": {
-    "@contember/database-tester": "2.2.0-alpha.4",
+    "@contember/database-tester": "2.2.0-alpha.6",
     "@types/bun": "^1.3.10",
     "@types/ws": "^8.18.1",
     "graphql": "^16.13.1"
@@ -51,7 +51,7 @@
     "pg": "^8.20.0"
   },
   "type": "module",
-  "version": "2.2.0-alpha.4",
+  "version": "2.2.0-alpha.6",
   "repository": {
     "type": "git",
     "url": "git+https://github.com/contember/contember.git",

package/src/graphql/http/ActionsWebsocketControllerFactory.ts

@@ -1,9 +1,7 @@
-import { HttpErrorResponse, WebSocketController } from '@contember/engine-http'
-import { ActionsContextResolver } from './ActionsContextResolver.js'
-import { DispatchWorkerSupervisorFactory } from '../../dispatch/DispatchWorkerSupervisor.js'
+import { HttpErrorResponse, ProjectGroupContainer, WebSocketController } from '@contember/engine-http'
 import * as Typesafe from '@contember/typesafe'
 import { TenantRole } from '@contember/engine-tenant-api'
-import { Running } from '@contember/engine-common'
+import { Runnable, Running } from '@contember/engine-common'
 
 const IncomingMessage = Typesafe.discriminatedUnion(
 	'type',
@@ -41,11 +39,14 @@ const OutgoingMessage = Typesafe.discriminatedUnion(
 
 type OutgoingMessage = ReturnType<typeof OutgoingMessage>
 
+type DispatchWorkerFactory = {
+	create(projectGroup: ProjectGroupContainer): Runnable
+}
+
 export class ActionsWebsocketControllerFactory {
 	constructor(
 		private readonly debug: boolean,
-		private readonly actionsContextResolver: ActionsContextResolver,
-		private readonly dispatchWorkerSupervisorFactory: DispatchWorkerSupervisorFactory,
+		private readonly dispatchWorkerSupervisorFactory: DispatchWorkerFactory,
 	) {
 	}
 
@@ -67,24 +68,44 @@ export class ActionsWebsocketControllerFactory {
 			}
 
 			let workers: { id: string; running: Promise<Running> }[] = []
+			let stopAllPromise: Promise<void> | undefined
+			let connectionClosing = false
 			const abortListener = async () => {
-				send({ type: 'message', message: 'shutting down' })
-				await stopAll()
-				send({ type: 'message', message: 'closing connection' })
-				ws.close(1012) // Service Restart
+				connectionClosing = true
+				try {
+					send({ type: 'message', message: 'shutting down' })
+					await stopAll()
+				} catch (error) {
+					logger.error(error, { message: 'Websocket worker cleanup failed' })
+				} finally {
+					try {
+						send({ type: 'message', message: 'closing connection' })
+					} finally {
+						ws.close(1012) // Service Restart
+					}
+				}
 			}
 			ctx.abortSignal.addEventListener('abort', abortListener)
 
-			const stopAll = async () => {
-				const currentWorkers = workers.map(async it => {
-					await (await it.running).end()
-					send({
-						type: 'workedStopped',
-						workerId: it.id,
-					})
+			const stopAll = (): Promise<void> => {
+				stopAllPromise ??= (async () => {
+					// a worker stays tracked until its own end() resolves, so a failed cleanup is not forgotten
+					const results = await Promise.allSettled(workers.map(async it => {
+						await (await it.running).end()
+						workers = workers.filter(worker => worker !== it)
+						send({
+							type: 'workedStopped',
+							workerId: it.id,
+						})
+					}))
+					const errors = results.flatMap(it => it.status === 'rejected' ? [it.reason] : [])
+					if (errors.length > 0) {
+						throw new AggregateError(errors, 'Worker cleanup failed')
+					}
+				})().finally(() => {
+					stopAllPromise = undefined
 				})
-				workers = []
-				await Promise.all(currentWorkers)
+				return stopAllPromise
 			}
 			send({ type: 'ready' })
 
@@ -110,27 +131,41 @@ export class ActionsWebsocketControllerFactory {
 
 				switch (data.type) {
 					case 'startWorker':
+						if (connectionClosing || stopAllPromise !== undefined) {
+							send({ type: 'error', message: 'Workers are stopping' })
+							break
+						}
 						const workerId = Math.random().toString().substring(2)
 						const dispatchSupervisor = this.dispatchWorkerSupervisorFactory.create(projectGroup)
-						try {
-							const running = dispatchSupervisor.run({
+						const running = Promise.resolve().then(() =>
+							dispatchSupervisor.run({
 								logger,
 								onError: () => {
 									send({ type: 'workerCrashed', workerId })
 								},
 							})
-							workers.push({ id: workerId, running })
+						)
+						const worker = { id: workerId, running }
+						workers.push(worker)
+						try {
 							await running
-							send({ type: 'workerStarted', workerId })
 						} catch (e) {
+							workers = workers.filter(it => it !== worker)
 							logger.error(e, { message: 'Worker failed to start' })
 							send({ type: 'workerFailedToStart', workerId })
+							break
 						}
+						send({ type: 'workerStarted', workerId })
 						break
 					case 'stopAllWorkers':
 						send({ type: 'message', message: 'stopping' })
-						await stopAll()
-						send({ type: 'message', message: 'all stopped' })
+						try {
+							await stopAll()
+							send({ type: 'message', message: 'all stopped' })
+						} catch (error) {
+							logger.error(error, { message: 'Websocket worker cleanup failed' })
+							send({ type: 'error', message: 'Worker cleanup failed' })
+						}
 						break
 				}
 			})
@@ -139,11 +174,19 @@ export class ActionsWebsocketControllerFactory {
 				ws.ping()
 			}, 5000)
 
-			ws.addEventListener('close', async () => {
-				ctx.abortSignal.removeEventListener('abort', abortListener)
-				clearInterval(pingHandle)
-				await stopAll()
-			})
+			ctx.waitUntil(
+				new Promise<void>(resolve => {
+					ws.addEventListener('close', () => {
+						connectionClosing = true
+						ctx.abortSignal.removeEventListener('abort', abortListener)
+						clearInterval(pingHandle)
+						void stopAll().then(resolve, error => {
+							logger.error(error, { message: 'Websocket worker cleanup failed' })
+							resolve()
+						})
+					})
+				}),
+			)
 		}
 	}
 }

package/src/index.ts

@@ -123,7 +123,6 @@ export default class ActionsPlugin implements Plugin {
 						)
 						const actionsWebsocketMiddlewareFactory = new ActionsWebsocketControllerFactory(
 							debugMode,
-							actionsContextResolver,
 							actions_dispatchWorkerSupervisorFactory,
 						)
 						it.addRoute('actions', '/actions/:projectSlug', actionsMiddlewareFactory.create())

package/tests/cases/unit/actionsWebsocketController.test.ts

@@ -0,0 +1,270 @@
+import { expect, test } from 'bun:test'
+import { Runnable, Running } from '@contember/engine-common'
+import { Application, Authenticator, ProjectGroupContainer, ProjectGroupResolver, serverConfigSchema } from '@contember/engine-http'
+import { createLogger, TestLoggerHandler } from '@contember/logger'
+import { TenantRole, VerifyResult } from '@contember/engine-tenant-api'
+import { WebSocket } from 'ws'
+import { ActionsWebsocketControllerFactory } from '../../../src/graphql/http/ActionsWebsocketControllerFactory.js'
+
+type Interface<T> = { [P in keyof T]: T[P] }
+
+const createMock = <T>(members: Interface<T>): T => members
+
+const unavailable = (): never => {
+	throw new Error('This service is not available in the actions WebSocket controller test')
+}
+
+const waitFor = async (condition: () => boolean): Promise<void> => {
+	for (let attempt = 0; attempt < 100; attempt++) {
+		if (condition()) {
+			return
+		}
+		await new Promise<void>(resolve => setTimeout(resolve, 1))
+	}
+	throw new Error('Condition was not met')
+}
+
+const listen = async (createWorker: () => Runnable) => {
+	const loggerHandler = new TestLoggerHandler()
+	const logger = createLogger(loggerHandler)
+	const authenticator = createMock<Authenticator>({
+		authenticate: async () => new VerifyResult('identity', 'api-key', [TenantRole.SUPER_ADMIN], null),
+	})
+	const projectGroup: ProjectGroupContainer = {
+		slug: undefined,
+		logger,
+		authenticator,
+		get projectMembershipResolver() {
+			return unavailable()
+		},
+		get projectContainerResolver() {
+			return unavailable()
+		},
+		get projectSchemaResolver() {
+			return unavailable()
+		},
+		get projectInitializer() {
+			return unavailable()
+		},
+		get tenantContainer() {
+			return unavailable()
+		},
+		get tenantGraphQLHandler() {
+			return unavailable()
+		},
+		get systemContainer() {
+			return unavailable()
+		},
+		get systemGraphQLHandler() {
+			return unavailable()
+		},
+	}
+	const projectGroupResolver = createMock<ProjectGroupResolver>({
+		resolveContainer: async () => projectGroup,
+	})
+	const application = new Application(projectGroupResolver, serverConfigSchema({ port: 0 }), false, undefined, logger)
+	application.addWebsocketRoute(
+		'actions',
+		'/actions/_worker',
+		new ActionsWebsocketControllerFactory(false, { create: createWorker }).create(),
+	)
+	const running = await application.listen()
+	const address = running.server.address()
+	if (address === null || typeof address === 'string') {
+		await running.close()
+		throw new Error('The test server did not bind to a TCP port')
+	}
+
+	const messages: string[] = []
+	const client = new WebSocket(`ws://127.0.0.1:${address.port}/actions/_worker`)
+	client.on('message', data => {
+		const message: unknown = JSON.parse(data.toString())
+		if (typeof message === 'object' && message !== null && 'type' in message && typeof message.type === 'string') {
+			const detail = 'message' in message && typeof message.message === 'string' ? `:${message.message}` : ''
+			messages.push(`${message.type}${detail}`)
+		}
+	})
+	const opened = new Promise<void>((resolve, reject) => {
+		client.once('open', () => resolve())
+		client.once('error', reject)
+	})
+	const closed = new Promise<number>(resolve => client.once('close', code => resolve(code)))
+	return { client, closed, loggerHandler, messages, opened, running }
+}
+
+test('a worker startup failure does not block WebSocket shutdown', async () => {
+	const worker: Runnable = {
+		run: (): Promise<Running> => Promise.reject(new Error('Worker startup failed')),
+	}
+	const { client, closed, loggerHandler, messages, opened, running } = await listen(() => worker)
+	let applicationClosed = false
+
+	try {
+		await opened
+		await waitFor(() => messages.includes('ready'))
+		client.send(JSON.stringify({ type: 'startWorker' }))
+		await waitFor(() => messages.includes('workerFailedToStart'))
+
+		await running.close()
+		applicationClosed = true
+		expect(await closed).toBe(1012)
+		expect(loggerHandler.messages.some(it => it.message.startsWith('Websocket worker cleanup failed'))).toBe(false)
+	} finally {
+		client.terminate()
+		if (!applicationClosed) {
+			await running.close()
+		}
+	}
+})
+
+test('shutdown waits for every worker cleanup after one cleanup fails', async () => {
+	let releaseCleanup = () => {}
+	const cleanupReleased = new Promise<void>(resolve => {
+		releaseCleanup = resolve
+	})
+	let cleanupStarted = false
+	const workers: Runnable[] = [
+		{
+			run: async () => ({
+				end: (): Promise<void> => Promise.reject(new Error('Worker cleanup failed')),
+			}),
+		},
+		{
+			run: async () => ({
+				end: async () => {
+					cleanupStarted = true
+					await cleanupReleased
+				},
+			}),
+		},
+	]
+	const { client, closed, loggerHandler, messages, opened, running } = await listen(() => workers.shift() ?? unavailable())
+	let shutdown: Promise<void> | undefined
+
+	try {
+		await opened
+		await waitFor(() => messages.includes('ready'))
+		client.send(JSON.stringify({ type: 'startWorker' }))
+		await waitFor(() => messages.filter(it => it === 'workerStarted').length === 1)
+		client.send(JSON.stringify({ type: 'startWorker' }))
+		await waitFor(() => messages.filter(it => it === 'workerStarted').length === 2)
+
+		let shutdownFinished = false
+		shutdown = running.close().then(() => {
+			shutdownFinished = true
+		})
+		await waitFor(() => cleanupStarted)
+		await new Promise<void>(resolve => setImmediate(resolve))
+		expect(shutdownFinished).toBe(false)
+		expect(messages).not.toContain('message:closing connection')
+		releaseCleanup()
+		await shutdown
+		expect(await closed).toBe(1012)
+		expect(loggerHandler.messages.some(it => it.message.startsWith('Websocket worker cleanup failed'))).toBe(true)
+	} finally {
+		releaseCleanup()
+		shutdown ??= running.close()
+		await shutdown
+		client.terminate()
+	}
+})
+
+test('shutdown waits for worker cleanup started by a peer disconnect', async () => {
+	let releaseCleanup = () => {}
+	const cleanupReleased = new Promise<void>(resolve => {
+		releaseCleanup = resolve
+	})
+	let cleanupStarted = false
+	const worker: Runnable = {
+		run: async () => ({
+			end: async () => {
+				cleanupStarted = true
+				await cleanupReleased
+			},
+		}),
+	}
+	const { client, messages, opened, running } = await listen(() => worker)
+	let shutdown: Promise<void> | undefined
+
+	try {
+		await opened
+		await waitFor(() => messages.includes('ready'))
+		client.send(JSON.stringify({ type: 'startWorker' }))
+		await waitFor(() => messages.includes('workerStarted'))
+		client.close()
+		await waitFor(() => cleanupStarted)
+
+		let shutdownFinished = false
+		shutdown = running.close().then(() => {
+			shutdownFinished = true
+		})
+		await new Promise<void>(resolve => setImmediate(resolve))
+		expect(shutdownFinished).toBe(false)
+		releaseCleanup()
+		await shutdown
+	} finally {
+		releaseCleanup()
+		shutdown ??= running.close()
+		await shutdown
+		client.terminate()
+	}
+})
+
+test('a worker whose cleanup fails stays tracked for the next stop', async () => {
+	const worker: Runnable = {
+		run: async () => ({
+			end: (): Promise<void> => Promise.reject(new Error('Worker cleanup failed')),
+		}),
+	}
+	const { client, closed, messages, opened, running } = await listen(() => worker)
+	let applicationClosed = false
+
+	try {
+		await opened
+		await waitFor(() => messages.includes('ready'))
+		client.send(JSON.stringify({ type: 'startWorker' }))
+		await waitFor(() => messages.includes('workerStarted'))
+		client.send(JSON.stringify({ type: 'stopAllWorkers' }))
+		await waitFor(() => messages.filter(it => it === 'error:Worker cleanup failed').length === 1)
+		client.send(JSON.stringify({ type: 'stopAllWorkers' }))
+		await waitFor(() => messages.filter(it => it === 'error:Worker cleanup failed').length === 2)
+		expect(messages).not.toContain('message:all stopped')
+
+		await running.close()
+		applicationClosed = true
+		expect(await closed).toBe(1012)
+	} finally {
+		client.terminate()
+		if (!applicationClosed) {
+			await running.close()
+		}
+	}
+})
+
+test('reports a manual worker cleanup failure without blocking shutdown', async () => {
+	const worker: Runnable = {
+		run: async () => ({
+			end: (): Promise<void> => Promise.reject(new Error('Worker cleanup failed')),
+		}),
+	}
+	const { client, closed, messages, opened, running } = await listen(() => worker)
+	let applicationClosed = false
+
+	try {
+		await opened
+		await waitFor(() => messages.includes('ready'))
+		client.send(JSON.stringify({ type: 'startWorker' }))
+		await waitFor(() => messages.includes('workerStarted'))
+		client.send(JSON.stringify({ type: 'stopAllWorkers' }))
+		await waitFor(() => messages.includes('error:Worker cleanup failed'))
+
+		await running.close()
+		applicationClosed = true
+		expect(await closed).toBe(1012)
+	} finally {
+		client.terminate()
+		if (!applicationClosed) {
+			await running.close()
+		}
+	}
+})