@contember/engine-actions 1.3.0-alpha.4

package/graphql-actions.codegen.yml

@@ -0,0 +1,21 @@
+schema:
+  - "./src/graphql/schema/actions.graphql.ts"
+documents: []
+config:
+  immutableTypes: "1"
+generates:
+  ./src/graphql/schema/index.ts:
+    config:
+      immutableTypes: "1"
+      enumsAsTypes: "1"
+      scalars:
+        DateTime: Date
+        Uuid: String
+        Json: any
+    plugins:
+      - "typescript"
+      - "typescript-operations"
+      - "typescript-resolvers"
+hooks:
+  afterAllFileWrite:
+    - eslint --fix

package/package.json

@@ -0,0 +1,41 @@
+{
+  "name": "@contember/engine-actions",
+  "version": "1.3.0-alpha.4",
+  "license": "proprietary",
+  "main": "dist/src/index.js",
+  "typings": "dist/src/index.d.ts",
+  "scripts": {
+    "generate": "gql-gen --config graphql-actions.codegen.yml",
+    "test": "vitest"
+  },
+  "dependencies": {
+    "@contember/database": "^1.3.0-alpha.4",
+    "@contember/database-migrations": "^1.3.0-alpha.4",
+    "@contember/engine-common": "^1.3.0-alpha.4",
+    "@contember/engine-content-api": "^1.3.0-alpha.4",
+    "@contember/engine-system-api": "^1.3.0-alpha.4",
+    "@contember/engine-tenant-api": "^1.3.0-alpha.4",
+    "@contember/engine-http": "^1.3.0-alpha.4",
+    "@contember/engine-plugins": "^1.3.0-alpha.4",
+    "@contember/graphql-utils": "^1.3.0-alpha.4",
+    "@contember/schema": "^1.3.0-alpha.4",
+    "@contember/schema-utils": "^1.3.0-alpha.4",
+    "@contember/typesafe": "^1.3.0-alpha.4",
+    "@contember/logger": "^1.3.0-alpha.4",
+    "@graphql-tools/schema": "^8.3.5",
+    "graphql-tag": "^2.12.5"
+  },
+  "devDependencies": {
+    "@contember/engine-api-tester": "^1.3.0-alpha.4",
+    "@graphql-codegen/cli": "^2.6.2",
+    "@graphql-codegen/typescript": "^2.5.1",
+    "@graphql-codegen/typescript-operations": "^2.4.2",
+    "@graphql-codegen/typescript-resolvers": "^2.6.6",
+    "@types/ws": "^8.5.3",
+    "graphql": "^16.5.0"
+  },
+  "peerDependencies": {
+    "graphql": ">= 14.6.0",
+    "pg": "^8.5.0"
+  }
+}

package/src/ActionsExecutionContainerHookFactory.ts

@@ -0,0 +1,28 @@
+import { TriggerHandler, TriggerIndirectChangesFetcher, TriggerPayloadManager } from './triggers'
+import { ListenerStoreProvider } from './ListenerStoreProvider'
+import { TriggerPayloadBuilder } from './triggers/TriggerPayloadBuilder'
+import { TriggerPayloadPersister } from './triggers/TriggerPayloadPersister'
+import { ExecutionContainerHook } from '@contember/engine-content-api'
+
+export class ActionsExecutionContainerHookFactory {
+	constructor(
+		private readonly listenerStoreProvider: ListenerStoreProvider,
+	) {
+	}
+
+	create(): ExecutionContainerHook {
+		return builder => {
+			return builder.setupService('mapperFactory', (mapperFactory, { whereBuilder, schema, pathFactory, systemSchema, providers, stage, project }) => {
+				mapperFactory.hooks.push(mapper => {
+					const triggerPayloadPersister = new TriggerPayloadPersister(mapper, mapper.db.forSchema(systemSchema), providers, project.slug, stage.id, schema.id)
+					const triggerPayloadBuilder = new TriggerPayloadBuilder(mapper)
+					const payloadManager = new TriggerPayloadManager(triggerPayloadBuilder, triggerPayloadPersister)
+					const changesFetcher = new TriggerIndirectChangesFetcher(mapper, whereBuilder, pathFactory)
+					const listenersStore = this.listenerStoreProvider.getListenerStore(schema)
+					const triggerHandler = new TriggerHandler(payloadManager, listenersStore, changesFetcher)
+					triggerHandler.attach(mapper.eventManager)
+				})
+			})
+		}
+	}
+}

package/src/ImplementationException.ts

@@ -0,0 +1,2 @@
+export class ImplementationException extends Error {
+}

package/src/ListenerStoreProvider.ts

@@ -0,0 +1,17 @@
+import { TriggerListenersFactory, TriggerListenersStore } from './triggers'
+import { Schema } from '@contember/schema'
+
+export class ListenerStoreProvider {
+	private listenersStoreCache = new WeakMap<Schema, TriggerListenersStore>()
+
+	public getListenerStore(schema: Schema): TriggerListenersStore {
+		const currentStore = this.listenersStoreCache.get(schema)
+		if (currentStore) {
+			return currentStore
+		}
+		const listenersFactory = new TriggerListenersFactory(schema)
+		const newStore = listenersFactory.create()
+		this.listenersStoreCache.set(schema, newStore)
+		return newStore
+	}
+}

package/src/dispatch/DispatchWorkerSupervisor.ts

@@ -0,0 +1,137 @@
+import { ProjectContainer, ProjectGroupContainer } from '@contember/engine-http'
+import { AcquiringListener } from '@contember/database'
+import { Runnable, RunnableArgs, Running, Supervisor } from '@contember/engine-common'
+import { ProjectDispatcherFactory } from './ProjectDispatcher'
+import { Logger } from '@contember/logger'
+import { ImplementationException } from '../ImplementationException'
+
+export class DispatchWorkerSupervisorFactory {
+
+	constructor(
+		private projectDispatcherFactory: ProjectDispatcherFactory,
+	) {
+	}
+
+	public create(projectGroup: ProjectGroupContainer): DispatchWorkerSupervisor {
+		return new DispatchWorkerSupervisor(this.projectDispatcherFactory, projectGroup)
+	}
+}
+
+export class DispatchWorkerSupervisor implements Runnable {
+	constructor(
+		private projectDispatcherFactory: ProjectDispatcherFactory,
+		private projectGroup: ProjectGroupContainer,
+	) {
+	}
+
+	public async run({ logger, onError }: RunnableArgs) {
+		const projectGroup = this.projectGroup
+		const tenantContainer = projectGroup.tenantContainer
+		const projectContainerResolver = projectGroup.projectContainerResolver
+
+		const running = new Map<string, { worker: Promise<Running>; container: ProjectContainer }>()
+
+		const cleanupListener = projectContainerResolver.on('destroy', async ({ container }) => {
+			const runningForProject = running.get(container.project.slug)
+			if (!runningForProject) {
+				return
+			}
+			running.delete(container.project.slug)
+			await (await runningForProject.worker).end()
+		})
+
+		const refreshProjectWorker = async (container: ProjectContainer, isReloading: boolean) => {
+			const db = container.systemDatabaseContextFactory.create()
+			const current = running.get(container.project.slug)
+			if (current && current.container === container) {
+				return
+			}
+			const dispatcher = this.projectDispatcherFactory.create({
+				db,
+				contentSchemaResolver: container.contentSchemaResolver,
+				projectSlug: container.project.slug,
+			})
+			const supervisedDispatcher = new Supervisor(
+				dispatcher,
+				{ startupMax: isReloading ? Number.MAX_VALUE : 10 },
+			)
+			const worker = supervisedDispatcher.run({
+				logger,
+				onError: () => {
+					throw new ImplementationException('supervised project dispatcher crashed')
+				},
+			})
+			running.set(container.project.slug, { worker, container })
+			try {
+				if (current) {
+					await (await current.worker).end()
+				}
+				await worker
+				logger.debug('Project dispatch worker started', { project: container.project.slug })
+			} catch (e) {
+				logger.error(e, { message: 'Project dispatch worker start failed', project: container.project.slug })
+				running.delete(container.project.slug)
+				throw e
+			}
+		}
+		const projectTracker = new Supervisor(new AcquiringListener(
+			tenantContainer.databaseContext.client,
+			'project_updated',
+			async () => {
+				logger.debug('Project config change detected, reloading dispatch workers')
+				await endProjectWorkers()
+				try {
+					await startWorkers(true)
+				} catch (e) {
+					logger.error(e, { message: 'Dispatch workers reload failed, terminating' })
+					await endAll()
+					onError(e)
+				}
+			},
+		))
+
+		const runningProjectTracker = await projectTracker.run({
+			logger,
+			onError: () => {
+				throw new ImplementationException('supervised project tracker crashed')
+			},
+		})
+		const fetchProjects = async () => await tenantContainer.projectManager.getProjects(tenantContainer.databaseContext)
+
+		const startWorkers = async (isReloading: boolean) => {
+			try {
+				const projects = await fetchProjects()
+				for (const project of projects) {
+					const container = await projectGroup.projectContainerResolver.getProjectContainer(project.slug)
+					if (container) {
+						await refreshProjectWorker(container, isReloading)
+					}
+				}
+			} catch (e) {
+				await endAll()
+				throw e
+			}
+		}
+		const endProjectWorkers = async () => {
+			const workers = [...running.values()]
+			running.clear()
+			await Promise.allSettled(workers.map(async ({ worker }) => await (await worker).end()))
+		}
+		const endAll = async () => {
+			logger.debug('Terminating project dispatch workers...')
+			cleanupListener()
+			try {
+				await runningProjectTracker.end()
+			} catch {
+			}
+			await endProjectWorkers()
+			logger.debug('Project dispatch workers terminated')
+		}
+
+		await startWorkers(false)
+
+		return {
+			end: endAll,
+		}
+	}
+}

package/src/dispatch/EventDispatcher.ts

@@ -0,0 +1,64 @@
+import { EventsRepository } from './EventsRepository'
+import { TargetHandlerResolver } from './TargetHandlerResolver'
+import { HandledEvent } from './types'
+import { Logger } from '@contember/logger'
+import { ContentSchemaResolver } from '@contember/engine-http'
+import { DatabaseContext } from '@contember/engine-system-api'
+import { VariablesManager } from '../model/VariablesManager'
+
+
+type ProcessBatchArgs = {
+	db: DatabaseContext
+	contentSchemaResolver: ContentSchemaResolver
+	logger: Logger
+}
+
+type ProcessBatchResult = {
+	succeed: number
+	failed: number
+	backoffMs: number | undefined
+}
+
+export class EventDispatcher {
+
+	constructor(
+		private readonly eventsRepository: EventsRepository,
+		private readonly variablesManager: VariablesManager,
+		private readonly invokeResolver: TargetHandlerResolver,
+	) {
+	}
+
+	async processBatch({ db, contentSchemaResolver, logger }: ProcessBatchArgs): Promise<ProcessBatchResult> {
+		const schema = await contentSchemaResolver.getSchema(db)
+		const batch = await this.eventsRepository.fetchBatch(schema.actions, db.client)
+		const batchId = Math.random().toString().substring(2)
+		const batchLogger = logger.child({ loc: 'Actions.EventDispatcher', batchId })
+		if (!batch.ok) {
+			batchLogger.debug('Nothing to process', {
+				backoffMs: batch.backoffMs ?? 'undefined',
+			})
+			return { failed: 0, succeed: 0, backoffMs: batch.backoffMs }
+		}
+		const { target, events } = batch
+		try {
+			const handler = this.invokeResolver.resolveHandler(target)
+			batchLogger.debug('Processing started', {
+				events: batch.events.map(it => it.id),
+			})
+			const variables = await this.variablesManager.fetchVariables(db)
+			const handledEvents = await handler.handle({ target, events, logger: batchLogger, variables })
+			const [succeed, failed] = await this.eventsRepository.persistProcessed(db.client, handledEvents)
+			batchLogger.debug('Processing done', { succeed, failed })
+			return { succeed, failed, backoffMs: 0 }
+		} catch (e) {
+			logger.error(e, { loc: 'EventDispatcher', batchId })
+			const failedEvents = events.map((it): HandledEvent => ({
+				row: it,
+				result: { ok: false, errorMessage: `Internal error` },
+			}))
+			await this.eventsRepository.persistProcessed(db.client, failedEvents)
+
+			return { succeed: 0, failed: batch.events.length, backoffMs: 0 }
+		}
+	}
+}

package/src/dispatch/EventsRepository.ts

@@ -0,0 +1,165 @@
+import {
+	Client,
+	Connection,
+	Literal,
+	LockModifier,
+	LockType,
+	Operator,
+	SelectBuilder,
+	UpdateBuilder,
+} from '@contember/database'
+import { EventRow, HandledEvent } from './types'
+import { Actions } from '@contember/schema'
+import { eventsToProcessSpecification, eventsToProcessStateSpecification } from '../model/EventsToProcessSpecification'
+
+
+const ACK_TIMEOUT_MS = 1_000 * 60 * 10 // 10 minutes
+const DEFAULT_REPEAT_INTERVAL_MS = 5_000 // 5 seconds
+const DEFAULT_MAX_ATTEMPTS = 10
+const DEFAULT_BATCH_SIZE = 20
+
+export class EventsRepository {
+
+	public async fetchBatch(actions: Actions.Schema, db: Client): Promise<FetchBatchResult> {
+		const primaryEvent = (await this.fetchInternal(db, 1))[0]
+		if (!primaryEvent) {
+			return { ok: false, backoffMs: await this.fetchBackOff(db) }
+		}
+		const target = actions.targets[primaryEvent.target]
+		if (!target) {
+			await this.markFailedOnUnknownTarget(db, primaryEvent.target, primaryEvent.id)
+			return await this.fetchBatch(actions, db)
+		}
+		const batchSize = (target.batchSize ?? DEFAULT_BATCH_SIZE) - 1
+		const batch = batchSize > 0 ? await this.fetchInternal(db, batchSize) : []
+		return { ok: true, events: [primaryEvent, ...batch], target }
+	}
+
+
+	private async fetchBackOff(db: Client): Promise<number | undefined> {
+		const result = await SelectBuilder.create<{ ms: number }>()
+			.from('actions_event')
+			.match(eventsToProcessStateSpecification)
+			.select(it => it.raw('ceil(extract(epoch from visible_at - now()) * 1000)'), 'ms')
+			.orderBy('visible_at', 'asc')
+			.limit(1)
+			.getResult(db)
+
+		return result.length ? Math.max(0, result[0].ms) : undefined
+	}
+
+
+	private async fetchInternal(db: Client, limit: number, targetName?: string): Promise<EventRow[]> {
+		const visibleAt = new Date()
+		visibleAt.setTime(visibleAt.getTime() + ACK_TIMEOUT_MS)
+		return await UpdateBuilder.create()
+			.table('actions_event')
+			.with('events', qb => qb
+				.select(it => it.raw('*'))
+				.from('actions_event')
+				.match(eventsToProcessSpecification)
+				.where(targetName ? { target: targetName } : {})
+				.limit(limit)
+				.lock(LockType.forNoKeyUpdate, LockModifier.skipLocked),
+			)
+			.values<EventRow>({
+				state: 'processing',
+				last_state_change: new Date(),
+				visible_at: visibleAt,
+			})
+			.from(it => it.from('events').where(it => it.columnsEq(['events', 'id'], ['actions_event', 'id'])))
+			.returning<EventRow>(new Literal('*'))
+			.execute(db)
+	}
+
+	public async persistProcessed(db: Client, events: HandledEvent[]): Promise<[number, number]> {
+		return await db.transaction(async trx => {
+			await trx.query(Connection.REPEATABLE_READ)
+			const succeed = events.filter(it => it.result.ok)
+			await this.markSucceed(trx, succeed.map(it => it.row))
+
+			for (const event of events) {
+				if (!event.result.ok) {
+					await this.markFailed(trx, event)
+				}
+			}
+			return [succeed.length, events.length - succeed.length]
+		})
+	}
+
+	public async requeue(db: Client, events: EventRow[]): Promise<void> {
+		await UpdateBuilder.create()
+			.table('actions_event')
+			.values<EventRow>({
+				state: it => it.raw('CASE WHEN num_retries = 0 THEN ? ELSE ? END', 'created', 'retrying'),
+				last_state_change: new Date(),
+				visible_at: new Date(),
+			})
+			.where(it => it.in('id', events.map(it => it.id)))
+			.execute(db)
+	}
+
+	private async markSucceed(db: Client, rows: EventRow[]): Promise<void> {
+		const now = new Date()
+		await UpdateBuilder.create()
+			.table('actions_event')
+			.values<EventRow>({
+				state: 'succeed',
+				num_retries: it => it.raw('num_retries + 1'),
+				resolved_at: now,
+				last_state_change: now,
+			})
+			.where(it => it.in('id', rows.map(it => it.id)))
+			.execute(db)
+	}
+
+	private async markFailed(db: Client, event: HandledEvent): Promise<void> {
+		const numRetries = event.row.num_retries
+		const now = new Date()
+		const nextAttempt = new Date()
+		nextAttempt.setTime(nextAttempt.getTime() + (event.target?.initialRepeatIntervalMs ?? DEFAULT_REPEAT_INTERVAL_MS) * Math.pow(2, event.row.num_retries))
+		await UpdateBuilder.create()
+			.table('actions_event')
+			.values<EventRow>({
+				last_state_change: now,
+				num_retries: event.row.num_retries + 1,
+				log: JSON.stringify([...event.row.log, event.result]) as any,
+				...(numRetries < (event.target?.maxAttempts ?? DEFAULT_MAX_ATTEMPTS) ? {
+					state: 'retrying',
+					visible_at: nextAttempt,
+				} : {
+					state: 'failed',
+				}),
+			})
+			.where({ id: event.row.id })
+			.execute(db)
+	}
+
+	private async markFailedOnUnknownTarget(db: Client, targetName: string, primaryEventId: string) {
+		await UpdateBuilder.create()
+			.table('actions_event')
+			.where(it => it.or(it => it
+				.and(it => it
+					.in('state', ['retrying', 'created', 'processing'])
+					.compare('visible_at', Operator.lte, 'now')
+					.compare('target', Operator.eq, targetName),
+				)
+				.and(it => it
+					.compare('id', Operator.eq, primaryEventId),
+				),
+			))
+			.values<EventRow>({
+				// intentionally not retrying
+				last_state_change: new Date(),
+				num_retries: it => it.raw('num_retries + 1'),
+				log: it => it.raw('log || ?::jsonb', { ok: false, errorMessage: `Event target ${targetName} not found, stopping.` }),
+				state: 'failed',
+				resolved_at: new Date(),
+			})
+			.execute(db)
+	}
+}
+
+export type FetchBatchResult =
+	| { ok: true; events: EventRow[]; target: Actions.AnyTarget }
+	| { ok: false; backoffMs?: number }

package/src/dispatch/ProjectDispatcher.ts

@@ -0,0 +1,119 @@
+import { DatabaseContext } from '@contember/engine-system-api'
+import { EventDispatcher } from './EventDispatcher'
+import { ContentSchemaResolver } from '@contember/engine-http'
+import { Runnable, RunnableArgs, Running } from '@contember/engine-common'
+import { Listener } from '@contember/database'
+import { NOTIFY_CHANNEL_NAME } from '../triggers/TriggerPayloadPersister'
+
+
+export class ProjectDispatcherFactory {
+	constructor(
+		private readonly dispatcher: EventDispatcher,
+	) {
+	}
+
+	public create({ db, contentSchemaResolver, projectSlug }: { db: DatabaseContext; contentSchemaResolver: ContentSchemaResolver; projectSlug: string}): ProjectDispatcher {
+		return new ProjectDispatcher(this.dispatcher, db, contentSchemaResolver, projectSlug)
+	}
+}
+
+
+
+export class ProjectDispatcher implements Runnable {
+	constructor(
+		private readonly dispatcher: EventDispatcher,
+		private readonly db: DatabaseContext,
+		private readonly contentSchemaResolver: ContentSchemaResolver,
+		private readonly projectSlug: string,
+	) {
+	}
+
+	public async run({ logger, onError, onClose }: RunnableArgs): Promise<Running> {
+
+		return await new Promise<Running>(async (resolve, reject) => {
+
+			let resolvePending = () => {}
+			let rejectPending = (e: any) => {}
+
+			const resolvePendingRef = () => {
+				resolvePending()
+			}
+			let pendingError: any = undefined
+			const rejectPendingRef = (e: any) => {
+				pendingError = e
+				rejectPending(e)
+			}
+
+			let succeedTotal = 0
+			let failedTotal = 0
+			let aborted = false
+
+			try {
+				await this.db.scope(async db => {
+					const listener = await new Listener<string>(db.client, NOTIFY_CHANNEL_NAME, project => {
+						if (project === this.projectSlug) {
+							resolvePendingRef()
+						}
+					})
+						.run({ onError: rejectPendingRef, onClose: resolvePendingRef })
+
+					resolve({
+						end: async () => {
+							aborted = true
+							await listener.end()
+							logger.info('Worker terminated', {
+								succeed: succeedTotal,
+								failed: failedTotal,
+							})
+						},
+					})
+
+					try {
+						while (!aborted) {
+							const { succeed, failed, backoffMs } = await this.dispatcher.processBatch({
+								db,
+								contentSchemaResolver: this.contentSchemaResolver,
+								logger,
+							})
+							succeedTotal += succeed
+							failedTotal += failed
+
+							// listener error occurred during batch processing, rethrow
+							if (pendingError) {
+								throw pendingError
+							}
+
+							// queue is empty, wait
+							if (backoffMs !== 0) {
+								await new Promise<void>((resolve, reject) => {
+									let cleanup = () => {
+									}
+									if (backoffMs !== undefined) {
+										const timeoutHandle = setTimeout(resolve, backoffMs)
+										cleanup = () => clearTimeout(timeoutHandle)
+									}
+									resolvePending = () => {
+										resolve()
+										cleanup()
+									}
+									rejectPending = e => {
+										reject(e)
+										cleanup()
+									}
+								})
+							}
+						}
+						onClose?.()
+					} catch (e) {
+						await listener.end()
+						onError(e)
+					}
+				})
+			} catch (e) {
+				reject(e)
+			}
+ 		})
+
+
+	}
+}

package/src/dispatch/TargetHandlerResolver.ts

@@ -0,0 +1,18 @@
+import { Actions } from '@contember/schema'
+import { WebhookTargetHandler } from './WebhookTargetHandler'
+import { ImplementationException } from '../ImplementationException'
+import { InvokeHandler } from './types'
+
+export class TargetHandlerResolver {
+	constructor(
+		private readonly webhookHandler: WebhookTargetHandler,
+	) {
+	}
+
+	public resolveHandler<T extends Actions.AnyTarget>(target: T): InvokeHandler<T> {
+		if (target.type !== 'webhook') {
+			throw new ImplementationException('Invalid trigger invocation type')
+		}
+		return this.webhookHandler
+	}
+}