@contember/engine-actions 1.3.0-alpha.4

package/src/dispatch/WebhookTargetHandler.ts

@@ -0,0 +1,88 @@
+import { Actions } from '@contember/schema'
+import { HandledEvent, InvokeHandler, InvokeHandlerArgs } from './types'
+import { VariablesMap } from '../model/VariablesManager'
+
+const DEFAULT_TIMEOUT_MS = 30_000 // 30 seconds
+
+export class WebhookTargetHandler implements InvokeHandler<Actions.WebhookTarget> {
+	public async handle({ target, events, logger, variables }: InvokeHandlerArgs<Actions.WebhookTarget>): Promise<HandledEvent[]> {
+		const abortController = new AbortController()
+		const timeoutMs = target.timeoutMs
+
+		const start = process.hrtime.bigint()
+		const getDuration = () => Math.floor(Number((process.hrtime.bigint() - start) / BigInt(1_000_000)))
+		try {
+			const resolvedUrl = this.resolveVariables(target.url, variables)
+			const resolvedHeaders = Object.fromEntries(Object.entries(target.headers ?? {})
+				.map(([key, value]) => [key, this.resolveVariables(value, variables)]))
+
+			const response = await withTimeout({ abortController, timeoutMs }, async () => {
+
+				return await fetch(resolvedUrl, {
+					method: 'POST',
+					headers: {
+						['User-Agent']: 'Contember Actions',
+						...resolvedHeaders,
+						['Content-type']: 'application/json',
+					},
+					signal: abortController.signal,
+					body: JSON.stringify({
+						events: events.map(it => it.payload),
+					}),
+				})
+			})
+			let responseText
+			try {
+				responseText = await withTimeout({ abortController, timeoutMs }, async () => {
+					return await response.text()
+				})
+			} catch {
+			}
+			const result = {
+				ok: response.ok,
+				code: response.status,
+				durationMs: getDuration(),
+				response: responseText,
+				errorMessage: !response.ok ? response.statusText : undefined,
+			}
+
+			// todo: per-event response format
+
+			return events.map(it => ({ target, row: it, result }))
+		} catch (e) {
+			logger.warn(e)
+			const errorMessages = []
+			let err = e
+			while (typeof err === 'object' && err !== null && 'message' in err && typeof (err as any).message === 'string') {
+				errorMessages.push((err as any).message)
+				err = (err as any).cause
+			}
+			const result = {
+				ok: false,
+				errorMessage: errorMessages.length ? errorMessages.join('; ') : undefined,
+				durationMs: getDuration(),
+			}
+			return events.map(it => ({ target, row: it, result }))
+		}
+	}
+
+	private resolveVariables(subject: string, variables: VariablesMap) {
+		return subject.replace(/\{\{([\w_]+)}}/gm, (_, name) => {
+			if (!variables[name]) {
+				throw new Error(`Undefined variable ${name}`)
+			}
+			return variables[name]
+		})
+	}
+}
+
+const withTimeout = async <T>({ abortController, timeoutMs }: { abortController: AbortController; timeoutMs?: number }, cb: () => Promise<T>): Promise<T> => {
+	const timeoutId = setTimeout(() => {
+		abortController.abort()
+	}, timeoutMs ?? DEFAULT_TIMEOUT_MS)
+	try {
+		return await cb()
+	} finally {
+		clearTimeout(timeoutId)
+	}
+}

package/src/dispatch/types.ts

@@ -0,0 +1,31 @@
+import { Logger } from '@contember/logger'
+import { Actions } from '@contember/schema'
+import { EventRow } from '../model/types'
+import { VariablesMap } from '../model/VariablesManager'
+
+export { EventRow }
+
+export interface InvokeHandler<Target extends Actions.AnyTarget> {
+	handle(args: InvokeHandlerArgs<Target>): Promise<HandledEvent[]>
+}
+
+export interface InvokeHandlerArgs<Target extends Actions.AnyTarget> {
+	target: Target
+	events: EventRow[]
+	logger: Logger
+	variables: VariablesMap
+}
+
+export type HandledEvent = {
+	row: EventRow
+	result: InvocationResult
+	target?: Actions.AnyTarget
+}
+
+export type InvocationResult = {
+	ok: boolean
+	durationMs?: number
+	errorMessage?: string
+	code?: number
+	response?: string
+}

package/src/graphql/http/ActionsApiMiddlewareFactory.ts

@@ -0,0 +1,36 @@
+import { HttpController } from '@contember/engine-http'
+import { ActionsGraphQLHandler } from './ActionsGraphQLHandlerFactory'
+import { ActionsContext } from '../resolvers/ActionsContext'
+import { ActionsContextResolver } from './ActionsContextResolver'
+
+export class ActionsApiMiddlewareFactory {
+	constructor(
+		private readonly actionsContextResolver: ActionsContextResolver,
+		private readonly actionsGraphQLHandler: ActionsGraphQLHandler,
+	) {
+	}
+
+	create(): HttpController {
+		return async ctx => {
+			const { timer, logger, koa } = ctx
+			const { projectContainer } = await this.actionsContextResolver.resolve(ctx)
+
+			const systemDatabase = projectContainer.systemDatabaseContextFactory.create()
+
+			logger.debug('Actions query processing started')
+			const graphqlContext: ActionsContext = {
+				db: systemDatabase,
+				logger,
+				contentSchemaResolver: projectContainer.contentSchemaResolver,
+			}
+			const handler = this.actionsGraphQLHandler
+
+			await timer('GraphQL', () => handler({
+				request: koa.request,
+				response: koa.response,
+				createContext: () => graphqlContext,
+			}))
+			logger.debug('Actions query finished')
+		}
+	}
+}

package/src/graphql/http/ActionsContextResolver.ts

@@ -0,0 +1,38 @@
+import { ApplicationContext, HttpErrorResponse, ProjectContextResolver } from '@contember/engine-http'
+import { ProjectRole } from '@contember/schema'
+
+export class ActionsContextResolver {
+	constructor(
+		private readonly debug: boolean,
+		private readonly projectContextResolver: ProjectContextResolver,
+	) {
+	}
+
+	async resolve(ctx: ApplicationContext) {
+		const { timer, projectGroup: { tenantContainer }, logger, authResult } = ctx
+		const projectContext = await this.projectContextResolver.resolve(ctx)
+		const { project } = projectContext
+		if (!authResult) {
+			throw new HttpErrorResponse(401, 'Authentication required')
+		}
+		const memberships = await timer('MembershipFetch', () =>
+			tenantContainer.projectMemberManager.getEffectiveProjectMemberships(
+				tenantContainer.databaseContext,
+				{ slug: project.slug },
+				{
+					id: authResult.identityId,
+					roles: authResult.roles,
+				},
+			),
+		)
+		logger.debug('Memberships fetched', { memberships })
+
+		const isProjectAdmin = memberships.some(it => it.role === ProjectRole.ADMIN)
+		if (!isProjectAdmin) {
+			throw this.debug
+				? new HttpErrorResponse(404, `You are not allowed to access project ${project.slug}`)
+				: new HttpErrorResponse(404, `Project ${project.slug} NOT found`)
+		}
+		return projectContext
+	}
+}

package/src/graphql/http/ActionsGraphQLHandlerFactory.ts

@@ -0,0 +1,23 @@
+import { makeExecutableSchema } from '@graphql-tools/schema'
+import { createGraphQLQueryHandler, GraphQLQueryHandler } from '@contember/engine-http'
+import { ResolversFactory } from '../resolvers/ResolversFactory'
+import { schema as graphqlDocument } from '../schema/actions.graphql'
+import { ActionsContext } from '../resolvers/ActionsContext'
+
+
+export type ActionsGraphQLHandler = GraphQLQueryHandler<ActionsContext>
+
+export class ActionsGraphQLHandlerFactory {
+	create(resolversFactory: ResolversFactory): ActionsGraphQLHandler {
+		const resolvers = resolversFactory.create()
+		const schema = makeExecutableSchema({
+			typeDefs: graphqlDocument,
+			resolvers,
+		})
+
+		return createGraphQLQueryHandler<ActionsContext>({
+			schema,
+			listeners: [],
+		})
+	}
+}

package/src/graphql/http/ActionsWebsocketControllerFactory.ts

@@ -0,0 +1,143 @@
+import { HttpErrorResponse, WebSocketController } from '@contember/engine-http'
+import { ActionsContextResolver } from './ActionsContextResolver'
+import { DispatchWorkerSupervisorFactory } from '../../dispatch/DispatchWorkerSupervisor'
+import * as Typesafe from '@contember/typesafe'
+import { TenantRole } from '@contember/engine-tenant-api'
+import { Running } from '@contember/engine-common'
+
+const IncomingMessage = Typesafe.discriminatedUnion(
+	'type',
+	{
+		startWorker: Typesafe.object({}),
+		stopAllWorkers: Typesafe.object({}),
+	},
+)
+type IncomingMessage = ReturnType<typeof IncomingMessage>
+const OutgoingMessage = Typesafe.discriminatedUnion(
+	'type',
+	{
+		error: Typesafe.object({
+			message: Typesafe.string,
+		}),
+		message: Typesafe.object({
+			message: Typesafe.string,
+		}),
+		ready: Typesafe.object({}),
+
+		workerStarted: Typesafe.object({
+			workerId: Typesafe.string,
+		}),
+		workedStopped: Typesafe.object({
+			workerId: Typesafe.string,
+		}),
+		workerFailedToStart: Typesafe.object({
+			workerId: Typesafe.string,
+		}),
+		workerCrashed: Typesafe.object({
+			workerId: Typesafe.string,
+		}),
+	},
+)
+
+type OutgoingMessage = ReturnType<typeof OutgoingMessage>
+
+export class ActionsWebsocketControllerFactory {
+	constructor(
+		private readonly debug: boolean,
+		private readonly actionsContextResolver: ActionsContextResolver,
+		private readonly dispatchWorkerSupervisorFactory: DispatchWorkerSupervisorFactory,
+	) {
+	}
+
+	create(): WebSocketController {
+		return async ctx => {
+			const { ws, logger, authResult, projectGroup } = ctx
+			if (!authResult) {
+				throw new HttpErrorResponse(401, 'Authentication required')
+			}
+			if (!authResult.roles.some(it => it === TenantRole.SUPER_ADMIN || it  === TenantRole.PROJECT_ADMIN)) {
+				throw new HttpErrorResponse(403, 'Not allowed to run actions worker')
+			}
+
+			const send = (message: OutgoingMessage) => {
+				if (this.debug) {
+					OutgoingMessage(message)
+				}
+				ws.send(JSON.stringify(message))
+			}
+
+			let workers: { id: string; running: Running }[] = []
+			const abortListener = async () => {
+				send({ type: 'message', message: 'shutting down' })
+				await stopAll()
+				send({ type: 'message', message: 'closing connection' })
+				ws.close(1012) // Service Restart
+			}
+			ctx.abortSignal.addEventListener('abort', abortListener)
+
+			const stopAll = async () => {
+				const currentWorkers = workers.map(it => it.running.end().then(() => send({
+					type: 'workedStopped',
+					workerId: it.id,
+				})))
+				workers = []
+				await Promise.all(currentWorkers)
+			}
+			send({ type: 'ready' })
+
+			ws.addEventListener('message', async message => {
+				let json
+				try {
+					json = JSON.parse(message.data.toString())
+				} catch {
+					send({ type: 'error', message: 'Invalid JSON' })
+					return
+				}
+				let data: IncomingMessage
+				try {
+					data = IncomingMessage(json)
+				} catch (e) {
+					if (e instanceof Typesafe.ParseError) {
+						send({ type: 'error', message: e.message })
+						return
+					} else {
+						throw e
+					}
+				}
+
+				switch (data.type) {
+					case 'startWorker':
+						const workerId = Math.random().toString().substring(2)
+						const dispatchSupervisor = this.dispatchWorkerSupervisorFactory.create(projectGroup)
+						try {
+							const running  = await dispatchSupervisor.run({ logger, onError: () => {
+								send({ type: 'workerCrashed', workerId })
+							} })
+							workers.push({ id: workerId, running })
+							send({ type: 'workerStarted', workerId })
+						} catch (e) {
+							logger.error(e, { message: 'Worker failed to start' })
+							send({ type: 'workerFailedToStart', workerId })
+						}
+						break
+					case 'stopAllWorkers':
+						send({ type: 'message', message: 'stopping' })
+						await stopAll()
+						send({ type: 'message', message: 'all stopped' })
+						break
+				}
+			})
+
+			const pingHandle = setInterval(() => {
+				ws.ping()
+			}, 5000)
+
+			ws.addEventListener('close', async () => {
+				ctx.abortSignal.removeEventListener('abort', abortListener)
+				clearInterval(pingHandle)
+				await stopAll()
+			})
+		}
+	}
+}
+

package/src/graphql/resolvers/ActionsContext.ts

@@ -0,0 +1,9 @@
+import { DatabaseContext } from '@contember/engine-system-api'
+import { ContentSchemaResolver } from '@contember/engine-http'
+import { Logger } from '@contember/logger'
+
+export interface ActionsContext {
+	logger: Logger
+	db: DatabaseContext
+	contentSchemaResolver: ContentSchemaResolver
+}

package/src/graphql/resolvers/ResolversFactory.ts

@@ -0,0 +1,35 @@
+import { DateTimeType, JSONType, UuidType } from '@contember/graphql-utils'
+import { Resolvers } from '../schema'
+import { EventsQueryResolver } from './query'
+import { ProcessBatchMutationResolver, SetVariablesMutationResolver } from './mutation'
+import { ActionsContext } from './ActionsContext'
+import { VariablesQueryResolver } from './query/VariablesQueryResolver'
+
+export class ResolversFactory {
+	constructor(
+		private readonly eventsQueryResolver: EventsQueryResolver,
+		private readonly processBatchMutationResolver: ProcessBatchMutationResolver,
+		private readonly variablesQueryResolver: VariablesQueryResolver,
+		private readonly setVariablesMutationResolver: SetVariablesMutationResolver,
+	) {
+	}
+
+	create(): Resolvers {
+		const resolvers: Resolvers<ActionsContext> & { Mutation: Required<Resolvers<ActionsContext>['Mutation']> } & { Query: Required<Resolvers<ActionsContext>['Query']> } = {
+			Json: JSONType,
+			DateTime: DateTimeType,
+			Uuid: UuidType,
+			Mutation: {
+				processBatch: this.processBatchMutationResolver.processBatch.bind(this.processBatchMutationResolver),
+				setVariables: this.setVariablesMutationResolver.setVariables.bind(this.setVariablesMutationResolver),
+			},
+			Query: {
+				eventsInProcessing: this.eventsQueryResolver.eventsInProcessing.bind(this.eventsQueryResolver),
+				eventsToProcess: this.eventsQueryResolver.eventsToProcess.bind(this.eventsQueryResolver),
+				failedEvents: this.eventsQueryResolver.failedEvents.bind(this.eventsQueryResolver),
+				variables: this.variablesQueryResolver.variables.bind(this.variablesQueryResolver),
+			},
+		}
+		return resolvers
+	}
+}

package/src/graphql/resolvers/index.ts

@@ -0,0 +1,2 @@
+export * from './mutation'
+export * from './query'

package/src/graphql/resolvers/mutation/ProcessBatchMutationResolver.ts

@@ -0,0 +1,16 @@
+import { MutationResolvers } from '../../schema'
+import { ActionsContext } from '../ActionsContext'
+import { EventDispatcher } from '../../../dispatch/EventDispatcher'
+
+export class ProcessBatchMutationResolver implements MutationResolvers<ActionsContext> {
+	constructor(
+		private eventDispatcher: EventDispatcher,
+	) {
+	}
+	async processBatch(parent: unknown, args: unknown, ctx: ActionsContext) {
+		await this.eventDispatcher.processBatch(ctx)
+		return {
+			ok: true,
+		}
+	}
+}

package/src/graphql/resolvers/mutation/SetVariablesMutationResolver.ts

@@ -0,0 +1,19 @@
+import { MutationResolvers, MutationSetVariablesArgs } from '../../schema'
+import { ActionsContext } from '../ActionsContext'
+import { VariablesManager } from '../../../model/VariablesManager'
+
+export class SetVariablesMutationResolver implements MutationResolvers<ActionsContext> {
+	constructor(
+		private readonly variablesManager: VariablesManager,
+	) {
+	}
+
+
+	async setVariables(parent: unknown, { args }: MutationSetVariablesArgs, ctx: ActionsContext) {
+		await this.variablesManager.setVariables(ctx.db, args)
+
+		return {
+			ok: true,
+		}
+	}
+}

package/src/graphql/resolvers/mutation/index.ts

@@ -0,0 +1,2 @@
+export * from './ProcessBatchMutationResolver'
+export * from './SetVariablesMutationResolver'

package/src/graphql/resolvers/query/EventsQueryResolver.ts

@@ -0,0 +1,55 @@
+import {
+	Event,
+	QueryEventsInProcessingArgs,
+	QueryEventsToProcessArgs,
+	QueryFailedEventsArgs,
+	QueryResolvers,
+} from '../../schema'
+import { ActionsContext } from '../ActionsContext'
+import { EventsToProcessQuery } from '../../../model/EventsToProcessQuery'
+import { EventRow } from '../../../model/types'
+import { DatabaseContext, StagesQuery } from '@contember/engine-system-api'
+import { Client, DatabaseQuery } from '@contember/database'
+import { EventsInProcessingQuery } from '../../../model/EventsInProcessingQuery'
+import { FailedEventsQuery } from '../../../model/FailedEventsQuery'
+
+export class EventsQueryResolver implements QueryResolvers<ActionsContext> {
+	async eventsInProcessing(parent: unknown, { args }: QueryEventsInProcessingArgs, ctx: ActionsContext) {
+		return await this.fetchEvents(ctx.db, new EventsInProcessingQuery(args ?? {}))
+	}
+
+	async eventsToProcess(parent: unknown, { args }: QueryEventsToProcessArgs, ctx: ActionsContext) {
+		return await this.fetchEvents(ctx.db, new EventsToProcessQuery(args ?? {}))
+	}
+
+	async failedEvents(parent: unknown, { args }: QueryFailedEventsArgs, ctx: ActionsContext) {
+		return await this.fetchEvents(ctx.db, new FailedEventsQuery(args ?? {}))
+	}
+
+	private async fetchEvents(db: DatabaseContext, query: DatabaseQuery<EventRow[]>) {
+		const stages = await this.fetchStagesMap(db)
+		return this.mapEventRows(await db.queryHandler.fetch(query), stages)
+	}
+
+	private async fetchStagesMap(db: DatabaseContext): Promise<Record<string, string>> {
+		return Object.fromEntries((await db.queryHandler.fetch(new StagesQuery())).map(it => [it.id, it.slug]))
+	}
+
+	private mapEventRows(rows: EventRow[], stagesMap: Record<string, string>): Event[] {
+
+		return rows.map(row => ({
+			id: row.id,
+			createdAt: row.created_at,
+			resolvedAt: row.created_at,
+			lastStateChange: row.last_state_change,
+			visibleAt: row.visible_at,
+			log: row.log,
+			numRetries: row.num_retries,
+			payload: row.payload,
+			stage: stagesMap[row.stage_id],
+			state: row.state,
+			target: row.target,
+			transactionId: row.transaction_id,
+		}))
+	}
+}

package/src/graphql/resolvers/query/VariablesQueryResolver.ts

@@ -0,0 +1,18 @@
+import { QueryResolvers } from '../../schema'
+import { ActionsContext } from '../ActionsContext'
+import { VariablesManager } from '../../../model/VariablesManager'
+
+export class VariablesQueryResolver implements QueryResolvers<ActionsContext> {
+
+	constructor(
+		private readonly variablesManager: VariablesManager,
+	) {
+	}
+
+	async variables(parent: unknown, args: unknown, ctx: ActionsContext) {
+		return Object.entries(await this.variablesManager.fetchVariables(ctx.db)).map(([name, value]) => ({
+			name,
+			value,
+		}))
+	}
+}

package/src/graphql/resolvers/query/index.ts

@@ -0,0 +1 @@
+export * from './EventsQueryResolver'

package/src/graphql/schema/actions.graphql.ts

@@ -0,0 +1,91 @@
+import { gql } from 'graphql-tag'
+import { DocumentNode } from 'graphql'
+
+export const schema: DocumentNode = gql`
+    scalar DateTime
+    scalar Json
+    scalar Uuid
+
+    schema {
+        query: Query
+        mutation: Mutation
+    }
+
+    type Query {
+        failedEvents(args: EventArgs): [Event!]!
+        eventsToProcess(args: EventArgs): [Event!]!
+        eventsInProcessing(args: EventArgs): [Event!]!
+        variables: [Variable!]!
+    }
+
+    type Mutation {
+        processBatch: ProcessBatchResponse!
+        setVariables(args: SetVariablesArgs!): SetVariablesResponse!
+    }
+
+    input EventArgs {
+        offset: Int
+        "Max 10000"
+        limit: Int
+    }
+
+    type Event {
+        id: Uuid!
+        transactionId: Uuid!
+        createdAt: DateTime!
+        lastStateChange: DateTime!
+        resolvedAt: DateTime
+        visibleAt: DateTime
+        numRetries: Int!
+        state: EventState!
+        stage: String!
+        target: String!
+        payload: Json!
+        log: Json!
+    }
+
+    enum EventState {
+        created
+        retrying
+        processing
+        succeed
+        failed
+        stopped
+    }
+
+    type ProcessBatchResponse {
+        ok: Boolean!
+    }
+
+    input SetVariablesArgs {
+        variables: [VariableInput!]!
+        mode: SetVariablesMode
+    }
+	
+	"""
+	Defines how it handles original variables. 
+	- MERGE merges with new values (default behaviour)
+	- SET replaces all variables
+	- APPEND_ONLY_MISSING appends values if not already exist 
+	"""
+	enum SetVariablesMode {
+		MERGE
+		SET
+		APPEND_ONLY_MISSING
+	}
+
+    input VariableInput {
+        name: String!
+        value: String!
+    }
+	
+	type SetVariablesResponse {
+		ok: Boolean!
+	}
+
+    type Variable {
+        name: String!
+        value: String!
+    }
+
+`