@contember/engine-actions 2.2.0-alpha.3 → 2.2.0-alpha.4

package/dist/types/utils/map.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"map.d.ts","sourceRoot":"","sources":["../../../src/utils/map.ts"],"names":[],"mappings":"AAAA,eAAO,MAAM,WAAW,GAAI,CAAC,EAAE,CAAC,EAAE,CAAC,SAAS,CAAC,EAC5C,KAAK,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC,EACd,KAAK,CAAC,EACN,cAAc,MAAM,CAAC,KACnB,CAQF,CAAA"}
+{"version":3,"file":"map.d.ts","sourceRoot":"","sources":["../../../src/utils/map.ts"],"names":[],"mappings":"AAAA,eAAO,MAAM,WAAW,GAAI,CAAC,EAAE,CAAC,EAAE,CAAC,SAAS,CAAC,OACvC,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC,OACT,CAAC,gBACQ,MAAM,CAAC,KACnB,CAQF,CAAA"}

package/dist/types/utils/notifyChannel.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"notifyChannel.d.ts","sourceRoot":"","sources":["../../../src/utils/notifyChannel.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,MAAM,qBAAqB,CAAA;AAE5C,eAAO,MAAM,mBAAmB,kBAAkB,CAAA;AAClD,eAAO,MAAM,MAAM,GAAU,IAAI,MAAM,EAAE,aAAa,MAAM,kBAE3D,CAAA"}
+{"version":3,"file":"notifyChannel.d.ts","sourceRoot":"","sources":["../../../src/utils/notifyChannel.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,MAAM,qBAAqB,CAAA;AAE5C,eAAO,MAAM,mBAAmB,kBAAkB,CAAA;AAClD,eAAO,MAAM,MAAM,OAAc,MAAM,eAAe,MAAM,kBAE3D,CAAA"}

package/graphql.codegen.yml

@@ -10,7 +10,7 @@ generates:
       enumsAsTypes: "1"
       scalars:
         DateTime: Date
-        Uuid: string
+        UUID: string
         Json: any
     plugins:
       - "typescript"

package/package.json

@@ -22,24 +22,26 @@
     }
   },
   "dependencies": {
-    "@contember/authorization": "2.2.0-alpha.3",
-    "@contember/database": "2.2.0-alpha.3",
-    "@contember/database-migrations": "2.2.0-alpha.3",
-    "@contember/engine-common": "2.2.0-alpha.3",
-    "@contember/engine-content-api": "2.2.0-alpha.3",
-    "@contember/engine-http": "2.2.0-alpha.3",
-    "@contember/engine-plugins": "2.2.0-alpha.3",
-    "@contember/engine-system-api": "2.2.0-alpha.3",
-    "@contember/engine-tenant-api": "2.2.0-alpha.3",
-    "@contember/graphql-utils": "2.2.0-alpha.3",
-    "@contember/logger": "2.2.0-alpha.3",
-    "@contember/schema": "2.2.0-alpha.3",
-    "@contember/schema-utils": "2.2.0-alpha.3",
-    "@contember/typesafe": "2.2.0-alpha.3",
+    "@contember/authorization": "2.2.0-alpha.4",
+    "@contember/database": "2.2.0-alpha.4",
+    "@contember/database-migrations": "2.2.0-alpha.4",
+    "@contember/engine-common": "2.2.0-alpha.4",
+    "@contember/engine-content-api": "2.2.0-alpha.4",
+    "@contember/engine-http": "2.2.0-alpha.4",
+    "@contember/engine-plugins": "2.2.0-alpha.4",
+    "@contember/engine-system-api": "2.2.0-alpha.4",
+    "@contember/engine-tenant-api": "2.2.0-alpha.4",
+    "@contember/graphql-utils": "2.2.0-alpha.4",
+    "@contember/logger": "2.2.0-alpha.4",
+    "@contember/schema": "2.2.0-alpha.4",
+    "@contember/schema-utils": "2.2.0-alpha.4",
+    "@contember/typesafe": "2.2.0-alpha.4",
     "@graphql-tools/schema": "^10.0.31",
-    "graphql-tag": "^2.12.6"
+    "graphql-tag": "^2.12.6",
+    "prom-client": "^12.0.0"
   },
   "devDependencies": {
+    "@contember/database-tester": "2.2.0-alpha.4",
     "@types/bun": "^1.3.10",
     "@types/ws": "^8.18.1",
     "graphql": "^16.13.1"
@@ -49,7 +51,7 @@
     "pg": "^8.20.0"
   },
   "type": "module",
-  "version": "2.2.0-alpha.3",
+  "version": "2.2.0-alpha.4",
   "repository": {
     "type": "git",
     "url": "git+https://github.com/contember/contember.git",

package/src/ActionsExecutionContainerHookFactory.ts

@@ -3,10 +3,14 @@ import { ListenerStoreProvider } from './ListenerStoreProvider.js'
 import { TriggerPayloadBuilder } from './triggers/TriggerPayloadBuilder.js'
 import { TriggerPayloadPersister } from './triggers/TriggerPayloadPersister.js'
 import { ExecutionContainerHook } from '@contember/engine-content-api'
+import { ActionsMetrics } from './ActionsMetrics.js'
 
 export class ActionsExecutionContainerHookFactory {
 	constructor(
 		private readonly listenerStoreProvider: ListenerStoreProvider,
+		// Resolved lazily: the metrics singleton lives in the master container and is wired up when the
+		// dispatch worker boots, which precedes any content request that could enqueue an event.
+		private readonly metricsProvider: () => ActionsMetrics | undefined,
 	) {
 	}
 
@@ -18,6 +22,7 @@ export class ActionsExecutionContainerHookFactory {
 					mapperFactory,
 					{ whereBuilder, schema, pathFactory, systemSchema, providers, stage, project, schemaMeta, joinBuilder, userInfo, triggeredActionsCollector },
 				) => {
+					const projectMetrics = this.metricsProvider()?.forProject(project.slug)
 					mapperFactory.hooks.push(mapper => {
 						const triggerPayloadPersister = new TriggerPayloadPersister(
 							mapper,
@@ -29,6 +34,7 @@ export class ActionsExecutionContainerHookFactory {
 							mapper.identityId,
 							userInfo,
 							triggeredActionsCollector,
+							projectMetrics,
 						)
 						const triggerPayloadBuilder = new TriggerPayloadBuilder(mapper)
 						const payloadManager = new TriggerPayloadManager(triggerPayloadBuilder, triggerPayloadPersister)

package/src/ActionsMetrics.ts

@@ -0,0 +1,135 @@
+import prom, { Registry } from 'prom-client'
+
+const LABEL_PROJECT = 'contember_project'
+type Label = typeof LABEL_PROJECT
+
+/**
+ * Prometheus metrics for the Actions dispatch subsystem.
+ *
+ * Two audiences, deliberately kept as cheap in-memory counters/gauges (no table polling):
+ *  - system / operator — is the dispatch loop alive and flowing, or stuck?
+ *      `workerHeartbeat` (liveness; works even when the queue is empty) + `workerCrashed`.
+ *  - userland — are webhook deliveries failing?
+ *      `eventsSucceeded` / `deliveryAttemptsFailed` / `eventsFailed`.
+ *
+ * Counting units matter: a *delivery attempt* and an *event* are different. A success is always
+ * terminal (one event = one successful attempt), but a failure splits — a failed attempt may be
+ * retried many times, while a terminally failed event gives up once. Hence the asymmetric set:
+ * one success counter, but both a per-attempt failure counter and a terminal-event failure counter.
+ *
+ * Queue backlog is *estimated* as a trend from the counters:
+ *   enqueued − succeeded − failed   (compare rates; the absolute value drifts across restarts and
+ *   ignores manual retry/stop, so for an exact depth query the `actions_event` table instead).
+ *
+ * Labelled only by `contember_project` to keep cardinality bounded; per-target / per-trigger
+ * detail belongs in the table-backed userland view, not in Prometheus.
+ */
+export class ActionsMetrics {
+	/** Events written to the queue (inflow). Unit: event. */
+	public readonly eventsEnqueued: prom.Counter<Label>
+	/** Events delivered successfully (terminal). Unit: event (= successful attempt). */
+	public readonly eventsSucceeded: prom.Counter<Label>
+	/** Failed webhook delivery attempts, including ones that will be retried. Unit: attempt. */
+	public readonly deliveryAttemptsFailed: prom.Counter<Label>
+	/** Events that reached a terminal failed state (retries exhausted or unknown target). Unit: event. */
+	public readonly eventsFailed: prom.Counter<Label>
+	/** Unix timestamp (seconds) of the last dispatch-loop iteration; staleness ⇒ stuck/dead worker. */
+	public readonly workerHeartbeat: prom.Gauge<Label>
+	/** Dispatch-loop crashes (each is auto-restarted by the supervisor). */
+	public readonly workerCrashed: prom.Counter<Label>
+
+	constructor(registry: Registry) {
+		const labelNames: Label[] = [LABEL_PROJECT]
+		this.eventsEnqueued = new prom.Counter({
+			registers: [registry],
+			name: 'contember_actions_events_enqueued_total',
+			help: 'Total number of action events written to the queue (inflow), by contember_project.',
+			labelNames,
+		})
+		this.eventsSucceeded = new prom.Counter({
+			registers: [registry],
+			name: 'contember_actions_events_succeeded_total',
+			help: 'Total number of action events delivered successfully, by contember_project.',
+			labelNames,
+		})
+		this.deliveryAttemptsFailed = new prom.Counter({
+			registers: [registry],
+			name: 'contember_actions_delivery_attempts_failed_total',
+			help: 'Total number of failed webhook delivery attempts (including attempts that will be retried), by contember_project.',
+			labelNames,
+		})
+		this.eventsFailed = new prom.Counter({
+			registers: [registry],
+			name: 'contember_actions_events_failed_total',
+			help: 'Total number of action events that reached a terminal failed state (retries exhausted or unknown target), by contember_project.',
+			labelNames,
+		})
+		this.workerHeartbeat = new prom.Gauge({
+			registers: [registry],
+			name: 'contember_actions_worker_heartbeat_timestamp_seconds',
+			help: 'Unix timestamp of the last dispatch-loop iteration per contember_project; staleness indicates a stuck or dead worker.',
+			labelNames,
+		})
+		this.workerCrashed = new prom.Counter({
+			registers: [registry],
+			name: 'contember_actions_worker_crashed_total',
+			help: 'Total number of dispatch-loop crashes per contember_project (each is auto-restarted by the supervisor).',
+			labelNames,
+		})
+	}
+
+	public forProject(projectSlug: string): ProjectActionsMetrics {
+		return new ProjectActionsMetrics(this, projectSlug)
+	}
+}
+
+/**
+ * A view of {@link ActionsMetrics} bound to a single project, so call sites don't repeat the label.
+ */
+export class ProjectActionsMetrics {
+	private readonly labels: { [LABEL_PROJECT]: string }
+
+	constructor(
+		private readonly metrics: ActionsMetrics,
+		private readonly projectSlug: string,
+	) {
+		this.labels = { [LABEL_PROJECT]: projectSlug }
+	}
+
+	public enqueued(count: number): void {
+		if (count > 0) {
+			this.metrics.eventsEnqueued.inc(this.labels, count)
+		}
+	}
+
+	public succeeded(count: number): void {
+		if (count > 0) {
+			this.metrics.eventsSucceeded.inc(this.labels, count)
+		}
+	}
+
+	public deliveryAttemptFailed(count: number): void {
+		if (count > 0) {
+			this.metrics.deliveryAttemptsFailed.inc(this.labels, count)
+		}
+	}
+
+	public failed(count: number): void {
+		if (count > 0) {
+			this.metrics.eventsFailed.inc(this.labels, count)
+		}
+	}
+
+	public heartbeat(): void {
+		this.metrics.workerHeartbeat.set(this.labels, Date.now() / 1000)
+	}
+
+	public crashed(): void {
+		this.metrics.workerCrashed.inc(this.labels, 1)
+	}
+
+	/** Drop the per-project heartbeat series when the worker stops, so a removed project doesn't look stuck. */
+	public dispose(): void {
+		this.metrics.workerHeartbeat.remove(this.projectSlug)
+	}
+}

package/src/dispatch/EventDispatcher.ts

@@ -10,11 +10,18 @@ type ProcessBatchArgs = {
 	db: DatabaseContext
 	contentSchemaResolver: ContentSchemaResolver
 	logger: Logger
+	project: { slug: string }
 }
 
 type ProcessBatchResult = {
-	succeed: number
-	failed: number
+	/** Events delivered successfully (terminal). */
+	succeeded: number
+	/** Failed delivery attempts that will be retried. */
+	retried: number
+	/** Events terminally failed after a delivery attempt (retries exhausted). */
+	failedAfterAttempt: number
+	/** Events terminally failed without a delivery attempt (target missing from schema). */
+	failedUnknownTarget: number
 	backoffMs: number | undefined
 }
 
@@ -26,16 +33,22 @@ export class EventDispatcher {
 	) {
 	}
 
-	async processBatch({ db, contentSchemaResolver, logger }: ProcessBatchArgs): Promise<ProcessBatchResult> {
+	async processBatch({ db, contentSchemaResolver, logger, project }: ProcessBatchArgs): Promise<ProcessBatchResult> {
 		const schema = (await contentSchemaResolver.getSchema({ db, normalize: true })).schema
-		const batch = await this.eventsRepository.fetchBatch(schema.actions, db.client)
 		const batchId = Math.random().toString().substring(2)
 		const batchLogger = logger.child({ loc: 'Actions.EventDispatcher', batchId })
+		const batch = await this.eventsRepository.fetchBatch(schema.actions, db.client, batchLogger)
 		if (!batch.ok) {
 			batchLogger.debug('Nothing to process', {
 				backoffMs: batch.backoffMs ?? 'undefined',
 			})
-			return { failed: 0, succeed: 0, backoffMs: batch.backoffMs }
+			return {
+				succeeded: 0,
+				retried: 0,
+				failedAfterAttempt: 0,
+				failedUnknownTarget: batch.unknownTargetFailed,
+				backoffMs: batch.backoffMs,
+			}
 		}
 		const { target, events } = batch
 		try {
@@ -43,20 +56,32 @@ export class EventDispatcher {
 			batchLogger.debug('Processing started', {
 				events: batch.events.map(it => it.id),
 			})
-			const variables = await this.variablesManager.fetchVariables(db)
+			const variables = await this.variablesManager.fetchVariables(db, project.slug)
 			const handledEvents = await handler.handle({ target, events, logger: batchLogger, variables })
-			const [succeed, failed] = await this.eventsRepository.persistProcessed(db.client, handledEvents)
-			batchLogger.debug('Processing done', { succeed, failed })
-			return { succeed, failed, backoffMs: 0 }
+			const { succeeded, retried, failed } = await this.eventsRepository.persistProcessed(db.client, handledEvents, batchLogger)
+			batchLogger.debug('Processing done', { succeed: succeeded, failed })
+			return {
+				succeeded,
+				retried,
+				failedAfterAttempt: failed,
+				failedUnknownTarget: batch.unknownTargetFailed,
+				backoffMs: 0,
+			}
 		} catch (e) {
 			logger.error(e, { loc: 'EventDispatcher', batchId })
 			const failedEvents = events.map((it): HandledEvent => ({
 				row: it,
 				result: { ok: false, errorMessage: `Internal error` },
 			}))
-			await this.eventsRepository.persistProcessed(db.client, failedEvents)
+			const { succeeded, retried, failed } = await this.eventsRepository.persistProcessed(db.client, failedEvents, batchLogger)
 
-			return { succeed: 0, failed: batch.events.length, backoffMs: 0 }
+			return {
+				succeeded,
+				retried,
+				failedAfterAttempt: failed,
+				failedUnknownTarget: batch.unknownTargetFailed,
+				backoffMs: 0,
+			}
 		}
 	}
 }

package/src/dispatch/EventsRepository.ts

@@ -3,6 +3,7 @@ import { EventRow, HandledEvent } from './types.js'
 import { Actions } from '@contember/schema'
 import { eventsToProcessSpecification, eventsToProcessStateSpecification } from '../model/EventsToProcessSpecification.js'
 import { notify } from '../utils/notifyChannel.js'
+import { Logger } from '@contember/logger'
 
 const ACK_TIMEOUT_MS = 1_000 * 60 * 10 // 10 minutes
 const DEFAULT_REPEAT_INTERVAL_MS = 5_000 // 5 seconds
@@ -10,19 +11,32 @@ const DEFAULT_MAX_ATTEMPTS = 10
 const DEFAULT_BATCH_SIZE = 1
 
 export class EventsRepository {
-	public async fetchBatch(actions: Actions.Schema, db: Client): Promise<FetchBatchResult> {
-		const primaryEvent = (await this.fetchInternal(db, 1))[0]
-		if (!primaryEvent) {
-			return { ok: false, backoffMs: await this.fetchBackOff(db) }
-		}
-		const target = actions.targets[primaryEvent.target]
-		if (!target) {
-			await this.markFailedOnUnknownTarget(db, primaryEvent.target, primaryEvent.id)
-			return await this.fetchBatch(actions, db)
+	public async fetchBatch(actions: Actions.Schema, db: Client, logger: Logger): Promise<FetchBatchResult> {
+		// Events referencing a target that no longer exists in the schema are terminally failed and
+		// skipped over; count them so the caller can report them as terminal failures.
+		let unknownTargetFailed = 0
+		while (true) {
+			const primaryEvent = (await this.fetchInternal(db, 1))[0]
+			if (!primaryEvent) {
+				return { ok: false, backoffMs: await this.fetchBackOff(db), unknownTargetFailed }
+			}
+			const target = actions.targets[primaryEvent.target]
+			if (!target) {
+				if (await this.markFailedOnUnknownTarget(db, primaryEvent.target, primaryEvent.id)) {
+					unknownTargetFailed++
+					// Never retried: the schema has to name the target again before such an event can be delivered.
+					logger.error('Action event failed permanently, its target is not in the schema', {
+						eventId: primaryEvent.id,
+						target: primaryEvent.target,
+						trigger: primaryEvent.trigger,
+					})
+				}
+				continue
+			}
+			const batchSize = (target.batchSize ?? DEFAULT_BATCH_SIZE) - 1
+			const batch = batchSize > 0 ? await this.fetchInternal(db, batchSize) : []
+			return { ok: true, events: [primaryEvent, ...batch], target, unknownTargetFailed }
 		}
-		const batchSize = (target.batchSize ?? DEFAULT_BATCH_SIZE) - 1
-		const batch = batchSize > 0 ? await this.fetchInternal(db, batchSize) : []
-		return { ok: true, events: [primaryEvent, ...batch], target }
 	}
 
 	private async fetchBackOff(db: Client): Promise<number | undefined> {
@@ -60,18 +74,34 @@ export class EventsRepository {
 			.execute(db)
 	}
 
-	public async persistProcessed(db: Client, events: HandledEvent[]): Promise<[number, number]> {
+	public async persistProcessed(db: Client, events: HandledEvent[], logger: Logger): Promise<PersistProcessedResult> {
 		return await db.transaction(async trx => {
 			await trx.query(Connection.REPEATABLE_READ)
 			const succeed = events.filter(it => it.result.ok)
 			await this.markSucceed(trx, succeed.map(it => it.row))
 
+			let retried = 0
+			let failed = 0
 			for (const event of events) {
 				if (!event.result.ok) {
-					await this.markFailed(trx, event)
+					const terminal = await this.markFailed(trx, event)
+					if (terminal) {
+						failed++
+						// Nothing else reports a given-up event; the target is named, never described by URL or headers.
+						logger.error('Action event failed permanently, no attempts left', {
+							eventId: event.row.id,
+							target: event.row.target,
+							trigger: event.row.trigger,
+							attempts: event.row.num_retries + 1,
+							code: event.result.code,
+							errorMessage: event.result.errorMessage,
+						})
+					} else {
+						retried++
+					}
 				}
 			}
-			return [succeed.length, events.length - succeed.length]
+			return { succeeded: succeed.length, retried, failed }
 		})
 	}
 
@@ -124,42 +154,44 @@ export class EventsRepository {
 			.execute(db)
 	}
 
-	private async markFailed(db: Client, event: HandledEvent): Promise<void> {
+	/** Returns `true` when the event reached a terminal `failed` state, `false` when it will be retried. */
+	private async markFailed(db: Client, event: HandledEvent): Promise<boolean> {
 		const numRetries = event.row.num_retries + 1
 		const now = new Date()
 		const nextAttempt = new Date()
 		nextAttempt.setTime(
 			nextAttempt.getTime() + (event.target?.initialRepeatIntervalMs ?? DEFAULT_REPEAT_INTERVAL_MS) * Math.pow(2, event.row.num_retries),
 		)
+		const terminal = numRetries >= (event.target?.maxAttempts ?? DEFAULT_MAX_ATTEMPTS)
 		await UpdateBuilder.create()
 			.table('actions_event')
 			.values<EventRow>({
 				last_state_change: now,
 				num_retries: numRetries,
 				log: JSON.stringify([...event.row.log, event.result]) as any,
-				...(numRetries < (event.target?.maxAttempts ?? DEFAULT_MAX_ATTEMPTS)
+				...(terminal
 					? {
-						state: 'retrying',
-						visible_at: nextAttempt,
+						state: 'failed',
 					}
 					: {
-						state: 'failed',
+						state: 'retrying',
+						visible_at: nextAttempt,
 					}),
 			})
 			.where({ id: event.row.id })
 			.execute(db)
+		return terminal
 	}
 
-	private async markFailedOnUnknownTarget(db: Client, targetName: string, primaryEventId: string) {
-		await UpdateBuilder.create()
+	private async markFailedOnUnknownTarget(db: Client, targetName: string, primaryEventId: string): Promise<boolean> {
+		const affectedRows = await UpdateBuilder.create()
 			.table('actions_event')
 			.where(it =>
 				it.or(it =>
 					it
 						.and(it =>
 							it
-								.in('state', ['retrying', 'created', 'processing'])
-								.compare('visible_at', Operator.lte, 'now')
+								.in('state', ['processing'])
 								.compare('target', Operator.eq, targetName)
 						)
 						.and(it =>
@@ -177,9 +209,19 @@ export class EventsRepository {
 				resolved_at: new Date(),
 			})
 			.execute(db)
+		return affectedRows > 0
 	}
 }
 
 export type FetchBatchResult =
-	| { ok: true; events: EventRow[]; target: Actions.AnyTarget }
-	| { ok: false; backoffMs?: number }
+	| { ok: true; events: EventRow[]; target: Actions.AnyTarget; unknownTargetFailed: number }
+	| { ok: false; backoffMs?: number; unknownTargetFailed: number }
+
+export type PersistProcessedResult = {
+	/** Events delivered successfully (terminal). */
+	succeeded: number
+	/** Events whose delivery attempt failed and that will be retried later. */
+	retried: number
+	/** Events whose delivery attempt failed and that reached a terminal `failed` state. */
+	failed: number
+}

package/src/dispatch/ProjectDispatcher.ts

@@ -4,17 +4,26 @@ import { ContentSchemaResolver } from '@contember/engine-http'
 import { Runnable, RunnableArgs, Running } from '@contember/engine-common'
 import { Listener } from '@contember/database'
 import { NOTIFY_CHANNEL_NAME } from '../utils/notifyChannel.js'
+import { ActionsMetrics, ProjectActionsMetrics } from '../ActionsMetrics.js'
+
+/**
+ * Upper bound on how long the loop parks while idle. Even when the queue is empty (and we'd
+ * otherwise wait indefinitely on a `pg_notify`), we re-check at least this often, so a lost
+ * notification self-heals and the heartbeat keeps refreshing while the worker is genuinely idle.
+ */
+const MAX_IDLE_SLEEP_MS = 30_000
 
 export class ProjectDispatcherFactory {
 	constructor(
 		private readonly dispatcher: EventDispatcher,
+		private readonly metrics: ActionsMetrics,
 	) {
 	}
 
 	public create(
 		{ db, contentSchemaResolver, projectSlug }: { db: DatabaseContext; contentSchemaResolver: ContentSchemaResolver; projectSlug: string },
 	): ProjectDispatcher {
-		return new ProjectDispatcher(this.dispatcher, db, contentSchemaResolver, projectSlug)
+		return new ProjectDispatcher(this.dispatcher, db, contentSchemaResolver, projectSlug, this.metrics.forProject(projectSlug))
 	}
 }
 
@@ -24,6 +33,7 @@ export class ProjectDispatcher implements Runnable {
 		private readonly db: DatabaseContext,
 		private readonly contentSchemaResolver: ContentSchemaResolver,
 		private readonly projectSlug: string,
+		private readonly metrics: ProjectActionsMetrics,
 	) {
 	}
 
@@ -58,6 +68,7 @@ export class ProjectDispatcher implements Runnable {
 						end: async () => {
 							aborted = true
 							await listener.end()
+							this.metrics.dispose()
 							logger.info('Worker terminated', {
 								succeed: succeedTotal,
 								failed: failedTotal,
@@ -67,28 +78,32 @@ export class ProjectDispatcher implements Runnable {
 
 					try {
 						while (!aborted) {
-							const { succeed, failed, backoffMs } = await this.dispatcher.processBatch({
+							this.metrics.heartbeat()
+							const { succeeded, retried, failedAfterAttempt, failedUnknownTarget, backoffMs } = await this.dispatcher.processBatch({
 								db,
 								contentSchemaResolver: this.contentSchemaResolver,
 								logger,
+								project: { slug: this.projectSlug },
 							})
-							succeedTotal += succeed
-							failedTotal += failed
+							const terminalFailed = failedAfterAttempt + failedUnknownTarget
+							this.metrics.succeeded(succeeded)
+							this.metrics.deliveryAttemptFailed(retried + failedAfterAttempt)
+							this.metrics.failed(terminalFailed)
+							succeedTotal += succeeded
+							failedTotal += terminalFailed
 
 							// listener error occurred during batch processing, rethrow
 							if (pendingError) {
 								throw pendingError
 							}
 
-							// queue is empty, wait
+							// queue is empty (or next retry is in the future), wait — but never longer than
+							// MAX_IDLE_SLEEP_MS, so a lost notification self-heals and the heartbeat stays fresh.
 							if (backoffMs !== 0) {
+								const waitMs = Math.min(backoffMs ?? MAX_IDLE_SLEEP_MS, MAX_IDLE_SLEEP_MS)
 								await new Promise<void>((resolve, reject) => {
-									let cleanup = () => {
-									}
-									if (backoffMs !== undefined) {
-										const timeoutHandle = setTimeout(resolve, backoffMs)
-										cleanup = () => clearTimeout(timeoutHandle)
-									}
+									const timeoutHandle = setTimeout(resolve, waitMs)
+									const cleanup = () => clearTimeout(timeoutHandle)
 									resolvePending = () => {
 										resolve()
 										cleanup()
@@ -103,6 +118,7 @@ export class ProjectDispatcher implements Runnable {
 						onClose?.()
 					} catch (e) {
 						await listener.end()
+						this.metrics.crashed()
 						onError(e)
 					}
 				})

package/src/dispatch/WebhookTargetHandler.ts

@@ -3,6 +3,7 @@ import { EventRow, HandledEvent, InvokeHandler, InvokeHandlerArgs } from './type
 import { VariablesMap } from '../model/VariablesManager.js'
 import * as Typesafe from '@contember/typesafe'
 import { FetcherResponse, WebhookFetcher } from './WebhookFetcher.js'
+import { Logger } from '@contember/logger'
 
 const DEFAULT_TIMEOUT_MS = 30_000 // 30 seconds
 
@@ -40,6 +41,8 @@ export class WebhookTargetHandler implements InvokeHandler<Actions.WebhookTarget
 			eventResponseFactory = this.createResponseFactory({
 				response: response,
 				events,
+				target,
+				logger,
 			})
 		} catch (e) {
 			logger.warn(e)
@@ -58,11 +61,20 @@ export class WebhookTargetHandler implements InvokeHandler<Actions.WebhookTarget
 		return events.map(it => ({ target, row: it, result: { durationMs, ...eventResponseFactory(it) } }))
 	}
 
-	private createResponseFactory({ response, events }: {
+	private createResponseFactory({ response, events, target, logger }: {
 		response: FetcherResponse
 		events: EventRow[]
+		target: Actions.WebhookTarget
+		logger: Logger
 	}): EventResponseFactory {
 		if (!response.ok) {
+			// The target is identified by name only: its URL and headers may carry credentials.
+			logger.warn('Webhook target responded with an error status', {
+				target: target.name,
+				status: response.status,
+				statusText: response.statusText,
+				events: events.map(it => it.id),
+			})
 			return () => ({
 				ok: false,
 				code: response.status,
@@ -95,6 +107,13 @@ export class WebhookTargetHandler implements InvokeHandler<Actions.WebhookTarget
 			const missingEvents = Object.keys(failedEvents).filter(it => !eventsInBatch.has(it))
 
 			if (missingEvents.length) {
+				// Only the count of unknown ids: they are strings the target chose, not ours.
+				logger.warn('Webhook target reported failures for event ids outside the batch', {
+					target: target.name,
+					status: response.status,
+					unknownEventIds: missingEvents.length,
+					events: events.map(it => it.id),
+				})
 				return () => ({
 					ok: false,
 					code: response.status,
@@ -103,6 +122,16 @@ export class WebhookTargetHandler implements InvokeHandler<Actions.WebhookTarget
 				})
 			}
 
+			const failedIds = events.filter(it => failedEvents[it.id]).map(it => it.id)
+			if (failedIds.length) {
+				// Per-event errors come from the target's payload and stay in the event log, not here.
+				logger.warn('Webhook target reported failed events', {
+					target: target.name,
+					status: response.status,
+					events: failedIds,
+				})
+			}
+
 			return it => {
 				if (!failedEvents[it.id]) {
 					return { ok: true, code: response.status }
@@ -114,6 +143,12 @@ export class WebhookTargetHandler implements InvokeHandler<Actions.WebhookTarget
 				}
 			}
 		} catch (e: any) {
+			// The parser message quotes the offending payload, so it stays in the event log only.
+			logger.warn('Webhook target returned a malformed response', {
+				target: target.name,
+				status: response.status,
+				events: events.map(it => it.id),
+			})
 			return () => ({
 				ok: false,
 				code: response.status,