@znight/messaging 0.0.0-stage → 0.1.1

package/src/plugin.ts

@@ -0,0 +1,236 @@
+import type { Plugin, PluginContext, PluginOptions } from '@znight/core'
+import type { MessageCodec } from './codec.ts'
+import type { MessagingDriver } from './driver.ts'
+import type { RetryOptions } from './retry.ts'
+
+import { resolveOptions } from '@znight/core'
+
+import { DriverMessageBroker, type MessageBroker } from './broker.ts'
+import { jsonCodec } from './codec.ts'
+import { type ConsumerDefaults, defaultRetryPolicy, prepareConsumers } from './consumers.ts'
+import { consumerError } from './errors.ts'
+import { isPositiveInteger } from './internal/async.ts'
+import { ConsumerRuntime, type ConsumerState } from './runtime.ts'
+import { MessagingTokens } from './tokens.ts'
+
+/** Defaults applied to every consumer (each `@Consumer` / `@Retry` may override them). */
+export interface MessagingDefaults {
+  /** Retry policy of handlers without `@Retry`. Default: 3 attempts, exponential from 1 s up to 30 s. `false`: no retry. */
+  readonly retry?: RetryOptions | false
+  /** Concurrency of consumers without `concurrency` option. Default: 1. */
+  readonly concurrency?: number
+  /**
+   * Dead-letter destination of a queue, or `false` to disable dead-lettering (messages are then
+   * rejected natively or discarded). Default: the driver naming (`orders.dlq`, SQS `orders-dlq`).
+   */
+  readonly deadLetter?: ((queue: string) => string) | false
+  /**
+   * Pause, in milliseconds, before releasing a message whose settlement failed (dead-letter queue
+   * unavailable...), to avoid hot redelivery loops. Default: 1000.
+   */
+  readonly requeueDelay?: number
+}
+
+/** Options of {@link MessagingPlugin.configure}. */
+export interface MessagingPluginOptions {
+  /** The broker driver: `rabbitmq()`, `sqs()`, `kafka()` or `memory()` (from the matching subpath). */
+  readonly driver: MessagingDriver
+  readonly defaults?: MessagingDefaults
+  /** Payload encoding. Default: {@link jsonCodec}. */
+  readonly codec?: MessageCodec
+  /** Subscribe the consumers on start. `false` for publish-only processes. Default: `true`. */
+  readonly consume?: boolean
+}
+
+/** Runtime status of a consumer. */
+export interface ConsumerStatus {
+  /** Display name of the consumer class. */
+  readonly consumer: string
+  readonly queue: string
+  readonly state: ConsumerState
+  readonly concurrency: number
+  readonly inFlight: number
+  readonly deadLetterQueue: string | undefined
+}
+
+/** The plugin returned by {@link MessagingPlugin.configure}. */
+export interface MessagingPluginInstance extends Plugin {
+  readonly name: 'messaging'
+  /** The broker (also injectable as `MessageBroker`). */
+  readonly broker: MessageBroker
+  /** The driver, once the plugin is bootstrapped. */
+  readonly driver: MessagingDriver | undefined
+  /** Status of every consumer, once the plugin is bootstrapped. */
+  consumers(): ConsumerStatus[]
+}
+
+const DEFAULT_REQUEUE_DELAY_MS = 1000
+/** Period of the keep-alive timer (it never does anything: it only keeps the event loop alive). */
+const KEEP_ALIVE_INTERVAL_MS = 2 ** 30
+
+/**
+ * Message consumers and publishing.
+ *
+ * ```ts
+ * import { MessagingPlugin } from '@znight/messaging'
+ * import { rabbitmq } from '@znight/messaging/rabbitmq'
+ *
+ * const app = await Application.create({
+ *   metadata: __ZNIGHT_METADATA__,
+ *   plugins: [MessagingPlugin.configure({ driver: rabbitmq({ url: 'amqp://localhost' }) })],
+ * })
+ * ```
+ *
+ * Options may be computed from the container (evaluated in `onBootstrap`):
+ * `MessagingPlugin.configure(c => ({ driver: rabbitmq({ url: c.get(AmqpConfig).url }) }))`.
+ */
+export const MessagingPlugin: {
+  configure(options: PluginOptions<MessagingPluginOptions>): MessagingPluginInstance
+} = Object.freeze({
+  configure(options: PluginOptions<MessagingPluginOptions>): MessagingPluginInstance {
+    return createPlugin(options)
+  },
+})
+
+function createPlugin(rawOptions: PluginOptions<MessagingPluginOptions>): MessagingPluginInstance {
+  const broker = new DriverMessageBroker()
+  let driver: MessagingDriver | undefined
+  let connected = false
+  let consume = true
+  let runtimes: ConsumerRuntime[] = []
+  /**
+   * Keeps the process alive while consumers run: they are its ingress, and some drivers hold no
+   * handle of their own (the memory driver only uses microtasks and short timers), so a process
+   * without other handle would exit right after `app.start()`, without graceful shutdown.
+   */
+  let keepAlive: ReturnType<typeof setInterval> | undefined
+  const stopKeepAlive = (): void => {
+    if (keepAlive !== undefined) clearInterval(keepAlive)
+    keepAlive = undefined
+  }
+
+  return {
+    name: 'messaging',
+    broker,
+    get driver(): MessagingDriver | undefined {
+      return driver
+    },
+    consumers(): ConsumerStatus[] {
+      return runtimes.map(runtime => ({
+        consumer: runtime.consumer.name,
+        queue: runtime.consumer.queue,
+        state: runtime.state,
+        concurrency: runtime.consumer.concurrency,
+        inFlight: runtime.inFlight,
+        deadLetterQueue: runtime.consumer.deadLetterQueue,
+      }))
+    },
+
+    register(ctx: PluginContext): void {
+      ctx.container.registerExternal({
+        token: MessagingTokens.InboundMessage,
+        scope: 'request',
+        providedBy: 'MessagingPlugin',
+      })
+      ctx.container.registerValue(MessagingTokens.MessageBroker, broker)
+    },
+
+    async onBootstrap(ctx: PluginContext): Promise<void> {
+      const options = validateOptions(resolveOptions(rawOptions, ctx.container))
+      driver = options.driver
+      consume = options.consume ?? true
+      const defaults = options.defaults ?? {}
+      const consumerDefaults: ConsumerDefaults = {
+        retry: defaultRetryPolicy(defaults.retry),
+        concurrency: defaults.concurrency ?? 1,
+        deadLetter: defaults.deadLetter,
+      }
+      const prepared = prepareConsumers(ctx.metadata.extensions?.messaging, ctx.container, driver, consumerDefaults)
+      const codec = options.codec ?? jsonCodec
+      connected = true // close() must run even if connect() failed half-way
+      await driver.connect({ logger: ctx.logger })
+      broker.bind(driver, codec, ctx.hooks)
+      const services = {
+        container: ctx.container,
+        hooks: ctx.hooks,
+        logger: ctx.logger,
+        codec,
+        driver,
+        requeueDelayMs: defaults.requeueDelay ?? DEFAULT_REQUEUE_DELAY_MS,
+      }
+      runtimes = prepared.map(consumer => new ConsumerRuntime(consumer, services))
+    },
+
+    async onStart(ctx: PluginContext): Promise<void> {
+      if (!consume || runtimes.length === 0) return
+      const results = await Promise.allSettled(runtimes.map(runtime => runtime.start()))
+      const failure = results.find(result => result.status === 'rejected')
+      if (failure !== undefined) {
+        // the application only stops plugins whose onStart succeeded: undo the subscriptions made
+        await Promise.all(runtimes.map(runtime => runtime.stop()))
+        throw failure.reason
+      }
+      keepAlive ??= setInterval(() => undefined, KEEP_ALIVE_INTERVAL_MS)
+      ctx.logger.info(`Messaging: ${runtimes.length} consumer(s) subscribed with the ${driver?.name} driver`, {
+        queues: runtimes.map(runtime => runtime.consumer.queue),
+      })
+    },
+
+    async onStop(): Promise<void> {
+      try {
+        await Promise.all(runtimes.map(runtime => runtime.stop()))
+      } finally {
+        stopKeepAlive()
+      }
+    },
+
+    async onDispose(): Promise<void> {
+      stopKeepAlive()
+      broker.unbind()
+      if (driver !== undefined && connected) {
+        connected = false
+        await driver.close()
+      }
+    },
+  }
+}
+
+function validateOptions(options: MessagingPluginOptions): MessagingPluginOptions {
+  if (options === null || typeof options !== 'object') {
+    throw consumerError('MessagingPlugin.configure(): options must be an object')
+  }
+  const driver = options.driver as Partial<MessagingDriver> | undefined
+  if (
+    driver === undefined ||
+    driver === null ||
+    typeof driver.connect !== 'function' ||
+    typeof driver.subscribe !== 'function' ||
+    typeof driver.publish !== 'function' ||
+    typeof driver.close !== 'function' ||
+    typeof driver.deadLetterName !== 'function' ||
+    typeof driver.capabilities !== 'object'
+  ) {
+    throw consumerError('MessagingPlugin.configure(): "driver" must be a messaging driver', [
+      "Use rabbitmq() from '@znight/messaging/rabbitmq', sqs(), kafka() or memory() from their subpaths",
+    ])
+  }
+  const defaults = options.defaults
+  if (defaults?.concurrency !== undefined && !isPositiveInteger(defaults.concurrency)) {
+    throw consumerError(
+      `MessagingPlugin defaults.concurrency must be a positive integer, got ${String(defaults.concurrency)}`,
+    )
+  }
+  if (defaults?.requeueDelay !== undefined && !(Number.isFinite(defaults.requeueDelay) && defaults.requeueDelay >= 0)) {
+    throw consumerError(
+      `MessagingPlugin defaults.requeueDelay must be a number of milliseconds >= 0, got ${String(defaults.requeueDelay)}`,
+    )
+  }
+  if (
+    defaults?.deadLetter !== undefined &&
+    defaults.deadLetter !== false &&
+    typeof defaults.deadLetter !== 'function'
+  ) {
+    throw consumerError('MessagingPlugin defaults.deadLetter must be a function (queue => name) or false')
+  }
+  return options
+}

package/src/retry.ts

@@ -0,0 +1,129 @@
+import type { DriverCapabilities } from './driver.ts'
+
+import { consumerError } from './errors.ts'
+
+/** Backoff strategy between attempts. */
+export type BackoffStrategy = 'fixed' | 'exponential'
+
+/** Options of `@Retry()` and of the plugin's `defaults.retry`. */
+export interface RetryOptions {
+  /** Total number of attempts, including the first delivery (`1`: no retry). */
+  readonly attempts: number
+  /** `fixed`: always `delay`; `exponential`: `delay * 2^(attempt - 1)`, capped by `maxDelay`. Default: `exponential`. */
+  readonly backoff?: BackoffStrategy
+  /** Base delay between attempts, in milliseconds. Default: 1000. */
+  readonly delay?: number
+  /** Upper bound of exponential delays, in milliseconds. */
+  readonly maxDelay?: number
+}
+
+/** A normalized retry policy, as stored in the metadata (`MessageHandlerDefinition.retry`). */
+export interface RetryPolicy {
+  readonly attempts: number
+  readonly backoff: BackoffStrategy
+  readonly delay: number
+  readonly maxDelay?: number
+}
+
+/** Default base delay of a retry policy, in milliseconds. */
+export const DEFAULT_RETRY_DELAY_MS = 1000
+
+/** Default backoff strategy. */
+export const DEFAULT_BACKOFF: BackoffStrategy = 'exponential'
+
+/** Policy applied to handlers without `@Retry` when the plugin sets no `defaults.retry`. */
+export const DEFAULT_RETRY_POLICY: RetryPolicy = Object.freeze({
+  attempts: 3,
+  backoff: 'exponential',
+  delay: DEFAULT_RETRY_DELAY_MS,
+  maxDelay: 30_000,
+})
+
+/** The policy of `retry: false` / `{ attempts: 1 }`. */
+export const NO_RETRY_POLICY: RetryPolicy = Object.freeze({ attempts: 1, backoff: 'fixed', delay: 0 })
+
+const MAX_ATTEMPTS = 10_000
+
+/**
+ * Validates retry options and fills defaults.
+ *
+ * @throws ZnightError ZNIGHT011 for invalid values (`where` names the handler in the message).
+ */
+export function normalizeRetry(options: RetryOptions | RetryPolicy, where = '@Retry()'): RetryPolicy {
+  if (options === null || typeof options !== 'object') {
+    throw consumerError(`${where}: retry options must be an object, got ${String(options)}`)
+  }
+  const { attempts, backoff = DEFAULT_BACKOFF, delay = DEFAULT_RETRY_DELAY_MS, maxDelay } = options
+  if (!Number.isInteger(attempts) || attempts < 1 || attempts > MAX_ATTEMPTS) {
+    throw consumerError(
+      `${where}: "attempts" must be an integer between 1 and ${MAX_ATTEMPTS}, got ${String(attempts)}`,
+      ['attempts counts the first delivery: { attempts: 3 } means 1 delivery + 2 retries'],
+    )
+  }
+  if (backoff !== 'fixed' && backoff !== 'exponential') {
+    throw consumerError(`${where}: "backoff" must be 'fixed' or 'exponential', got ${String(backoff)}`)
+  }
+  if (!isDuration(delay)) {
+    throw consumerError(`${where}: "delay" must be a finite number of milliseconds >= 0, got ${String(delay)}`)
+  }
+  if (maxDelay !== undefined && (!isDuration(maxDelay) || maxDelay < delay)) {
+    throw consumerError(
+      `${where}: "maxDelay" must be a finite number of milliseconds >= delay (${delay}), got ${String(maxDelay)}`,
+    )
+  }
+  return maxDelay === undefined ? { attempts, backoff, delay } : { attempts, backoff, delay, maxDelay }
+}
+
+function isDuration(value: unknown): value is number {
+  return typeof value === 'number' && Number.isFinite(value) && value >= 0
+}
+
+/**
+ * Delay before the next attempt once `attempt` (1-based) failed: `delay` for `fixed`,
+ * `delay * 2^(attempt - 1)` capped by `maxDelay` for `exponential`.
+ */
+export function computeRetryDelay(policy: RetryPolicy, attempt: number): number {
+  if (policy.backoff === 'fixed') return policy.delay
+  const exponent = Math.max(0, attempt - 1)
+  const raw = policy.delay * 2 ** exponent
+  return Math.min(raw, policy.maxDelay ?? Number.MAX_SAFE_INTEGER)
+}
+
+/** Every delay the policy may request (one per retry), in attempt order. */
+export function retryDelays(policy: RetryPolicy): number[] {
+  const delays: number[] = []
+  for (let attempt = 1; attempt < policy.attempts; attempt++) delays.push(computeRetryDelay(policy, attempt))
+  return delays
+}
+
+/** Whether a delay is waited in-process (`true`) or delegated to the broker (`false`). */
+export function isInProcessDelay(delayMs: number, capabilities: DriverCapabilities): boolean {
+  return delayMs <= capabilities.maxInProcessDelayMs
+}
+
+/**
+ * Checks that the driver can honour every delay of `policy`.
+ *
+ * @throws ZnightError ZNIGHT011 naming `where` and the offending delay.
+ */
+export function assertRetrySupported(
+  policy: RetryPolicy,
+  capabilities: DriverCapabilities,
+  driver: string,
+  where: string,
+): void {
+  for (const delay of new Set(retryDelays(policy))) {
+    if (isInProcessDelay(delay, capabilities)) continue
+    if (capabilities.nativeDelay && delay <= capabilities.maxNativeDelayMs) continue
+    const limits = capabilities.nativeDelay
+      ? `in-process delays up to ${capabilities.maxInProcessDelayMs} ms and native delays up to ${capabilities.maxNativeDelayMs} ms`
+      : `in-process delays up to ${capabilities.maxInProcessDelayMs} ms and no native delay`
+    throw consumerError(
+      `${where}: the ${driver} driver cannot wait ${delay} ms between attempts (it supports ${limits})`,
+      [
+        'Lower "delay"/"maxDelay" of the retry policy',
+        'Or raise the driver budget (e.g. maxInProcessDelayMs) if holding messages that long is acceptable',
+      ],
+    )
+  }
+}