@motiadev/adapter-bullmq-events 0.13.0-beta.162-717198

package/src/config-builder.ts

@@ -0,0 +1,41 @@
+import {
+  DEFAULT_ATTEMPTS,
+  DEFAULT_BACKOFF_DELAY,
+  DEFAULT_CONCURRENCY,
+  DEFAULT_DLQ_SUFFIX,
+  DEFAULT_DLQ_TTL,
+  DEFAULT_PREFIX,
+  DEFAULT_REMOVE_ON_COMPLETE_COUNT,
+  DEFAULT_REMOVE_ON_FAIL_COUNT,
+} from './constants'
+import type { BullMQEventAdapterConfig } from './types'
+
+export type MergedConfig = Required<Pick<BullMQEventAdapterConfig, 'defaultJobOptions' | 'prefix' | 'concurrency'>> & {
+  dlq: Required<NonNullable<BullMQEventAdapterConfig['dlq']>>
+}
+
+export function buildConfig(config: BullMQEventAdapterConfig): MergedConfig {
+  return {
+    concurrency: config.concurrency ?? DEFAULT_CONCURRENCY,
+    defaultJobOptions: {
+      attempts: DEFAULT_ATTEMPTS,
+      backoff: {
+        type: 'fixed',
+        delay: DEFAULT_BACKOFF_DELAY,
+      },
+      removeOnComplete: {
+        count: DEFAULT_REMOVE_ON_COMPLETE_COUNT,
+      },
+      removeOnFail: {
+        count: DEFAULT_REMOVE_ON_FAIL_COUNT,
+      },
+      ...config.defaultJobOptions,
+    },
+    prefix: config.prefix ?? DEFAULT_PREFIX,
+    dlq: {
+      enabled: config.dlq?.enabled ?? true,
+      ttl: config.dlq?.ttl ?? DEFAULT_DLQ_TTL,
+      suffix: config.dlq?.suffix ?? DEFAULT_DLQ_SUFFIX,
+    },
+  }
+}

package/src/connection-manager.ts

@@ -0,0 +1,40 @@
+import IORedis, { type Redis } from 'ioredis'
+import { ConnectionError } from './errors'
+import type { BullMQConnectionConfig } from './types'
+
+export class ConnectionManager {
+  readonly connection: Redis
+  readonly ownsConnection: boolean
+
+  constructor(config: BullMQConnectionConfig) {
+    if (config instanceof IORedis) {
+      this.connection = config
+      this.ownsConnection = false
+    } else {
+      this.connection = new IORedis({
+        maxRetriesPerRequest: null,
+        ...config,
+      })
+      this.ownsConnection = true
+    }
+
+    this.setupEventHandlers()
+  }
+
+  private setupEventHandlers(): void {
+    this.connection.on('error', (err: Error) => {
+      const error = new ConnectionError(err.message, err)
+      console.error('[BullMQ] Connection error:', error)
+    })
+
+    this.connection.on('close', () => {
+      console.warn('[BullMQ] Connection closed')
+    })
+  }
+
+  async close(): Promise<void> {
+    if (this.ownsConnection && this.connection.status !== 'end') {
+      await this.connection.quit()
+    }
+  }
+}

package/src/constants.ts

@@ -0,0 +1,14 @@
+export const DEFAULT_CONCURRENCY = 5
+export const DEFAULT_ATTEMPTS = 3
+export const DEFAULT_BACKOFF_DELAY = 2000
+export const DEFAULT_REMOVE_ON_COMPLETE_COUNT = 1000
+export const DEFAULT_REMOVE_ON_FAIL_COUNT = 5000
+export const DEFAULT_PREFIX = 'motia'
+export const FIFO_CONCURRENCY = 1
+export const MILLISECONDS_PER_SECOND = 1000
+export const SECONDS_PER_DAY = 86400
+export const DEFAULT_DLQ_TTL = 30 * SECONDS_PER_DAY
+export const DEFAULT_DLQ_SUFFIX = '.dlq'
+export const DLQ_JOB_PREFIX = 'dlq-'
+
+export const LOG_PREFIX = '[BullMQ]'

package/src/dlq-manager.ts

@@ -0,0 +1,151 @@
+import type { Event } from '@motiadev/core'
+import { Queue } from 'bullmq'
+import type { Redis } from 'ioredis'
+import type { MergedConfig } from './config-builder'
+import { DLQ_JOB_PREFIX, LOG_PREFIX, MILLISECONDS_PER_SECOND } from './constants'
+import { QueueCreationError } from './errors'
+
+type DLQJobData<TData> = {
+  originalEvent: Event<TData>
+  failureReason: string
+  failureTimestamp: number
+  attemptsMade: number
+  originalJobId?: string
+}
+
+export class DLQManager {
+  private readonly dlqQueues: Map<string, Queue> = new Map()
+  private readonly connection: Redis
+  private readonly config: MergedConfig
+
+  constructor(connection: Redis, config: MergedConfig) {
+    this.connection = connection
+    this.config = config
+  }
+
+  getDLQQueueName(topic: string, stepName: string): string {
+    const baseQueueName = `${topic}.${stepName}`
+    return `${baseQueueName}${this.config.dlq.suffix}`
+  }
+
+  private getOrCreateDLQQueue(queueName: string): Queue {
+    if (!this.dlqQueues.has(queueName)) {
+      const ttlMs = this.config.dlq.ttl * MILLISECONDS_PER_SECOND
+      const queue = new Queue(queueName, {
+        connection: this.connection,
+        prefix: this.config.prefix,
+        defaultJobOptions: {
+          removeOnComplete: {
+            age: ttlMs,
+          },
+          removeOnFail: {
+            age: ttlMs,
+          },
+        },
+      })
+
+      queue.on('error', (err: Error) => {
+        console.error(`${LOG_PREFIX} DLQ error for ${queueName}:`, err)
+      })
+
+      this.dlqQueues.set(queueName, queue)
+    }
+
+    const queue = this.dlqQueues.get(queueName)
+    if (!queue) {
+      throw new QueueCreationError(queueName)
+    }
+    return queue
+  }
+
+  async moveToDLQ<TData>(
+    topic: string,
+    stepName: string,
+    event: Event<TData>,
+    error: Error,
+    attemptsMade: number,
+    originalJobId?: string,
+  ): Promise<void> {
+    if (!this.config.dlq!.enabled) {
+      console.warn(`${LOG_PREFIX} DLQ is disabled, skipping move to DLQ for topic ${topic}, step ${stepName}`)
+      return
+    }
+
+    try {
+      const dlqQueueName = this.getDLQQueueName(topic, stepName)
+      const dlqQueue = this.getOrCreateDLQQueue(dlqQueueName)
+
+      const sanitizedEvent = {
+        topic: event.topic,
+        data: event.data,
+        traceId: event.traceId || 'unknown',
+        ...(event.flows && { flows: event.flows }),
+        ...(event.messageGroupId && { messageGroupId: event.messageGroupId }),
+      } as Event<TData>
+
+      const dlqJobData: DLQJobData<TData> = {
+        originalEvent: sanitizedEvent,
+        failureReason: error.message || 'Unknown error',
+        failureTimestamp: Date.now(),
+        attemptsMade,
+        ...(originalJobId && { originalJobId }),
+      }
+
+      const jobOptions = originalJobId ? { jobId: `${DLQ_JOB_PREFIX}${originalJobId}` } : {}
+
+      await dlqQueue.add(`${topic}.dlq`, dlqJobData, jobOptions)
+
+      console.warn(`${LOG_PREFIX} Moved failed job to DLQ: ${dlqQueueName}`, {
+        topic,
+        stepName,
+        attemptsMade,
+        error: error.message,
+      })
+    } catch (err) {
+      const dlqError = err instanceof Error ? err : new Error(String(err))
+      console.error(`${LOG_PREFIX} Failed to move job to DLQ for topic ${topic}, step ${stepName}:`, dlqError)
+    }
+  }
+
+  async closeDLQQueue(queueName: string): Promise<void> {
+    const queue = this.dlqQueues.get(queueName)
+    if (queue) {
+      await queue.close()
+      this.dlqQueues.delete(queueName)
+    }
+  }
+
+  async closeAll(): Promise<void> {
+    const promises = Array.from(this.dlqQueues.values()).map((queue) =>
+      queue.close().catch((err) => {
+        console.error(`${LOG_PREFIX} Error closing DLQ queue:`, err)
+      }),
+    )
+    await Promise.allSettled(promises)
+    this.dlqQueues.clear()
+  }
+
+  getDLQQueue(queueName: string): Queue | undefined {
+    return this.dlqQueues.get(queueName)
+  }
+
+  getOrCreateDLQ(queueName: string): Queue {
+    return this.getOrCreateDLQQueue(queueName)
+  }
+
+  listDLQQueueNames(): string[] {
+    return Array.from(this.dlqQueues.keys())
+  }
+
+  getDLQSuffix(): string {
+    return this.config.dlq.suffix
+  }
+
+  getPrefix(): string {
+    return this.config.prefix
+  }
+
+  getConnection(): Redis {
+    return this.connection
+  }
+}

package/src/errors.ts

@@ -0,0 +1,33 @@
+export class BullMQAdapterError extends Error {
+  constructor(
+    message: string,
+    public readonly cause?: Error,
+  ) {
+    super(message)
+    this.name = 'BullMQAdapterError'
+    if (cause) {
+      this.stack = `${this.stack}\nCaused by: ${cause.stack}`
+    }
+  }
+}
+
+export class QueueCreationError extends BullMQAdapterError {
+  constructor(queueName: string, cause?: Error) {
+    super(`Failed to create queue: ${queueName}`, cause)
+    this.name = 'QueueCreationError'
+  }
+}
+
+export class WorkerCreationError extends BullMQAdapterError {
+  constructor(topic: string, stepName: string, cause?: Error) {
+    super(`Failed to create worker for topic ${topic}, step ${stepName}`, cause)
+    this.name = 'WorkerCreationError'
+  }
+}
+
+export class ConnectionError extends BullMQAdapterError {
+  constructor(message: string, cause?: Error) {
+    super(`Connection error: ${message}`, cause)
+    this.name = 'ConnectionError'
+  }
+}

package/src/index.ts

@@ -0,0 +1,6 @@
+export { BullMQEventAdapter } from './bullmq-event-adapter'
+export { DLQManager } from './dlq-manager'
+export { QueueManager } from './queue-manager'
+export type { BullMQConnectionConfig, BullMQEventAdapterConfig } from './types'
+export type { SubscriberInfo } from './worker-manager'
+export { WorkerManager } from './worker-manager'

package/src/queue-manager.ts

@@ -0,0 +1,107 @@
+import type { Event } from '@motiadev/core'
+import { Queue } from 'bullmq'
+import type { Redis } from 'ioredis'
+import type { MergedConfig } from './config-builder'
+import { MILLISECONDS_PER_SECOND } from './constants'
+import { QueueCreationError } from './errors'
+import type { SubscriberInfo } from './worker-manager'
+
+export class QueueManager {
+  private readonly queues: Map<string, Queue> = new Map()
+  private readonly connection: Redis
+  private readonly config: MergedConfig
+
+  constructor(connection: Redis, config: MergedConfig) {
+    this.connection = connection
+    this.config = config
+  }
+
+  getQueue(queueName: string): Queue {
+    if (!this.queues.has(queueName)) {
+      const queue = new Queue(queueName, {
+        connection: this.connection,
+        prefix: this.config.prefix,
+        defaultJobOptions: this.config.defaultJobOptions,
+      })
+
+      queue.on('error', (err: Error) => {
+        console.error(`[BullMQ] Queue error for ${queueName}:`, err)
+      })
+
+      this.queues.set(queueName, queue)
+    }
+
+    const queue = this.queues.get(queueName)
+    if (!queue) {
+      throw new QueueCreationError(queueName)
+    }
+    return queue
+  }
+
+  getQueueName(topic: string, stepName: string): string {
+    return `${topic}.${stepName}`
+  }
+
+  async enqueueToAll<TData>(event: Event<TData>, subscribers: SubscriberInfo[]): Promise<void> {
+    const promises = subscribers.map((subscriber) => {
+      const queueName = this.getQueueName(subscriber.topic, subscriber.stepName)
+      const queue = this.getQueue(queueName)
+      const jobId = event.messageGroupId ? `${queueName}.${event.messageGroupId}` : undefined
+
+      const jobData = {
+        topic: event.topic,
+        data: event.data,
+        traceId: event.traceId,
+        flows: event.flows,
+        messageGroupId: event.messageGroupId,
+      }
+
+      const maxRetries = subscriber.queueConfig?.maxRetries
+      const attempts = maxRetries != null ? maxRetries + 1 : this.config.defaultJobOptions.attempts
+      const delay = subscriber.queueConfig?.delaySeconds
+        ? subscriber.queueConfig.delaySeconds * MILLISECONDS_PER_SECOND
+        : undefined
+
+      const jobOptions = {
+        jobId,
+        attempts,
+        backoff: this.config.defaultJobOptions.backoff,
+        delay,
+      }
+
+      return queue.add(event.topic, jobData, jobOptions).then(() => undefined)
+    })
+
+    await Promise.all(promises)
+  }
+
+  async closeQueue(queueName: string): Promise<void> {
+    const queue = this.queues.get(queueName)
+    if (queue) {
+      await queue.close()
+      this.queues.delete(queueName)
+    }
+  }
+
+  async closeAll(): Promise<void> {
+    const promises = Array.from(this.queues.values()).map((queue) =>
+      queue.close().catch((err) => {
+        console.error(`[BullMQ] Error closing queue:`, err)
+      }),
+    )
+    await Promise.allSettled(promises)
+    this.queues.clear()
+  }
+
+  listQueueNames(): string[] {
+    return Array.from(this.queues.keys())
+  }
+
+  getPrefix(): string {
+    return this.config.prefix
+  }
+
+  getConnection(): Redis {
+    return this.connection
+  }
+}

package/src/types.ts

@@ -0,0 +1,24 @@
+import type { KeepJobs } from 'bullmq'
+import type { Redis, RedisOptions } from 'ioredis'
+
+export type BullMQConnectionConfig = Redis | RedisOptions
+
+export interface BullMQEventAdapterConfig {
+  connection: BullMQConnectionConfig
+  concurrency?: number
+  defaultJobOptions?: {
+    attempts?: number
+    backoff?: {
+      type: 'fixed' | 'exponential'
+      delay: number
+    }
+    removeOnComplete?: KeepJobs
+    removeOnFail?: KeepJobs
+  }
+  prefix?: string
+  dlq?: {
+    enabled?: boolean
+    ttl?: number
+    suffix?: string
+  }
+}

package/src/worker-manager.ts

@@ -0,0 +1,200 @@
+import type { Event, QueueConfig, SubscriptionHandle } from '@motiadev/core'
+import { type Job, Worker } from 'bullmq'
+import type { Redis } from 'ioredis'
+import { v4 as uuidv4 } from 'uuid'
+import type { MergedConfig } from './config-builder'
+import { FIFO_CONCURRENCY, MILLISECONDS_PER_SECOND } from './constants'
+import type { DLQManager } from './dlq-manager'
+import { WorkerCreationError } from './errors'
+
+export type SubscriberInfo = {
+  topic: string
+  stepName: string
+  queueConfig?: QueueConfig
+}
+
+type WorkerInfo = {
+  worker: Worker
+  topic: string
+  stepName: string
+  handle: SubscriptionHandle
+  queueConfig?: QueueConfig
+}
+
+type JobData<TData> = {
+  topic: string
+  data: TData
+  traceId: string
+  flows?: string[]
+  messageGroupId?: string
+}
+
+export class WorkerManager {
+  private readonly workers: Map<string, WorkerInfo> = new Map()
+  private readonly topicSubscriptions: Map<string, Set<string>> = new Map()
+  private readonly connection: Redis
+  private readonly config: MergedConfig
+  private readonly getQueueName: (topic: string, stepName: string) => string
+  private readonly dlqManager: DLQManager | null
+
+  constructor(
+    connection: Redis,
+    config: MergedConfig,
+    getQueueName: (topic: string, stepName: string) => string,
+    dlqManager?: DLQManager,
+  ) {
+    this.connection = connection
+    this.config = config
+    this.getQueueName = getQueueName
+    this.dlqManager = dlqManager ?? null
+  }
+
+  createWorker<TData>(
+    topic: string,
+    stepName: string,
+    handler: (event: Event<TData>) => void | Promise<void>,
+    options?: QueueConfig,
+  ): SubscriptionHandle {
+    const id = uuidv4()
+    const queueName = this.getQueueName(topic, stepName)
+
+    this.addTopicSubscription(topic, id)
+
+    const concurrency = options?.type === 'fifo' ? FIFO_CONCURRENCY : this.config.concurrency
+    const attempts = options?.maxRetries != null ? options.maxRetries + 1 : this.config.defaultJobOptions.attempts
+    const lockDuration = options?.visibilityTimeout ? options.visibilityTimeout * MILLISECONDS_PER_SECOND : undefined
+
+    const worker = new Worker(
+      queueName,
+      async (job: Job<JobData<TData>>) => {
+        const eventData = job.data
+        const event = {
+          topic: eventData.topic,
+          data: eventData.data,
+          traceId: eventData.traceId,
+          flows: eventData.flows,
+          messageGroupId: eventData.messageGroupId,
+        } as Event<TData>
+        await handler(event)
+      },
+      {
+        connection: this.connection,
+        prefix: this.config.prefix,
+        concurrency,
+        lockDuration,
+        removeOnComplete: this.config.defaultJobOptions.removeOnComplete,
+        removeOnFail: this.config.defaultJobOptions.removeOnFail,
+      },
+    )
+
+    this.setupWorkerHandlers(worker, topic, stepName, attempts ?? 3)
+
+    const handle: SubscriptionHandle = {
+      topic,
+      id,
+      unsubscribe: async () => {
+        await this.removeWorker(handle.id)
+      },
+    }
+
+    const workerInfo: WorkerInfo = {
+      worker,
+      topic,
+      stepName,
+      handle,
+      queueConfig: options,
+    }
+
+    this.workers.set(id, workerInfo)
+    return handle
+  }
+
+  getSubscribers(topic: string): SubscriberInfo[] {
+    const subscriptionIds = this.topicSubscriptions.get(topic)
+    if (!subscriptionIds || subscriptionIds.size === 0) {
+      return []
+    }
+
+    return Array.from(subscriptionIds)
+      .map((id) => this.workers.get(id))
+      .filter((info): info is WorkerInfo => info !== undefined)
+      .map((info) => ({ topic: info.topic, stepName: info.stepName, queueConfig: info.queueConfig }))
+  }
+
+  getWorkerInfo(id: string): WorkerInfo | undefined {
+    return this.workers.get(id)
+  }
+
+  async removeWorker(id: string): Promise<void> {
+    const workerInfo = this.workers.get(id)
+    if (!workerInfo) {
+      return
+    }
+
+    this.removeTopicSubscription(workerInfo.topic, id)
+    await workerInfo.worker.close()
+    this.workers.delete(id)
+  }
+
+  async closeAll(): Promise<void> {
+    const promises = Array.from(this.workers.values()).map((info) =>
+      info.worker.close().catch((err) => {
+        console.error(`[BullMQ] Error closing worker for topic ${info.topic}, step ${info.stepName}:`, err)
+      }),
+    )
+    await Promise.allSettled(promises)
+    this.workers.clear()
+    this.topicSubscriptions.clear()
+  }
+
+  getSubscriptionCount(topic: string): number {
+    return Array.from(this.workers.values()).filter((w) => w.topic === topic).length
+  }
+
+  listTopics(): string[] {
+    return Array.from(new Set(Array.from(this.workers.values()).map((w) => w.topic)))
+  }
+
+  private addTopicSubscription(topic: string, id: string): void {
+    if (!this.topicSubscriptions.has(topic)) {
+      this.topicSubscriptions.set(topic, new Set())
+    }
+    this.topicSubscriptions.get(topic)?.add(id)
+  }
+
+  private removeTopicSubscription(topic: string, id: string): void {
+    const subscriptions = this.topicSubscriptions.get(topic)
+    if (subscriptions) {
+      subscriptions.delete(id)
+      if (subscriptions.size === 0) {
+        this.topicSubscriptions.delete(topic)
+      }
+    }
+  }
+
+  private setupWorkerHandlers(worker: Worker, topic: string, stepName: string, attempts: number): void {
+    worker.on('error', (err: Error) => {
+      const error = new WorkerCreationError(topic, stepName, err)
+      console.error(`[BullMQ] Worker error for topic ${topic}, step ${stepName}:`, error)
+    })
+    worker.on('failed', async (job: Job<JobData<unknown>> | undefined, err: Error) => {
+      if (job) {
+        const attemptsMade = job.attemptsMade || 0
+        if (attemptsMade >= attempts) {
+          if (this.dlqManager) {
+            const eventData = job.data
+            const event = {
+              topic: eventData.topic || topic,
+              data: eventData.data,
+              traceId: eventData.traceId || 'unknown',
+              ...(eventData.flows && { flows: eventData.flows }),
+              ...(eventData.messageGroupId && { messageGroupId: eventData.messageGroupId }),
+            } as Event<unknown>
+
+            await this.dlqManager.moveToDLQ(topic, stepName, event, err, attemptsMade, job.id)
+          }
+        }
+      }
+    })
+  }
+}

package/tsconfig.json

@@ -0,0 +1,19 @@
+{
+  "compilerOptions": {
+    "target": "ES2020",
+    "module": "commonjs",
+    "lib": ["ES2020"],
+    "declaration": true,
+    "declarationMap": true,
+    "outDir": "./dist",
+    "rootDir": "./src",
+    "strict": true,
+    "esModuleInterop": true,
+    "skipLibCheck": true,
+    "forceConsistentCasingInFileNames": true,
+    "resolveJsonModule": true,
+    "moduleResolution": "node"
+  },
+  "include": ["src/**/*"],
+  "exclude": ["node_modules", "dist"]
+}