@pikku/queue-bullmq 0.7.0

package/package.json

@@ -0,0 +1,25 @@
+{
+  "name": "@pikku/queue-bullmq",
+  "version": "0.7.0",
+  "author": "yasser.fadl@gmail.com",
+  "license": "MIT",
+  "module": "dist/index.js",
+  "main": "dist/index.js",
+  "type": "module",
+  "scripts": {
+    "tsc": "tsc",
+    "ncu": "npx npm-check-updates",
+    "build": "tsc -b",
+    "release": "npm run build && npm test"
+  },
+  "peerDependencies": {
+    "@pikku/core": "^0.7.0"
+  },
+  "dependencies": {
+    "bullmq": "^5.56.0",
+    "ioredis": "^5.6.1"
+  },
+  "devDependencies": {
+    "typescript": "^5.6"
+  }
+}

package/run-tests.sh

@@ -0,0 +1,3 @@
+#!/bin/bash
+cd "$(dirname "$0")"
+exec node --test --experimental-test-coverage --loader=tsx/esm dist/**/*.test.js

package/src/bull-queue-service.ts

@@ -0,0 +1,118 @@
+import Bull, {
+  ConnectionOptions,
+  JobsOptions,
+  Queue,
+  RedisConnection,
+} from 'bullmq'
+import type { QueueService, QueueJob, JobOptions } from '@pikku/core/queue'
+import { mapBullJobToQueueJob } from './utils.js'
+
+export const mapPikkuJobToBull = (options?: JobOptions): JobsOptions => {
+  const bullOptions: JobsOptions = {}
+
+  if (options?.priority !== undefined) {
+    bullOptions.priority = options.priority
+  }
+
+  if (options?.delay !== undefined) {
+    bullOptions.delay = options.delay
+  }
+
+  if (options?.attempts !== undefined) {
+    bullOptions.attempts = options.attempts
+  }
+
+  if (options?.removeOnComplete !== undefined) {
+    bullOptions.removeOnComplete = options.removeOnComplete
+  }
+
+  if (options?.removeOnFail !== undefined) {
+    bullOptions.removeOnFail = options.removeOnFail
+  }
+
+  if (options?.jobId !== undefined) {
+    bullOptions.jobId = options.jobId
+  }
+
+  return bullOptions
+}
+
+class PikkuRedisConnection extends RedisConnection {
+  constructor(options: ConnectionOptions) {
+    super(options)
+    this.on('error', (error) => {
+      console.error('Redis connection error:', error)
+    })
+  }
+}
+
+/**
+ * Bull/Redis queue service implementation
+ * Supports job results, retries, and most queue features
+ */
+export class BullQueueService implements QueueService {
+  readonly name = 'bull'
+  readonly supportsResults = true
+  protected queues = new Map<string, Bull.Queue>()
+  protected queueEvents = new Map<string, Bull.QueueEvents>()
+
+  constructor(private redisConnectionOptions: ConnectionOptions = {}) {}
+
+  /**
+   * Create a Bull queue with translated configuration
+   */
+  protected async createQueue(
+    queueName: string,
+    config?: Bull.QueueOptions
+  ): Promise<Bull.Queue> {
+    if (this.queues.has(queueName)) {
+      return this.queues.get(queueName)!
+    }
+
+    const queue = new Queue(queueName, config, PikkuRedisConnection)
+    await queue.waitUntilReady()
+
+    this.queues.set(queueName, queue)
+    return queue
+  }
+
+  /**
+   * Add a job to the queue
+   */
+  public async add<In>(
+    queueName: string,
+    data: In,
+    options?: JobOptions
+  ): Promise<string> {
+    const queue = await this.createQueue(queueName)
+    const bullJob = await queue.add(queueName, data, mapPikkuJobToBull(options))
+    if (!bullJob.id) {
+      throw new Error(`Failed to add job to queue '${queueName}'.`)
+    }
+    return bullJob.id
+  }
+
+  /**
+   * Get job by ID
+   */
+  async getJob<T, R>(
+    queueName: string,
+    jobId: string
+  ): Promise<QueueJob<T, R> | null> {
+    const queue = this.queues.get(queueName)
+    if (!queue) {
+      throw new Error(`Queue '${queueName}' not found.`)
+    }
+
+    const bullJob = await queue.getJob(jobId)
+    if (!bullJob) {
+      return null
+    }
+
+    return mapBullJobToQueueJob<T, R>(
+      bullJob,
+      this.redisConnectionOptions,
+      this.queueEvents
+    )
+  }
+}

package/src/bull-queue-worker.ts

@@ -0,0 +1,195 @@
+import Bull, {
+  ConnectionOptions,
+  WorkerOptions,
+  Worker,
+  QueueEvents,
+} from 'bullmq'
+import type {
+  QueueWorkers,
+  PikkuWorkerConfig,
+  QueueConfigMapping,
+  ConfigValidationResult,
+} from '@pikku/core/queue'
+import { runQueueJob, registerqueueWorkers } from '@pikku/core/queue'
+import {
+  CoreServices,
+  CoreSingletonServices,
+  CreateSessionServices,
+} from '@pikku/core'
+import { mapBullJobToQueueJob } from './utils.js'
+
+export const mapPikkuWorkerToBull = (
+  workerConfig?: PikkuWorkerConfig
+): Omit<WorkerOptions, 'connection'> => {
+  const workerOptions: Omit<WorkerOptions, 'connection'> = {}
+
+  // Direct mappings
+  if (workerConfig?.name !== undefined) {
+    workerOptions.name = workerConfig.name
+  }
+
+  if (workerConfig?.batchSize !== undefined) {
+    workerOptions.concurrency = workerConfig.batchSize
+  }
+
+  if (workerConfig?.autorun !== undefined) {
+    workerOptions.autorun = workerConfig.autorun
+  }
+
+  if (workerConfig?.lockDuration !== undefined) {
+    workerOptions.lockDuration = workerConfig.lockDuration
+  }
+
+  if (workerConfig?.drainDelay !== undefined) {
+    workerOptions.drainDelay = workerConfig.drainDelay
+  }
+
+  if (workerConfig?.maxStalledCount !== undefined) {
+    workerOptions.maxStalledCount = workerConfig.maxStalledCount
+  }
+
+  // Job retention
+  if (workerConfig?.removeOnComplete !== undefined) {
+    workerOptions.removeOnComplete = { count: workerConfig.removeOnComplete }
+  }
+
+  if (workerConfig?.removeOnFail !== undefined) {
+    workerOptions.removeOnFail = { count: workerConfig.removeOnFail }
+  }
+
+  return workerOptions
+}
+
+/**
+ * Bull/Redis queue service implementation
+ * Supports job results, retries, and most queue features
+ */
+export class BullQueueWorkers implements QueueWorkers {
+  readonly name = 'bull'
+  readonly supportsResults = true
+
+  /**
+   * Configuration mapping rules for BullMQ
+   * This defines how Pikku worker configs map to BullMQ configurations
+   */
+  readonly configMappings: QueueConfigMapping = {
+    // Configurations that are directly supported
+    supported: {
+      name: {
+        queueProperty: 'name',
+        description: 'Worker name for identification and monitoring',
+      },
+      batchSize: {
+        queueProperty: 'concurrency',
+        description:
+          "Number of jobs to process concurrently, this is Bull's concurrency setting",
+      },
+      autorun: {
+        queueProperty: 'autorun',
+        description: 'Whether to start processing jobs automatically',
+      },
+      lockDuration: {
+        queueProperty: 'lockDuration',
+        description: 'Duration of job lock in milliseconds',
+      },
+      drainDelay: {
+        queueProperty: 'drainDelay',
+        description: 'Delay when queue is empty before polling again',
+      },
+      maxStalledCount: {
+        queueProperty: 'maxStalledCount',
+        description:
+          'Maximum number of times a job can be recovered from stalled state',
+      },
+      removeOnComplete: {
+        queueProperty: 'removeOnComplete',
+        description: 'Number of completed jobs to keep for inspection',
+      },
+      removeOnFail: {
+        queueProperty: 'removeOnFail',
+        description: 'Number of failed jobs to keep for inspection',
+      },
+    },
+
+    // Configurations that are not supported
+    unsupported: {
+      visibilityTimeout: {
+        reason: 'Bull does not use visibility timeout',
+        explanation:
+          'BullMQ locks jobs during processing instead of using visibility timeouts',
+      },
+      pollInterval: {
+        reason: 'Bull is push-based and does not use polling intervals',
+        explanation:
+          'BullMQ uses Redis pub/sub for real-time job notifications',
+      },
+      prefetch: {
+        reason: 'Bull manages job prefetching internally',
+        explanation:
+          'BullMQ optimizes job fetching automatically based on concurrency settings',
+      },
+    },
+
+    // No fallback configurations for BullMQ currently
+    fallbacks: {},
+  }
+
+  private workers = new Map<string, Bull.Worker>()
+  private queueEvents = new Map<string, QueueEvents>()
+
+  constructor(
+    private redisConnectionOptions: ConnectionOptions,
+    private singletonServices: CoreSingletonServices,
+    private createSessionServices?: CreateSessionServices<
+      CoreSingletonServices,
+      CoreServices,
+      any
+    >
+  ) {}
+
+  /**
+   * Scan state and register all compatible processors
+   */
+  async registerQueues(): Promise<Record<string, ConfigValidationResult[]>> {
+    return await registerqueueWorkers(
+      this.configMappings,
+      this.singletonServices.logger,
+      async (queueName, processor) => {
+        const worker = new Worker(
+          processor.queueName,
+          async (job: Bull.Job) => {
+            return await runQueueJob({
+              singletonServices: this.singletonServices,
+              createSessionServices: this.createSessionServices,
+              job: await mapBullJobToQueueJob(
+                job,
+                this.redisConnectionOptions,
+                this.queueEvents
+              ),
+            })
+          },
+          {
+            connection: this.redisConnectionOptions,
+            ...mapPikkuWorkerToBull(processor.config),
+          }
+        )
+        worker.on('error', console.error)
+        this.workers.set(queueName, worker)
+      }
+    )
+  }
+
+  /**
+   * Close all queues and connections
+   */
+  async close(): Promise<void> {
+    await Promise.all([
+      ...Array.from(this.workers.values()).map((worker) => worker.close()),
+      ...Array.from(this.queueEvents.values()).map((queueEvents) =>
+        queueEvents.close()
+      ),
+    ])
+    this.workers.clear()
+    this.queueEvents.clear()
+  }
+}

package/src/index.ts

@@ -0,0 +1,18 @@
+/**
+ * @module @pikku/queue-bullmq
+ *
+ * Bull/Redis queue service for Pikku
+ * Provides job results, retries, priorities, and most queue features
+ */
+
+export { BullQueueService } from './bull-queue-service.js'
+export { BullQueueWorkers } from './bull-queue-worker.js'
+
+// Re-export core queue types for convenience
+export type {
+  QueueService,
+  PikkuWorkerConfig,
+  PikkuJobConfig,
+  QueueJob,
+  JobOptions,
+} from '@pikku/core/queue'

package/src/utils.ts

@@ -0,0 +1,64 @@
+import { QueueJob, QueueJobStatus } from '@pikku/core/queue'
+import { ConnectionOptions, Job, QueueEvents } from 'bullmq'
+
+/**
+ * Map Bull job state to our queue job status
+ */
+const mapBullStateToStatus = (state: string): QueueJobStatus => {
+  switch (state) {
+    case 'waiting':
+      return 'waiting'
+    case 'active':
+      return 'active'
+    case 'completed':
+      return 'completed'
+    case 'failed':
+      return 'failed'
+    case 'delayed':
+      return 'delayed'
+    default:
+      return 'waiting'
+  }
+}
+
+/**
+ * Map Bull job to QueueJob interface
+ */
+export const mapBullJobToQueueJob = async <In, Out>(
+  bullJob: Job,
+  redisConnection: ConnectionOptions | undefined,
+  queueEventsMap: Map<string, QueueEvents>
+): Promise<QueueJob<In, Out>> => {
+  return {
+    queueName: bullJob.name,
+    id: bullJob.id || '',
+    data: bullJob.data,
+    result: bullJob.returnvalue as Out,
+    status: async () => mapBullStateToStatus(await bullJob.getState()),
+    waitForCompletion: async (ttl?: number): Promise<Out> => {
+      let queueEvents = queueEventsMap.get(bullJob.name)
+      if (!queueEvents) {
+        queueEvents = new QueueEvents(
+          bullJob.name,
+          redisConnection ? { connection: redisConnection } : undefined
+        )
+        queueEventsMap.set(bullJob.name, queueEvents)
+      }
+      return await bullJob.waitUntilFinished(queueEvents, ttl)
+    },
+    metadata: () => ({
+      progress: bullJob.progress,
+      attemptsMade: bullJob.attemptsMade,
+      maxAttempts: bullJob.opts.attempts || 1,
+      createdAt: new Date(bullJob.timestamp),
+      processedAt: bullJob.processedOn
+        ? new Date(bullJob.processedOn)
+        : undefined,
+      completedAt: bullJob.finishedOn
+        ? new Date(bullJob.finishedOn)
+        : undefined,
+      failedAt: bullJob.finishedOn ? new Date(bullJob.finishedOn) : undefined,
+      error: bullJob.failedReason,
+    }),
+  }
+}

package/tsconfig.json

@@ -0,0 +1,17 @@
+{
+  "extends": "../../tsconfig.json",
+  "compilerOptions": {
+    "rootDir": "./src",
+    "module": "Node16",
+    "outDir": "dist",
+    "target": "esnext",
+    "declaration": true
+  },
+  "include": ["src/**/*.ts"],
+  "exclude": ["**/*.test.ts", "node_modules"],
+  "references": [
+    {
+      "path": "../../core/tsconfig.json"
+    }
+  ]
+}