@checkstack/queue-bullmq-backend 0.0.2

package/CHANGELOG.md

@@ -0,0 +1,53 @@
+# @checkstack/queue-bullmq-backend
+
+## 0.0.2
+
+### Patch Changes
+
+- d20d274: Initial release of all @checkstack packages. Rebranded from Checkmate to Checkstack with new npm organization @checkstack and domain checkstack.dev.
+- Updated dependencies [d20d274]
+  - @checkstack/backend-api@0.0.2
+  - @checkstack/common@0.0.2
+  - @checkstack/queue-api@0.0.2
+  - @checkstack/queue-bullmq-common@0.0.2
+
+## 0.2.1
+
+### Patch Changes
+
+- Updated dependencies [b4eb432]
+- Updated dependencies [a65e002]
+  - @checkstack/backend-api@1.1.0
+  - @checkstack/common@0.2.0
+  - @checkstack/queue-api@1.0.1
+  - @checkstack/queue-bullmq-common@0.2.1
+
+## 0.2.0
+
+### Minor Changes
+
+- e4d83fc: Add BullMQ queue plugin with orphaned job cleanup
+
+  - **queue-api**: Added `listRecurringJobs()` method to Queue interface for detecting orphaned jobs
+  - **queue-bullmq-backend**: New plugin implementing BullMQ (Redis) queue backend with job schedulers, consumer groups, and distributed job persistence
+  - **queue-bullmq-common**: New common package with queue permissions
+  - **queue-memory-backend**: Implemented `listRecurringJobs()` for in-memory queue
+  - **healthcheck-backend**: Enhanced `bootstrapHealthChecks` to clean up orphaned job schedulers using `listRecurringJobs()`
+  - **test-utils-backend**: Added `listRecurringJobs()` to mock queue factory
+
+  This enables production-ready distributed queue processing with Redis persistence and automatic cleanup of orphaned jobs when health checks are deleted.
+
+### Patch Changes
+
+- Updated dependencies [ffc28f6]
+- Updated dependencies [e4d83fc]
+- Updated dependencies [71275dd]
+- Updated dependencies [ae19ff6]
+- Updated dependencies [b55fae6]
+- Updated dependencies [b354ab3]
+- Updated dependencies [8e889b4]
+- Updated dependencies [81f3f85]
+  - @checkstack/common@0.1.0
+  - @checkstack/backend-api@1.0.0
+  - @checkstack/queue-api@1.0.0
+  - @checkstack/queue-bullmq-common@0.2.0

package/package.json

@@ -0,0 +1,25 @@
+{
+  "name": "@checkstack/queue-bullmq-backend",
+  "version": "0.0.2",
+  "type": "module",
+  "main": "src/index.ts",
+  "scripts": {
+    "typecheck": "tsc --noEmit",
+    "lint": "bun run lint:code",
+    "lint:code": "eslint . --max-warnings 0"
+  },
+  "dependencies": {
+    "@checkstack/backend-api": "workspace:*",
+    "@checkstack/queue-api": "workspace:*",
+    "@checkstack/queue-bullmq-common": "workspace:*",
+    "bullmq": "^5.66.4",
+    "zod": "^4.0.0",
+    "@checkstack/common": "workspace:*"
+  },
+  "devDependencies": {
+    "@types/bun": "latest",
+    "@checkstack/tsconfig": "workspace:*",
+    "@checkstack/scripts": "workspace:*",
+    "ioredis-mock": "^8.9.0"
+  }
+}

package/src/bullmq-queue.ts

@@ -0,0 +1,281 @@
+import {
+  Queue,
+  QueueJob,
+  QueueConsumer,
+  QueueStats,
+  ConsumeOptions,
+  RecurringJobDetails,
+} from "@checkstack/queue-api";
+import { Queue as BullQueue, Worker, JobsOptions } from "bullmq";
+import type { BullMQConfig } from "./plugin";
+
+/**
+ * Consumer group state tracking
+ */
+interface ConsumerGroupState {
+  worker: Worker;
+  consumerCount: number;
+}
+
+/**
+ * BullMQ-based queue implementation
+ */
+export class BullMQQueue<T = unknown> implements Queue<T> {
+  private queue: BullQueue;
+  private consumerGroups = new Map<string, ConsumerGroupState>();
+  private stopped = false;
+
+  constructor(private name: string, private config: BullMQConfig) {
+    // Initialize BullMQ Queue with Redis connection
+    this.queue = new BullQueue(name, {
+      connection: {
+        host: config.host,
+        port: config.port,
+        password: config.password,
+        db: config.db,
+        // Disable automatic reconnection and retries for immediate failure
+        // eslint-disable-next-line unicorn/no-null
+        retryStrategy: () => null, // Don't retry, fail immediately
+        maxRetriesPerRequest: 1,
+        enableReadyCheck: true,
+        connectTimeout: 5000, // 5 second connection timeout
+      },
+      prefix: config.keyPrefix,
+    });
+  }
+
+  /**
+   * Test Redis connection by attempting a simple operation
+   * @throws Error if connection fails
+   */
+  async testConnection(): Promise<void> {
+    try {
+      // Try to get job counts - this will fail if Redis is not accessible
+      await this.queue.getJobCounts();
+    } catch (error) {
+      const message = error instanceof Error ? error.message : String(error);
+      throw new Error(
+        `Failed to connect to Redis at ${this.config.host}:${this.config.port}: ${message}`
+      );
+    }
+  }
+
+  async enqueue(
+    data: T,
+    options?: {
+      priority?: number;
+      startDelay?: number;
+      jobId?: string;
+    }
+  ): Promise<string> {
+    if (this.stopped) {
+      throw new Error("Queue has been stopped");
+    }
+
+    const jobOptions: JobsOptions = {};
+
+    if (options?.priority !== undefined) {
+      jobOptions.priority = options.priority;
+    }
+
+    if (options?.startDelay !== undefined) {
+      // Convert seconds to milliseconds
+      jobOptions.delay = options.startDelay * 1000;
+    }
+
+    if (options?.jobId) {
+      jobOptions.jobId = options.jobId;
+    }
+
+    const job = await this.queue.add(this.name, data, jobOptions);
+    return job.id!;
+  }
+
+  async consume(
+    consumer: QueueConsumer<T>,
+    options: ConsumeOptions
+  ): Promise<void> {
+    if (this.stopped) {
+      throw new Error("Queue has been stopped");
+    }
+
+    const { consumerGroup, maxRetries = 3 } = options;
+
+    // Check if worker already exists for this consumer group
+    let groupState = this.consumerGroups.get(consumerGroup);
+
+    if (groupState) {
+      // Increment consumer count for existing group
+      groupState.consumerCount++;
+    } else {
+      // Create new worker for this consumer group
+      const worker = new Worker(
+        this.name,
+        async (job) => {
+          const queueJob: QueueJob<T> = {
+            id: job.id!,
+            data: job.data as T,
+            priority: job.opts.priority,
+            timestamp: new Date(job.timestamp),
+            attempts: job.attemptsMade,
+          };
+
+          await consumer(queueJob);
+        },
+        {
+          connection: {
+            host: this.config.host,
+            port: this.config.port,
+            password: this.config.password,
+            db: this.config.db,
+          },
+          prefix: this.config.keyPrefix,
+          concurrency: this.config.concurrency,
+          // BullMQ's built-in retry mechanism
+          settings: {
+            backoffStrategy: (attemptsMade: number) => {
+              // Exponential backoff: 2^attemptsMade * 1000ms
+              return Math.pow(2, attemptsMade) * 1000;
+            },
+          },
+        }
+      );
+
+      // Configure retries at job level via job options
+      worker.on("failed", async (job, err) => {
+        if (job && job.attemptsMade >= maxRetries) {
+          // Max retries exhausted
+          console.debug(
+            `Job ${job.id} exhausted retries (${job.attemptsMade}/${maxRetries}):`,
+            err
+          );
+        }
+      });
+
+      groupState = {
+        worker,
+        consumerCount: 1,
+      };
+      this.consumerGroups.set(consumerGroup, groupState);
+    }
+  }
+
+  async scheduleRecurring(
+    data: T,
+    options: {
+      jobId: string;
+      intervalSeconds: number;
+      startDelay?: number;
+      priority?: number;
+    }
+  ): Promise<string> {
+    if (this.stopped) {
+      throw new Error("Queue has been stopped");
+    }
+
+    const { jobId, intervalSeconds, startDelay, priority } = options;
+
+    // Use upsertJobScheduler for create-or-update semantics
+    await this.queue.upsertJobScheduler(
+      jobId,
+      {
+        every: intervalSeconds * 1000,
+        startDate: startDelay
+          ? new Date(Date.now() + startDelay * 1000)
+          : undefined,
+      },
+      {
+        name: this.name,
+        data,
+        opts: {
+          priority,
+        },
+      }
+    );
+
+    return jobId;
+  }
+
+  async cancelRecurring(jobId: string): Promise<void> {
+    if (this.stopped) {
+      throw new Error("Queue has been stopped");
+    }
+
+    await this.queue.removeJobScheduler(jobId);
+  }
+
+  async listRecurringJobs(): Promise<string[]> {
+    if (this.stopped) {
+      throw new Error("Queue has been stopped");
+    }
+
+    const schedulers = await this.queue.getJobSchedulers();
+    return schedulers.map((scheduler) => scheduler.key);
+  }
+
+  async getRecurringJobDetails(
+    jobId: string
+  ): Promise<RecurringJobDetails<T> | undefined> {
+    if (this.stopped) {
+      throw new Error("Queue has been stopped");
+    }
+
+    const schedulers = await this.queue.getJobSchedulers();
+    const scheduler = schedulers.find((s) => s.key === jobId);
+
+    if (!scheduler) {
+      return undefined;
+    }
+
+    // BullMQ scheduler template contains the data
+    return {
+      jobId,
+      data: scheduler.template?.data as T,
+      intervalSeconds: scheduler.every ? scheduler.every / 1000 : 0,
+      priority: scheduler.template?.opts?.priority,
+      nextRunAt: scheduler.next ? new Date(scheduler.next) : undefined,
+    };
+  }
+
+  async getInFlightCount(): Promise<number> {
+    const counts = await this.queue.getJobCounts("active");
+    return counts.active || 0;
+  }
+
+  async stop(): Promise<void> {
+    if (this.stopped) {
+      return;
+    }
+
+    this.stopped = true;
+
+    // Close all workers gracefully
+    const closePromises: Promise<void>[] = [];
+    for (const groupState of this.consumerGroups.values()) {
+      closePromises.push(groupState.worker.close());
+    }
+    await Promise.all(closePromises);
+
+    // Close queue connection
+    await this.queue.close();
+
+    this.consumerGroups.clear();
+  }
+
+  async getStats(): Promise<QueueStats> {
+    const counts = await this.queue.getJobCounts(
+      "waiting",
+      "active",
+      "completed",
+      "failed"
+    );
+
+    return {
+      pending: counts.waiting || 0,
+      processing: counts.active || 0,
+      completed: counts.completed || 0,
+      failed: counts.failed || 0,
+      consumerGroups: this.consumerGroups.size,
+    };
+  }
+}

package/src/index.ts

@@ -0,0 +1,26 @@
+import {
+  createBackendPlugin,
+  coreServices,
+} from "@checkstack/backend-api";
+import { BullMQPlugin } from "./plugin";
+import { permissionList } from "@checkstack/queue-bullmq-common";
+import { pluginMetadata } from "./plugin-metadata";
+
+export default createBackendPlugin({
+  metadata: pluginMetadata,
+  register(env) {
+    env.registerPermissions(permissionList);
+
+    env.registerInit({
+      deps: {
+        queuePluginRegistry: coreServices.queuePluginRegistry,
+        logger: coreServices.logger,
+      },
+      init: async ({ queuePluginRegistry, logger }) => {
+        logger.debug("🔌 Registering BullMQ Queue Plugin...");
+        const plugin = new BullMQPlugin();
+        queuePluginRegistry.register(plugin);
+      },
+    });
+  },
+});

package/src/plugin-metadata.ts

@@ -0,0 +1,9 @@
+import { definePluginMetadata } from "@checkstack/common";
+
+/**
+ * Plugin metadata for the Queue BullMQ backend.
+ * This is the single source of truth for the plugin ID.
+ */
+export const pluginMetadata = definePluginMetadata({
+  pluginId: "queue-bullmq",
+});

package/src/plugin.ts

@@ -0,0 +1,36 @@
+import { QueuePlugin, Queue } from "@checkstack/queue-api";
+import { configString, configNumber } from "@checkstack/backend-api";
+import { z } from "zod";
+import { BullMQQueue } from "./bullmq-queue";
+
+const configSchema = z.object({
+  host: z.string().default("localhost").describe("Redis host"),
+  port: z.number().min(1).max(65_535).default(6379).describe("Redis port"),
+  password: configString({ "x-secret": true })
+    .describe("Redis password (optional)")
+    .optional(),
+  db: configNumber({}).min(0).default(0).describe("Redis database number"),
+  keyPrefix: configString({})
+    .default("checkstack:")
+    .describe("Key prefix for queue names"),
+  concurrency: configNumber({})
+    .min(1)
+    .max(100)
+    .default(10)
+    .describe("Maximum number of concurrent jobs to process"),
+});
+
+export type BullMQConfig = z.infer<typeof configSchema>;
+
+export class BullMQPlugin implements QueuePlugin<BullMQConfig> {
+  id = "bullmq";
+  displayName = "BullMQ (Redis)";
+  description =
+    "Production-grade distributed queue with Redis backend supporting multi-instance deployments";
+  configVersion = 1;
+  configSchema = configSchema;
+
+  createQueue<T>(name: string, config: BullMQConfig): Queue<T> {
+    return new BullMQQueue<T>(name, config);
+  }
+}

package/tsconfig.json

@@ -0,0 +1,3 @@
+{
+  "extends": "@checkstack/tsconfig/base.json"
+}