@pikku/queue-bullmq 0.9.4 → 0.9.5-next.0

package/CHANGELOG.md

@@ -1,5 +1,12 @@
 # @pikku/queue-bullmq
 
+## 0.9.5-next.0
+
+### Patch Changes
+
+- Updated dependencies
+  - @pikku/core@0.9.12-next.0
+
 ## 0.9.4
 
 ### Patch Changes

package/dist/bull-queue-service.d.ts

@@ -0,0 +1,27 @@
+import Bull, { ConnectionOptions, JobsOptions } from 'bullmq';
+import type { QueueService, QueueJob, JobOptions } from '@pikku/core/queue';
+export declare const mapPikkuJobToBull: (options?: JobOptions) => JobsOptions;
+/**
+ * Bull/Redis queue service implementation
+ * Supports job results, retries, and most queue features
+ */
+export declare class BullQueueService implements QueueService {
+    private redisConnectionOptions;
+    readonly name = "bull";
+    readonly supportsResults = true;
+    protected queues: Map<string, Bull.Queue<any, any, string, any, any, string>>;
+    protected queueEvents: Map<string, Bull.QueueEvents>;
+    constructor(redisConnectionOptions?: ConnectionOptions);
+    /**
+     * Create a Bull queue with translated configuration
+     */
+    protected createQueue(queueName: string, config?: Bull.QueueOptions): Promise<Bull.Queue>;
+    /**
+     * Add a job to the queue
+     */
+    add<In>(queueName: string, data: In, options?: JobOptions): Promise<string>;
+    /**
+     * Get job by ID
+     */
+    getJob<T, R>(queueName: string, jobId: string): Promise<QueueJob<T, R> | null>;
+}

package/dist/bull-queue-service.js

@@ -0,0 +1,83 @@
+import { Queue, RedisConnection, } from 'bullmq';
+import { mapBullJobToQueueJob } from './utils.js';
+export const mapPikkuJobToBull = (options) => {
+    const bullOptions = {};
+    if (options?.priority !== undefined) {
+        bullOptions.priority = options.priority;
+    }
+    if (options?.delay !== undefined) {
+        bullOptions.delay = options.delay;
+    }
+    if (options?.attempts !== undefined) {
+        bullOptions.attempts = options.attempts;
+    }
+    if (options?.removeOnComplete !== undefined) {
+        bullOptions.removeOnComplete = options.removeOnComplete;
+    }
+    if (options?.removeOnFail !== undefined) {
+        bullOptions.removeOnFail = options.removeOnFail;
+    }
+    if (options?.jobId !== undefined) {
+        bullOptions.jobId = options.jobId;
+    }
+    return bullOptions;
+};
+class PikkuRedisConnection extends RedisConnection {
+    constructor(options) {
+        super(options);
+        this.on('error', (error) => {
+            console.error('Redis connection error:', error);
+        });
+    }
+}
+/**
+ * Bull/Redis queue service implementation
+ * Supports job results, retries, and most queue features
+ */
+export class BullQueueService {
+    redisConnectionOptions;
+    name = 'bull';
+    supportsResults = true;
+    queues = new Map();
+    queueEvents = new Map();
+    constructor(redisConnectionOptions = {}) {
+        this.redisConnectionOptions = redisConnectionOptions;
+    }
+    /**
+     * Create a Bull queue with translated configuration
+     */
+    async createQueue(queueName, config) {
+        if (this.queues.has(queueName)) {
+            return this.queues.get(queueName);
+        }
+        const queue = new Queue(queueName, config, PikkuRedisConnection);
+        await queue.waitUntilReady();
+        this.queues.set(queueName, queue);
+        return queue;
+    }
+    /**
+     * Add a job to the queue
+     */
+    async add(queueName, data, options) {
+        const queue = await this.createQueue(queueName);
+        const bullJob = await queue.add(queueName, data, mapPikkuJobToBull(options));
+        if (!bullJob.id) {
+            throw new Error(`Failed to add job to queue '${queueName}'.`);
+        }
+        return bullJob.id;
+    }
+    /**
+     * Get job by ID
+     */
+    async getJob(queueName, jobId) {
+        const queue = this.queues.get(queueName);
+        if (!queue) {
+            throw new Error(`Queue '${queueName}' not found.`);
+        }
+        const bullJob = await queue.getJob(jobId);
+        if (!bullJob) {
+            return null;
+        }
+        return mapBullJobToQueueJob(bullJob, this.redisConnectionOptions, this.queueEvents);
+    }
+}

package/dist/bull-queue-worker.d.ts

@@ -0,0 +1,31 @@
+import { ConnectionOptions, WorkerOptions } from 'bullmq';
+import type { QueueWorkers, PikkuWorkerConfig, QueueConfigMapping, ConfigValidationResult } from '@pikku/core/queue';
+import { CoreServices, CoreSingletonServices, CreateSessionServices } from '@pikku/core';
+export declare const mapPikkuWorkerToBull: (workerConfig?: PikkuWorkerConfig) => Omit<WorkerOptions, "connection">;
+/**
+ * Bull/Redis queue service implementation
+ * Supports job results, retries, and most queue features
+ */
+export declare class BullQueueWorkers implements QueueWorkers {
+    private redisConnectionOptions;
+    private singletonServices;
+    private createSessionServices?;
+    readonly name = "bull";
+    readonly supportsResults = true;
+    /**
+     * Configuration mapping rules for BullMQ
+     * This defines how Pikku worker configs map to BullMQ configurations
+     */
+    readonly configMappings: QueueConfigMapping;
+    private workers;
+    private queueEvents;
+    constructor(redisConnectionOptions: ConnectionOptions, singletonServices: CoreSingletonServices, createSessionServices?: CreateSessionServices<CoreSingletonServices, CoreServices, any> | undefined);
+    /**
+     * Scan state and register all compatible processors
+     */
+    registerQueues(): Promise<Record<string, ConfigValidationResult[]>>;
+    /**
+     * Close all queues and connections
+     */
+    close(): Promise<void>;
+}

package/dist/bull-queue-worker.js

@@ -0,0 +1,155 @@
+import { Worker, } from 'bullmq';
+import { runQueueJob, registerQueueWorkers, QueueJobFailedError, QueueJobDiscardedError, } from '@pikku/core/queue';
+import { mapBullJobToQueueJob } from './utils.js';
+export const mapPikkuWorkerToBull = (workerConfig) => {
+    const workerOptions = {};
+    // Direct mappings
+    if (workerConfig?.name !== undefined) {
+        workerOptions.name = workerConfig.name;
+    }
+    if (workerConfig?.batchSize !== undefined) {
+        workerOptions.concurrency = workerConfig.batchSize;
+    }
+    if (workerConfig?.autorun !== undefined) {
+        workerOptions.autorun = workerConfig.autorun;
+    }
+    if (workerConfig?.lockDuration !== undefined) {
+        workerOptions.lockDuration = workerConfig.lockDuration;
+    }
+    if (workerConfig?.drainDelay !== undefined) {
+        workerOptions.drainDelay = workerConfig.drainDelay;
+    }
+    if (workerConfig?.maxStalledCount !== undefined) {
+        workerOptions.maxStalledCount = workerConfig.maxStalledCount;
+    }
+    // Job retention
+    if (workerConfig?.removeOnComplete !== undefined) {
+        workerOptions.removeOnComplete = { count: workerConfig.removeOnComplete };
+    }
+    if (workerConfig?.removeOnFail !== undefined) {
+        workerOptions.removeOnFail = { count: workerConfig.removeOnFail };
+    }
+    return workerOptions;
+};
+/**
+ * Bull/Redis queue service implementation
+ * Supports job results, retries, and most queue features
+ */
+export class BullQueueWorkers {
+    redisConnectionOptions;
+    singletonServices;
+    createSessionServices;
+    name = 'bull';
+    supportsResults = true;
+    /**
+     * Configuration mapping rules for BullMQ
+     * This defines how Pikku worker configs map to BullMQ configurations
+     */
+    configMappings = {
+        // Configurations that are directly supported
+        supported: {
+            name: {
+                queueProperty: 'name',
+                description: 'Worker name for identification and monitoring',
+            },
+            batchSize: {
+                queueProperty: 'concurrency',
+                description: "Number of jobs to process concurrently, this is Bull's concurrency setting",
+            },
+            autorun: {
+                queueProperty: 'autorun',
+                description: 'Whether to start processing jobs automatically',
+            },
+            lockDuration: {
+                queueProperty: 'lockDuration',
+                description: 'Duration of job lock in milliseconds',
+            },
+            drainDelay: {
+                queueProperty: 'drainDelay',
+                description: 'Delay when queue is empty before polling again',
+            },
+            maxStalledCount: {
+                queueProperty: 'maxStalledCount',
+                description: 'Maximum number of times a job can be recovered from stalled state',
+            },
+            removeOnComplete: {
+                queueProperty: 'removeOnComplete',
+                description: 'Number of completed jobs to keep for inspection',
+            },
+            removeOnFail: {
+                queueProperty: 'removeOnFail',
+                description: 'Number of failed jobs to keep for inspection',
+            },
+        },
+        // Configurations that are not supported
+        unsupported: {
+            visibilityTimeout: {
+                reason: 'Bull does not use visibility timeout',
+                explanation: 'BullMQ locks jobs during processing instead of using visibility timeouts',
+            },
+            pollInterval: {
+                reason: 'Bull is push-based and does not use polling intervals',
+                explanation: 'BullMQ uses Redis pub/sub for real-time job notifications',
+            },
+            prefetch: {
+                reason: 'Bull manages job prefetching internally',
+                explanation: 'BullMQ optimizes job fetching automatically based on concurrency settings',
+            },
+        },
+        // No fallback configurations for BullMQ currently
+        fallbacks: {},
+    };
+    workers = new Map();
+    queueEvents = new Map();
+    constructor(redisConnectionOptions, singletonServices, createSessionServices) {
+        this.redisConnectionOptions = redisConnectionOptions;
+        this.singletonServices = singletonServices;
+        this.createSessionServices = createSessionServices;
+    }
+    /**
+     * Scan state and register all compatible processors
+     */
+    async registerQueues() {
+        return await registerQueueWorkers(this.configMappings, this.singletonServices.logger, async (queueName, processor) => {
+            const worker = new Worker(processor.queueName, async (job) => {
+                try {
+                    return await runQueueJob({
+                        singletonServices: this.singletonServices,
+                        createSessionServices: this.createSessionServices,
+                        job: await mapBullJobToQueueJob(job, this.redisConnectionOptions, this.queueEvents),
+                        updateProgress: async (progress) => {
+                            await job.updateProgress(progress);
+                        },
+                    });
+                }
+                catch (error) {
+                    if (error instanceof QueueJobFailedError) {
+                        // Let BullMQ handle this as a failed job
+                        throw new Error(error.message);
+                    }
+                    else if (error instanceof QueueJobDiscardedError) {
+                        return; // Don't throw, job is discarded, which I guess is considered
+                        // a successful job removal
+                    }
+                    throw error;
+                }
+            }, {
+                connection: this.redisConnectionOptions,
+                ...mapPikkuWorkerToBull(processor.config),
+            });
+            worker.on('error', console.error);
+            this.workers.set(queueName, worker);
+        });
+    }
+    /**
+     * Close all queues and connections
+     */
+    async close() {
+        await Promise.all([
+            ...Array.from(this.workers.values()).map((worker) => worker.close()),
+            ...Array.from(this.queueEvents.values()).map((queueEvents) => queueEvents.close()),
+        ]);
+        this.workers.clear();
+        this.queueEvents.clear();
+    }
+}

package/dist/index.d.ts

@@ -0,0 +1,9 @@
+/**
+ * @module @pikku/queue-bullmq
+ *
+ * Bull/Redis queue service for Pikku
+ * Provides job results, retries, priorities, and most queue features
+ */
+export { BullQueueService } from './bull-queue-service.js';
+export { BullQueueWorkers } from './bull-queue-worker.js';
+export type { QueueService, PikkuWorkerConfig, PikkuJobConfig, QueueJob, JobOptions, } from '@pikku/core/queue';

package/dist/index.js

@@ -0,0 +1,8 @@
+/**
+ * @module @pikku/queue-bullmq
+ *
+ * Bull/Redis queue service for Pikku
+ * Provides job results, retries, priorities, and most queue features
+ */
+export { BullQueueService } from './bull-queue-service.js';
+export { BullQueueWorkers } from './bull-queue-worker.js';

package/dist/utils.d.ts

@@ -0,0 +1,6 @@
+import { QueueJob } from '@pikku/core/queue';
+import { ConnectionOptions, Job, QueueEvents } from 'bullmq';
+/**
+ * Map Bull job to QueueJob interface
+ */
+export declare const mapBullJobToQueueJob: <In, Out>(bullJob: Job, redisConnection: ConnectionOptions | undefined, queueEventsMap: Map<string, QueueEvents>) => Promise<QueueJob<In, Out>>;

package/dist/utils.js

@@ -0,0 +1,54 @@
+import { QueueEvents } from 'bullmq';
+/**
+ * Map Bull job state to our queue job status
+ */
+const mapBullStateToStatus = (state) => {
+    switch (state) {
+        case 'waiting':
+            return 'waiting';
+        case 'active':
+            return 'active';
+        case 'completed':
+            return 'completed';
+        case 'failed':
+            return 'failed';
+        case 'delayed':
+            return 'delayed';
+        default:
+            return 'waiting';
+    }
+};
+/**
+ * Map Bull job to QueueJob interface
+ */
+export const mapBullJobToQueueJob = async (bullJob, redisConnection, queueEventsMap) => {
+    return {
+        queueName: bullJob.name,
+        id: bullJob.id || '',
+        data: bullJob.data,
+        result: bullJob.returnvalue,
+        status: async () => mapBullStateToStatus(await bullJob.getState()),
+        waitForCompletion: async (ttl) => {
+            let queueEvents = queueEventsMap.get(bullJob.name);
+            if (!queueEvents) {
+                queueEvents = new QueueEvents(bullJob.name, redisConnection ? { connection: redisConnection } : undefined);
+                queueEventsMap.set(bullJob.name, queueEvents);
+            }
+            return await bullJob.waitUntilFinished(queueEvents, ttl);
+        },
+        metadata: () => ({
+            progress: bullJob.progress,
+            attemptsMade: bullJob.attemptsMade,
+            maxAttempts: bullJob.opts.attempts || 1,
+            createdAt: new Date(bullJob.timestamp),
+            processedAt: bullJob.processedOn
+                ? new Date(bullJob.processedOn)
+                : undefined,
+            completedAt: bullJob.finishedOn
+                ? new Date(bullJob.finishedOn)
+                : undefined,
+            failedAt: bullJob.finishedOn ? new Date(bullJob.finishedOn) : undefined,
+            error: bullJob.failedReason,
+        }),
+    };
+};

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@pikku/queue-bullmq",
-  "version": "0.9.4",
+  "version": "0.9.5-next.0",
   "author": "yasser.fadl@gmail.com",
   "license": "MIT",
   "module": "dist/index.js",
@@ -13,7 +13,7 @@
     "release": "npm run build && npm test"
   },
   "peerDependencies": {
-    "@pikku/core": "^0.9.9"
+    "@pikku/core": "^0.9.12-next.0"
   },
   "dependencies": {
     "bullmq": "^5.57.0",

package/tsconfig.tsbuildinfo

@@ -0,0 +1 @@
+{"root":["./src/bull-queue-service.ts","./src/bull-queue-worker.ts","./src/index.ts","./src/utils.ts"],"version":"5.9.2"}