@pikku/queue-bullmq 0.11.1 → 0.12.0

package/CHANGELOG.md

@@ -1,5 +1,24 @@
+## 0.12.0
+
+### New Features
+
+- Recurring cron scheduled tasks support via `BullSchedulerService`
+- `setServices()`, `start()`, `stop()` lifecycle methods on scheduler and queue workers
+
 ## 0.11.0
 
+## 0.11.2
+
+### Patch Changes
+
+- db9c7bf: Simplify CreateWireServices type signature for custom Config compatibility
+- Updated dependencies [db9c7bf]
+  - @pikku/core@0.11.2
+
+### Fixes
+
+- ddd87eaf: Simplify CreateWireServices type signature for custom Config compatibility
+
 ## 0.11.1
 
 ### Patch Changes

package/dist/bull-queue-worker.d.ts

@@ -1,6 +1,6 @@
 import { ConnectionOptions, WorkerOptions } from 'bullmq';
-import type { QueueWorkers, PikkuWorkerConfig, QueueConfigMapping, ConfigValidationResult } from '@pikku/core/queue';
-import type { CoreServices, CoreSingletonServices, CreateWireServices } from '@pikku/core';
+import type { QueueWorkers, PikkuWorkerConfig, QueueConfigMapping, ConfigValidationResult, QueueJob } from '@pikku/core/queue';
+import type { Logger } from '@pikku/core/services';
 export declare const mapPikkuWorkerToBull: (workerConfig?: PikkuWorkerConfig) => Omit<WorkerOptions, "connection">;
 /**
  * Bull/Redis queue service implementation
@@ -8,8 +8,6 @@ export declare const mapPikkuWorkerToBull: (workerConfig?: PikkuWorkerConfig) =>
  */
 export declare class BullQueueWorkers implements QueueWorkers {
     private redisConnectionOptions;
-    private singletonServices;
-    private createWireServices?;
     readonly name = "bull";
     readonly supportsResults = true;
     /**
@@ -19,7 +17,13 @@ export declare class BullQueueWorkers implements QueueWorkers {
     readonly configMappings: QueueConfigMapping;
     private workers;
     private queueEvents;
-    constructor(redisConnectionOptions: ConnectionOptions, singletonServices: CoreSingletonServices, createWireServices?: CreateWireServices<CoreSingletonServices, CoreServices, any> | undefined);
+    private runJob?;
+    private logger?;
+    constructor(redisConnectionOptions: ConnectionOptions);
+    setJobRunner(runJob: (params: {
+        job: QueueJob;
+        updateProgress?: (progress: number | string | object) => Promise<void>;
+    }) => Promise<void>, logger: Logger): void;
     /**
      * Scan state and register all compatible processors
      */

package/dist/bull-queue-worker.js

@@ -1,5 +1,5 @@
 import { Worker, } from 'bullmq';
-import { runQueueJob, registerQueueWorkers, QueueJobFailedError, QueueJobDiscardedError, } from '@pikku/core/queue';
+import { registerQueueWorkers, QueueJobFailedError, QueueJobDiscardedError, } from '@pikku/core/queue';
 import { mapBullJobToQueueJob } from './utils.js';
 export const mapPikkuWorkerToBull = (workerConfig) => {
     const workerOptions = {};
@@ -37,8 +37,6 @@ export const mapPikkuWorkerToBull = (workerConfig) => {
  */
 export class BullQueueWorkers {
     redisConnectionOptions;
-    singletonServices;
-    createWireServices;
     name = 'bull';
     supportsResults = true;
     /**
@@ -101,21 +99,26 @@ export class BullQueueWorkers {
     };
     workers = new Map();
     queueEvents = new Map();
-    constructor(redisConnectionOptions, singletonServices, createWireServices) {
+    runJob;
+    logger;
+    constructor(redisConnectionOptions) {
         this.redisConnectionOptions = redisConnectionOptions;
-        this.singletonServices = singletonServices;
-        this.createWireServices = createWireServices;
+    }
+    setJobRunner(runJob, logger) {
+        this.runJob = runJob;
+        this.logger = logger;
     }
     /**
      * Scan state and register all compatible processors
      */
     async registerQueues() {
-        return await registerQueueWorkers(this.configMappings, this.singletonServices.logger, async (queueName, processor) => {
-            const worker = new Worker(processor.queueName, async (job) => {
+        if (!this.runJob || !this.logger) {
+            throw new Error('BullQueueWorkers requires setJobRunner() before registerQueues()');
+        }
+        return await registerQueueWorkers(this.configMappings, this.logger, async (queueName, processor) => {
+            const worker = new Worker(processor.name, async (job) => {
                 try {
-                    return await runQueueJob({
-                        singletonServices: this.singletonServices,
-                        createWireServices: this.createWireServices,
+                    return await this.runJob({
                         job: await mapBullJobToQueueJob(job, this.redisConnectionOptions, this.queueEvents),
                         updateProgress: async (progress) => {
                             await job.updateProgress(progress);

package/dist/bull-scheduler-service.d.ts

@@ -1,12 +1,19 @@
 import { ConnectionOptions } from 'bullmq';
-import { SchedulerService, ScheduledTaskInfo, ScheduledTaskSummary, CoreUserSession } from '@pikku/core';
-/**
- * BullMQ scheduler service implementation
- * Uses BullMQ's delayed jobs for one-time execution
- */
+import { SchedulerService, SchedulerRuntimeHandlers, ScheduledTaskInfo, ScheduledTaskSummary, CoreUserSession } from '@pikku/core';
 export declare class BullSchedulerService extends SchedulerService {
+    private redisConnectionOptions;
     private queue;
+    private recurringQueue;
+    private recurringWorker?;
+    private repeatJobKeys;
+    private handlers?;
     constructor(redisConnectionOptions: ConnectionOptions);
+    /**
+     * Set services needed for processing recurring tasks.
+     * Must be called before start() since the scheduler is typically
+     * created before singletonServices are available.
+     */
+    setServices(handlers: SchedulerRuntimeHandlers): void;
     /**
      * Initialize - wait for queue to be ready
      */
@@ -32,4 +39,13 @@ export declare class BullSchedulerService extends SchedulerService {
      * Close the queue connection
      */
     close(): Promise<void>;
+    /**
+     * Start recurring scheduled tasks.
+     * Creates a BullMQ Worker to process repeat jobs via runScheduledTask.
+     */
+    start(): Promise<void>;
+    /**
+     * Stop recurring scheduled tasks by removing repeat jobs.
+     */
+    stop(): Promise<void>;
 }

package/dist/bull-scheduler-service.js

@@ -1,16 +1,31 @@
-import { Queue } from 'bullmq';
+import { Queue, Worker } from 'bullmq';
 import { SchedulerService, parseDurationString, } from '@pikku/core';
-/**
- * BullMQ scheduler service implementation
- * Uses BullMQ's delayed jobs for one-time execution
- */
+import { getScheduledTasks } from '@pikku/core/scheduler';
+const RECURRING_QUEUE_NAME = 'pikku-recurring-scheduled-task';
 export class BullSchedulerService extends SchedulerService {
+    redisConnectionOptions;
     queue;
+    recurringQueue;
+    recurringWorker;
+    repeatJobKeys = [];
+    handlers;
     constructor(redisConnectionOptions) {
         super();
+        this.redisConnectionOptions = redisConnectionOptions;
         this.queue = new Queue('pikku-remote-internal-rpc', {
             connection: redisConnectionOptions,
         });
+        this.recurringQueue = new Queue(RECURRING_QUEUE_NAME, {
+            connection: redisConnectionOptions,
+        });
+    }
+    /**
+     * Set services needed for processing recurring tasks.
+     * Must be called before start() since the scheduler is typically
+     * created before singletonServices are available.
+     */
+    setServices(handlers) {
+        this.handlers = handlers;
     }
     /**
      * Initialize - wait for queue to be ready
@@ -95,6 +110,51 @@ export class BullSchedulerService extends SchedulerService {
      * Close the queue connection
      */
     async close() {
+        if (this.recurringWorker) {
+            await this.recurringWorker.close();
+        }
+        await this.recurringQueue.close();
         await this.queue.close();
     }
+    /**
+     * Start recurring scheduled tasks.
+     * Creates a BullMQ Worker to process repeat jobs via runScheduledTask.
+     */
+    async start() {
+        if (!this.handlers) {
+            throw new Error('BullSchedulerService requires setServices() before start()');
+        }
+        const scheduledTasks = getScheduledTasks();
+        // Create a worker to process recurring scheduled task jobs
+        this.recurringWorker = new Worker(RECURRING_QUEUE_NAME, async (job) => {
+            const { rpcName } = job.data;
+            this.handlers.logger.info(`Running scheduled task: ${rpcName}`);
+            await this.handlers.runScheduledTask(rpcName);
+        }, { connection: this.redisConnectionOptions });
+        this.recurringWorker.on('error', (err) => {
+            this.handlers.logger.error(`Recurring task worker error: ${err}`);
+        });
+        for (const [name, task] of scheduledTasks) {
+            const job = await this.recurringQueue.add(name, { rpcName: name }, {
+                repeat: { pattern: task.schedule },
+                jobId: `recurring:${name}`,
+            });
+            if (job.repeatJobKey) {
+                this.repeatJobKeys.push(job.repeatJobKey);
+            }
+        }
+    }
+    /**
+     * Stop recurring scheduled tasks by removing repeat jobs.
+     */
+    async stop() {
+        for (const key of this.repeatJobKeys) {
+            await this.recurringQueue.removeRepeatableByKey(key);
+        }
+        this.repeatJobKeys = [];
+        if (this.recurringWorker) {
+            await this.recurringWorker.close();
+            this.recurringWorker = undefined;
+        }
+    }
 }

package/dist/bull-service-factory.d.ts

@@ -2,7 +2,7 @@ import { ConnectionOptions } from 'bullmq';
 import { BullQueueService } from './bull-queue-service.js';
 import { BullQueueWorkers } from './bull-queue-worker.js';
 import { BullSchedulerService } from './bull-scheduler-service.js';
-import type { CoreServices, CoreSingletonServices, CreateWireServices } from '@pikku/core';
+import type { CoreSingletonServices, CreateWireServices } from '@pikku/core';
 /**
  * Factory class for BullMQ services
  * Manages Redis connection options shared across queue, worker, and scheduler services
@@ -24,13 +24,18 @@ export declare class BullServiceFactory {
     /**
      * Get the queue workers for processing jobs
      */
-    getQueueWorkers(singletonServices: CoreSingletonServices, createWireServices?: CreateWireServices<CoreSingletonServices, CoreServices, any>): BullQueueWorkers;
+    getQueueWorkers(singletonServices: CoreSingletonServices, createWireServices?: CreateWireServices): BullQueueWorkers;
     /**
      * Get the scheduler service for managing delayed tasks
      */
     getSchedulerService(): BullSchedulerService;
+    setSchedulerRuntime(singletonServices: CoreSingletonServices, createWireServices?: CreateWireServices): void;
     /**
      * Close all services
      */
     close(): Promise<void>;
+    /**
+     * Alias for close() - used by stopSingletonServices
+     */
+    stop(): Promise<void>;
 }

package/dist/bull-service-factory.js

@@ -1,6 +1,8 @@
 import { BullQueueService } from './bull-queue-service.js';
 import { BullQueueWorkers } from './bull-queue-worker.js';
 import { BullSchedulerService } from './bull-scheduler-service.js';
+import { createQueueJobRunner } from '@pikku/core/queue';
+import { createSchedulerRuntimeHandlers } from '@pikku/core/scheduler';
 /**
  * Factory class for BullMQ services
  * Manages Redis connection options shared across queue, worker, and scheduler services
@@ -33,8 +35,9 @@ export class BullServiceFactory {
      */
     getQueueWorkers(singletonServices, createWireServices) {
         if (!this.queueWorkers) {
-            this.queueWorkers = new BullQueueWorkers(this.redisConnectionOptions, singletonServices, createWireServices);
+            this.queueWorkers = new BullQueueWorkers(this.redisConnectionOptions);
         }
+        this.queueWorkers.setJobRunner(createQueueJobRunner({ singletonServices, createWireServices }), singletonServices.logger);
         return this.queueWorkers;
     }
     /**
@@ -46,6 +49,12 @@ export class BullServiceFactory {
         }
         return this.schedulerService;
     }
+    setSchedulerRuntime(singletonServices, createWireServices) {
+        this.getSchedulerService().setServices(createSchedulerRuntimeHandlers({
+            singletonServices,
+            createWireServices,
+        }));
+    }
     /**
      * Close all services
      */
@@ -62,4 +71,10 @@ export class BullServiceFactory {
         }
         await Promise.all(closePromises);
     }
+    /**
+     * Alias for close() - used by stopSingletonServices
+     */
+    async stop() {
+        await this.close();
+    }
 }

package/dist/utils.js

@@ -29,10 +29,11 @@ export const mapBullJobToQueueJob = async (bullJob, redisConnection, queueEvents
         result: bullJob.returnvalue,
         status: async () => mapBullStateToStatus(await bullJob.getState()),
         waitForCompletion: async (ttl) => {
-            let queueEvents = queueEventsMap.get(bullJob.name);
+            let queueEvents = queueEventsMap.get(bullJob.queueName);
             if (!queueEvents) {
-                queueEvents = new QueueEvents(bullJob.name, redisConnection ? { connection: redisConnection } : undefined);
-                queueEventsMap.set(bullJob.name, queueEvents);
+                queueEvents = new QueueEvents(bullJob.queueName, redisConnection ? { connection: redisConnection } : undefined);
+                queueEventsMap.set(bullJob.queueName, queueEvents);
+                await queueEvents.waitUntilReady();
             }
             return await bullJob.waitUntilFinished(queueEvents, ttl);
         },

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@pikku/queue-bullmq",
-  "version": "0.11.1",
+  "version": "0.12.0",
   "author": "yasser.fadl@gmail.com",
   "license": "MIT",
   "module": "dist/index.js",
@@ -13,11 +13,11 @@
     "release": "npm run build && npm test"
   },
   "peerDependencies": {
-    "@pikku/core": "^0.11.1"
+    "@pikku/core": "^0.12.0"
   },
   "dependencies": {
-    "bullmq": "^5.63.2",
-    "ioredis": "^5.8.2"
+    "bullmq": "^5.67.3",
+    "ioredis": "^5.9.2"
   },
   "devDependencies": {
     "typescript": "^5.9"

package/src/bull-queue-worker.test.ts

@@ -0,0 +1,15 @@
+import { describe, test } from 'node:test'
+import assert from 'node:assert/strict'
+
+import { BullQueueWorkers } from './bull-queue-worker.js'
+
+describe('BullQueueWorkers', () => {
+  test('requires setJobRunner before registerQueues', async () => {
+    const workers = new BullQueueWorkers({})
+
+    await assert.rejects(() => workers.registerQueues(), {
+      message:
+        'BullQueueWorkers requires setJobRunner() before registerQueues()',
+    })
+  })
+})

package/src/bull-queue-worker.ts

@@ -9,18 +9,14 @@ import type {
   PikkuWorkerConfig,
   QueueConfigMapping,
   ConfigValidationResult,
+  QueueJob,
 } from '@pikku/core/queue'
 import {
-  runQueueJob,
   registerQueueWorkers,
   QueueJobFailedError,
   QueueJobDiscardedError,
 } from '@pikku/core/queue'
-import type {
-  CoreServices,
-  CoreSingletonServices,
-  CreateWireServices,
-} from '@pikku/core'
+import type { Logger } from '@pikku/core/services'
 import { mapBullJobToQueueJob } from './utils.js'
 
 export const mapPikkuWorkerToBull = (
@@ -141,32 +137,46 @@ export class BullQueueWorkers implements QueueWorkers {
 
   private workers = new Map<string, Bull.Worker>()
   private queueEvents = new Map<string, QueueEvents>()
-
-  constructor(
-    private redisConnectionOptions: ConnectionOptions,
-    private singletonServices: CoreSingletonServices,
-    private createWireServices?: CreateWireServices<
-      CoreSingletonServices,
-      CoreServices,
-      any
-    >
-  ) {}
+  private runJob?:
+    | ((params: {
+        job: QueueJob
+        updateProgress?: (progress: number | string | object) => Promise<void>
+      }) => Promise<void>)
+    | undefined
+  private logger?: Logger
+
+  constructor(private redisConnectionOptions: ConnectionOptions) {}
+
+  setJobRunner(
+    runJob: (params: {
+      job: QueueJob
+      updateProgress?: (progress: number | string | object) => Promise<void>
+    }) => Promise<void>,
+    logger: Logger
+  ): void {
+    this.runJob = runJob
+    this.logger = logger
+  }
 
   /**
    * Scan state and register all compatible processors
    */
   async registerQueues(): Promise<Record<string, ConfigValidationResult[]>> {
+    if (!this.runJob || !this.logger) {
+      throw new Error(
+        'BullQueueWorkers requires setJobRunner() before registerQueues()'
+      )
+    }
+
     return await registerQueueWorkers(
       this.configMappings,
-      this.singletonServices.logger,
+      this.logger,
       async (queueName, processor) => {
         const worker = new Worker(
-          processor.queueName,
+          processor.name,
           async (job: Bull.Job) => {
             try {
-              return await runQueueJob({
-                singletonServices: this.singletonServices,
-                createWireServices: this.createWireServices,
+              return await this.runJob!({
                 job: await mapBullJobToQueueJob(
                   job,
                   this.redisConnectionOptions,

package/src/bull-scheduler-service.ts

@@ -1,11 +1,13 @@
-import { ConnectionOptions, Queue } from 'bullmq'
+import { ConnectionOptions, Queue, Worker } from 'bullmq'
 import {
   SchedulerService,
+  SchedulerRuntimeHandlers,
   ScheduledTaskInfo,
   ScheduledTaskSummary,
   CoreUserSession,
   parseDurationString,
 } from '@pikku/core'
+import { getScheduledTasks } from '@pikku/core/scheduler'
 
 /**
  * Data stored in scheduled job
@@ -16,18 +18,32 @@ interface ScheduledJobData {
   session?: CoreUserSession
 }
 
-/**
- * BullMQ scheduler service implementation
- * Uses BullMQ's delayed jobs for one-time execution
- */
+const RECURRING_QUEUE_NAME = 'pikku-recurring-scheduled-task'
+
 export class BullSchedulerService extends SchedulerService {
   private queue: Queue
+  private recurringQueue: Queue
+  private recurringWorker?: Worker
+  private repeatJobKeys: string[] = []
+  private handlers?: SchedulerRuntimeHandlers
 
-  constructor(redisConnectionOptions: ConnectionOptions) {
+  constructor(private redisConnectionOptions: ConnectionOptions) {
     super()
     this.queue = new Queue('pikku-remote-internal-rpc', {
       connection: redisConnectionOptions,
     })
+    this.recurringQueue = new Queue(RECURRING_QUEUE_NAME, {
+      connection: redisConnectionOptions,
+    })
+  }
+
+  /**
+   * Set services needed for processing recurring tasks.
+   * Must be called before start() since the scheduler is typically
+   * created before singletonServices are available.
+   */
+  setServices(handlers: SchedulerRuntimeHandlers): void {
+    this.handlers = handlers
   }
 
   /**
@@ -134,6 +150,66 @@ export class BullSchedulerService extends SchedulerService {
    * Close the queue connection
    */
   async close(): Promise<void> {
+    if (this.recurringWorker) {
+      await this.recurringWorker.close()
+    }
+    await this.recurringQueue.close()
     await this.queue.close()
   }
+
+  /**
+   * Start recurring scheduled tasks.
+   * Creates a BullMQ Worker to process repeat jobs via runScheduledTask.
+   */
+  async start(): Promise<void> {
+    if (!this.handlers) {
+      throw new Error(
+        'BullSchedulerService requires setServices() before start()'
+      )
+    }
+
+    const scheduledTasks = getScheduledTasks()
+
+    // Create a worker to process recurring scheduled task jobs
+    this.recurringWorker = new Worker(
+      RECURRING_QUEUE_NAME,
+      async (job) => {
+        const { rpcName } = job.data as ScheduledJobData
+        this.handlers!.logger.info(`Running scheduled task: ${rpcName}`)
+        await this.handlers!.runScheduledTask(rpcName)
+      },
+      { connection: this.redisConnectionOptions }
+    )
+    this.recurringWorker.on('error', (err) => {
+      this.handlers!.logger.error(`Recurring task worker error: ${err}`)
+    })
+
+    for (const [name, task] of scheduledTasks) {
+      const job = await this.recurringQueue.add(
+        name,
+        { rpcName: name } as ScheduledJobData,
+        {
+          repeat: { pattern: task.schedule },
+          jobId: `recurring:${name}`,
+        }
+      )
+      if (job.repeatJobKey) {
+        this.repeatJobKeys.push(job.repeatJobKey)
+      }
+    }
+  }
+
+  /**
+   * Stop recurring scheduled tasks by removing repeat jobs.
+   */
+  async stop(): Promise<void> {
+    for (const key of this.repeatJobKeys) {
+      await this.recurringQueue.removeRepeatableByKey(key)
+    }
+    this.repeatJobKeys = []
+    if (this.recurringWorker) {
+      await this.recurringWorker.close()
+      this.recurringWorker = undefined
+    }
+  }
 }

package/src/bull-service-factory.ts

@@ -2,11 +2,9 @@ import { ConnectionOptions } from 'bullmq'
 import { BullQueueService } from './bull-queue-service.js'
 import { BullQueueWorkers } from './bull-queue-worker.js'
 import { BullSchedulerService } from './bull-scheduler-service.js'
-import type {
-  CoreServices,
-  CoreSingletonServices,
-  CreateWireServices,
-} from '@pikku/core'
+import type { CoreSingletonServices, CreateWireServices } from '@pikku/core'
+import { createQueueJobRunner } from '@pikku/core/queue'
+import { createSchedulerRuntimeHandlers } from '@pikku/core/scheduler'
 
 /**
  * Factory class for BullMQ services
@@ -41,19 +39,15 @@ export class BullServiceFactory {
    */
   getQueueWorkers(
     singletonServices: CoreSingletonServices,
-    createWireServices?: CreateWireServices<
-      CoreSingletonServices,
-      CoreServices,
-      any
-    >
+    createWireServices?: CreateWireServices
   ): BullQueueWorkers {
     if (!this.queueWorkers) {
-      this.queueWorkers = new BullQueueWorkers(
-        this.redisConnectionOptions,
-        singletonServices,
-        createWireServices
-      )
+      this.queueWorkers = new BullQueueWorkers(this.redisConnectionOptions)
     }
+    this.queueWorkers.setJobRunner(
+      createQueueJobRunner({ singletonServices, createWireServices }),
+      singletonServices.logger
+    )
     return this.queueWorkers
   }
 
@@ -69,6 +63,18 @@ export class BullServiceFactory {
     return this.schedulerService
   }
 
+  setSchedulerRuntime(
+    singletonServices: CoreSingletonServices,
+    createWireServices?: CreateWireServices
+  ): void {
+    this.getSchedulerService().setServices(
+      createSchedulerRuntimeHandlers({
+        singletonServices,
+        createWireServices,
+      })
+    )
+  }
+
   /**
    * Close all services
    */
@@ -89,4 +95,11 @@ export class BullServiceFactory {
 
     await Promise.all(closePromises)
   }
+
+  /**
+   * Alias for close() - used by stopSingletonServices
+   */
+  async stop(): Promise<void> {
+    await this.close()
+  }
 }

package/src/utils.ts

@@ -36,13 +36,14 @@ export const mapBullJobToQueueJob = async <In, Out>(
     result: bullJob.returnvalue as Out,
     status: async () => mapBullStateToStatus(await bullJob.getState()),
     waitForCompletion: async (ttl?: number): Promise<Out> => {
-      let queueEvents = queueEventsMap.get(bullJob.name)
+      let queueEvents = queueEventsMap.get(bullJob.queueName)
       if (!queueEvents) {
         queueEvents = new QueueEvents(
-          bullJob.name,
+          bullJob.queueName,
           redisConnection ? { connection: redisConnection } : undefined
         )
-        queueEventsMap.set(bullJob.name, queueEvents)
+        queueEventsMap.set(bullJob.queueName, queueEvents)
+        await queueEvents.waitUntilReady()
       }
       return await bullJob.waitUntilFinished(queueEvents, ttl)
     },