@boringnode/queue 0.0.1-alpha → 0.0.1-alpha.1

package/README.md

@@ -0,0 +1,334 @@
+# @boringnode/queue
+
+A simple and efficient queue system for Node.js applications. Built for simplicity and ease of use, `@boringnode/queue` allows you to dispatch background jobs and process them asynchronously with support for multiple queue adapters.
+
+## Installation
+
+```bash
+npm install @boringnode/queue
+```
+
+## Features
+
+- **Multiple Queue Adapters**: Support for Redis (production) and Sync (testing/development)
+- **Type-Safe Jobs**: Define jobs as TypeScript classes with typed payloads
+- **Delayed Jobs**: Schedule jobs to run after a specific delay
+- **Multiple Queues**: Organize jobs into different queues for better organization
+- **Worker Management**: Process jobs with configurable concurrency
+- **Auto-Discovery**: Automatically discover and register jobs from specified locations
+- **Priority Queues**: Process high-priority jobs first
+- **Retry with Backoff**: Automatic retries with exponential, linear, or fixed backoff strategies
+- **Job Timeout**: Automatically fail or retry jobs that exceed a time limit
+
+## Quick Start
+
+### 1. Define a Job
+
+Create a job by extending the `Job` class:
+
+```typescript
+import { Job } from '@boringnode/queue'
+import type { JobOptions } from '@boringnode/queue/types/main'
+
+interface SendEmailPayload {
+  to: string
+}
+
+export default class SendEmailJob extends Job<SendEmailPayload> {
+  static readonly jobName = 'SendEmailJob'
+
+  static options: JobOptions = {
+    queue: 'email',
+  }
+
+  async execute(): Promise<void> {
+    console.log(`Sending email to: ${this.payload.to}`)
+  }
+}
+```
+
+### 2. Configure the Queue Manager
+
+```typescript
+import { QueueManager } from '@boringnode/queue'
+import { redis } from '@boringnode/queue/drivers/redis_adapter'
+import { sync } from '@boringnode/queue/drivers/sync_adapter'
+import { Redis } from 'ioredis'
+
+const redisConnection = new Redis({
+  host: 'localhost',
+  port: 6379,
+  keyPrefix: 'boringnode::queue::',
+  db: 0,
+})
+
+const config = {
+  default: 'redis',
+
+  adapters: {
+    sync: sync(),
+    redis: redis(redisConnection),
+  },
+
+  worker: {
+    concurrency: 5,
+    pollingInterval: '10ms',
+  },
+
+  locations: ['./app/jobs/**/*.ts'],
+}
+
+await QueueManager.init(config)
+```
+
+### 3. Dispatch Jobs
+
+```typescript
+import SendEmailJob from './jobs/send_email_job.ts'
+
+// Dispatch immediately
+await SendEmailJob.dispatch({ to: 'user@example.com' })
+
+// Dispatch with delay
+await SendEmailJob.dispatch({ to: 'user@example.com' }).in('5m')
+```
+
+### 4. Start a Worker
+
+Create a worker to process jobs:
+
+```typescript
+import { Worker } from '@boringnode/queue'
+
+const worker = new Worker(config)
+await worker.start(['default', 'email', 'reports'])
+```
+
+## Configuration
+
+### Queue Manager Options
+
+```typescript
+interface QueueManagerConfig {
+  // Default adapter to use
+  default: string
+
+  // Available queue adapters
+  adapters: {
+    [key: string]: QueueAdapter
+  }
+
+  // Worker configuration
+  worker: {
+    concurrency: number
+    pollingInterval: string
+  }
+
+  // Job discovery locations
+  locations: string[]
+}
+```
+
+### Job Options
+
+Configure individual jobs with the `options` property:
+
+```typescript
+static options: JobOptions = {
+  queue: 'email',       // Queue name (default: 'default')
+  adapter: 'redis',     // Override default adapter
+  priority: 1,          // Lower number = higher priority (default: 5)
+  maxRetries: 3,        // Maximum retry attempts
+  timeout: '30s',       // Job timeout duration
+  failOnTimeout: true,  // Fail permanently on timeout (default: false, will retry)
+}
+```
+
+## Adapters
+
+### Redis Adapter
+
+For production use with distributed systems:
+
+```typescript
+import { redis } from '@boringnode/queue/drivers/redis_adapter'
+import { Redis } from 'ioredis'
+
+const redisConnection = new Redis({
+  host: 'localhost',
+  port: 6379,
+  keyPrefix: 'boringnode::queue::',
+})
+
+const adapter = redis(redisConnection)
+```
+
+### Sync Adapter
+
+For testing and development:
+
+```typescript
+import { sync } from '@boringnode/queue/drivers/sync_adapter'
+
+const adapter = sync()
+```
+
+## Worker Configuration
+
+Workers process jobs from one or more queues:
+
+```typescript
+const worker = new Worker(config)
+
+// Process specific queues
+await worker.start(['default', 'email', 'reports'])
+
+// Worker will:
+// - Process jobs with configured concurrency
+// - Poll queues at the configured interval
+// - Execute jobs in the order they were queued
+```
+
+## Delayed Jobs
+
+Schedule jobs to run in the future:
+
+```typescript
+// Various time formats
+await SendEmailJob.dispatch(payload).in('30s')  // 30 seconds
+await SendEmailJob.dispatch(payload).in('5m')   // 5 minutes
+await SendEmailJob.dispatch(payload).in('2h')   // 2 hours
+await SendEmailJob.dispatch(payload).in('1d')   // 1 day
+```
+
+## Priority
+
+Jobs with lower priority numbers are processed first:
+
+```typescript
+export default class UrgentJob extends Job<Payload> {
+  static readonly jobName = 'UrgentJob'
+
+  static options: JobOptions = {
+    priority: 1,  // Processed before default priority (5)
+  }
+
+  async execute(): Promise<void> {
+    // ...
+  }
+}
+```
+
+## Retry and Backoff
+
+Configure automatic retries with backoff strategies:
+
+```typescript
+import { exponentialBackoff, linearBackoff, fixedBackoff } from '@boringnode/queue'
+
+export default class ReliableJob extends Job<Payload> {
+  static readonly jobName = 'ReliableJob'
+
+  static options: JobOptions = {
+    maxRetries: 5,
+    retry: {
+      backoff: () => exponentialBackoff({
+        baseDelay: '1s',
+        maxDelay: '1m',
+        multiplier: 2,
+        jitter: true,
+      }),
+    },
+  }
+
+  async execute(): Promise<void> {
+    // ...
+  }
+}
+```
+
+Available backoff strategies:
+
+- `exponentialBackoff({ baseDelay, maxDelay, multiplier, jitter })` - Exponential increase
+- `linearBackoff({ baseDelay, maxDelay, multiplier })` - Linear increase
+- `fixedBackoff({ baseDelay, jitter })` - Fixed delay between retries
+
+## Job Timeout
+
+Set a maximum execution time for jobs:
+
+```typescript
+export default class LimitedJob extends Job<Payload> {
+  static readonly jobName = 'LimitedJob'
+
+  static options: JobOptions = {
+    timeout: '30s',       // Maximum execution time
+    failOnTimeout: false, // Retry on timeout (default)
+  }
+
+  async execute(): Promise<void> {
+    // Long running operation...
+  }
+}
+```
+
+You can also set a global timeout in the worker configuration:
+
+```typescript
+const config = {
+  worker: {
+    timeout: '1m',  // Default timeout for all jobs
+  },
+}
+```
+
+## Job Discovery
+
+The queue manager automatically discovers and registers jobs from the specified locations:
+
+```typescript
+const config = {
+  locations: [
+    './app/jobs/**/*.ts',
+    './modules/**/jobs/**/*.ts',
+  ],
+}
+```
+
+Jobs must:
+- Extend the `Job` class
+- Have a static `jobName` property
+- Implement the `execute` method
+- Be exported as default
+
+## Benchmarks
+
+Performance comparison with BullMQ using realistic jobs (5ms simulated work per job):
+
+| Jobs | Concurrency | @boringnode/queue | BullMQ | Diff         |
+|------|-------------|-------------------|--------|--------------|
+| 100  | 1           | 562ms             | 596ms  | 5.7% faster  |
+| 100  | 5           | 116ms             | 117ms  | ~same        |
+| 100  | 10          | 62ms              | 62ms   | ~same        |
+| 500  | 1           | 2728ms            | 2798ms | 2.5% faster  |
+| 500  | 5           | 565ms             | 565ms  | ~same        |
+| 500  | 10          | 287ms             | 288ms  | ~same        |
+| 1000 | 1           | 5450ms            | 5547ms | 1.7% faster  |
+| 1000 | 5           | 1096ms            | 1116ms | 1.8% faster  |
+| 1000 | 10          | 565ms             | 579ms  | 2.4% faster  |
+| 100K | 5           | 110.5s            | 112.3s | 1.5% faster  |
+| 100K | 10          | 56.2s             | 57.5s  | 2.1% faster  |
+| 100K | 20          | 29.1s             | 29.6s  | 1.7% faster  |
+
+Run benchmarks yourself:
+
+```bash
+# Realistic benchmark (5ms job duration)
+npm run benchmark -- --realistic
+
+# Pure dequeue overhead (no-op jobs)
+npm run benchmark
+
+# Custom job duration
+npm run benchmark -- --duration=10
+```

package/build/index.d.ts

@@ -1,6 +1,5 @@
-import { Q as QueueManagerConfig, W as WorkerCycle, A as Adapter, R as RetryConfig } from './job-CcAUWe8j.js';
-export { J as Job, c as customBackoff, e as exponentialBackoff, f as fixedBackoff, l as linearBackoff } from './job-CcAUWe8j.js';
-import './src/contracts/lease_manager.js';
+import { Q as QueueManagerConfig, W as WorkerCycle, A as Adapter, R as RetryConfig } from './job-Bd_c2lFK.js';
+export { J as Job, c as customBackoff, e as exponentialBackoff, f as fixedBackoff, l as linearBackoff } from './job-Bd_c2lFK.js';
 
 declare class Worker {
     #private;

package/build/index.js

@@ -34,15 +34,16 @@ import debug from "#src/debug";
 import { parse } from "#src/utils";
 import * as errors from "#src/exceptions";
 import { QueueManager } from "#src/queue_manager";
+import { JobPool } from "#src/job_pool";
 import { Locator } from "#src/locator";
 var Worker = class {
   #id;
   #config;
   #adapter;
-  #leaseManager;
   #running = false;
   #initialized = false;
   #generator;
+  #pool;
   get id() {
     return this.#id;
   }
@@ -58,11 +59,7 @@ var Worker = class {
     debug("initializing worker %s", this.#id);
     await QueueManager.init(this.#config);
     this.#adapter = QueueManager.use();
-    this.#leaseManager = this.#adapter.createLeaseManager({
-      workerId: this.#id,
-      leaseTimeout: parse(this.#config.worker?.leaseTimeout || "5m"),
-      renewalInterval: parse(this.#config.worker?.renewalInterval || "5m")
-    });
+    this.#adapter.setWorkerId(this.#id);
     this.#initialized = true;
     debug("worker %s initialized", this.#id);
   }
@@ -93,8 +90,9 @@ var Worker = class {
   async stop() {
     debug("stopping worker %s", this.#id);
     this.#running = false;
-    if (this.#leaseManager) {
-      await this.#leaseManager.destroy();
+    if (this.#pool) {
+      debug("worker %s: waiting for %d running jobs to complete", this.#id, this.#pool.size);
+      await this.#pool.drain();
     }
     if (this.#adapter) {
       await this.#adapter.destroy();
@@ -114,56 +112,58 @@ var Worker = class {
     return result.value;
   }
   async *process(queues) {
-    const concurrency = this.#config.worker?.concurrency || 1;
-    runningLoop: while (this.#running) {
+    const pollingInterval = parse(this.#config.worker?.pollingInterval || "2s");
+    this.#pool = new JobPool();
+    while (this.#running) {
       try {
-        for (const queue of queues) {
-          const jobs = await this.#acquireJobs(queue, concurrency);
-          if (jobs.length > 0) {
-            for (const job of jobs) {
-              yield { type: "started", queue, job };
-            }
-            const results = await Promise.allSettled(jobs.map((job) => this.#execute(job, queue)));
-            for (const job of jobs) {
-              yield { type: "completed", queue, job };
-            }
-            const hasError = results.some((r) => r.status === "rejected");
-            if (hasError) {
-              const error = results.find((r) => r.status === "rejected");
-              yield { type: "error", error: error.reason, suggestedDelay: parse("5s") };
-            }
-            continue runningLoop;
-          }
+        yield* this.#fillPool(queues);
+        if (this.#pool.isEmpty()) {
+          yield { type: "idle", suggestedDelay: pollingInterval };
+          continue;
         }
-        const pollingInterval = parse(this.#config.worker?.pollingInterval || "2s");
-        yield { type: "idle", suggestedDelay: pollingInterval };
+        const completed = await this.#pool.waitForNextCompletion();
+        yield { type: "completed", queue: completed.queue, job: completed.job };
       } catch (error) {
         yield { type: "error", error, suggestedDelay: parse("5s") };
       }
     }
   }
+  async *#fillPool(queues) {
+    const concurrency = this.#config.worker?.concurrency || 1;
+    const slotsAvailable = concurrency - this.#pool.size;
+    if (slotsAvailable <= 0) return;
+    const popPromises = Array.from({ length: slotsAvailable }, () => this.#acquireNextJob(queues));
+    const results = await Promise.all(popPromises);
+    for (const result of results) {
+      if (!result) continue;
+      const { job, queue } = result;
+      const promise = this.#execute(job, queue);
+      this.#pool.add(job, queue, promise);
+      yield { type: "started", queue, job };
+    }
+  }
   async #execute(job, queue) {
     const startTime = performance.now();
     debug("worker %s: executing job %s (%s)", this.#id, job.id, job.name);
-    let JobClass;
-    try {
-      JobClass = Locator.getOrThrow(job.name);
-    } catch (error) {
-      debug("worker %s: job class %s not found for job %s", this.#id, job.name, job.id);
-      throw error;
-    }
-    const instance = new JobClass(job.payload);
-    const options = JobClass.options || {};
+    const { instance, options, timeout } = await this.#initJob(job, queue);
     try {
-      await instance.execute();
-      await job._lease.commit();
+      await this.#executeWithTimeout(instance, timeout);
+      await this.#adapter.completeJob(job.id, queue);
       const duration = (performance.now() - startTime).toFixed(2);
       debug("worker %s: successfully executed job %s in %dms", this.#id, job.id, duration);
     } catch (e) {
+      const isTimeout = e instanceof errors.E_JOB_TIMEOUT;
+      if (isTimeout && options.failOnTimeout) {
+        debug("worker %s: job %s timed out and failOnTimeout is set", this.#id, job.id);
+        await this.#adapter.failJob(job.id, queue, e);
+        await instance.failed?.(e);
+        return;
+      }
       const mergedConfig = QueueManager.getMergedRetryConfig(queue, options.retry);
       if (typeof mergedConfig.maxRetries === "undefined" || mergedConfig.maxRetries <= 0) {
         debug("worker %s: job %s has no retries configured, marking as failed", this.#id, job.id);
-        await instance.failed(e);
+        await this.#adapter.failJob(job.id, queue, e);
+        await instance.failed?.(e);
         return;
       }
       if (job.attempts >= mergedConfig.maxRetries) {
@@ -173,76 +173,65 @@ var Worker = class {
           job.id,
           mergedConfig.maxRetries
         );
+        await this.#adapter.failJob(job.id, queue, e);
         const exception = new errors.E_JOB_MAX_ATTEMPTS_REACHED([job.name]);
-        await instance.failed(exception);
+        await instance.failed?.(exception);
         return;
       }
       if (mergedConfig.backoff) {
         const strategy = mergedConfig.backoff();
         const nextRetryAt = strategy.getNextRetryAt(job.attempts + 1);
         debug("worker %s: job %s will retry at %s", this.#id, job.id, nextRetryAt.toISOString());
-        await this.#rollbackJobWithBackoff(job, queue, nextRetryAt);
+        await this.#adapter.retryJob(job.id, queue, nextRetryAt);
         return;
       }
-      await this.#rollbackJob(job, queue);
+      await this.#adapter.retryJob(job.id, queue);
     }
   }
-  async #acquireJobs(queue, count) {
-    const jobs = [];
-    for (let i = 0; i < count; i++) {
-      const job = await this.#adapter.popFrom(queue);
-      if (!job) {
-        break;
-      }
-      debug("worker %s: attempting to acquire lease for job %s", this.#id, job.id);
-      try {
-        const acquired = await this.#leaseManager.acquire(job.id);
-        if (!acquired) {
-          debug("worker %s: failed to acquire lease for job %s", this.#id, job.id);
-          await this.#adapter.pushOn(queue, job);
-          continue;
-        }
-        debug("worker %s: acquired lease for job %s", this.#id, job.id);
-        jobs.push({
-          ...job,
-          _lease: {
-            commit: () => this.#commitJob(job.id),
-            rollback: () => this.#rollbackJob(job, queue)
-          }
-        });
-      } catch (error) {
-        console.log(error);
-        throw error;
-      }
+  async #initJob(job, queue) {
+    try {
+      const JobClass = Locator.getOrThrow(job.name);
+      const instance = new JobClass(job.payload);
+      const options = JobClass.options || {};
+      const timeout = this.#getJobTimeout(options);
+      return { instance, options, timeout };
+    } catch (error) {
+      debug("worker %s: failed to initialize job %s (%s)", this.#id, job.id, job.name);
+      await this.#adapter.failJob(job.id, queue, error);
+      throw error;
     }
-    return jobs;
   }
-  #commitJob(jobId) {
-    debug("worker %s: committing job %s", this.#id, jobId);
-    return this.#leaseManager.release(jobId);
+  #getJobTimeout(options) {
+    if (options.timeout !== void 0) {
+      return parse(options.timeout);
+    }
+    if (this.#config.worker?.timeout !== void 0) {
+      return parse(this.#config.worker.timeout);
+    }
+    return void 0;
   }
-  async #rollbackJob(job, queue) {
-    debug("worker %s: rolling back job %s", this.#id, job.id);
-    const updatedJob = {
-      ...job,
-      attempts: (job.attempts || 0) + 1
-    };
-    await Promise.all([this.#leaseManager.release(job.id), this.#adapter.pushOn(queue, updatedJob)]);
+  async #executeWithTimeout(instance, timeout) {
+    if (!timeout) {
+      return instance.execute();
+    }
+    const signal = AbortSignal.timeout(timeout);
+    const abortPromise = new Promise((_, reject) => {
+      signal.addEventListener("abort", () => {
+        reject(new errors.E_JOB_TIMEOUT([instance.constructor.name, timeout]));
+      });
+    });
+    await Promise.race([instance.execute(signal), abortPromise]);
   }
-  async #rollbackJobWithBackoff(job, queue, nextRetryAt) {
-    debug("worker %s: rolling back job %s with backoff", this.#id, job.id);
-    const updatedJob = {
-      ...job,
-      attempts: (job.attempts || 0) + 1,
-      nextRetryAt
-    };
-    await this.#leaseManager.release(job.id);
-    const delay = nextRetryAt.getTime() - Date.now();
-    if (delay > 0) {
-      await this.#adapter.pushLaterOn(queue, updatedJob, delay);
-    } else {
-      await this.#adapter.pushOn(queue, updatedJob);
+  async #acquireNextJob(queues) {
+    for (const queue of queues) {
+      const job = await this.#adapter.popFrom(queue);
+      if (!job) {
+        continue;
+      }
+      debug("worker %s: acquired job %s", this.#id, job.id);
+      return { job, queue };
     }
+    return null;
   }
   async #setupGracefulShutdown() {
     const shutdown = async () => {
@@ -262,11 +251,13 @@ import { Locator as Locator2 } from "#src/locator";
 var QueueManagerSingleton = class {
   #defaultAdapter;
   #adapters = {};
+  #adapterInstances = /* @__PURE__ */ new Map();
   #globalRetryConfig;
   #queueConfigs = /* @__PURE__ */ new Map();
   async init(config) {
     debug2("initializing queue manager with config: %O", config);
     this.#validateConfig(config);
+    this.#adapterInstances.clear();
     this.#defaultAdapter = config.default;
     this.#adapters = config.adapters;
     this.#globalRetryConfig = config.retry;
@@ -282,13 +273,19 @@ var QueueManagerSingleton = class {
     if (!adapter) {
       adapter = this.#defaultAdapter;
     }
-    const adapterInstance = this.#adapters[adapter];
-    if (!adapterInstance) {
+    const cached = this.#adapterInstances.get(adapter);
+    if (cached) {
+      return cached;
+    }
+    const adapterFactory = this.#adapters[adapter];
+    if (!adapterFactory) {
       throw new errors2.E_CONFIGURATION_ERROR([`Adapter "${adapter}" is not registered`]);
     }
     debug2('using adapter "%s"', adapter);
     try {
-      return adapterInstance();
+      const instance = adapterFactory();
+      this.#adapterInstances.set(adapter, instance);
+      return instance;
     } catch (error) {
       throw new Error();
     }
@@ -325,10 +322,11 @@ var QueueManagerSingleton = class {
     }
   }
   async destroy() {
-    for (const adapterName in this.#adapters) {
-      const adapter = this.#adapters[adapterName]();
+    for (const [name, adapter] of this.#adapterInstances) {
+      debug2('destroying adapter "%s"', name);
       await adapter.destroy();
     }
+    this.#adapterInstances.clear();
   }
 };
 var QueueManager2 = new QueueManagerSingleton();