light-async-queue 1.0.1 â 2.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +334 -40
- package/dist/src/constants.d.ts +68 -0
- package/dist/src/constants.d.ts.map +1 -0
- package/dist/src/constants.js +75 -0
- package/dist/src/constants.js.map +1 -0
- package/dist/src/dashboard/Dashboard.d.ts +70 -0
- package/dist/src/dashboard/Dashboard.d.ts.map +1 -0
- package/dist/src/dashboard/Dashboard.js +308 -0
- package/dist/src/dashboard/Dashboard.js.map +1 -0
- package/dist/src/dashboard/index.d.ts +3 -0
- package/dist/src/dashboard/index.d.ts.map +1 -0
- package/dist/src/dashboard/index.js +2 -0
- package/dist/src/dashboard/index.js.map +1 -0
- package/dist/src/index.d.ts +13 -1
- package/dist/src/index.d.ts.map +1 -1
- package/dist/src/index.js +11 -0
- package/dist/src/index.js.map +1 -1
- package/dist/src/queue/Backoff.d.ts.map +1 -1
- package/dist/src/queue/Backoff.js +2 -1
- package/dist/src/queue/Backoff.js.map +1 -1
- package/dist/src/queue/Job.d.ts +35 -4
- package/dist/src/queue/Job.d.ts.map +1 -1
- package/dist/src/queue/Job.js +97 -12
- package/dist/src/queue/Job.js.map +1 -1
- package/dist/src/queue/Queue.d.ts +73 -3
- package/dist/src/queue/Queue.d.ts.map +1 -1
- package/dist/src/queue/Queue.js +359 -36
- package/dist/src/queue/Queue.js.map +1 -1
- package/dist/src/queue/Scheduler.d.ts.map +1 -1
- package/dist/src/queue/Scheduler.js +8 -1
- package/dist/src/queue/Scheduler.js.map +1 -1
- package/dist/src/storage/FileStore.d.ts.map +1 -1
- package/dist/src/storage/FileStore.js +4 -3
- package/dist/src/storage/FileStore.js.map +1 -1
- package/dist/src/storage/MemoryStore.d.ts.map +1 -1
- package/dist/src/storage/MemoryStore.js +2 -1
- package/dist/src/storage/MemoryStore.js.map +1 -1
- package/dist/src/types.d.ts +85 -10
- package/dist/src/types.d.ts.map +1 -1
- package/dist/src/types.js +4 -1
- package/dist/src/types.js.map +1 -1
- package/dist/src/utils/CronParser.d.ts +12 -0
- package/dist/src/utils/CronParser.d.ts.map +1 -0
- package/dist/src/utils/CronParser.js +28 -0
- package/dist/src/utils/CronParser.js.map +1 -0
- package/dist/src/utils/RateLimiter.d.ts +37 -0
- package/dist/src/utils/RateLimiter.d.ts.map +1 -0
- package/dist/src/utils/RateLimiter.js +68 -0
- package/dist/src/utils/RateLimiter.js.map +1 -0
- package/dist/src/utils/WebhookManager.d.ts +29 -0
- package/dist/src/utils/WebhookManager.d.ts.map +1 -0
- package/dist/src/utils/WebhookManager.js +82 -0
- package/dist/src/utils/WebhookManager.js.map +1 -0
- package/dist/src/worker/Worker.d.ts +2 -2
- package/dist/src/worker/Worker.d.ts.map +1 -1
- package/dist/src/worker/Worker.js +60 -38
- package/dist/src/worker/Worker.js.map +1 -1
- package/dist/src/worker/childProcessor.js +23 -7
- package/dist/src/worker/childProcessor.js.map +1 -1
- package/package.json +27 -5
package/README.md
CHANGED
|
@@ -7,19 +7,41 @@
|
|
|
7
7
|
[](https://nodejs.org)
|
|
8
8
|
[](https://www.typescriptlang.org/)
|
|
9
9
|
|
|
10
|
-
A production-ready, Redis-free async job queue for Node.js with TypeScript.
|
|
10
|
+
A production-ready, **Redis-free** async job queue for Node.js with TypeScript. A powerful BullMQ alternative designed for single-node reliability with file-based persistence, worker process isolation, and enterprise-grade features.
|
|
11
|
+
|
|
12
|
+
## ðĨ Demo Video
|
|
13
|
+
|
|
14
|
+
GitHub README often does not render local `.mp4` files inline. Use the direct raw link:
|
|
15
|
+
|
|
16
|
+
- [âķïļ Watch dashboard demo (streamable MP4)](https://raw.githubusercontent.com/gaikwadakshay79/light-async-queue/main/Light-Async-Queue-Dashboard.mp4)
|
|
17
|
+
- [ð Repo file link](./Light-Async-Queue-Dashboard.mp4)
|
|
11
18
|
|
|
12
19
|
## âĻ Features
|
|
13
20
|
|
|
21
|
+
### Core Features
|
|
22
|
+
|
|
14
23
|
- **ð Reliable Job Processing** - File-based persistence with crash recovery
|
|
15
24
|
- **ð· Worker Isolation** - Jobs execute in separate child processes using `child_process.fork()`
|
|
16
25
|
- **ð Smart Retry Logic** - Exponential backoff with configurable attempts
|
|
17
26
|
- **ð Dead Letter Queue** - Failed jobs are preserved and can be reprocessed
|
|
18
27
|
- **⥠Concurrency Control** - Configurable parallel job execution
|
|
19
28
|
- **ðĄïļ Graceful Shutdown** - Waits for active jobs before exiting
|
|
20
|
-
- **ð Queue Statistics** - Monitor active, pending, completed, and failed jobs
|
|
21
29
|
- **ðŊ TypeScript First** - Full type safety with no `any` types
|
|
22
|
-
- **ðŠķ
|
|
30
|
+
- **ðŠķ Minimal Dependencies** - Only 2 runtime dependencies (`cron-parser`, `ws`)
|
|
31
|
+
|
|
32
|
+
### Advanced Features (BullMQ Compatible)
|
|
33
|
+
|
|
34
|
+
- **ðĒ Job Events** - Listen to waiting, active, completed, failed, progress, stalled events
|
|
35
|
+
- **â° Delayed Jobs** - Schedule jobs to run after a delay
|
|
36
|
+
- **ð Repeating Jobs** - Interval-based and cron-pattern recurring jobs
|
|
37
|
+
- **ðŊ Job Priorities** - High-priority jobs run first
|
|
38
|
+
- **ð Progress Tracking** - Real-time job progress updates
|
|
39
|
+
- **ð Job Dependencies** - Jobs can wait for other jobs to complete
|
|
40
|
+
- **⥠Rate Limiting** - Control job execution rate
|
|
41
|
+
- **ð Webhooks** - HTTP callbacks for job events
|
|
42
|
+
- **âąïļ Stalled Job Detection** - Automatically detect and handle stuck jobs
|
|
43
|
+
- **ð Enhanced Metrics** - Detailed queue statistics by job status
|
|
44
|
+
- **ðĨïļ HTML Dashboard** - Real-time web UI for monitoring (Zookeeper-style)
|
|
23
45
|
|
|
24
46
|
## ðĶ Installation
|
|
25
47
|
|
|
@@ -49,17 +71,17 @@ The queue follows a producer-consumer pattern with the following components:
|
|
|
49
71
|
## ð Quick Start
|
|
50
72
|
|
|
51
73
|
```typescript
|
|
52
|
-
import { Queue } from "light-async-queue";
|
|
74
|
+
import { Queue, StorageType, BackoffStrategyType } from "light-async-queue";
|
|
53
75
|
|
|
54
76
|
// Create a queue
|
|
55
77
|
const queue = new Queue({
|
|
56
|
-
storage:
|
|
78
|
+
storage: StorageType.FILE,
|
|
57
79
|
filePath: "./jobs.log",
|
|
58
80
|
concurrency: 3,
|
|
59
81
|
retry: {
|
|
60
82
|
maxAttempts: 5,
|
|
61
83
|
backoff: {
|
|
62
|
-
type:
|
|
84
|
+
type: BackoffStrategyType.EXPONENTIAL,
|
|
63
85
|
delay: 1000, // 1 second base delay
|
|
64
86
|
},
|
|
65
87
|
},
|
|
@@ -91,43 +113,200 @@ Create a new queue instance.
|
|
|
91
113
|
**Config Options:**
|
|
92
114
|
|
|
93
115
|
```typescript
|
|
116
|
+
import {
|
|
117
|
+
StorageType,
|
|
118
|
+
BackoffStrategyType,
|
|
119
|
+
QueueEventType,
|
|
120
|
+
} from "light-async-queue";
|
|
121
|
+
|
|
94
122
|
interface QueueConfig {
|
|
95
|
-
storage:
|
|
96
|
-
filePath?: string; // Required if storage is
|
|
123
|
+
storage: StorageType;
|
|
124
|
+
filePath?: string; // Required if storage is StorageType.FILE
|
|
97
125
|
concurrency: number; // Max parallel jobs
|
|
98
126
|
retry: {
|
|
99
127
|
maxAttempts: number;
|
|
100
128
|
backoff: {
|
|
101
|
-
type:
|
|
129
|
+
type: BackoffStrategyType;
|
|
102
130
|
delay: number; // Base delay in ms
|
|
103
131
|
};
|
|
104
132
|
};
|
|
133
|
+
rateLimiter?: {
|
|
134
|
+
max: number; // Max jobs
|
|
135
|
+
duration: number; // Per duration in ms
|
|
136
|
+
};
|
|
137
|
+
webhooks?: Array<{
|
|
138
|
+
url: string;
|
|
139
|
+
events: QueueEventType[];
|
|
140
|
+
headers?: Record<string, string>;
|
|
141
|
+
}>;
|
|
142
|
+
stalledInterval?: number; // Check for stalled jobs every X ms (default: 30000)
|
|
105
143
|
}
|
|
106
144
|
```
|
|
107
145
|
|
|
108
146
|
### `queue.process(processor)`
|
|
109
147
|
|
|
110
|
-
Set the job processor function.
|
|
148
|
+
Set the job processor function with progress tracking support.
|
|
111
149
|
|
|
112
150
|
```typescript
|
|
113
|
-
queue.process(async (job
|
|
114
|
-
//
|
|
115
|
-
|
|
151
|
+
queue.process(async (job) => {
|
|
152
|
+
// Access job data
|
|
153
|
+
console.log(job.payload);
|
|
154
|
+
|
|
155
|
+
// Report progress
|
|
156
|
+
await job.updateProgress(50);
|
|
157
|
+
|
|
158
|
+
// Log messages
|
|
159
|
+
job.log("Processing step 1");
|
|
160
|
+
|
|
161
|
+
// Return result
|
|
162
|
+
return { success: true };
|
|
116
163
|
});
|
|
117
164
|
```
|
|
118
165
|
|
|
119
|
-
### `queue.add(payload)`
|
|
166
|
+
### `queue.add(payload, options?)`
|
|
120
167
|
|
|
121
|
-
Add a job to the queue.
|
|
168
|
+
Add a job to the queue with advanced options.
|
|
122
169
|
|
|
123
170
|
```typescript
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
171
|
+
// Simple job
|
|
172
|
+
const jobId = await queue.add({ userId: 123 });
|
|
173
|
+
|
|
174
|
+
// Job with priority (higher = more important)
|
|
175
|
+
await queue.add({ urgent: true }, { priority: 10 });
|
|
176
|
+
|
|
177
|
+
// Delayed job (runs after delay)
|
|
178
|
+
await queue.add({ task: "cleanup" }, { delay: 5000 });
|
|
179
|
+
|
|
180
|
+
// Repeating job (every X milliseconds)
|
|
181
|
+
await queue.add(
|
|
182
|
+
{ type: "heartbeat" },
|
|
183
|
+
{
|
|
184
|
+
repeat: {
|
|
185
|
+
every: 60000, // Every minute
|
|
186
|
+
limit: 100, // Max 100 repetitions
|
|
187
|
+
},
|
|
188
|
+
},
|
|
189
|
+
);
|
|
190
|
+
|
|
191
|
+
// Cron-style repeating job
|
|
192
|
+
await queue.add(
|
|
193
|
+
{ type: "daily-report" },
|
|
194
|
+
{
|
|
195
|
+
repeat: {
|
|
196
|
+
pattern: "0 0 * * *", // Every day at midnight
|
|
197
|
+
},
|
|
198
|
+
},
|
|
199
|
+
);
|
|
200
|
+
|
|
201
|
+
// Job with dependencies
|
|
202
|
+
const job1 = await queue.add({ step: 1 });
|
|
203
|
+
await queue.add({ step: 2 }, { dependsOn: [job1] });
|
|
204
|
+
|
|
205
|
+
// Custom job ID
|
|
206
|
+
await queue.add({ data: "test" }, { jobId: "custom-id-123" });
|
|
207
|
+
```
|
|
208
|
+
|
|
209
|
+
### Queue Events
|
|
210
|
+
|
|
211
|
+
Listen to job lifecycle events:
|
|
212
|
+
|
|
213
|
+
```typescript
|
|
214
|
+
import { QueueEventType } from "light-async-queue";
|
|
215
|
+
|
|
216
|
+
queue.on(QueueEventType.WAITING, (job) => {
|
|
217
|
+
console.log("Job waiting for dependencies:", job.id);
|
|
218
|
+
});
|
|
219
|
+
|
|
220
|
+
queue.on(QueueEventType.DELAYED, (job) => {
|
|
221
|
+
console.log("Job delayed until:", new Date(job.nextRunAt));
|
|
222
|
+
});
|
|
223
|
+
|
|
224
|
+
queue.on(QueueEventType.ACTIVE, (job) => {
|
|
225
|
+
console.log("Job started:", job.id);
|
|
127
226
|
});
|
|
227
|
+
|
|
228
|
+
queue.on(QueueEventType.PROGRESS, (job, progress) => {
|
|
229
|
+
console.log(`Job ${job.id} progress: ${progress}%`);
|
|
230
|
+
});
|
|
231
|
+
|
|
232
|
+
queue.on(QueueEventType.COMPLETED, (job, result) => {
|
|
233
|
+
console.log("Job completed:", job.id, result);
|
|
234
|
+
});
|
|
235
|
+
|
|
236
|
+
queue.on(QueueEventType.FAILED, (job, error) => {
|
|
237
|
+
console.error("Job failed:", job.id, error.message);
|
|
238
|
+
});
|
|
239
|
+
|
|
240
|
+
queue.on(QueueEventType.STALLED, (job) => {
|
|
241
|
+
console.warn("Job appears stalled:", job.id);
|
|
242
|
+
});
|
|
243
|
+
|
|
244
|
+
queue.on(QueueEventType.DRAINED, () => {
|
|
245
|
+
console.log("Queue drained - all jobs processed");
|
|
246
|
+
});
|
|
247
|
+
|
|
248
|
+
queue.on(QueueEventType.ERROR, (error) => {
|
|
249
|
+
console.error("Queue error:", error);
|
|
250
|
+
});
|
|
251
|
+
```
|
|
252
|
+
|
|
253
|
+
### Queue Methods
|
|
254
|
+
|
|
255
|
+
#### `queue.getJob(jobId)`
|
|
256
|
+
|
|
257
|
+
Get a specific job by ID.
|
|
258
|
+
|
|
259
|
+
```typescript
|
|
260
|
+
const job = await queue.getJob("job-id-123");
|
|
261
|
+
if (job) {
|
|
262
|
+
console.log(job.status, job.progress);
|
|
263
|
+
}
|
|
128
264
|
```
|
|
129
265
|
|
|
130
|
-
|
|
266
|
+
#### `queue.removeJob(jobId)`
|
|
267
|
+
|
|
268
|
+
Remove a specific job (only if not currently active).
|
|
269
|
+
|
|
270
|
+
```typescript
|
|
271
|
+
const removed = await queue.removeJob("job-id-123");
|
|
272
|
+
```
|
|
273
|
+
|
|
274
|
+
#### `queue.pause()`
|
|
275
|
+
|
|
276
|
+
Pause job processing.
|
|
277
|
+
|
|
278
|
+
```typescript
|
|
279
|
+
queue.pause();
|
|
280
|
+
```
|
|
281
|
+
|
|
282
|
+
#### `queue.resume()`
|
|
283
|
+
|
|
284
|
+
Resume job processing.
|
|
285
|
+
|
|
286
|
+
```typescript
|
|
287
|
+
queue.resume();
|
|
288
|
+
```
|
|
289
|
+
|
|
290
|
+
#### `queue.drain()`
|
|
291
|
+
|
|
292
|
+
Wait for all pending jobs to be processed.
|
|
293
|
+
|
|
294
|
+
```typescript
|
|
295
|
+
await queue.drain();
|
|
296
|
+
console.log("All jobs completed!");
|
|
297
|
+
```
|
|
298
|
+
|
|
299
|
+
#### `queue.clean(maxAge)`
|
|
300
|
+
|
|
301
|
+
Remove completed jobs older than maxAge (in milliseconds).
|
|
302
|
+
|
|
303
|
+
```typescript
|
|
304
|
+
// Clean jobs older than 24 hours
|
|
305
|
+
const cleaned = await queue.clean(24 * 60 * 60 * 1000);
|
|
306
|
+
console.log(`Cleaned ${cleaned} old jobs`);
|
|
307
|
+
```
|
|
308
|
+
|
|
309
|
+
#### `queue.getFailedJobs()`
|
|
131
310
|
|
|
132
311
|
Get all jobs in the Dead Letter Queue.
|
|
133
312
|
|
|
@@ -135,7 +314,7 @@ Get all jobs in the Dead Letter Queue.
|
|
|
135
314
|
const failedJobs = await queue.getFailedJobs();
|
|
136
315
|
```
|
|
137
316
|
|
|
138
|
-
|
|
317
|
+
#### `queue.reprocessFailed(jobId)`
|
|
139
318
|
|
|
140
319
|
Reprocess a failed job from the DLQ.
|
|
141
320
|
|
|
@@ -143,21 +322,24 @@ Reprocess a failed job from the DLQ.
|
|
|
143
322
|
await queue.reprocessFailed("job-id-here");
|
|
144
323
|
```
|
|
145
324
|
|
|
146
|
-
|
|
325
|
+
#### `queue.getStats()`
|
|
147
326
|
|
|
148
|
-
Get queue statistics.
|
|
327
|
+
Get enhanced queue statistics.
|
|
149
328
|
|
|
150
329
|
```typescript
|
|
151
330
|
const stats = await queue.getStats();
|
|
152
331
|
// {
|
|
153
332
|
// active: 2,
|
|
333
|
+
// waiting: 1,
|
|
334
|
+
// delayed: 3,
|
|
154
335
|
// pending: 5,
|
|
155
336
|
// completed: 100,
|
|
156
|
-
// failed: 3
|
|
337
|
+
// failed: 3,
|
|
338
|
+
// stalled: 0
|
|
157
339
|
// }
|
|
158
340
|
```
|
|
159
341
|
|
|
160
|
-
|
|
342
|
+
#### `queue.shutdown()`
|
|
161
343
|
|
|
162
344
|
Gracefully shutdown the queue.
|
|
163
345
|
|
|
@@ -190,8 +372,10 @@ After `maxAttempts`, jobs move to the Dead Letter Queue.
|
|
|
190
372
|
Fast, in-memory storage for development:
|
|
191
373
|
|
|
192
374
|
```typescript
|
|
375
|
+
import { Queue, StorageType } from "light-async-queue";
|
|
376
|
+
|
|
193
377
|
const queue = new Queue({
|
|
194
|
-
storage:
|
|
378
|
+
storage: StorageType.MEMORY,
|
|
195
379
|
concurrency: 5,
|
|
196
380
|
retry: {
|
|
197
381
|
/* ... */
|
|
@@ -204,8 +388,10 @@ const queue = new Queue({
|
|
|
204
388
|
Persistent, crash-recoverable storage for production:
|
|
205
389
|
|
|
206
390
|
```typescript
|
|
391
|
+
import { Queue, StorageType } from "light-async-queue";
|
|
392
|
+
|
|
207
393
|
const queue = new Queue({
|
|
208
|
-
storage:
|
|
394
|
+
storage: StorageType.FILE,
|
|
209
395
|
filePath: "./jobs.log",
|
|
210
396
|
concurrency: 5,
|
|
211
397
|
retry: {
|
|
@@ -257,16 +443,103 @@ The queue handles `SIGINT` and `SIGTERM` signals:
|
|
|
257
443
|
await queue.shutdown();
|
|
258
444
|
```
|
|
259
445
|
|
|
260
|
-
##
|
|
446
|
+
## ïŋ―ïļ HTML Dashboard - Real-Time Monitoring
|
|
447
|
+
|
|
448
|
+
Light Async Queue includes a built-in HTML dashboard for real-time monitoring, similar to Zookeeper. The dashboard provides a modern, responsive web interface for tracking job statuses and managing your queue.
|
|
449
|
+
|
|
450
|
+
### Quick Start
|
|
451
|
+
|
|
452
|
+
```typescript
|
|
453
|
+
import { Queue, Dashboard, StorageType } from "light-async-queue";
|
|
454
|
+
|
|
455
|
+
const queue = new Queue({
|
|
456
|
+
storage: StorageType.FILE,
|
|
457
|
+
filePath: "./jobs.log",
|
|
458
|
+
concurrency: 3,
|
|
459
|
+
retry: {
|
|
460
|
+
/* ... */
|
|
461
|
+
},
|
|
462
|
+
});
|
|
463
|
+
|
|
464
|
+
// Create and start dashboard
|
|
465
|
+
const dashboard = new Dashboard(queue, {
|
|
466
|
+
port: 3000,
|
|
467
|
+
host: "localhost",
|
|
468
|
+
updateInterval: 1000, // Update every 1 second
|
|
469
|
+
});
|
|
470
|
+
|
|
471
|
+
await dashboard.start();
|
|
472
|
+
console.log("ð Dashboard: http://localhost:3000");
|
|
473
|
+
```
|
|
474
|
+
|
|
475
|
+
### Dashboard Features
|
|
476
|
+
|
|
477
|
+
- **ð Real-time Statistics** - Live job counts (active, waiting, delayed, pending, completed, failed, stalled)
|
|
478
|
+
- **ð Job Tracking** - View active/waiting jobs with progress bars
|
|
479
|
+
- **â ïļ Dead Letter Queue** - Monitor and retry failed jobs from the UI
|
|
480
|
+
- **ð WebSocket Updates** - Fast, real-time data synchronization
|
|
481
|
+
- **ðĻ Modern UI** - Responsive design with color-coded status badges
|
|
482
|
+
- **ð Progress Visualization** - Track job completion with visual indicators
|
|
483
|
+
|
|
484
|
+
### API Endpoints
|
|
261
485
|
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
|
|
486
|
+
The dashboard exposes REST API endpoints:
|
|
487
|
+
|
|
488
|
+
- `GET /` - HTML dashboard interface
|
|
489
|
+
- `GET /api/stats` - Queue statistics (JSON)
|
|
490
|
+
- `GET /api/jobs` - Active and waiting jobs
|
|
491
|
+
- `GET /api/failed-jobs` - Failed jobs in DLQ
|
|
492
|
+
- `POST /api/reprocess-failed` - Retry a failed job
|
|
493
|
+
|
|
494
|
+
### Example Usage
|
|
495
|
+
|
|
496
|
+
See the [complete dashboard example](./example/dashboard-example.ts) for a working implementation with:
|
|
497
|
+
|
|
498
|
+
- Real-time job processing
|
|
499
|
+
- Progress tracking
|
|
500
|
+
- Event handling
|
|
501
|
+
- Failed job retry from UI
|
|
502
|
+
|
|
503
|
+
```bash
|
|
504
|
+
# Run the example
|
|
505
|
+
npm run build:examples
|
|
506
|
+
node dist/example/dashboard-example.js
|
|
507
|
+
# Open http://localhost:3000
|
|
508
|
+
```
|
|
509
|
+
|
|
510
|
+
For detailed dashboard documentation, see [Dashboard README](./src/dashboard/README.md).
|
|
511
|
+
|
|
512
|
+
## ïŋ―ð Comparison with BullMQ and Bull
|
|
513
|
+
|
|
514
|
+
| Feature | light-async-queue | BullMQ | Bull |
|
|
515
|
+
| ----------------- | ----------------------------- | ---------------- | --------------- |
|
|
516
|
+
| Redis Required | â No | â
Yes | â
Yes |
|
|
517
|
+
| File Persistence | â
Yes | â No | â No |
|
|
518
|
+
| Worker Isolation | â
Child Process | â ïļ Same Process | â ïļ Same Process |
|
|
519
|
+
| Crash Recovery | â
Built-in | â ïļ Needs Redis | â ïļ Needs Redis |
|
|
520
|
+
| Job Events | â
Yes | â
Yes | â
Yes |
|
|
521
|
+
| Job Priorities | â
Yes | â
Yes | â
Yes |
|
|
522
|
+
| Delayed Jobs | â
Yes | â
Yes | â
Yes |
|
|
523
|
+
| Repeating Jobs | â
Yes | â
Yes | â
Yes |
|
|
524
|
+
| Cron Patterns | â
Yes | â
Yes | â
Yes |
|
|
525
|
+
| Job Dependencies | â
Yes | â
Yes | â No |
|
|
526
|
+
| Progress Tracking | â
Yes | â
Yes | â
Yes |
|
|
527
|
+
| Rate Limiting | â
Yes | â
Yes | â
Yes |
|
|
528
|
+
| Webhooks | â
Yes | â No | â No |
|
|
529
|
+
| Stalled Detection | â
Yes | â
Yes | â
Yes |
|
|
530
|
+
| HTML Dashboard | â
Built-in | â ïļ Bull Board | â ïļ Bull Board |
|
|
531
|
+
| Setup Complexity | ðĒ Low | ðĄ Medium | ðĄ Medium |
|
|
532
|
+
| Dependencies | ðĒ Minimal (cron-parser + ws) | ðī Redis + deps | ðī Redis + deps |
|
|
533
|
+
| Best For | Single-node apps | Distributed apps | Legacy apps |
|
|
534
|
+
|
|
535
|
+
**Why choose light-async-queue?**
|
|
536
|
+
|
|
537
|
+
- â
No Redis infrastructure or maintenance
|
|
538
|
+
- â
Built-in crash recovery with file persistence
|
|
539
|
+
- â
True process isolation for better fault tolerance
|
|
540
|
+
- â
Minimal runtime dependencies (`cron-parser`, `ws`)
|
|
541
|
+
- â
Perfect for edge deployments, serverless, or single-server apps
|
|
542
|
+
- â
All BullMQ features without the complexity
|
|
270
543
|
|
|
271
544
|
## ðŊ Use Cases
|
|
272
545
|
|
|
@@ -275,22 +548,22 @@ Perfect for:
|
|
|
275
548
|
- **Single-server applications** that don't need Redis
|
|
276
549
|
- **Background job processing** (emails, reports, etc.)
|
|
277
550
|
- **Reliable task queues** with crash recovery
|
|
278
|
-
- **Development environments**
|
|
551
|
+
- **Development environments** with minimal external dependencies
|
|
279
552
|
- **Edge deployments** where Redis isn't available
|
|
280
553
|
|
|
281
554
|
## ð§ Advanced Example
|
|
282
555
|
|
|
283
556
|
```typescript
|
|
284
|
-
import { Queue } from "light-async-queue";
|
|
557
|
+
import { Queue, StorageType, BackoffStrategyType } from "light-async-queue";
|
|
285
558
|
|
|
286
559
|
const queue = new Queue({
|
|
287
|
-
storage:
|
|
560
|
+
storage: StorageType.FILE,
|
|
288
561
|
filePath: "./production-jobs.log",
|
|
289
562
|
concurrency: 10,
|
|
290
563
|
retry: {
|
|
291
564
|
maxAttempts: 3,
|
|
292
565
|
backoff: {
|
|
293
|
-
type:
|
|
566
|
+
type: BackoffStrategyType.EXPONENTIAL,
|
|
294
567
|
delay: 2000,
|
|
295
568
|
},
|
|
296
569
|
},
|
|
@@ -351,10 +624,27 @@ npm run build
|
|
|
351
624
|
npm run example
|
|
352
625
|
```
|
|
353
626
|
|
|
354
|
-
**Test Results:** â
|
|
627
|
+
**Test Results:** â
85+ tests passing across 8 test suites (powered by Vitest)
|
|
355
628
|
|
|
356
629
|
See [TEST_SUITE.md](./TEST_SUITE.md) for detailed test documentation.
|
|
357
630
|
|
|
631
|
+
## ð Examples
|
|
632
|
+
|
|
633
|
+
Check out the `example/` directory for comprehensive examples:
|
|
634
|
+
|
|
635
|
+
- **[basic.ts](./example/basic.ts)** - Simple queue setup and job processing
|
|
636
|
+
- **[concurrency.ts](./example/concurrency.ts)** - Concurrent job processing
|
|
637
|
+
- **[crash-recovery.ts](./example/crash-recovery.ts)** - Crash recovery demonstration
|
|
638
|
+
- **[advanced-features.ts](./example/advanced-features.ts)** - All BullMQ-compatible features:
|
|
639
|
+
- Job events and listeners
|
|
640
|
+
- Job priorities
|
|
641
|
+
- Delayed and repeating jobs
|
|
642
|
+
- Cron patterns
|
|
643
|
+
- Job dependencies
|
|
644
|
+
- Progress tracking
|
|
645
|
+
- Rate limiting
|
|
646
|
+
- Webhooks
|
|
647
|
+
|
|
358
648
|
## ð License
|
|
359
649
|
|
|
360
650
|
MIT
|
|
@@ -363,6 +653,10 @@ MIT
|
|
|
363
653
|
|
|
364
654
|
Contributions welcome! This is a production-ready implementation focused on reliability and simplicity.
|
|
365
655
|
|
|
656
|
+
## ðĶ Publishing
|
|
657
|
+
|
|
658
|
+
This package uses [npm trusted publishing](https://docs.npmjs.com/generating-provenance-statements) for secure, token-free releases from GitHub Actions. Publishes are automatically triggered when version tags are pushed (e.g., `v1.0.0`). See [TRUSTED_PUBLISHING.md](./TRUSTED_PUBLISHING.md) for detailed setup instructions.
|
|
659
|
+
|
|
366
660
|
---
|
|
367
661
|
|
|
368
662
|
Built with âĪïļ for Node.js developers who need reliable job queues without Redis.
|
|
@@ -0,0 +1,68 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Job Status Enum
|
|
3
|
+
* Represents the different states a job can be in
|
|
4
|
+
*/
|
|
5
|
+
export declare enum JobStatus {
|
|
6
|
+
WAITING = "waiting",// Waiting to be picked up (new default)
|
|
7
|
+
DELAYED = "delayed",// Scheduled for future execution
|
|
8
|
+
PENDING = "pending",// Ready to be processed
|
|
9
|
+
PROCESSING = "processing",// Currently being processed
|
|
10
|
+
COMPLETED = "completed",// Successfully completed
|
|
11
|
+
FAILED = "failed",// Failed after all retries
|
|
12
|
+
STALLED = "stalled"
|
|
13
|
+
}
|
|
14
|
+
/**
|
|
15
|
+
* Backoff Strategy Type Enum
|
|
16
|
+
* Determines how to calculate delay between retries
|
|
17
|
+
*/
|
|
18
|
+
export declare enum BackoffStrategyType {
|
|
19
|
+
EXPONENTIAL = "exponential",
|
|
20
|
+
FIXED = "fixed"
|
|
21
|
+
}
|
|
22
|
+
/**
|
|
23
|
+
* Storage Type Enum
|
|
24
|
+
* Determines the storage backend for the queue
|
|
25
|
+
*/
|
|
26
|
+
export declare enum StorageType {
|
|
27
|
+
MEMORY = "memory",
|
|
28
|
+
FILE = "file"
|
|
29
|
+
}
|
|
30
|
+
/**
|
|
31
|
+
* Worker Message Type Enum
|
|
32
|
+
* Types of messages sent from parent to child worker process
|
|
33
|
+
*/
|
|
34
|
+
export declare enum WorkerMessageType {
|
|
35
|
+
EXECUTE = "execute",
|
|
36
|
+
SET_PROCESSOR = "setProcessor"
|
|
37
|
+
}
|
|
38
|
+
/**
|
|
39
|
+
* Worker Response Type Enum
|
|
40
|
+
* Types of responses sent from child to parent worker process
|
|
41
|
+
*/
|
|
42
|
+
export declare enum WorkerResponseType {
|
|
43
|
+
RESULT = "result"
|
|
44
|
+
}
|
|
45
|
+
/**
|
|
46
|
+
* Worker Signal Type Enum
|
|
47
|
+
* Special signals for worker communication
|
|
48
|
+
*/
|
|
49
|
+
export declare enum WorkerSignalType {
|
|
50
|
+
READY = "ready",
|
|
51
|
+
PROGRESS = "progress"
|
|
52
|
+
}
|
|
53
|
+
/**
|
|
54
|
+
* Queue Event Type Enum
|
|
55
|
+
* Events emitted by the queue
|
|
56
|
+
*/
|
|
57
|
+
export declare enum QueueEventType {
|
|
58
|
+
WAITING = "waiting",
|
|
59
|
+
DELAYED = "delayed",
|
|
60
|
+
ACTIVE = "active",
|
|
61
|
+
PROGRESS = "progress",
|
|
62
|
+
COMPLETED = "completed",
|
|
63
|
+
FAILED = "failed",
|
|
64
|
+
STALLED = "stalled",
|
|
65
|
+
DRAINED = "drained",
|
|
66
|
+
ERROR = "error"
|
|
67
|
+
}
|
|
68
|
+
//# sourceMappingURL=constants.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"constants.d.ts","sourceRoot":"","sources":["../../src/constants.ts"],"names":[],"mappings":"AAAA;;;GAGG;AACH,oBAAY,SAAS;IACnB,OAAO,YAAY,CAAO,wCAAwC;IAClE,OAAO,YAAY,CAAO,iCAAiC;IAC3D,OAAO,YAAY,CAAO,wBAAwB;IAClD,UAAU,eAAe,CAAE,4BAA4B;IACvD,SAAS,cAAc,CAAI,yBAAyB;IACpD,MAAM,WAAW,CAAU,2BAA2B;IACtD,OAAO,YAAY;CACpB;AAED;;;GAGG;AACH,oBAAY,mBAAmB;IAC7B,WAAW,gBAAgB;IAC3B,KAAK,UAAU;CAChB;AAED;;;GAGG;AACH,oBAAY,WAAW;IACrB,MAAM,WAAW;IACjB,IAAI,SAAS;CACd;AAED;;;GAGG;AACH,oBAAY,iBAAiB;IAC3B,OAAO,YAAY;IACnB,aAAa,iBAAiB;CAC/B;AAED;;;GAGG;AACH,oBAAY,kBAAkB;IAC5B,MAAM,WAAW;CAClB;AAED;;;GAGG;AACH,oBAAY,gBAAgB;IAC1B,KAAK,UAAU;IACf,QAAQ,aAAa;CACtB;AAED;;;GAGG;AACH,oBAAY,cAAc;IACxB,OAAO,YAAY;IACnB,OAAO,YAAY;IACnB,MAAM,WAAW;IACjB,QAAQ,aAAa;IACrB,SAAS,cAAc;IACvB,MAAM,WAAW;IACjB,OAAO,YAAY;IACnB,OAAO,YAAY;IACnB,KAAK,UAAU;CAChB"}
|
|
@@ -0,0 +1,75 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Job Status Enum
|
|
3
|
+
* Represents the different states a job can be in
|
|
4
|
+
*/
|
|
5
|
+
export var JobStatus;
|
|
6
|
+
(function (JobStatus) {
|
|
7
|
+
JobStatus["WAITING"] = "waiting";
|
|
8
|
+
JobStatus["DELAYED"] = "delayed";
|
|
9
|
+
JobStatus["PENDING"] = "pending";
|
|
10
|
+
JobStatus["PROCESSING"] = "processing";
|
|
11
|
+
JobStatus["COMPLETED"] = "completed";
|
|
12
|
+
JobStatus["FAILED"] = "failed";
|
|
13
|
+
JobStatus["STALLED"] = "stalled";
|
|
14
|
+
})(JobStatus || (JobStatus = {}));
|
|
15
|
+
/**
|
|
16
|
+
* Backoff Strategy Type Enum
|
|
17
|
+
* Determines how to calculate delay between retries
|
|
18
|
+
*/
|
|
19
|
+
export var BackoffStrategyType;
|
|
20
|
+
(function (BackoffStrategyType) {
|
|
21
|
+
BackoffStrategyType["EXPONENTIAL"] = "exponential";
|
|
22
|
+
BackoffStrategyType["FIXED"] = "fixed";
|
|
23
|
+
})(BackoffStrategyType || (BackoffStrategyType = {}));
|
|
24
|
+
/**
|
|
25
|
+
* Storage Type Enum
|
|
26
|
+
* Determines the storage backend for the queue
|
|
27
|
+
*/
|
|
28
|
+
export var StorageType;
|
|
29
|
+
(function (StorageType) {
|
|
30
|
+
StorageType["MEMORY"] = "memory";
|
|
31
|
+
StorageType["FILE"] = "file";
|
|
32
|
+
})(StorageType || (StorageType = {}));
|
|
33
|
+
/**
|
|
34
|
+
* Worker Message Type Enum
|
|
35
|
+
* Types of messages sent from parent to child worker process
|
|
36
|
+
*/
|
|
37
|
+
export var WorkerMessageType;
|
|
38
|
+
(function (WorkerMessageType) {
|
|
39
|
+
WorkerMessageType["EXECUTE"] = "execute";
|
|
40
|
+
WorkerMessageType["SET_PROCESSOR"] = "setProcessor";
|
|
41
|
+
})(WorkerMessageType || (WorkerMessageType = {}));
|
|
42
|
+
/**
|
|
43
|
+
* Worker Response Type Enum
|
|
44
|
+
* Types of responses sent from child to parent worker process
|
|
45
|
+
*/
|
|
46
|
+
export var WorkerResponseType;
|
|
47
|
+
(function (WorkerResponseType) {
|
|
48
|
+
WorkerResponseType["RESULT"] = "result";
|
|
49
|
+
})(WorkerResponseType || (WorkerResponseType = {}));
|
|
50
|
+
/**
|
|
51
|
+
* Worker Signal Type Enum
|
|
52
|
+
* Special signals for worker communication
|
|
53
|
+
*/
|
|
54
|
+
export var WorkerSignalType;
|
|
55
|
+
(function (WorkerSignalType) {
|
|
56
|
+
WorkerSignalType["READY"] = "ready";
|
|
57
|
+
WorkerSignalType["PROGRESS"] = "progress";
|
|
58
|
+
})(WorkerSignalType || (WorkerSignalType = {}));
|
|
59
|
+
/**
|
|
60
|
+
* Queue Event Type Enum
|
|
61
|
+
* Events emitted by the queue
|
|
62
|
+
*/
|
|
63
|
+
export var QueueEventType;
|
|
64
|
+
(function (QueueEventType) {
|
|
65
|
+
QueueEventType["WAITING"] = "waiting";
|
|
66
|
+
QueueEventType["DELAYED"] = "delayed";
|
|
67
|
+
QueueEventType["ACTIVE"] = "active";
|
|
68
|
+
QueueEventType["PROGRESS"] = "progress";
|
|
69
|
+
QueueEventType["COMPLETED"] = "completed";
|
|
70
|
+
QueueEventType["FAILED"] = "failed";
|
|
71
|
+
QueueEventType["STALLED"] = "stalled";
|
|
72
|
+
QueueEventType["DRAINED"] = "drained";
|
|
73
|
+
QueueEventType["ERROR"] = "error";
|
|
74
|
+
})(QueueEventType || (QueueEventType = {}));
|
|
75
|
+
//# sourceMappingURL=constants.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"constants.js","sourceRoot":"","sources":["../../src/constants.ts"],"names":[],"mappings":"AAAA;;;GAGG;AACH,MAAM,CAAN,IAAY,SAQX;AARD,WAAY,SAAS;IACnB,gCAAmB,CAAA;IACnB,gCAAmB,CAAA;IACnB,gCAAmB,CAAA;IACnB,sCAAyB,CAAA;IACzB,oCAAuB,CAAA;IACvB,8BAAiB,CAAA;IACjB,gCAAmB,CAAA;AACrB,CAAC,EARW,SAAS,KAAT,SAAS,QAQpB;AAED;;;GAGG;AACH,MAAM,CAAN,IAAY,mBAGX;AAHD,WAAY,mBAAmB;IAC7B,kDAA2B,CAAA;IAC3B,sCAAe,CAAA;AACjB,CAAC,EAHW,mBAAmB,KAAnB,mBAAmB,QAG9B;AAED;;;GAGG;AACH,MAAM,CAAN,IAAY,WAGX;AAHD,WAAY,WAAW;IACrB,gCAAiB,CAAA;IACjB,4BAAa,CAAA;AACf,CAAC,EAHW,WAAW,KAAX,WAAW,QAGtB;AAED;;;GAGG;AACH,MAAM,CAAN,IAAY,iBAGX;AAHD,WAAY,iBAAiB;IAC3B,wCAAmB,CAAA;IACnB,mDAA8B,CAAA;AAChC,CAAC,EAHW,iBAAiB,KAAjB,iBAAiB,QAG5B;AAED;;;GAGG;AACH,MAAM,CAAN,IAAY,kBAEX;AAFD,WAAY,kBAAkB;IAC5B,uCAAiB,CAAA;AACnB,CAAC,EAFW,kBAAkB,KAAlB,kBAAkB,QAE7B;AAED;;;GAGG;AACH,MAAM,CAAN,IAAY,gBAGX;AAHD,WAAY,gBAAgB;IAC1B,mCAAe,CAAA;IACf,yCAAqB,CAAA;AACvB,CAAC,EAHW,gBAAgB,KAAhB,gBAAgB,QAG3B;AAED;;;GAGG;AACH,MAAM,CAAN,IAAY,cAUX;AAVD,WAAY,cAAc;IACxB,qCAAmB,CAAA;IACnB,qCAAmB,CAAA;IACnB,mCAAiB,CAAA;IACjB,uCAAqB,CAAA;IACrB,yCAAuB,CAAA;IACvB,mCAAiB,CAAA;IACjB,qCAAmB,CAAA;IACnB,qCAAmB,CAAA;IACnB,iCAAe,CAAA;AACjB,CAAC,EAVW,cAAc,KAAd,cAAc,QAUzB"}
|