light-async-queue 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +385 -0
- package/dist/src/dlq/DeadLetterQueue.d.ts +34 -0
- package/dist/src/dlq/DeadLetterQueue.d.ts.map +1 -0
- package/dist/src/dlq/DeadLetterQueue.js +60 -0
- package/dist/src/dlq/DeadLetterQueue.js.map +1 -0
- package/dist/src/index.d.ts +14 -0
- package/dist/src/index.d.ts.map +1 -0
- package/dist/src/index.js +13 -0
- package/dist/src/index.js.map +1 -0
- package/dist/src/queue/Backoff.d.ts +24 -0
- package/dist/src/queue/Backoff.d.ts.map +1 -0
- package/dist/src/queue/Backoff.js +37 -0
- package/dist/src/queue/Backoff.js.map +1 -0
- package/dist/src/queue/Job.d.ts +44 -0
- package/dist/src/queue/Job.d.ts.map +1 -0
- package/dist/src/queue/Job.js +89 -0
- package/dist/src/queue/Job.js.map +1 -0
- package/dist/src/queue/Queue.d.ts +67 -0
- package/dist/src/queue/Queue.d.ts.map +1 -0
- package/dist/src/queue/Queue.js +261 -0
- package/dist/src/queue/Queue.js.map +1 -0
- package/dist/src/queue/Scheduler.d.ts +30 -0
- package/dist/src/queue/Scheduler.d.ts.map +1 -0
- package/dist/src/queue/Scheduler.js +62 -0
- package/dist/src/queue/Scheduler.js.map +1 -0
- package/dist/src/storage/FileStore.d.ts +55 -0
- package/dist/src/storage/FileStore.d.ts.map +1 -0
- package/dist/src/storage/FileStore.js +247 -0
- package/dist/src/storage/FileStore.js.map +1 -0
- package/dist/src/storage/MemoryStore.d.ts +21 -0
- package/dist/src/storage/MemoryStore.d.ts.map +1 -0
- package/dist/src/storage/MemoryStore.js +55 -0
- package/dist/src/storage/MemoryStore.js.map +1 -0
- package/dist/src/types.d.ts +126 -0
- package/dist/src/types.d.ts.map +1 -0
- package/dist/src/types.js +2 -0
- package/dist/src/types.js.map +1 -0
- package/dist/src/worker/Worker.d.ts +36 -0
- package/dist/src/worker/Worker.d.ts.map +1 -0
- package/dist/src/worker/Worker.js +170 -0
- package/dist/src/worker/Worker.js.map +1 -0
- package/dist/src/worker/childProcessor.d.ts +2 -0
- package/dist/src/worker/childProcessor.d.ts.map +1 -0
- package/dist/src/worker/childProcessor.js +70 -0
- package/dist/src/worker/childProcessor.js.map +1 -0
- package/package.json +71 -0
package/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Akshay Gaikwad
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
package/README.md
ADDED
|
@@ -0,0 +1,385 @@
|
|
|
1
|
+
# ð light-async-queue
|
|
2
|
+
|
|
3
|
+
[](https://www.npmjs.com/package/light-async-queue)
|
|
4
|
+
[](https://www.npmjs.com/package/light-async-queue)
|
|
5
|
+
[](https://github.com/gaikwadakshay79/light-async-queue/actions/workflows/ci.yml)
|
|
6
|
+
[](https://opensource.org/licenses/MIT)
|
|
7
|
+
[](https://nodejs.org)
|
|
8
|
+
[](https://www.typescriptlang.org/)
|
|
9
|
+
|
|
10
|
+
A production-ready, Redis-free async job queue for Node.js with TypeScript. Designed for single-node reliability with file-based persistence, worker process isolation, and crash recovery.
|
|
11
|
+
|
|
12
|
+
## âĻ Features
|
|
13
|
+
|
|
14
|
+
- **ð Reliable Job Processing** - File-based persistence with crash recovery
|
|
15
|
+
- **ð· Worker Isolation** - Jobs execute in separate child processes using `child_process.fork()`
|
|
16
|
+
- **ð Smart Retry Logic** - Exponential backoff with configurable attempts
|
|
17
|
+
- **ð Dead Letter Queue** - Failed jobs are preserved and can be reprocessed
|
|
18
|
+
- **⥠Concurrency Control** - Configurable parallel job execution
|
|
19
|
+
- **ðĄïļ Graceful Shutdown** - Waits for active jobs before exiting
|
|
20
|
+
- **ð Queue Statistics** - Monitor active, pending, completed, and failed jobs
|
|
21
|
+
- **ðŊ TypeScript First** - Full type safety with no `any` types
|
|
22
|
+
- **ðŠķ Zero Dependencies** - Uses only Node.js built-in modules
|
|
23
|
+
|
|
24
|
+
## ðĶ Installation
|
|
25
|
+
|
|
26
|
+
```bash
|
|
27
|
+
npm install light-async-queue
|
|
28
|
+
```
|
|
29
|
+
|
|
30
|
+
## ðïļ Architecture
|
|
31
|
+
|
|
32
|
+
```
|
|
33
|
+
âââââââââââââââââââââââââââââââââââââââââââââââââââââââââââââââ
|
|
34
|
+
â Queue API â
|
|
35
|
+
â (add jobs, process, getStats, shutdown) â
|
|
36
|
+
ââââââââââââââââââŽâââââââââââââââââââââââââââââââââââââââââââââ
|
|
37
|
+
â
|
|
38
|
+
ââââââââââââââīâââââââââââââ
|
|
39
|
+
â â
|
|
40
|
+
âââââžâââââââââ ââââââââžâââââââ
|
|
41
|
+
â Scheduler â â Storage â
|
|
42
|
+
â (200ms) ââââââââââĪ (Memory/ â
|
|
43
|
+
â â â File) â
|
|
44
|
+
âââââââŽâââââââ ââââââââŽâââââââ
|
|
45
|
+
â â
|
|
46
|
+
â job-ready â persist
|
|
47
|
+
â â
|
|
48
|
+
âââââââžâââââââââââââââââââââââžâââââââ
|
|
49
|
+
â Worker Pool â
|
|
50
|
+
â ââââââââ ââââââââ ââââââââ â
|
|
51
|
+
â âWorkerâ âWorkerâ âWorkerâ â
|
|
52
|
+
â â (CP) â â (CP) â â (CP) â â
|
|
53
|
+
â ââââââââ ââââââââ ââââââââ â
|
|
54
|
+
ââââââââââââââââââŽââââââââââââââââââââ
|
|
55
|
+
â
|
|
56
|
+
â on failure
|
|
57
|
+
â
|
|
58
|
+
âââââââââžâââââââââ
|
|
59
|
+
â Dead Letter â
|
|
60
|
+
â Queue (DLQ) â
|
|
61
|
+
ââââââââââââââââââ
|
|
62
|
+
|
|
63
|
+
CP = Child Process (isolated execution)
|
|
64
|
+
```
|
|
65
|
+
|
|
66
|
+
## ð Quick Start
|
|
67
|
+
|
|
68
|
+
```typescript
|
|
69
|
+
import { Queue } from "light-async-queue";
|
|
70
|
+
|
|
71
|
+
// Create a queue
|
|
72
|
+
const queue = new Queue({
|
|
73
|
+
storage: "file",
|
|
74
|
+
filePath: "./jobs.log",
|
|
75
|
+
concurrency: 3,
|
|
76
|
+
retry: {
|
|
77
|
+
maxAttempts: 5,
|
|
78
|
+
backoff: {
|
|
79
|
+
type: "exponential",
|
|
80
|
+
delay: 1000, // 1 second base delay
|
|
81
|
+
},
|
|
82
|
+
},
|
|
83
|
+
});
|
|
84
|
+
|
|
85
|
+
// Define job processor
|
|
86
|
+
queue.process(async (job) => {
|
|
87
|
+
console.log("Processing:", job.payload);
|
|
88
|
+
|
|
89
|
+
// Your job logic here
|
|
90
|
+
await sendEmail(job.payload.email);
|
|
91
|
+
|
|
92
|
+
return { success: true };
|
|
93
|
+
});
|
|
94
|
+
|
|
95
|
+
// Add jobs
|
|
96
|
+
await queue.add({
|
|
97
|
+
email: "user@example.com",
|
|
98
|
+
template: "welcome",
|
|
99
|
+
});
|
|
100
|
+
```
|
|
101
|
+
|
|
102
|
+
## ð API Reference
|
|
103
|
+
|
|
104
|
+
### `new Queue(config)`
|
|
105
|
+
|
|
106
|
+
Create a new queue instance.
|
|
107
|
+
|
|
108
|
+
**Config Options:**
|
|
109
|
+
|
|
110
|
+
```typescript
|
|
111
|
+
interface QueueConfig {
|
|
112
|
+
storage: "memory" | "file";
|
|
113
|
+
filePath?: string; // Required if storage is 'file'
|
|
114
|
+
concurrency: number; // Max parallel jobs
|
|
115
|
+
retry: {
|
|
116
|
+
maxAttempts: number;
|
|
117
|
+
backoff: {
|
|
118
|
+
type: "exponential";
|
|
119
|
+
delay: number; // Base delay in ms
|
|
120
|
+
};
|
|
121
|
+
};
|
|
122
|
+
}
|
|
123
|
+
```
|
|
124
|
+
|
|
125
|
+
### `queue.process(processor)`
|
|
126
|
+
|
|
127
|
+
Set the job processor function.
|
|
128
|
+
|
|
129
|
+
```typescript
|
|
130
|
+
queue.process(async (job: JobData) => {
|
|
131
|
+
// Process job
|
|
132
|
+
return result;
|
|
133
|
+
});
|
|
134
|
+
```
|
|
135
|
+
|
|
136
|
+
### `queue.add(payload)`
|
|
137
|
+
|
|
138
|
+
Add a job to the queue.
|
|
139
|
+
|
|
140
|
+
```typescript
|
|
141
|
+
const jobId = await queue.add({
|
|
142
|
+
userId: 123,
|
|
143
|
+
action: "send-email",
|
|
144
|
+
});
|
|
145
|
+
```
|
|
146
|
+
|
|
147
|
+
### `queue.getFailedJobs()`
|
|
148
|
+
|
|
149
|
+
Get all jobs in the Dead Letter Queue.
|
|
150
|
+
|
|
151
|
+
```typescript
|
|
152
|
+
const failedJobs = await queue.getFailedJobs();
|
|
153
|
+
```
|
|
154
|
+
|
|
155
|
+
### `queue.reprocessFailed(jobId)`
|
|
156
|
+
|
|
157
|
+
Reprocess a failed job from the DLQ.
|
|
158
|
+
|
|
159
|
+
```typescript
|
|
160
|
+
await queue.reprocessFailed("job-id-here");
|
|
161
|
+
```
|
|
162
|
+
|
|
163
|
+
### `queue.getStats()`
|
|
164
|
+
|
|
165
|
+
Get queue statistics.
|
|
166
|
+
|
|
167
|
+
```typescript
|
|
168
|
+
const stats = await queue.getStats();
|
|
169
|
+
// {
|
|
170
|
+
// active: 2,
|
|
171
|
+
// pending: 5,
|
|
172
|
+
// completed: 100,
|
|
173
|
+
// failed: 3
|
|
174
|
+
// }
|
|
175
|
+
```
|
|
176
|
+
|
|
177
|
+
### `queue.shutdown()`
|
|
178
|
+
|
|
179
|
+
Gracefully shutdown the queue.
|
|
180
|
+
|
|
181
|
+
```typescript
|
|
182
|
+
await queue.shutdown();
|
|
183
|
+
```
|
|
184
|
+
|
|
185
|
+
## ð Retry & Backoff
|
|
186
|
+
|
|
187
|
+
Jobs are retried with exponential backoff:
|
|
188
|
+
|
|
189
|
+
```
|
|
190
|
+
delay = baseDelay * (2 ^ (attempt - 1))
|
|
191
|
+
```
|
|
192
|
+
|
|
193
|
+
**Example with 1000ms base delay:**
|
|
194
|
+
|
|
195
|
+
- Attempt 1: Immediate
|
|
196
|
+
- Attempt 2: 1 second delay
|
|
197
|
+
- Attempt 3: 2 seconds delay
|
|
198
|
+
- Attempt 4: 4 seconds delay
|
|
199
|
+
- Attempt 5: 8 seconds delay
|
|
200
|
+
|
|
201
|
+
After `maxAttempts`, jobs move to the Dead Letter Queue.
|
|
202
|
+
|
|
203
|
+
## ðū Storage Options
|
|
204
|
+
|
|
205
|
+
### Memory Storage
|
|
206
|
+
|
|
207
|
+
Fast, in-memory storage for development:
|
|
208
|
+
|
|
209
|
+
```typescript
|
|
210
|
+
const queue = new Queue({
|
|
211
|
+
storage: "memory",
|
|
212
|
+
concurrency: 5,
|
|
213
|
+
retry: {
|
|
214
|
+
/* ... */
|
|
215
|
+
},
|
|
216
|
+
});
|
|
217
|
+
```
|
|
218
|
+
|
|
219
|
+
### File Storage
|
|
220
|
+
|
|
221
|
+
Persistent, crash-recoverable storage for production:
|
|
222
|
+
|
|
223
|
+
```typescript
|
|
224
|
+
const queue = new Queue({
|
|
225
|
+
storage: "file",
|
|
226
|
+
filePath: "./jobs.log",
|
|
227
|
+
concurrency: 5,
|
|
228
|
+
retry: {
|
|
229
|
+
/* ... */
|
|
230
|
+
},
|
|
231
|
+
});
|
|
232
|
+
```
|
|
233
|
+
|
|
234
|
+
**File Format:**
|
|
235
|
+
|
|
236
|
+
- Append-only log
|
|
237
|
+
- One JSON object per line
|
|
238
|
+
- Atomic writes
|
|
239
|
+
- Separate `dead-letter.log` for failed jobs
|
|
240
|
+
|
|
241
|
+
## ðĄïļ Crash Recovery
|
|
242
|
+
|
|
243
|
+
When using file storage, the queue automatically recovers from crashes:
|
|
244
|
+
|
|
245
|
+
1. **On startup**, the queue reads the job log
|
|
246
|
+
2. Any job with status `"processing"` is marked as `"pending"`
|
|
247
|
+
3. The job's `attempts` counter is incremented
|
|
248
|
+
4. The job is scheduled for immediate retry
|
|
249
|
+
|
|
250
|
+
This ensures no jobs are lost during unexpected shutdowns.
|
|
251
|
+
|
|
252
|
+
## ð· Worker Isolation
|
|
253
|
+
|
|
254
|
+
Jobs execute in isolated child processes:
|
|
255
|
+
|
|
256
|
+
- **Process Isolation**: Each job runs in a separate Node.js process
|
|
257
|
+
- **Crash Detection**: Parent detects worker crashes and retries the job
|
|
258
|
+
- **IPC Communication**: Results are sent back via inter-process communication
|
|
259
|
+
- **Resource Cleanup**: Workers are properly terminated on shutdown
|
|
260
|
+
|
|
261
|
+
## ð Graceful Shutdown
|
|
262
|
+
|
|
263
|
+
The queue handles `SIGINT` and `SIGTERM` signals:
|
|
264
|
+
|
|
265
|
+
1. Stop accepting new jobs
|
|
266
|
+
2. Wait for active jobs to complete
|
|
267
|
+
3. Terminate all worker processes
|
|
268
|
+
4. Persist final state to disk
|
|
269
|
+
5. Exit cleanly
|
|
270
|
+
|
|
271
|
+
```typescript
|
|
272
|
+
// Automatic on SIGINT/SIGTERM
|
|
273
|
+
// Or manual:
|
|
274
|
+
await queue.shutdown();
|
|
275
|
+
```
|
|
276
|
+
|
|
277
|
+
## ð Comparison with Bull
|
|
278
|
+
|
|
279
|
+
| Feature | light-queue | Bull |
|
|
280
|
+
| ---------------- | ---------------- | ------------------- |
|
|
281
|
+
| Redis Required | â No | â
Yes |
|
|
282
|
+
| File Persistence | â
Yes | â No |
|
|
283
|
+
| Worker Isolation | â
Child Process | â ïļ Same Process |
|
|
284
|
+
| Crash Recovery | â
Built-in | â ïļ Requires Redis |
|
|
285
|
+
| Setup Complexity | ðĒ Low | ðĄ Medium |
|
|
286
|
+
| Best For | Single-node apps | Distributed systems |
|
|
287
|
+
|
|
288
|
+
## ðŊ Use Cases
|
|
289
|
+
|
|
290
|
+
Perfect for:
|
|
291
|
+
|
|
292
|
+
- **Single-server applications** that don't need Redis
|
|
293
|
+
- **Background job processing** (emails, reports, etc.)
|
|
294
|
+
- **Reliable task queues** with crash recovery
|
|
295
|
+
- **Development environments** without external dependencies
|
|
296
|
+
- **Edge deployments** where Redis isn't available
|
|
297
|
+
|
|
298
|
+
## ð§ Advanced Example
|
|
299
|
+
|
|
300
|
+
```typescript
|
|
301
|
+
import { Queue } from "light-async-queue";
|
|
302
|
+
|
|
303
|
+
const queue = new Queue({
|
|
304
|
+
storage: "file",
|
|
305
|
+
filePath: "./production-jobs.log",
|
|
306
|
+
concurrency: 10,
|
|
307
|
+
retry: {
|
|
308
|
+
maxAttempts: 3,
|
|
309
|
+
backoff: {
|
|
310
|
+
type: "exponential",
|
|
311
|
+
delay: 2000,
|
|
312
|
+
},
|
|
313
|
+
},
|
|
314
|
+
});
|
|
315
|
+
|
|
316
|
+
// Email sending processor
|
|
317
|
+
queue.process(async (job) => {
|
|
318
|
+
const { email, template, data } = job.payload;
|
|
319
|
+
|
|
320
|
+
try {
|
|
321
|
+
await emailService.send({
|
|
322
|
+
to: email,
|
|
323
|
+
template,
|
|
324
|
+
data,
|
|
325
|
+
});
|
|
326
|
+
|
|
327
|
+
return { sent: true, timestamp: Date.now() };
|
|
328
|
+
} catch (error) {
|
|
329
|
+
// Will retry with exponential backoff
|
|
330
|
+
throw error;
|
|
331
|
+
}
|
|
332
|
+
});
|
|
333
|
+
|
|
334
|
+
// Add jobs
|
|
335
|
+
await queue.add({
|
|
336
|
+
email: "user@example.com",
|
|
337
|
+
template: "welcome",
|
|
338
|
+
data: { name: "John" },
|
|
339
|
+
});
|
|
340
|
+
|
|
341
|
+
// Monitor failed jobs
|
|
342
|
+
setInterval(async () => {
|
|
343
|
+
const stats = await queue.getStats();
|
|
344
|
+
console.log("Queue stats:", stats);
|
|
345
|
+
|
|
346
|
+
if (stats.failed > 0) {
|
|
347
|
+
const failed = await queue.getFailedJobs();
|
|
348
|
+
console.log("Failed jobs:", failed);
|
|
349
|
+
}
|
|
350
|
+
}, 60000);
|
|
351
|
+
```
|
|
352
|
+
|
|
353
|
+
## ð§Š Testing
|
|
354
|
+
|
|
355
|
+
```bash
|
|
356
|
+
# Run all tests
|
|
357
|
+
npm test
|
|
358
|
+
|
|
359
|
+
# Run tests in watch mode
|
|
360
|
+
npm run test:watch
|
|
361
|
+
|
|
362
|
+
# Run tests with coverage
|
|
363
|
+
npm run test:coverage
|
|
364
|
+
|
|
365
|
+
# Run examples
|
|
366
|
+
npm install
|
|
367
|
+
npm run build
|
|
368
|
+
npm run example
|
|
369
|
+
```
|
|
370
|
+
|
|
371
|
+
**Test Results:** â
25 tests passing across 4 test suites (powered by Vitest)
|
|
372
|
+
|
|
373
|
+
See [TEST_SUITE.md](./TEST_SUITE.md) for detailed test documentation.
|
|
374
|
+
|
|
375
|
+
## ð License
|
|
376
|
+
|
|
377
|
+
MIT
|
|
378
|
+
|
|
379
|
+
## ðĪ Contributing
|
|
380
|
+
|
|
381
|
+
Contributions welcome! This is a production-ready implementation focused on reliability and simplicity.
|
|
382
|
+
|
|
383
|
+
---
|
|
384
|
+
|
|
385
|
+
Built with âĪïļ for Node.js developers who need reliable job queues without Redis.
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
import { StorageInterface, JobData } from '../types.js';
|
|
2
|
+
import { Job } from '../queue/Job.js';
|
|
3
|
+
/**
|
|
4
|
+
* Dead Letter Queue for managing failed jobs
|
|
5
|
+
*/
|
|
6
|
+
export declare class DeadLetterQueue {
|
|
7
|
+
private storage;
|
|
8
|
+
constructor(storage: StorageInterface);
|
|
9
|
+
/**
|
|
10
|
+
* Move a job to the dead letter queue
|
|
11
|
+
*/
|
|
12
|
+
add(job: Job): Promise<void>;
|
|
13
|
+
/**
|
|
14
|
+
* Get all failed jobs
|
|
15
|
+
*/
|
|
16
|
+
getAll(): Promise<JobData[]>;
|
|
17
|
+
/**
|
|
18
|
+
* Get a specific failed job by ID
|
|
19
|
+
*/
|
|
20
|
+
get(jobId: string): Promise<JobData | null>;
|
|
21
|
+
/**
|
|
22
|
+
* Remove a job from DLQ and return it for reprocessing
|
|
23
|
+
*/
|
|
24
|
+
remove(jobId: string): Promise<Job | null>;
|
|
25
|
+
/**
|
|
26
|
+
* Get count of failed jobs
|
|
27
|
+
*/
|
|
28
|
+
count(): Promise<number>;
|
|
29
|
+
/**
|
|
30
|
+
* Clear all failed jobs (use with caution)
|
|
31
|
+
*/
|
|
32
|
+
clear(): Promise<void>;
|
|
33
|
+
}
|
|
34
|
+
//# sourceMappingURL=DeadLetterQueue.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"DeadLetterQueue.d.ts","sourceRoot":"","sources":["../../../src/dlq/DeadLetterQueue.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,gBAAgB,EAAE,OAAO,EAAE,MAAM,aAAa,CAAC;AACxD,OAAO,EAAE,GAAG,EAAE,MAAM,iBAAiB,CAAC;AAEtC;;GAEG;AACH,qBAAa,eAAe;IAC1B,OAAO,CAAC,OAAO,CAAmB;gBAEtB,OAAO,EAAE,gBAAgB;IAIrC;;OAEG;IACG,GAAG,CAAC,GAAG,EAAE,GAAG,GAAG,OAAO,CAAC,IAAI,CAAC;IAIlC;;OAEG;IACG,MAAM,IAAI,OAAO,CAAC,OAAO,EAAE,CAAC;IAIlC;;OAEG;IACG,GAAG,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,GAAG,IAAI,CAAC;IAKjD;;OAEG;IACG,MAAM,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,GAAG,GAAG,IAAI,CAAC;IAgBhD;;OAEG;IACG,KAAK,IAAI,OAAO,CAAC,MAAM,CAAC;IAK9B;;OAEG;IACG,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;CAM7B"}
|
|
@@ -0,0 +1,60 @@
|
|
|
1
|
+
import { Job } from '../queue/Job.js';
|
|
2
|
+
/**
|
|
3
|
+
* Dead Letter Queue for managing failed jobs
|
|
4
|
+
*/
|
|
5
|
+
export class DeadLetterQueue {
|
|
6
|
+
storage;
|
|
7
|
+
constructor(storage) {
|
|
8
|
+
this.storage = storage;
|
|
9
|
+
}
|
|
10
|
+
/**
|
|
11
|
+
* Move a job to the dead letter queue
|
|
12
|
+
*/
|
|
13
|
+
async add(job) {
|
|
14
|
+
await this.storage.moveToDeadLetter(job.toData());
|
|
15
|
+
}
|
|
16
|
+
/**
|
|
17
|
+
* Get all failed jobs
|
|
18
|
+
*/
|
|
19
|
+
async getAll() {
|
|
20
|
+
return this.storage.getFailedJobs();
|
|
21
|
+
}
|
|
22
|
+
/**
|
|
23
|
+
* Get a specific failed job by ID
|
|
24
|
+
*/
|
|
25
|
+
async get(jobId) {
|
|
26
|
+
const failedJobs = await this.storage.getFailedJobs();
|
|
27
|
+
return failedJobs.find(job => job.id === jobId) || null;
|
|
28
|
+
}
|
|
29
|
+
/**
|
|
30
|
+
* Remove a job from DLQ and return it for reprocessing
|
|
31
|
+
*/
|
|
32
|
+
async remove(jobId) {
|
|
33
|
+
const jobData = await this.get(jobId);
|
|
34
|
+
if (!jobData) {
|
|
35
|
+
return null;
|
|
36
|
+
}
|
|
37
|
+
await this.storage.removeFromDeadLetter(jobId);
|
|
38
|
+
// Create a Job instance and reset it for reprocessing
|
|
39
|
+
const job = Job.fromData(jobData);
|
|
40
|
+
job.reset();
|
|
41
|
+
return job;
|
|
42
|
+
}
|
|
43
|
+
/**
|
|
44
|
+
* Get count of failed jobs
|
|
45
|
+
*/
|
|
46
|
+
async count() {
|
|
47
|
+
const jobs = await this.storage.getFailedJobs();
|
|
48
|
+
return jobs.length;
|
|
49
|
+
}
|
|
50
|
+
/**
|
|
51
|
+
* Clear all failed jobs (use with caution)
|
|
52
|
+
*/
|
|
53
|
+
async clear() {
|
|
54
|
+
const jobs = await this.storage.getFailedJobs();
|
|
55
|
+
for (const job of jobs) {
|
|
56
|
+
await this.storage.removeFromDeadLetter(job.id);
|
|
57
|
+
}
|
|
58
|
+
}
|
|
59
|
+
}
|
|
60
|
+
//# sourceMappingURL=DeadLetterQueue.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"DeadLetterQueue.js","sourceRoot":"","sources":["../../../src/dlq/DeadLetterQueue.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,GAAG,EAAE,MAAM,iBAAiB,CAAC;AAEtC;;GAEG;AACH,MAAM,OAAO,eAAe;IAClB,OAAO,CAAmB;IAElC,YAAY,OAAyB;QACnC,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC;IACzB,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,GAAG,CAAC,GAAQ;QAChB,MAAM,IAAI,CAAC,OAAO,CAAC,gBAAgB,CAAC,GAAG,CAAC,MAAM,EAAE,CAAC,CAAC;IACpD,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,MAAM;QACV,OAAO,IAAI,CAAC,OAAO,CAAC,aAAa,EAAE,CAAC;IACtC,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,GAAG,CAAC,KAAa;QACrB,MAAM,UAAU,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,aAAa,EAAE,CAAC;QACtD,OAAO,UAAU,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,EAAE,KAAK,KAAK,CAAC,IAAI,IAAI,CAAC;IAC1D,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,MAAM,CAAC,KAAa;QACxB,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;QAEtC,IAAI,CAAC,OAAO,EAAE,CAAC;YACb,OAAO,IAAI,CAAC;QACd,CAAC;QAED,MAAM,IAAI,CAAC,OAAO,CAAC,oBAAoB,CAAC,KAAK,CAAC,CAAC;QAE/C,sDAAsD;QACtD,MAAM,GAAG,GAAG,GAAG,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC;QAClC,GAAG,CAAC,KAAK,EAAE,CAAC;QAEZ,OAAO,GAAG,CAAC;IACb,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,KAAK;QACT,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,aAAa,EAAE,CAAC;QAChD,OAAO,IAAI,CAAC,MAAM,CAAC;IACrB,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,KAAK;QACT,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,aAAa,EAAE,CAAC;QAChD,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;YACvB,MAAM,IAAI,CAAC,OAAO,CAAC,oBAAoB,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QAClD,CAAC;IACH,CAAC;CACF"}
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* light-queue - Production-ready Redis-free async job queue
|
|
3
|
+
*
|
|
4
|
+
* A reliable job queue for single-node applications with:
|
|
5
|
+
* - File-based persistence with crash recovery
|
|
6
|
+
* - Worker process isolation
|
|
7
|
+
* - Retry with exponential backoff
|
|
8
|
+
* - Dead letter queue for failed jobs
|
|
9
|
+
* - Graceful shutdown handling
|
|
10
|
+
*/
|
|
11
|
+
export { Queue } from './queue/Queue.js';
|
|
12
|
+
export { Job } from './queue/Job.js';
|
|
13
|
+
export type { QueueConfig, JobData, JobStatus, JobProcessor, RetryConfig, BackoffConfig, } from './types.js';
|
|
14
|
+
//# sourceMappingURL=index.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AAEH,OAAO,EAAE,KAAK,EAAE,MAAM,kBAAkB,CAAC;AACzC,OAAO,EAAE,GAAG,EAAE,MAAM,gBAAgB,CAAC;AACrC,YAAY,EACV,WAAW,EACX,OAAO,EACP,SAAS,EACT,YAAY,EACZ,WAAW,EACX,aAAa,GACd,MAAM,YAAY,CAAC"}
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* light-queue - Production-ready Redis-free async job queue
|
|
3
|
+
*
|
|
4
|
+
* A reliable job queue for single-node applications with:
|
|
5
|
+
* - File-based persistence with crash recovery
|
|
6
|
+
* - Worker process isolation
|
|
7
|
+
* - Retry with exponential backoff
|
|
8
|
+
* - Dead letter queue for failed jobs
|
|
9
|
+
* - Graceful shutdown handling
|
|
10
|
+
*/
|
|
11
|
+
export { Queue } from './queue/Queue.js';
|
|
12
|
+
export { Job } from './queue/Job.js';
|
|
13
|
+
//# sourceMappingURL=index.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AAEH,OAAO,EAAE,KAAK,EAAE,MAAM,kBAAkB,CAAC;AACzC,OAAO,EAAE,GAAG,EAAE,MAAM,gBAAgB,CAAC"}
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
import { BackoffConfig } from '../types.js';
|
|
2
|
+
/**
|
|
3
|
+
* Backoff calculator for retry delays
|
|
4
|
+
*/
|
|
5
|
+
export declare class Backoff {
|
|
6
|
+
private config;
|
|
7
|
+
constructor(config: BackoffConfig);
|
|
8
|
+
/**
|
|
9
|
+
* Calculate the next retry delay based on attempt number
|
|
10
|
+
* Formula: delay = baseDelay * (2 ^ (attempt - 1))
|
|
11
|
+
*
|
|
12
|
+
* @param attempt - Current attempt number (1-based)
|
|
13
|
+
* @returns Delay in milliseconds
|
|
14
|
+
*/
|
|
15
|
+
calculateDelay(attempt: number): number;
|
|
16
|
+
/**
|
|
17
|
+
* Calculate the next run timestamp for a job
|
|
18
|
+
*
|
|
19
|
+
* @param attempt - Current attempt number (1-based)
|
|
20
|
+
* @returns Unix timestamp in milliseconds
|
|
21
|
+
*/
|
|
22
|
+
getNextRunAt(attempt: number): number;
|
|
23
|
+
}
|
|
24
|
+
//# sourceMappingURL=Backoff.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"Backoff.d.ts","sourceRoot":"","sources":["../../../src/queue/Backoff.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AAE5C;;GAEG;AACH,qBAAa,OAAO;IAClB,OAAO,CAAC,MAAM,CAAgB;gBAElB,MAAM,EAAE,aAAa;IAIjC;;;;;;OAMG;IACH,cAAc,CAAC,OAAO,EAAE,MAAM,GAAG,MAAM;IAavC;;;;;OAKG;IACH,YAAY,CAAC,OAAO,EAAE,MAAM,GAAG,MAAM;CAItC"}
|
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Backoff calculator for retry delays
|
|
3
|
+
*/
|
|
4
|
+
export class Backoff {
|
|
5
|
+
config;
|
|
6
|
+
constructor(config) {
|
|
7
|
+
this.config = config;
|
|
8
|
+
}
|
|
9
|
+
/**
|
|
10
|
+
* Calculate the next retry delay based on attempt number
|
|
11
|
+
* Formula: delay = baseDelay * (2 ^ (attempt - 1))
|
|
12
|
+
*
|
|
13
|
+
* @param attempt - Current attempt number (1-based)
|
|
14
|
+
* @returns Delay in milliseconds
|
|
15
|
+
*/
|
|
16
|
+
calculateDelay(attempt) {
|
|
17
|
+
if (this.config.type === 'exponential') {
|
|
18
|
+
// Exponential backoff: delay * 2^(attempt-1)
|
|
19
|
+
const exponentialDelay = this.config.delay * Math.pow(2, attempt - 1);
|
|
20
|
+
// Cap at 1 hour to prevent extremely long delays
|
|
21
|
+
const maxDelay = 60 * 60 * 1000; // 1 hour
|
|
22
|
+
return Math.min(exponentialDelay, maxDelay);
|
|
23
|
+
}
|
|
24
|
+
return this.config.delay;
|
|
25
|
+
}
|
|
26
|
+
/**
|
|
27
|
+
* Calculate the next run timestamp for a job
|
|
28
|
+
*
|
|
29
|
+
* @param attempt - Current attempt number (1-based)
|
|
30
|
+
* @returns Unix timestamp in milliseconds
|
|
31
|
+
*/
|
|
32
|
+
getNextRunAt(attempt) {
|
|
33
|
+
const delay = this.calculateDelay(attempt);
|
|
34
|
+
return Date.now() + delay;
|
|
35
|
+
}
|
|
36
|
+
}
|
|
37
|
+
//# sourceMappingURL=Backoff.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"Backoff.js","sourceRoot":"","sources":["../../../src/queue/Backoff.ts"],"names":[],"mappings":"AAEA;;GAEG;AACH,MAAM,OAAO,OAAO;IACV,MAAM,CAAgB;IAE9B,YAAY,MAAqB;QAC/B,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;IACvB,CAAC;IAED;;;;;;OAMG;IACH,cAAc,CAAC,OAAe;QAC5B,IAAI,IAAI,CAAC,MAAM,CAAC,IAAI,KAAK,aAAa,EAAE,CAAC;YACvC,6CAA6C;YAC7C,MAAM,gBAAgB,GAAG,IAAI,CAAC,MAAM,CAAC,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,OAAO,GAAG,CAAC,CAAC,CAAC;YAEtE,iDAAiD;YACjD,MAAM,QAAQ,GAAG,EAAE,GAAG,EAAE,GAAG,IAAI,CAAC,CAAC,SAAS;YAC1C,OAAO,IAAI,CAAC,GAAG,CAAC,gBAAgB,EAAE,QAAQ,CAAC,CAAC;QAC9C,CAAC;QAED,OAAO,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC;IAC3B,CAAC;IAED;;;;;OAKG;IACH,YAAY,CAAC,OAAe;QAC1B,MAAM,KAAK,GAAG,IAAI,CAAC,cAAc,CAAC,OAAO,CAAC,CAAC;QAC3C,OAAO,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK,CAAC;IAC5B,CAAC;CACF"}
|
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
import { JobData, JobStatus } from '../types.js';
|
|
2
|
+
/**
|
|
3
|
+
* Job class representing a single unit of work in the queue
|
|
4
|
+
*/
|
|
5
|
+
export declare class Job {
|
|
6
|
+
readonly id: string;
|
|
7
|
+
payload: unknown;
|
|
8
|
+
attempts: number;
|
|
9
|
+
maxAttempts: number;
|
|
10
|
+
status: JobStatus;
|
|
11
|
+
nextRunAt: number;
|
|
12
|
+
readonly createdAt: number;
|
|
13
|
+
updatedAt: number;
|
|
14
|
+
constructor(payload: unknown, maxAttempts: number, nextRunAt?: number);
|
|
15
|
+
/**
|
|
16
|
+
* Create a Job instance from stored data
|
|
17
|
+
*/
|
|
18
|
+
static fromData(data: JobData): Job;
|
|
19
|
+
/**
|
|
20
|
+
* Convert job to plain data object for storage
|
|
21
|
+
*/
|
|
22
|
+
toData(): JobData;
|
|
23
|
+
/**
|
|
24
|
+
* Mark job as processing
|
|
25
|
+
*/
|
|
26
|
+
markProcessing(): void;
|
|
27
|
+
/**
|
|
28
|
+
* Mark job as completed
|
|
29
|
+
*/
|
|
30
|
+
markCompleted(): void;
|
|
31
|
+
/**
|
|
32
|
+
* Mark job as failed and increment attempts
|
|
33
|
+
*/
|
|
34
|
+
markFailed(nextRunAt?: number): void;
|
|
35
|
+
/**
|
|
36
|
+
* Check if job has exceeded max attempts
|
|
37
|
+
*/
|
|
38
|
+
hasExceededMaxAttempts(): boolean;
|
|
39
|
+
/**
|
|
40
|
+
* Reset job for reprocessing (used when recovering from DLQ)
|
|
41
|
+
*/
|
|
42
|
+
reset(): void;
|
|
43
|
+
}
|
|
44
|
+
//# sourceMappingURL=Job.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"Job.d.ts","sourceRoot":"","sources":["../../../src/queue/Job.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,SAAS,EAAE,MAAM,aAAa,CAAC;AAGjD;;GAEG;AACH,qBAAa,GAAG;IACd,SAAgB,EAAE,EAAE,MAAM,CAAC;IACpB,OAAO,EAAE,OAAO,CAAC;IACjB,QAAQ,EAAE,MAAM,CAAC;IACjB,WAAW,EAAE,MAAM,CAAC;IACpB,MAAM,EAAE,SAAS,CAAC;IAClB,SAAS,EAAE,MAAM,CAAC;IACzB,SAAgB,SAAS,EAAE,MAAM,CAAC;IAC3B,SAAS,EAAE,MAAM,CAAC;gBAEb,OAAO,EAAE,OAAO,EAAE,WAAW,EAAE,MAAM,EAAE,SAAS,CAAC,EAAE,MAAM;IAYrE;;OAEG;IACH,MAAM,CAAC,QAAQ,CAAC,IAAI,EAAE,OAAO,GAAG,GAAG;IAMnC;;OAEG;IACH,MAAM,IAAI,OAAO;IAajB;;OAEG;IACH,cAAc,IAAI,IAAI;IAKtB;;OAEG;IACH,aAAa,IAAI,IAAI;IAKrB;;OAEG;IACH,UAAU,CAAC,SAAS,CAAC,EAAE,MAAM,GAAG,IAAI;IASpC;;OAEG;IACH,sBAAsB,IAAI,OAAO;IAIjC;;OAEG;IACH,KAAK,IAAI,IAAI;CAMd"}
|