@riktajs/queue 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +476 -0
- package/dist/config/queue.config.d.ts +137 -0
- package/dist/config/queue.config.d.ts.map +1 -0
- package/dist/config/queue.config.js +82 -0
- package/dist/config/queue.config.js.map +1 -0
- package/dist/constants.d.ts +33 -0
- package/dist/constants.d.ts.map +1 -0
- package/dist/constants.js +37 -0
- package/dist/constants.js.map +1 -0
- package/dist/decorators/events.decorator.d.ts +85 -0
- package/dist/decorators/events.decorator.d.ts.map +1 -0
- package/dist/decorators/events.decorator.js +120 -0
- package/dist/decorators/events.decorator.js.map +1 -0
- package/dist/decorators/index.d.ts +8 -0
- package/dist/decorators/index.d.ts.map +1 -0
- package/dist/decorators/index.js +8 -0
- package/dist/decorators/index.js.map +1 -0
- package/dist/decorators/process.decorator.d.ts +41 -0
- package/dist/decorators/process.decorator.d.ts.map +1 -0
- package/dist/decorators/process.decorator.js +61 -0
- package/dist/decorators/process.decorator.js.map +1 -0
- package/dist/decorators/processor.decorator.d.ts +41 -0
- package/dist/decorators/processor.decorator.d.ts.map +1 -0
- package/dist/decorators/processor.decorator.js +59 -0
- package/dist/decorators/processor.decorator.js.map +1 -0
- package/dist/decorators/queue.decorator.d.ts +35 -0
- package/dist/decorators/queue.decorator.d.ts.map +1 -0
- package/dist/decorators/queue.decorator.js +49 -0
- package/dist/decorators/queue.decorator.js.map +1 -0
- package/dist/events/queue-events.d.ts +32 -0
- package/dist/events/queue-events.d.ts.map +1 -0
- package/dist/events/queue-events.js +103 -0
- package/dist/events/queue-events.js.map +1 -0
- package/dist/index.d.ts +22 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +30 -0
- package/dist/index.js.map +1 -0
- package/dist/monitoring/bull-board.d.ts +77 -0
- package/dist/monitoring/bull-board.d.ts.map +1 -0
- package/dist/monitoring/bull-board.js +112 -0
- package/dist/monitoring/bull-board.js.map +1 -0
- package/dist/providers/queue.provider.d.ts +94 -0
- package/dist/providers/queue.provider.d.ts.map +1 -0
- package/dist/providers/queue.provider.js +333 -0
- package/dist/providers/queue.provider.js.map +1 -0
- package/dist/services/queue.service.d.ts +133 -0
- package/dist/services/queue.service.d.ts.map +1 -0
- package/dist/services/queue.service.js +192 -0
- package/dist/services/queue.service.js.map +1 -0
- package/dist/types.d.ts +133 -0
- package/dist/types.d.ts.map +1 -0
- package/dist/types.js +5 -0
- package/dist/types.js.map +1 -0
- package/dist/utils/connection.d.ts +47 -0
- package/dist/utils/connection.d.ts.map +1 -0
- package/dist/utils/connection.js +104 -0
- package/dist/utils/connection.js.map +1 -0
- package/dist/utils/validation.d.ts +187 -0
- package/dist/utils/validation.d.ts.map +1 -0
- package/dist/utils/validation.js +156 -0
- package/dist/utils/validation.js.map +1 -0
- package/package.json +69 -0
package/README.md
ADDED
|
@@ -0,0 +1,476 @@
|
|
|
1
|
+
# @riktajs/queue
|
|
2
|
+
|
|
3
|
+
BullMQ-based job queue integration for Rikta Framework with lifecycle management, event-driven processing, and optional Bull Board monitoring.
|
|
4
|
+
|
|
5
|
+
## Features
|
|
6
|
+
|
|
7
|
+
- 🚀 **High Performance** - Built on BullMQ for distributed job processing
|
|
8
|
+
- 🎯 **Decorator-based API** - `@Processor`, `@Process`, `@OnJobComplete`, etc.
|
|
9
|
+
- 🔄 **Lifecycle Integration** - Seamless integration with Rikta's lifecycle hooks
|
|
10
|
+
- 📡 **Event System** - Queue events emitted via Rikta's EventBus
|
|
11
|
+
- ⚡ **Connection Pooling** - Shared Redis connections for optimal performance
|
|
12
|
+
- 📊 **Optional Monitoring** - Bull Board integration (bring your own dependency)
|
|
13
|
+
- 🛡️ **Type-safe** - Full TypeScript support with generics and Zod validation
|
|
14
|
+
- ⏰ **Scheduling** - Delayed jobs, repeatable jobs, cron patterns
|
|
15
|
+
|
|
16
|
+
## Installation
|
|
17
|
+
|
|
18
|
+
```bash
|
|
19
|
+
npm install @riktajs/queue bullmq
|
|
20
|
+
```
|
|
21
|
+
|
|
22
|
+
> **Note:** `ioredis` is included as a direct dependency and will be installed automatically.
|
|
23
|
+
|
|
24
|
+
### Optional: Bull Board Monitoring
|
|
25
|
+
|
|
26
|
+
```bash
|
|
27
|
+
npm install @bull-board/api @bull-board/fastify
|
|
28
|
+
```
|
|
29
|
+
|
|
30
|
+
## Quick Start
|
|
31
|
+
|
|
32
|
+
### 1. Create a Processor
|
|
33
|
+
|
|
34
|
+
```typescript
|
|
35
|
+
import { Processor, Process, OnJobComplete, OnJobFailed } from '@riktajs/queue';
|
|
36
|
+
import { Job } from 'bullmq';
|
|
37
|
+
|
|
38
|
+
interface EmailJobData {
|
|
39
|
+
to: string;
|
|
40
|
+
subject: string;
|
|
41
|
+
body: string;
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
@Processor('email-queue', { concurrency: 5 })
|
|
45
|
+
class EmailProcessor {
|
|
46
|
+
@Process('send')
|
|
47
|
+
async handleSendEmail(job: Job<EmailJobData>) {
|
|
48
|
+
console.log(`📧 Sending email to ${job.data.to}`);
|
|
49
|
+
|
|
50
|
+
// Your email sending logic here
|
|
51
|
+
await this.sendEmail(job.data);
|
|
52
|
+
|
|
53
|
+
return { sent: true, messageId: `msg-${job.id}` };
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
@Process('bulk-send')
|
|
57
|
+
async handleBulkSend(job: Job<{ emails: EmailJobData[] }>) {
|
|
58
|
+
for (const email of job.data.emails) {
|
|
59
|
+
await this.sendEmail(email);
|
|
60
|
+
await job.updateProgress(/* calculate progress */);
|
|
61
|
+
}
|
|
62
|
+
return { sent: job.data.emails.length };
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
@OnJobComplete()
|
|
66
|
+
async onComplete(job: Job, result: unknown) {
|
|
67
|
+
console.log(`✅ Job ${job.id} completed:`, result);
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
@OnJobFailed()
|
|
71
|
+
async onFailed(job: Job | undefined, error: Error) {
|
|
72
|
+
console.error(`❌ Job ${job?.id} failed:`, error.message);
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
private async sendEmail(data: EmailJobData): Promise<void> {
|
|
76
|
+
// Implementation
|
|
77
|
+
}
|
|
78
|
+
}
|
|
79
|
+
```
|
|
80
|
+
|
|
81
|
+
### 2. Configure the Provider
|
|
82
|
+
|
|
83
|
+
```typescript
|
|
84
|
+
import { Rikta } from '@riktajs/core';
|
|
85
|
+
import { createQueueProvider } from '@riktajs/queue';
|
|
86
|
+
|
|
87
|
+
// Create and configure provider
|
|
88
|
+
const queueProvider = createQueueProvider({
|
|
89
|
+
config: {
|
|
90
|
+
redis: {
|
|
91
|
+
host: process.env.REDIS_HOST || 'localhost',
|
|
92
|
+
port: parseInt(process.env.REDIS_PORT || '6379'),
|
|
93
|
+
password: process.env.REDIS_PASSWORD,
|
|
94
|
+
},
|
|
95
|
+
defaultConcurrency: 3,
|
|
96
|
+
shutdownTimeout: 30000,
|
|
97
|
+
},
|
|
98
|
+
});
|
|
99
|
+
|
|
100
|
+
// Register your processors
|
|
101
|
+
queueProvider.registerProcessors(EmailProcessor);
|
|
102
|
+
|
|
103
|
+
// Bootstrap your app
|
|
104
|
+
const app = await Rikta.create();
|
|
105
|
+
// Register the provider for lifecycle management
|
|
106
|
+
```
|
|
107
|
+
|
|
108
|
+
### 3. Add Jobs from Services
|
|
109
|
+
|
|
110
|
+
```typescript
|
|
111
|
+
import { Injectable, Autowired } from '@riktajs/core';
|
|
112
|
+
import { QueueService } from '@riktajs/queue';
|
|
113
|
+
|
|
114
|
+
@Injectable()
|
|
115
|
+
class NotificationService {
|
|
116
|
+
@Autowired()
|
|
117
|
+
private queueService!: QueueService;
|
|
118
|
+
|
|
119
|
+
async sendWelcomeEmail(userEmail: string) {
|
|
120
|
+
// Add a single job
|
|
121
|
+
await this.queueService.addJob('email-queue', 'send', {
|
|
122
|
+
to: userEmail,
|
|
123
|
+
subject: 'Welcome!',
|
|
124
|
+
body: 'Thanks for signing up!',
|
|
125
|
+
});
|
|
126
|
+
}
|
|
127
|
+
|
|
128
|
+
async sendDelayedReminder(userEmail: string) {
|
|
129
|
+
// Add a delayed job (sends after 1 hour)
|
|
130
|
+
await this.queueService.addDelayedJob(
|
|
131
|
+
'email-queue',
|
|
132
|
+
'send',
|
|
133
|
+
{
|
|
134
|
+
to: userEmail,
|
|
135
|
+
subject: 'Don\'t forget!',
|
|
136
|
+
body: 'Complete your profile.',
|
|
137
|
+
},
|
|
138
|
+
60 * 60 * 1000 // 1 hour
|
|
139
|
+
);
|
|
140
|
+
}
|
|
141
|
+
|
|
142
|
+
async sendDailyDigest() {
|
|
143
|
+
// Add a repeatable job (runs daily at 9am)
|
|
144
|
+
await this.queueService.addRepeatableJob(
|
|
145
|
+
'email-queue',
|
|
146
|
+
'bulk-send',
|
|
147
|
+
{ emails: [] }, // Data populated at runtime
|
|
148
|
+
{ pattern: '0 9 * * *' } // Cron pattern
|
|
149
|
+
);
|
|
150
|
+
}
|
|
151
|
+
|
|
152
|
+
async sendBulkEmails(emails: EmailJobData[]) {
|
|
153
|
+
// Add multiple jobs in bulk
|
|
154
|
+
const jobs = emails.map(email => ({
|
|
155
|
+
name: 'send',
|
|
156
|
+
data: email,
|
|
157
|
+
}));
|
|
158
|
+
|
|
159
|
+
await this.queueService.addJobs('email-queue', jobs);
|
|
160
|
+
}
|
|
161
|
+
}
|
|
162
|
+
```
|
|
163
|
+
|
|
164
|
+
## Configuration
|
|
165
|
+
|
|
166
|
+
### Environment Variables
|
|
167
|
+
|
|
168
|
+
| Variable | Description | Default |
|
|
169
|
+
|----------|-------------|---------|
|
|
170
|
+
| `QUEUE_REDIS_HOST` | Redis host | `localhost` |
|
|
171
|
+
| `QUEUE_REDIS_PORT` | Redis port | `6379` |
|
|
172
|
+
| `QUEUE_REDIS_PASSWORD` | Redis password | - |
|
|
173
|
+
| `QUEUE_REDIS_DB` | Redis database number | `0` |
|
|
174
|
+
| `QUEUE_REDIS_USERNAME` | Redis username (ACL) | - |
|
|
175
|
+
| `QUEUE_DEFAULT_CONCURRENCY` | Default worker concurrency | `1` |
|
|
176
|
+
| `QUEUE_SHUTDOWN_TIMEOUT` | Graceful shutdown timeout (ms) | `30000` |
|
|
177
|
+
| `QUEUE_DASHBOARD_PATH` | Bull Board path | `/admin/queues` |
|
|
178
|
+
| `QUEUE_DASHBOARD_ENABLED` | Enable Bull Board | `false` |
|
|
179
|
+
|
|
180
|
+
### Programmatic Configuration
|
|
181
|
+
|
|
182
|
+
```typescript
|
|
183
|
+
const provider = createQueueProvider({
|
|
184
|
+
config: {
|
|
185
|
+
redis: {
|
|
186
|
+
host: 'redis.example.com',
|
|
187
|
+
port: 6379,
|
|
188
|
+
password: 'secret',
|
|
189
|
+
tls: true,
|
|
190
|
+
},
|
|
191
|
+
defaultConcurrency: 5,
|
|
192
|
+
defaultRateLimiter: {
|
|
193
|
+
max: 100,
|
|
194
|
+
duration: 60000, // 100 jobs per minute
|
|
195
|
+
},
|
|
196
|
+
shutdownTimeout: 60000,
|
|
197
|
+
},
|
|
198
|
+
retryAttempts: 3,
|
|
199
|
+
retryDelay: 5000,
|
|
200
|
+
});
|
|
201
|
+
```
|
|
202
|
+
|
|
203
|
+
## Decorators
|
|
204
|
+
|
|
205
|
+
### `@Processor(queueName, options?)`
|
|
206
|
+
|
|
207
|
+
Marks a class as a job processor for a specific queue.
|
|
208
|
+
|
|
209
|
+
```typescript
|
|
210
|
+
@Processor('my-queue', {
|
|
211
|
+
concurrency: 10,
|
|
212
|
+
rateLimiter: { max: 100, duration: 60000 },
|
|
213
|
+
})
|
|
214
|
+
class MyProcessor { }
|
|
215
|
+
```
|
|
216
|
+
|
|
217
|
+
### `@Process(jobName?)`
|
|
218
|
+
|
|
219
|
+
Marks a method as a job handler. If no name is provided, uses the method name.
|
|
220
|
+
|
|
221
|
+
```typescript
|
|
222
|
+
@Process('send-email')
|
|
223
|
+
async handleSendEmail(job: Job) { }
|
|
224
|
+
|
|
225
|
+
@Process() // Uses 'processOrder' as job name
|
|
226
|
+
async processOrder(job: Job) { }
|
|
227
|
+
```
|
|
228
|
+
|
|
229
|
+
### Event Decorators
|
|
230
|
+
|
|
231
|
+
| Decorator | Event | Signature |
|
|
232
|
+
|-----------|-------|-----------|
|
|
233
|
+
| `@OnJobComplete()` | Job completed | `(job: Job, result: unknown)` |
|
|
234
|
+
| `@OnJobFailed()` | Job failed | `(job: Job \| undefined, error: Error)` |
|
|
235
|
+
| `@OnJobProgress()` | Job progress updated | `(job: Job, progress: number \| object)` |
|
|
236
|
+
| `@OnJobStalled()` | Job stalled | `(jobId: string)` |
|
|
237
|
+
| `@OnWorkerReady()` | Worker ready | `()` |
|
|
238
|
+
| `@OnWorkerError()` | Worker error | `(error: Error)` |
|
|
239
|
+
|
|
240
|
+
## Validation with Zod
|
|
241
|
+
|
|
242
|
+
Use built-in Zod utilities for type-safe job validation:
|
|
243
|
+
|
|
244
|
+
```typescript
|
|
245
|
+
import { createJobSchema, z, CommonJobSchemas } from '@riktajs/queue';
|
|
246
|
+
|
|
247
|
+
// Create custom schema
|
|
248
|
+
const OrderJobSchema = createJobSchema(z.object({
|
|
249
|
+
orderId: z.string().uuid(),
|
|
250
|
+
items: z.array(z.object({
|
|
251
|
+
productId: z.string(),
|
|
252
|
+
quantity: z.number().positive(),
|
|
253
|
+
})),
|
|
254
|
+
total: z.number().positive(),
|
|
255
|
+
}));
|
|
256
|
+
|
|
257
|
+
// Validate in processor
|
|
258
|
+
@Processor('orders')
|
|
259
|
+
class OrderProcessor {
|
|
260
|
+
@Process('process')
|
|
261
|
+
async handleOrder(job: Job) {
|
|
262
|
+
const data = OrderJobSchema.validate(job.data);
|
|
263
|
+
// data is now typed as { orderId: string, items: [...], total: number }
|
|
264
|
+
}
|
|
265
|
+
}
|
|
266
|
+
|
|
267
|
+
// Use common schemas
|
|
268
|
+
const emailData = CommonJobSchemas.email.parse({
|
|
269
|
+
to: 'user@example.com',
|
|
270
|
+
subject: 'Hello',
|
|
271
|
+
body: 'World',
|
|
272
|
+
});
|
|
273
|
+
```
|
|
274
|
+
|
|
275
|
+
### Common Job Schemas
|
|
276
|
+
|
|
277
|
+
- `CommonJobSchemas.email` - Email job with to, subject, body, attachments
|
|
278
|
+
- `CommonJobSchemas.notification` - User notifications
|
|
279
|
+
- `CommonJobSchemas.fileProcessing` - File operations
|
|
280
|
+
- `CommonJobSchemas.webhook` - HTTP webhook calls
|
|
281
|
+
|
|
282
|
+
## Event System
|
|
283
|
+
|
|
284
|
+
Queue events are emitted to Rikta's EventBus:
|
|
285
|
+
|
|
286
|
+
```typescript
|
|
287
|
+
import { EventBus } from '@riktajs/core';
|
|
288
|
+
import { QUEUE_EVENTS } from '@riktajs/queue';
|
|
289
|
+
|
|
290
|
+
@Injectable()
|
|
291
|
+
class MonitoringService {
|
|
292
|
+
constructor(private eventBus: EventBus) {
|
|
293
|
+
// Listen to queue events
|
|
294
|
+
eventBus.on(QUEUE_EVENTS.JOB_COMPLETED, (payload) => {
|
|
295
|
+
console.log(`Job ${payload.jobId} completed in ${payload.queueName}`);
|
|
296
|
+
});
|
|
297
|
+
|
|
298
|
+
eventBus.on(QUEUE_EVENTS.JOB_FAILED, (payload) => {
|
|
299
|
+
console.error(`Job ${payload.jobId} failed: ${payload.error}`);
|
|
300
|
+
});
|
|
301
|
+
}
|
|
302
|
+
}
|
|
303
|
+
```
|
|
304
|
+
|
|
305
|
+
### Available Events
|
|
306
|
+
|
|
307
|
+
| Event | Description |
|
|
308
|
+
|-------|-------------|
|
|
309
|
+
| `queue:job:added` | Job added to queue |
|
|
310
|
+
| `queue:job:completed` | Job completed successfully |
|
|
311
|
+
| `queue:job:failed` | Job failed |
|
|
312
|
+
| `queue:job:progress` | Job progress updated |
|
|
313
|
+
| `queue:job:stalled` | Job stalled |
|
|
314
|
+
| `queue:job:delayed` | Job delayed |
|
|
315
|
+
| `queue:worker:ready` | Worker ready |
|
|
316
|
+
| `queue:worker:error` | Worker error |
|
|
317
|
+
|
|
318
|
+
## Bull Board Dashboard (Optional)
|
|
319
|
+
|
|
320
|
+
```typescript
|
|
321
|
+
import { registerBullBoard } from '@riktajs/queue';
|
|
322
|
+
|
|
323
|
+
// After app is created and queue provider initialized
|
|
324
|
+
await registerBullBoard(app.server, {
|
|
325
|
+
queues: queueProvider.getAllQueues(),
|
|
326
|
+
path: '/admin/queues',
|
|
327
|
+
readOnly: false,
|
|
328
|
+
auth: async (request) => {
|
|
329
|
+
// Your authentication logic
|
|
330
|
+
const token = request.headers.authorization;
|
|
331
|
+
return validateAdminToken(token);
|
|
332
|
+
},
|
|
333
|
+
});
|
|
334
|
+
```
|
|
335
|
+
|
|
336
|
+
**Note:** Bull Board packages must be installed separately:
|
|
337
|
+
|
|
338
|
+
```bash
|
|
339
|
+
npm install @bull-board/api @bull-board/fastify
|
|
340
|
+
```
|
|
341
|
+
|
|
342
|
+
## QueueService API
|
|
343
|
+
|
|
344
|
+
### Adding Jobs
|
|
345
|
+
|
|
346
|
+
```typescript
|
|
347
|
+
// Single job
|
|
348
|
+
await queueService.addJob(queueName, jobName, data, options?);
|
|
349
|
+
|
|
350
|
+
// Multiple jobs (bulk)
|
|
351
|
+
await queueService.addJobs(queueName, [{ name, data, options? }]);
|
|
352
|
+
|
|
353
|
+
// Delayed job
|
|
354
|
+
await queueService.addDelayedJob(queueName, jobName, data, delayMs, options?);
|
|
355
|
+
|
|
356
|
+
// Repeatable job
|
|
357
|
+
await queueService.addRepeatableJob(queueName, jobName, data, repeatOptions);
|
|
358
|
+
```
|
|
359
|
+
|
|
360
|
+
### Job Options
|
|
361
|
+
|
|
362
|
+
```typescript
|
|
363
|
+
await queueService.addJob('queue', 'job', data, {
|
|
364
|
+
attempts: 3, // Retry attempts
|
|
365
|
+
backoff: {
|
|
366
|
+
type: 'exponential', // 'fixed' | 'exponential'
|
|
367
|
+
delay: 1000,
|
|
368
|
+
},
|
|
369
|
+
priority: 1, // Lower = higher priority
|
|
370
|
+
delay: 5000, // Delay in ms
|
|
371
|
+
deduplicationKey: 'id', // Prevent duplicates
|
|
372
|
+
removeOnComplete: true, // Clean up completed jobs
|
|
373
|
+
removeOnFail: false, // Keep failed jobs for debugging
|
|
374
|
+
});
|
|
375
|
+
```
|
|
376
|
+
|
|
377
|
+
### Queue Management
|
|
378
|
+
|
|
379
|
+
```typescript
|
|
380
|
+
// Get job by ID
|
|
381
|
+
const job = await queueService.getJob(queueName, jobId);
|
|
382
|
+
|
|
383
|
+
// Get queue statistics
|
|
384
|
+
const stats = await queueService.getQueueStats(queueName);
|
|
385
|
+
// { waiting: 5, active: 2, completed: 100, failed: 3, delayed: 1, paused: 0 }
|
|
386
|
+
|
|
387
|
+
// Pause/Resume
|
|
388
|
+
await queueService.pauseQueue(queueName);
|
|
389
|
+
await queueService.resumeQueue(queueName);
|
|
390
|
+
|
|
391
|
+
// Clear jobs
|
|
392
|
+
await queueService.clearQueue(queueName, 'completed');
|
|
393
|
+
await queueService.clearQueue(queueName); // Clear all
|
|
394
|
+
|
|
395
|
+
// Get all queue names
|
|
396
|
+
const names = queueService.getQueueNames();
|
|
397
|
+
```
|
|
398
|
+
|
|
399
|
+
## Error Handling
|
|
400
|
+
|
|
401
|
+
```typescript
|
|
402
|
+
import {
|
|
403
|
+
QueueNotFoundError,
|
|
404
|
+
QueueConnectionError,
|
|
405
|
+
QueueInitializationError,
|
|
406
|
+
JobSchemaValidationError,
|
|
407
|
+
} from '@riktajs/queue';
|
|
408
|
+
|
|
409
|
+
try {
|
|
410
|
+
await queueService.addJob('unknown-queue', 'job', {});
|
|
411
|
+
} catch (error) {
|
|
412
|
+
if (error instanceof QueueNotFoundError) {
|
|
413
|
+
console.error('Queue does not exist:', error.message);
|
|
414
|
+
}
|
|
415
|
+
}
|
|
416
|
+
```
|
|
417
|
+
|
|
418
|
+
## Best Practices
|
|
419
|
+
|
|
420
|
+
### 1. Use Type-Safe Job Data
|
|
421
|
+
|
|
422
|
+
```typescript
|
|
423
|
+
interface MyJobData {
|
|
424
|
+
userId: string;
|
|
425
|
+
action: 'create' | 'update' | 'delete';
|
|
426
|
+
}
|
|
427
|
+
|
|
428
|
+
@Process('my-job')
|
|
429
|
+
async handle(job: Job<MyJobData>) {
|
|
430
|
+
const { userId, action } = job.data; // Fully typed
|
|
431
|
+
}
|
|
432
|
+
```
|
|
433
|
+
|
|
434
|
+
### 2. Handle Failures Gracefully
|
|
435
|
+
|
|
436
|
+
```typescript
|
|
437
|
+
@Process('risky-job')
|
|
438
|
+
async handle(job: Job) {
|
|
439
|
+
try {
|
|
440
|
+
await this.riskyOperation(job.data);
|
|
441
|
+
} catch (error) {
|
|
442
|
+
// Log for debugging
|
|
443
|
+
console.error('Job failed:', error);
|
|
444
|
+
// Re-throw to trigger retry
|
|
445
|
+
throw error;
|
|
446
|
+
}
|
|
447
|
+
}
|
|
448
|
+
```
|
|
449
|
+
|
|
450
|
+
### 3. Use Progress Updates for Long Jobs
|
|
451
|
+
|
|
452
|
+
```typescript
|
|
453
|
+
@Process('long-job')
|
|
454
|
+
async handle(job: Job<{ items: string[] }>) {
|
|
455
|
+
const { items } = job.data;
|
|
456
|
+
|
|
457
|
+
for (let i = 0; i < items.length; i++) {
|
|
458
|
+
await this.processItem(items[i]);
|
|
459
|
+
await job.updateProgress(Math.round((i + 1) / items.length * 100));
|
|
460
|
+
}
|
|
461
|
+
}
|
|
462
|
+
```
|
|
463
|
+
|
|
464
|
+
### 4. Configure Appropriate Concurrency
|
|
465
|
+
|
|
466
|
+
```typescript
|
|
467
|
+
// CPU-intensive tasks: lower concurrency
|
|
468
|
+
@Processor('image-processing', { concurrency: 2 })
|
|
469
|
+
|
|
470
|
+
// I/O-bound tasks: higher concurrency
|
|
471
|
+
@Processor('api-calls', { concurrency: 20 })
|
|
472
|
+
```
|
|
473
|
+
|
|
474
|
+
## License
|
|
475
|
+
|
|
476
|
+
MIT
|
|
@@ -0,0 +1,137 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Queue configuration schema and loader
|
|
3
|
+
*/
|
|
4
|
+
import { z } from 'zod';
|
|
5
|
+
import type { QueueConfig } from '../types.js';
|
|
6
|
+
/** Zod schema for queue configuration */
|
|
7
|
+
export declare const QueueConfigSchema: z.ZodObject<{
|
|
8
|
+
redis: z.ZodObject<{
|
|
9
|
+
host: z.ZodDefault<z.ZodString>;
|
|
10
|
+
port: z.ZodDefault<z.ZodNumber>;
|
|
11
|
+
password: z.ZodOptional<z.ZodString>;
|
|
12
|
+
db: z.ZodDefault<z.ZodNumber>;
|
|
13
|
+
username: z.ZodOptional<z.ZodString>;
|
|
14
|
+
tls: z.ZodOptional<z.ZodBoolean>;
|
|
15
|
+
cluster: z.ZodOptional<z.ZodObject<{
|
|
16
|
+
nodes: z.ZodArray<z.ZodObject<{
|
|
17
|
+
host: z.ZodString;
|
|
18
|
+
port: z.ZodNumber;
|
|
19
|
+
}, "strip", z.ZodTypeAny, {
|
|
20
|
+
host: string;
|
|
21
|
+
port: number;
|
|
22
|
+
}, {
|
|
23
|
+
host: string;
|
|
24
|
+
port: number;
|
|
25
|
+
}>, "many">;
|
|
26
|
+
}, "strip", z.ZodTypeAny, {
|
|
27
|
+
nodes: {
|
|
28
|
+
host: string;
|
|
29
|
+
port: number;
|
|
30
|
+
}[];
|
|
31
|
+
}, {
|
|
32
|
+
nodes: {
|
|
33
|
+
host: string;
|
|
34
|
+
port: number;
|
|
35
|
+
}[];
|
|
36
|
+
}>>;
|
|
37
|
+
}, "strip", z.ZodTypeAny, {
|
|
38
|
+
host: string;
|
|
39
|
+
port: number;
|
|
40
|
+
db: number;
|
|
41
|
+
password?: string | undefined;
|
|
42
|
+
username?: string | undefined;
|
|
43
|
+
tls?: boolean | undefined;
|
|
44
|
+
cluster?: {
|
|
45
|
+
nodes: {
|
|
46
|
+
host: string;
|
|
47
|
+
port: number;
|
|
48
|
+
}[];
|
|
49
|
+
} | undefined;
|
|
50
|
+
}, {
|
|
51
|
+
host?: string | undefined;
|
|
52
|
+
port?: number | undefined;
|
|
53
|
+
password?: string | undefined;
|
|
54
|
+
db?: number | undefined;
|
|
55
|
+
username?: string | undefined;
|
|
56
|
+
tls?: boolean | undefined;
|
|
57
|
+
cluster?: {
|
|
58
|
+
nodes: {
|
|
59
|
+
host: string;
|
|
60
|
+
port: number;
|
|
61
|
+
}[];
|
|
62
|
+
} | undefined;
|
|
63
|
+
}>;
|
|
64
|
+
defaultConcurrency: z.ZodDefault<z.ZodNumber>;
|
|
65
|
+
defaultRateLimiter: z.ZodOptional<z.ZodObject<{
|
|
66
|
+
max: z.ZodNumber;
|
|
67
|
+
duration: z.ZodNumber;
|
|
68
|
+
}, "strip", z.ZodTypeAny, {
|
|
69
|
+
max: number;
|
|
70
|
+
duration: number;
|
|
71
|
+
}, {
|
|
72
|
+
max: number;
|
|
73
|
+
duration: number;
|
|
74
|
+
}>>;
|
|
75
|
+
dashboardPath: z.ZodDefault<z.ZodString>;
|
|
76
|
+
dashboardEnabled: z.ZodDefault<z.ZodBoolean>;
|
|
77
|
+
shutdownTimeout: z.ZodDefault<z.ZodNumber>;
|
|
78
|
+
}, "strip", z.ZodTypeAny, {
|
|
79
|
+
redis: {
|
|
80
|
+
host: string;
|
|
81
|
+
port: number;
|
|
82
|
+
db: number;
|
|
83
|
+
password?: string | undefined;
|
|
84
|
+
username?: string | undefined;
|
|
85
|
+
tls?: boolean | undefined;
|
|
86
|
+
cluster?: {
|
|
87
|
+
nodes: {
|
|
88
|
+
host: string;
|
|
89
|
+
port: number;
|
|
90
|
+
}[];
|
|
91
|
+
} | undefined;
|
|
92
|
+
};
|
|
93
|
+
defaultConcurrency: number;
|
|
94
|
+
dashboardPath: string;
|
|
95
|
+
dashboardEnabled: boolean;
|
|
96
|
+
shutdownTimeout: number;
|
|
97
|
+
defaultRateLimiter?: {
|
|
98
|
+
max: number;
|
|
99
|
+
duration: number;
|
|
100
|
+
} | undefined;
|
|
101
|
+
}, {
|
|
102
|
+
redis: {
|
|
103
|
+
host?: string | undefined;
|
|
104
|
+
port?: number | undefined;
|
|
105
|
+
password?: string | undefined;
|
|
106
|
+
db?: number | undefined;
|
|
107
|
+
username?: string | undefined;
|
|
108
|
+
tls?: boolean | undefined;
|
|
109
|
+
cluster?: {
|
|
110
|
+
nodes: {
|
|
111
|
+
host: string;
|
|
112
|
+
port: number;
|
|
113
|
+
}[];
|
|
114
|
+
} | undefined;
|
|
115
|
+
};
|
|
116
|
+
defaultConcurrency?: number | undefined;
|
|
117
|
+
defaultRateLimiter?: {
|
|
118
|
+
max: number;
|
|
119
|
+
duration: number;
|
|
120
|
+
} | undefined;
|
|
121
|
+
dashboardPath?: string | undefined;
|
|
122
|
+
dashboardEnabled?: boolean | undefined;
|
|
123
|
+
shutdownTimeout?: number | undefined;
|
|
124
|
+
}>;
|
|
125
|
+
export type QueueConfigInput = z.input<typeof QueueConfigSchema>;
|
|
126
|
+
/**
|
|
127
|
+
* Load queue configuration from environment variables
|
|
128
|
+
* @param overrides - Optional configuration overrides
|
|
129
|
+
*/
|
|
130
|
+
export declare function loadQueueConfig(overrides?: Partial<QueueConfigInput>): QueueConfig;
|
|
131
|
+
/**
|
|
132
|
+
* Error thrown when queue configuration is invalid
|
|
133
|
+
*/
|
|
134
|
+
export declare class QueueConfigError extends Error {
|
|
135
|
+
constructor(message: string);
|
|
136
|
+
}
|
|
137
|
+
//# sourceMappingURL=queue.config.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"queue.config.d.ts","sourceRoot":"","sources":["../../src/config/queue.config.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AACxB,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,aAAa,CAAC;AA2B/C,yCAAyC;AACzC,eAAO,MAAM,iBAAiB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;EAO5B,CAAC;AAEH,MAAM,MAAM,gBAAgB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,iBAAiB,CAAC,CAAC;AAEjE;;;GAGG;AACH,wBAAgB,eAAe,CAAC,SAAS,CAAC,EAAE,OAAO,CAAC,gBAAgB,CAAC,GAAG,WAAW,CAoClF;AAED;;GAEG;AACH,qBAAa,gBAAiB,SAAQ,KAAK;gBAC7B,OAAO,EAAE,MAAM;CAI5B"}
|
|
@@ -0,0 +1,82 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Queue configuration schema and loader
|
|
3
|
+
*/
|
|
4
|
+
import { z } from 'zod';
|
|
5
|
+
/** Zod schema for Redis cluster node */
|
|
6
|
+
const RedisClusterNodeSchema = z.object({
|
|
7
|
+
host: z.string(),
|
|
8
|
+
port: z.number().int().positive(),
|
|
9
|
+
});
|
|
10
|
+
/** Zod schema for Redis connection options */
|
|
11
|
+
const RedisConnectionSchema = z.object({
|
|
12
|
+
host: z.string().default('localhost'),
|
|
13
|
+
port: z.number().int().positive().default(6379),
|
|
14
|
+
password: z.string().optional(),
|
|
15
|
+
db: z.number().int().min(0).default(0),
|
|
16
|
+
username: z.string().optional(),
|
|
17
|
+
tls: z.boolean().optional(),
|
|
18
|
+
cluster: z.object({
|
|
19
|
+
nodes: z.array(RedisClusterNodeSchema).min(1),
|
|
20
|
+
}).optional(),
|
|
21
|
+
});
|
|
22
|
+
/** Zod schema for rate limiter */
|
|
23
|
+
const RateLimiterSchema = z.object({
|
|
24
|
+
max: z.number().int().positive(),
|
|
25
|
+
duration: z.number().int().positive(),
|
|
26
|
+
});
|
|
27
|
+
/** Zod schema for queue configuration */
|
|
28
|
+
export const QueueConfigSchema = z.object({
|
|
29
|
+
redis: RedisConnectionSchema,
|
|
30
|
+
defaultConcurrency: z.number().int().positive().default(1),
|
|
31
|
+
defaultRateLimiter: RateLimiterSchema.optional(),
|
|
32
|
+
dashboardPath: z.string().default('/admin/queues'),
|
|
33
|
+
dashboardEnabled: z.boolean().default(false),
|
|
34
|
+
shutdownTimeout: z.number().int().positive().default(30000),
|
|
35
|
+
});
|
|
36
|
+
/**
|
|
37
|
+
* Load queue configuration from environment variables
|
|
38
|
+
* @param overrides - Optional configuration overrides
|
|
39
|
+
*/
|
|
40
|
+
export function loadQueueConfig(overrides) {
|
|
41
|
+
const envConfig = {
|
|
42
|
+
redis: {
|
|
43
|
+
host: process.env['QUEUE_REDIS_HOST'] || 'localhost',
|
|
44
|
+
port: parseInt(process.env['QUEUE_REDIS_PORT'] || '6379', 10),
|
|
45
|
+
password: process.env['QUEUE_REDIS_PASSWORD'] || undefined,
|
|
46
|
+
db: parseInt(process.env['QUEUE_REDIS_DB'] || '0', 10),
|
|
47
|
+
username: process.env['QUEUE_REDIS_USERNAME'] || undefined,
|
|
48
|
+
},
|
|
49
|
+
defaultConcurrency: parseInt(process.env['QUEUE_DEFAULT_CONCURRENCY'] || '1', 10),
|
|
50
|
+
dashboardPath: process.env['QUEUE_DASHBOARD_PATH'] || '/admin/queues',
|
|
51
|
+
dashboardEnabled: process.env['QUEUE_DASHBOARD_ENABLED'] === 'true',
|
|
52
|
+
shutdownTimeout: parseInt(process.env['QUEUE_SHUTDOWN_TIMEOUT'] || '30000', 10),
|
|
53
|
+
};
|
|
54
|
+
// Merge with overrides
|
|
55
|
+
const merged = {
|
|
56
|
+
...envConfig,
|
|
57
|
+
...overrides,
|
|
58
|
+
redis: {
|
|
59
|
+
...envConfig.redis,
|
|
60
|
+
...overrides?.redis,
|
|
61
|
+
},
|
|
62
|
+
};
|
|
63
|
+
// Validate and return
|
|
64
|
+
const result = QueueConfigSchema.safeParse(merged);
|
|
65
|
+
if (!result.success) {
|
|
66
|
+
const errors = result.error.errors
|
|
67
|
+
.map(e => `${e.path.join('.')}: ${e.message}`)
|
|
68
|
+
.join(', ');
|
|
69
|
+
throw new QueueConfigError(`Invalid queue configuration: ${errors}`);
|
|
70
|
+
}
|
|
71
|
+
return result.data;
|
|
72
|
+
}
|
|
73
|
+
/**
|
|
74
|
+
* Error thrown when queue configuration is invalid
|
|
75
|
+
*/
|
|
76
|
+
export class QueueConfigError extends Error {
|
|
77
|
+
constructor(message) {
|
|
78
|
+
super(message);
|
|
79
|
+
this.name = 'QueueConfigError';
|
|
80
|
+
}
|
|
81
|
+
}
|
|
82
|
+
//# sourceMappingURL=queue.config.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"queue.config.js","sourceRoot":"","sources":["../../src/config/queue.config.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAGxB,wCAAwC;AACxC,MAAM,sBAAsB,GAAG,CAAC,CAAC,MAAM,CAAC;IACtC,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE;IAChB,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,QAAQ,EAAE;CAClC,CAAC,CAAC;AAEH,8CAA8C;AAC9C,MAAM,qBAAqB,GAAG,CAAC,CAAC,MAAM,CAAC;IACrC,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,OAAO,CAAC,WAAW,CAAC;IACrC,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,QAAQ,EAAE,CAAC,OAAO,CAAC,IAAI,CAAC;IAC/C,QAAQ,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IAC/B,EAAE,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC;IACtC,QAAQ,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IAC/B,GAAG,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,QAAQ,EAAE;IAC3B,OAAO,EAAE,CAAC,CAAC,MAAM,CAAC;QAChB,KAAK,EAAE,CAAC,CAAC,KAAK,CAAC,sBAAsB,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;KAC9C,CAAC,CAAC,QAAQ,EAAE;CACd,CAAC,CAAC;AAEH,kCAAkC;AAClC,MAAM,iBAAiB,GAAG,CAAC,CAAC,MAAM,CAAC;IACjC,GAAG,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,QAAQ,EAAE;IAChC,QAAQ,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,QAAQ,EAAE;CACtC,CAAC,CAAC;AAEH,yCAAyC;AACzC,MAAM,CAAC,MAAM,iBAAiB,GAAG,CAAC,CAAC,MAAM,CAAC;IACxC,KAAK,EAAE,qBAAqB;IAC5B,kBAAkB,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,QAAQ,EAAE,CAAC,OAAO,CAAC,CAAC,CAAC;IAC1D,kBAAkB,EAAE,iBAAiB,CAAC,QAAQ,EAAE;IAChD,aAAa,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,OAAO,CAAC,eAAe,CAAC;IAClD,gBAAgB,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,OAAO,CAAC,KAAK,CAAC;IAC5C,eAAe,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,QAAQ,EAAE,CAAC,OAAO,CAAC,KAAK,CAAC;CAC5D,CAAC,CAAC;AAIH;;;GAGG;AACH,MAAM,UAAU,eAAe,CAAC,SAAqC;IACnE,MAAM,SAAS,GAAqB;QAClC,KAAK,EAAE;YACL,IAAI,EAAE,OAAO,CAAC,GAAG,CAAC,kBAAkB,CAAC,IAAI,WAAW;YACpD,IAAI,EAAE,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,kBAAkB,CAAC,IAAI,MAAM,EAAE,EAAE,CAAC;YAC7D,QAAQ,EAAE,OAAO,CAAC,GAAG,CAAC,sBAAsB,CAAC,IAAI,SAAS;YAC1D,EAAE,EAAE,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,gBAAgB,CAAC,IAAI,GAAG,EAAE,EAAE,CAAC;YACtD,QAAQ,EAAE,OAAO,CAAC,GAAG,CAAC,sBAAsB,CAAC,IAAI,SAAS;SAC3D;QACD,kBAAkB,EAAE,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,2BAA2B,CAAC,IAAI,GAAG,EAAE,EAAE,CAAC;QACjF,aAAa,EAAE,OAAO,CAAC,GAAG,CAAC,sBAAsB,CAAC,IAAI,eAAe;QACrE,gBAAgB,EAAE,OAAO,CAAC,GAAG,CAAC,yBAAyB,CAAC,KAAK,MAAM;QACnE,eAAe,EAAE,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,wBAAwB,CAAC,IAAI,OAAO,EAAE,EAAE,CAAC;KAChF,CAAC;IAEF,uBAAuB;IACvB,MAAM,MAAM,GAAG;QACb,GAAG,SAAS;QACZ,GAAG,SAAS;QACZ,KAAK,EAAE;YACL,GAAG,SAAS,CAAC,KAAK;YAClB,GAAG,SAAS,EAAE,KAAK;SACpB;KACF,CAAC;IAEF,sBAAsB;IACtB,MAAM,MAAM,GAAG,iBAAiB,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC;IAEnD,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC;QACpB,MAAM,MAAM,GAAG,MAAM,CAAC,KAAK,CAAC,MAAM;aAC/B,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,OAAO,EAAE,CAAC;aAC7C,IAAI,CAAC,IAAI,CAAC,CAAC;QACd,MAAM,IAAI,gBAAgB,CAAC,gCAAgC,MAAM,EAAE,CAAC,CAAC;IACvE,CAAC;IAED,OAAO,MAAM,CAAC,IAAmB,CAAC;AACpC,CAAC;AAED;;GAEG;AACH,MAAM,OAAO,gBAAiB,SAAQ,KAAK;IACzC,YAAY,OAAe;QACzB,KAAK,CAAC,OAAO,CAAC,CAAC;QACf,IAAI,CAAC,IAAI,GAAG,kBAAkB,CAAC;IACjC,CAAC;CACF"}
|