atomic-queues 1.2.2 → 1.2.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +371 -833
- package/dist/domain/interfaces.d.ts +8 -0
- package/dist/domain/interfaces.d.ts.map +1 -1
- package/dist/services/command-discovery/command-discovery.service.d.ts.map +1 -1
- package/dist/services/command-discovery/command-discovery.service.js +4 -1
- package/dist/services/command-discovery/command-discovery.service.js.map +1 -1
- package/dist/services/processor-discovery/processor-discovery.service.d.ts +4 -0
- package/dist/services/processor-discovery/processor-discovery.service.d.ts.map +1 -1
- package/dist/services/processor-discovery/processor-discovery.service.js +13 -0
- package/dist/services/processor-discovery/processor-discovery.service.js.map +1 -1
- package/dist/services/queue-bus/queue-bus.service.d.ts +28 -0
- package/dist/services/queue-bus/queue-bus.service.d.ts.map +1 -1
- package/dist/services/queue-bus/queue-bus.service.js +54 -0
- package/dist/services/queue-bus/queue-bus.service.js.map +1 -1
- package/dist/services/service-queue/service-queue.service.js +3 -1
- package/dist/services/service-queue/service-queue.service.js.map +1 -1
- package/package.json +1 -1
package/README.md
CHANGED
|
@@ -1,34 +1,46 @@
|
|
|
1
1
|
# atomic-queues
|
|
2
2
|
|
|
3
|
-
A
|
|
4
|
-
|
|
5
|
-
##
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
3
|
+
A NestJS library for atomic, sequential job processing per entity with BullMQ and Redis.
|
|
4
|
+
|
|
5
|
+
## What It Does
|
|
6
|
+
|
|
7
|
+
```
|
|
8
|
+
╔═══════════════════════════════════════════════════════════════════════════════╗
|
|
9
|
+
║ THE PROBLEM ║
|
|
10
|
+
╠═══════════════════════════════════════════════════════════════════════════════╣
|
|
11
|
+
║ ║
|
|
12
|
+
║ Multiple requests for the same entity arrive simultaneously: ║
|
|
13
|
+
║ ║
|
|
14
|
+
║ ┌──────────┐ ║
|
|
15
|
+
║ │ Request A │──┐ ║
|
|
16
|
+
║ └──────────┘ │ ║
|
|
17
|
+
║ ┌──────────┐ │ ┌─────────────┐ ║
|
|
18
|
+
║ │ Request B │──┼───▶│ Entity 123 │───▶ 💥 RACE CONDITION! ║
|
|
19
|
+
║ └──────────┘ │ └─────────────┘ ║
|
|
20
|
+
║ ┌──────────┐ │ ║
|
|
21
|
+
║ │ Request C │──┘ ║
|
|
22
|
+
║ └──────────┘ ║
|
|
23
|
+
║ ║
|
|
24
|
+
╚═══════════════════════════════════════════════════════════════════════════════╝
|
|
25
|
+
|
|
26
|
+
╔═══════════════════════════════════════════════════════════════════════════════╗
|
|
27
|
+
║ THE SOLUTION ║
|
|
28
|
+
╠═══════════════════════════════════════════════════════════════════════════════╣
|
|
29
|
+
║ ║
|
|
30
|
+
║ atomic-queues ensures sequential processing per entity: ║
|
|
31
|
+
║ ║
|
|
32
|
+
║ ┌──────────┐ ┌─────────────────┐ ┌──────────┐ ║
|
|
33
|
+
║ │ Request A │──┐ │ │ │ │ ║
|
|
34
|
+
║ └──────────┘ │ │ Redis Queue │ │ Worker │ ┌───────────┐ ║
|
|
35
|
+
║ ┌──────────┐ │ │ ┌───┬───┬───┐ │ │ │ │ │ ║
|
|
36
|
+
║ │ Request B │──┼──▶│ │ A │ B │ C │ │─────▶│ (1 job │─▶│Entity 123 │ ║
|
|
37
|
+
║ └──────────┘ │ │ └───┴───┴───┘ │ │ at a time│ │ │ ║
|
|
38
|
+
║ ┌──────────┐ │ │ │ │ │ └───────────┘ ║
|
|
39
|
+
║ │ Request C │──┘ └─────────────────┘ └──────────┘ ║
|
|
40
|
+
║ └──────────┘ ║
|
|
41
|
+
║ ║
|
|
42
|
+
╚═══════════════════════════════════════════════════════════════════════════════╝
|
|
43
|
+
```
|
|
32
44
|
|
|
33
45
|
## Installation
|
|
34
46
|
|
|
@@ -36,13 +48,9 @@ This library solves all of these with a single, cohesive module.
|
|
|
36
48
|
npm install atomic-queues bullmq ioredis
|
|
37
49
|
```
|
|
38
50
|
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
## Quick Start (Decorator-based API) ✨
|
|
42
|
-
|
|
43
|
-
The recommended way to use `atomic-queues` is with the decorator-based API for clean, declarative code.
|
|
51
|
+
## Quick Start
|
|
44
52
|
|
|
45
|
-
### 1.
|
|
53
|
+
### 1. Configure the Module
|
|
46
54
|
|
|
47
55
|
```typescript
|
|
48
56
|
import { Module } from '@nestjs/common';
|
|
@@ -50,936 +58,466 @@ import { AtomicQueuesModule } from 'atomic-queues';
|
|
|
50
58
|
|
|
51
59
|
@Module({
|
|
52
60
|
imports: [
|
|
53
|
-
AtomicQueuesModule.
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
redis: {
|
|
57
|
-
url: configService.get('REDIS_URL'),
|
|
58
|
-
},
|
|
59
|
-
keyPrefix: 'myapp',
|
|
60
|
-
enableCronManager: true,
|
|
61
|
-
workerDefaults: {
|
|
62
|
-
concurrency: 1,
|
|
63
|
-
heartbeatTTL: 3,
|
|
64
|
-
},
|
|
65
|
-
}),
|
|
66
|
-
inject: [ConfigService],
|
|
61
|
+
AtomicQueuesModule.forRoot({
|
|
62
|
+
redis: { host: 'localhost', port: 6379 },
|
|
63
|
+
keyPrefix: 'myapp',
|
|
67
64
|
}),
|
|
68
65
|
],
|
|
69
66
|
})
|
|
70
67
|
export class AppModule {}
|
|
71
68
|
```
|
|
72
69
|
|
|
73
|
-
### 2. Create
|
|
70
|
+
### 2. Create Your Commands
|
|
74
71
|
|
|
75
|
-
|
|
72
|
+
Plain classes - no decorators needed:
|
|
76
73
|
|
|
77
74
|
```typescript
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
workerName: (orderId) => `order-${orderId}-worker`,
|
|
87
|
-
workerConfig: {
|
|
88
|
-
concurrency: 1,
|
|
89
|
-
heartbeatTTL: 3,
|
|
90
|
-
},
|
|
91
|
-
})
|
|
92
|
-
@Injectable()
|
|
93
|
-
export class OrderWorkerProcessor {
|
|
94
|
-
constructor(private readonly commandBus: CommandBus) {}
|
|
95
|
-
|
|
96
|
-
@JobHandler('validate')
|
|
97
|
-
async handleValidate(job: Job, orderId: string) {
|
|
98
|
-
const { items } = job.data;
|
|
99
|
-
return this.commandBus.execute(new ValidateOrderCommand(orderId, items));
|
|
100
|
-
}
|
|
101
|
-
|
|
102
|
-
@JobHandler('process-payment')
|
|
103
|
-
async handlePayment(job: Job, orderId: string) {
|
|
104
|
-
const { amount } = job.data;
|
|
105
|
-
return this.commandBus.execute(new ProcessPaymentCommand(orderId, amount));
|
|
106
|
-
}
|
|
107
|
-
|
|
108
|
-
@JobHandler('ship')
|
|
109
|
-
async handleShip(job: Job, orderId: string) {
|
|
110
|
-
return this.commandBus.execute(new ShipOrderCommand(orderId));
|
|
111
|
-
}
|
|
75
|
+
// commands/process-order.command.ts
|
|
76
|
+
export class ProcessOrderCommand {
|
|
77
|
+
constructor(
|
|
78
|
+
public readonly orderId: string,
|
|
79
|
+
public readonly items: string[],
|
|
80
|
+
public readonly amount: number,
|
|
81
|
+
) {}
|
|
82
|
+
}
|
|
112
83
|
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
84
|
+
// commands/ship-order.command.ts
|
|
85
|
+
export class ShipOrderCommand {
|
|
86
|
+
constructor(
|
|
87
|
+
public readonly orderId: string,
|
|
88
|
+
public readonly address: string,
|
|
89
|
+
) {}
|
|
118
90
|
}
|
|
119
91
|
```
|
|
120
92
|
|
|
121
|
-
### 3. Create
|
|
122
|
-
|
|
123
|
-
Use `@EntityScaler` to define scaling logic with decorated methods:
|
|
93
|
+
### 3. Create a Worker Processor
|
|
124
94
|
|
|
125
95
|
```typescript
|
|
126
96
|
import { Injectable } from '@nestjs/common';
|
|
127
|
-
import {
|
|
97
|
+
import { WorkerProcessor } from 'atomic-queues';
|
|
128
98
|
|
|
129
|
-
@
|
|
99
|
+
@WorkerProcessor({
|
|
130
100
|
entityType: 'order',
|
|
131
|
-
|
|
101
|
+
queueName: (orderId) => `order-${orderId}-queue`,
|
|
102
|
+
workerName: (orderId) => `order-${orderId}-worker`,
|
|
132
103
|
})
|
|
133
104
|
@Injectable()
|
|
134
|
-
export class
|
|
135
|
-
constructor(private readonly orderRepository: OrderRepository) {}
|
|
136
|
-
|
|
137
|
-
@GetActiveEntities()
|
|
138
|
-
async getActiveOrders(): Promise<string[]> {
|
|
139
|
-
// Return order IDs that have pending work
|
|
140
|
-
return this.orderRepository.findPendingOrderIds();
|
|
141
|
-
}
|
|
142
|
-
|
|
143
|
-
@GetDesiredWorkerCount()
|
|
144
|
-
async getWorkerCount(orderId: string): Promise<number> {
|
|
145
|
-
// Each order gets exactly 1 worker
|
|
146
|
-
return 1;
|
|
147
|
-
}
|
|
148
|
-
}
|
|
105
|
+
export class OrderProcessor {}
|
|
149
106
|
```
|
|
150
107
|
|
|
151
|
-
### 4.
|
|
152
|
-
|
|
153
|
-
```typescript
|
|
154
|
-
@Module({
|
|
155
|
-
imports: [AtomicQueuesModule.forRootAsync({ ... })],
|
|
156
|
-
providers: [
|
|
157
|
-
OrderWorkerProcessor, // Auto-discovered by @WorkerProcessor
|
|
158
|
-
OrderEntityScaler, // Auto-discovered by @EntityScaler
|
|
159
|
-
],
|
|
160
|
-
})
|
|
161
|
-
export class OrderModule {}
|
|
162
|
-
```
|
|
108
|
+
### 4. Queue Jobs with the Fluent API
|
|
163
109
|
|
|
164
|
-
|
|
110
|
+
Commands are **automatically registered** from your `@CommandHandler` classes - no manual registration needed!
|
|
165
111
|
|
|
166
112
|
```typescript
|
|
167
113
|
import { Injectable } from '@nestjs/common';
|
|
168
|
-
import {
|
|
114
|
+
import { QueueBus } from 'atomic-queues';
|
|
115
|
+
import { OrderProcessor } from './order.processor';
|
|
116
|
+
import { ProcessOrderCommand, ShipOrderCommand } from './commands';
|
|
169
117
|
|
|
170
118
|
@Injectable()
|
|
171
119
|
export class OrderService {
|
|
172
|
-
constructor(private readonly
|
|
120
|
+
constructor(private readonly queueBus: QueueBus) {}
|
|
173
121
|
|
|
174
|
-
async createOrder(orderId: string, items:
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
await
|
|
181
|
-
|
|
182
|
-
|
|
122
|
+
async createOrder(orderId: string, items: string[], amount: number) {
|
|
123
|
+
// Jobs are queued and processed sequentially per orderId
|
|
124
|
+
await this.queueBus
|
|
125
|
+
.forProcessor(OrderProcessor)
|
|
126
|
+
.enqueue(new ProcessOrderCommand(orderId, items, amount));
|
|
127
|
+
|
|
128
|
+
await this.queueBus
|
|
129
|
+
.forProcessor(OrderProcessor)
|
|
130
|
+
.enqueue(new ShipOrderCommand(orderId, '123 Main St'));
|
|
183
131
|
}
|
|
184
132
|
}
|
|
185
133
|
```
|
|
186
134
|
|
|
187
|
-
That's it! The library
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
135
|
+
That's it! The library automatically:
|
|
136
|
+
- Discovers commands from `@CommandHandler` decorators
|
|
137
|
+
- Creates a queue for each `orderId`
|
|
138
|
+
- Spawns a worker to process jobs sequentially
|
|
139
|
+
- Routes jobs to the correct command handlers
|
|
192
140
|
|
|
193
141
|
---
|
|
194
142
|
|
|
195
|
-
##
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
}
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
143
|
+
## How It Works
|
|
144
|
+
|
|
145
|
+
```
|
|
146
|
+
╔═══════════════════════════════════════════════════════════════════════════════╗
|
|
147
|
+
║ ARCHITECTURE ║
|
|
148
|
+
╚═══════════════════════════════════════════════════════════════════════════════╝
|
|
149
|
+
|
|
150
|
+
YOUR CODE ATOMIC-QUEUES EXECUTION
|
|
151
|
+
───────── ───────────── ─────────
|
|
152
|
+
|
|
153
|
+
┌─────────────────────────┐
|
|
154
|
+
│ queueBus │
|
|
155
|
+
│ .forProcessor(...) │
|
|
156
|
+
│ .enqueue(command) │
|
|
157
|
+
└───────────┬─────────────┘
|
|
158
|
+
│
|
|
159
|
+
│ ① Extract queue config from @WorkerProcessor
|
|
160
|
+
│ ② Extract entityId from command properties
|
|
161
|
+
│ ③ Build queue name: {prefix}-{entityId}-queue
|
|
162
|
+
▼
|
|
163
|
+
┌───────────────────┐
|
|
164
|
+
│ │
|
|
165
|
+
│ Redis Queue │◀─── Job { name: "MyCommand", data: {...} }
|
|
166
|
+
│ (per entity) │
|
|
167
|
+
│ │
|
|
168
|
+
└─────────┬─────────┘
|
|
169
|
+
│
|
|
170
|
+
│ ④ Worker pulls job (one at a time)
|
|
171
|
+
▼
|
|
172
|
+
┌───────────────────┐
|
|
173
|
+
│ │
|
|
174
|
+
│ BullMQ Worker │
|
|
175
|
+
│ (1 per entity) │
|
|
176
|
+
│ │
|
|
177
|
+
└─────────┬─────────┘
|
|
178
|
+
│
|
|
179
|
+
│ ⑤ Lookup command class in registry
|
|
180
|
+
│ ⑥ Instantiate from job.data
|
|
181
|
+
│ ⑦ Execute via CQRS CommandBus
|
|
182
|
+
▼
|
|
183
|
+
┌───────────────────┐ ┌─────────────────────────┐
|
|
184
|
+
│ │ │ │
|
|
185
|
+
│ CommandBus │─────▶│ MyCommandHandler │
|
|
186
|
+
│ │ │ .execute(command) │
|
|
187
|
+
└───────────────────┘ └─────────────────────────┘
|
|
226
188
|
```
|
|
227
189
|
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
Class decorator for entity scaling configuration.
|
|
190
|
+
---
|
|
231
191
|
|
|
232
|
-
|
|
233
|
-
@EntityScaler({
|
|
234
|
-
entityType: string; // Required: Entity type to scale
|
|
235
|
-
maxWorkersPerEntity?: number; // Default: 1
|
|
236
|
-
})
|
|
237
|
-
```
|
|
192
|
+
## API Reference
|
|
238
193
|
|
|
239
|
-
###
|
|
194
|
+
### QueueBus
|
|
240
195
|
|
|
241
|
-
|
|
196
|
+
The main way to add jobs to queues:
|
|
242
197
|
|
|
243
198
|
```typescript
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
```
|
|
199
|
+
// Enqueue a single command
|
|
200
|
+
await queueBus
|
|
201
|
+
.forProcessor(MyProcessor)
|
|
202
|
+
.enqueue(new MyCommand(entityId, data));
|
|
249
203
|
|
|
250
|
-
|
|
204
|
+
// Enqueue and wait for result
|
|
205
|
+
const result = await queueBus
|
|
206
|
+
.forProcessor(MyProcessor)
|
|
207
|
+
.enqueueAndWait(new MyQuery(entityId));
|
|
251
208
|
|
|
252
|
-
|
|
209
|
+
// Enqueue multiple commands
|
|
210
|
+
await queueBus
|
|
211
|
+
.forProcessor(MyProcessor)
|
|
212
|
+
.enqueueBulk([
|
|
213
|
+
new CommandA(entityId),
|
|
214
|
+
new CommandB(entityId),
|
|
215
|
+
]);
|
|
253
216
|
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
}
|
|
217
|
+
// With job options (delay, priority, etc.)
|
|
218
|
+
await queueBus
|
|
219
|
+
.forProcessor(MyProcessor)
|
|
220
|
+
.enqueue(new MyCommand(entityId), {
|
|
221
|
+
jobOptions: { delay: 5000, priority: 1 }
|
|
222
|
+
});
|
|
259
223
|
```
|
|
260
224
|
|
|
261
|
-
### @
|
|
225
|
+
### @WorkerProcessor
|
|
262
226
|
|
|
263
|
-
|
|
227
|
+
Defines how workers are created for an entity type:
|
|
264
228
|
|
|
265
229
|
```typescript
|
|
266
|
-
@
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
}
|
|
270
|
-
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
|
|
274
|
-
}
|
|
230
|
+
@WorkerProcessor({
|
|
231
|
+
entityType: 'order', // Required
|
|
232
|
+
queueName: (id) => `order-${id}-queue`, // Optional
|
|
233
|
+
workerName: (id) => `order-${id}-worker`, // Optional
|
|
234
|
+
workerConfig: {
|
|
235
|
+
concurrency: 1, // Jobs per worker (default: 1)
|
|
236
|
+
stalledInterval: 1000, // Check stalled jobs (ms)
|
|
237
|
+
lockDuration: 30000, // Job lock duration (ms)
|
|
238
|
+
},
|
|
239
|
+
})
|
|
275
240
|
```
|
|
276
241
|
|
|
277
242
|
---
|
|
278
243
|
|
|
279
|
-
##
|
|
280
|
-
|
|
281
|
-
### Migrating from Manual Registration to Decorators
|
|
244
|
+
## Entity ID Extraction
|
|
282
245
|
|
|
283
|
-
|
|
246
|
+
The `entityId` is automatically extracted from your command's properties:
|
|
284
247
|
|
|
285
248
|
```typescript
|
|
286
|
-
//
|
|
287
|
-
|
|
288
|
-
@JobProcessor('validate-order')
|
|
289
|
-
export class ValidateOrderProcessor {
|
|
290
|
-
async process(job: Job) {
|
|
291
|
-
// validation logic
|
|
292
|
-
}
|
|
293
|
-
}
|
|
249
|
+
// These property names are checked in order:
|
|
250
|
+
// entityId, tableId, userId, id, gameId, playerId
|
|
294
251
|
|
|
295
|
-
|
|
296
|
-
@Injectable()
|
|
297
|
-
export class OrderWorkerService {
|
|
252
|
+
export class ProcessOrderCommand {
|
|
298
253
|
constructor(
|
|
299
|
-
|
|
300
|
-
|
|
254
|
+
public readonly orderId: string, // ✓ 'orderId' contains 'Id' → entityId
|
|
255
|
+
public readonly items: string[],
|
|
301
256
|
) {}
|
|
302
|
-
|
|
303
|
-
async createOrderWorker(orderId: string) {
|
|
304
|
-
await this.workerManager.createWorker({
|
|
305
|
-
workerName: `order-${orderId}-worker`,
|
|
306
|
-
queueName: `order-${orderId}-queue`,
|
|
307
|
-
processor: async (job) => {
|
|
308
|
-
const processor = this.jobRegistry.getProcessor(job.name);
|
|
309
|
-
await processor.process(job);
|
|
310
|
-
},
|
|
311
|
-
});
|
|
312
|
-
}
|
|
313
257
|
}
|
|
314
258
|
|
|
315
|
-
//
|
|
316
|
-
|
|
317
|
-
|
|
318
|
-
|
|
319
|
-
|
|
320
|
-
|
|
321
|
-
}
|
|
259
|
+
// Or use standard names
|
|
260
|
+
export class UpdateUserCommand {
|
|
261
|
+
constructor(
|
|
262
|
+
public readonly userId: string, // ✓ Matches 'userId' → entityId
|
|
263
|
+
public readonly name: string,
|
|
264
|
+
) {}
|
|
265
|
+
}
|
|
322
266
|
```
|
|
323
267
|
|
|
324
|
-
|
|
268
|
+
---
|
|
269
|
+
|
|
270
|
+
## Scaling with Entity Scalers
|
|
271
|
+
|
|
272
|
+
For dynamic worker management based on demand:
|
|
325
273
|
|
|
326
274
|
```typescript
|
|
327
|
-
|
|
328
|
-
|
|
275
|
+
import { Injectable } from '@nestjs/common';
|
|
276
|
+
import { EntityScaler, GetActiveEntities, GetDesiredWorkerCount } from 'atomic-queues';
|
|
277
|
+
|
|
278
|
+
@EntityScaler({
|
|
329
279
|
entityType: 'order',
|
|
330
|
-
|
|
331
|
-
workerName: (id) => `order-${id}-worker`,
|
|
280
|
+
maxWorkersPerEntity: 1,
|
|
332
281
|
})
|
|
333
282
|
@Injectable()
|
|
334
|
-
export class
|
|
335
|
-
|
|
336
|
-
async handleValidate(job: Job, orderId: string) {
|
|
337
|
-
// validation logic
|
|
338
|
-
}
|
|
339
|
-
|
|
340
|
-
@JobHandler('process-payment')
|
|
341
|
-
async handlePayment(job: Job, orderId: string) {
|
|
342
|
-
// payment logic
|
|
343
|
-
}
|
|
344
|
-
}
|
|
283
|
+
export class OrderScaler {
|
|
284
|
+
constructor(private readonly orderRepo: OrderRepository) {}
|
|
345
285
|
|
|
346
|
-
// table-entity.scaler.ts (scaling config in one place)
|
|
347
|
-
@EntityScaler({ entityType: 'order', maxWorkersPerEntity: 1 })
|
|
348
|
-
@Injectable()
|
|
349
|
-
export class OrderEntityScaler {
|
|
350
286
|
@GetActiveEntities()
|
|
351
|
-
async getActiveOrders(): Promise<string[]> {
|
|
352
|
-
|
|
287
|
+
async getActiveOrders(): Promise<string[]> {
|
|
288
|
+
// Return IDs that need workers
|
|
289
|
+
return this.orderRepo.findPendingOrderIds();
|
|
290
|
+
}
|
|
291
|
+
|
|
353
292
|
@GetDesiredWorkerCount()
|
|
354
|
-
async getWorkerCount(
|
|
293
|
+
async getWorkerCount(orderId: string): Promise<number> {
|
|
294
|
+
return 1; // One worker per order
|
|
295
|
+
}
|
|
355
296
|
}
|
|
356
|
-
|
|
357
|
-
// app.module.ts (just provide the classes, auto-discovery handles the rest)
|
|
358
|
-
@Module({
|
|
359
|
-
providers: [OrderWorkerProcessor, OrderEntityScaler],
|
|
360
|
-
})
|
|
361
|
-
export class OrderModule {}
|
|
362
|
-
```
|
|
363
|
-
|
|
364
|
-
### Key Benefits of Migration
|
|
365
|
-
|
|
366
|
-
| Aspect | Manual API | Decorator API |
|
|
367
|
-
|--------|-----------|---------------|
|
|
368
|
-
| **Job routing** | Manual switch/case or registry lookup | Automatic via `@JobHandler` |
|
|
369
|
-
| **Worker creation** | Explicit service method | Auto-generated by library |
|
|
370
|
-
| **Scaling config** | Imperative `registerEntityType()` call | Declarative `@EntityScaler` class |
|
|
371
|
-
| **Entity ID access** | Manual parsing from job data | Injected as method parameter |
|
|
372
|
-
| **Code organization** | Multiple files and services | Single processor class per entity type |
|
|
373
|
-
| **Registration** | Manual in `onModuleInit` | Auto-discovered at startup |
|
|
374
|
-
|
|
375
|
-
---
|
|
376
|
-
|
|
377
|
-
## Architecture
|
|
378
|
-
|
|
379
|
-
### High-Level Flow
|
|
380
|
-
|
|
381
|
-
```
|
|
382
|
-
┌─────────────────────────────────────────────────────────────────────────────────────────────┐
|
|
383
|
-
│ atomic-queues ARCHITECTURE │
|
|
384
|
-
└─────────────────────────────────────────────────────────────────────────────────────────────┘
|
|
385
|
-
|
|
386
|
-
┌─────────────────────┐
|
|
387
|
-
│ External Triggers │
|
|
388
|
-
│ (WebSocket, HTTP, │
|
|
389
|
-
│ Cron, Pub/Sub) │
|
|
390
|
-
└──────────┬──────────┘
|
|
391
|
-
│
|
|
392
|
-
▼
|
|
393
|
-
┌──────────────────────────────────────────────────────────────────────────────────────────────┐
|
|
394
|
-
│ APPLICATION LAYER │
|
|
395
|
-
│ ┌────────────────────────────────────────────────────────────────────────────────────────┐ │
|
|
396
|
-
│ │ QueueManagerService │ │
|
|
397
|
-
│ │ │ │
|
|
398
|
-
│ │ queueManager.addJob(entityQueue, jobName, { entityId, action, payload }) │ │
|
|
399
|
-
│ │ │ │
|
|
400
|
-
│ └────────────────────────────────────────────────────────────────────────────────────────┘ │
|
|
401
|
-
└──────────────────────────────────────────────────────────────────────────────────────────────┘
|
|
402
|
-
│
|
|
403
|
-
▼
|
|
404
|
-
┌──────────────────────────────────────────────────────────────────────────────────────────────┐
|
|
405
|
-
│ REDIS (BullMQ) │
|
|
406
|
-
│ │
|
|
407
|
-
│ ┌───────────────┐ ┌───────────────┐ ┌───────────────┐ ┌───────────────┐ │
|
|
408
|
-
│ │ entity-A-q │ │ entity-B-q │ │ entity-C-q │ │ entity-N-q │ │
|
|
409
|
-
│ │ │ │ │ │ │ │ │ │
|
|
410
|
-
│ │ ┌─────────┐ │ │ ┌─────────┐ │ │ ┌─────────┐ │ │ ┌─────────┐ │ │
|
|
411
|
-
│ │ │ Job 1 │ │ │ │ Job 1 │ │ │ │ Job 1 │ │ │ │ Job 1 │ │ │
|
|
412
|
-
│ │ │ Job 2 │ │ │ │ Job 2 │ │ │ └─────────┘ │ │ │ Job 2 │ │ │
|
|
413
|
-
│ │ │ Job 3 │ │ │ └─────────┘ │ │ │ │ │ Job 3 │ │ │
|
|
414
|
-
│ │ │ ... │ │ │ │ │ │ │ │ ... │ │ │
|
|
415
|
-
│ │ └─────────┘ │ │ │ │ │ │ └─────────┘ │ │
|
|
416
|
-
│ └───────┬───────┘ └───────┬───────┘ └───────┬───────┘ └───────┬───────┘ │
|
|
417
|
-
│ │ │ │ │ │
|
|
418
|
-
└───────────┼────────────────────┼────────────────────┼────────────────────┼───────────────────┘
|
|
419
|
-
│ │ │ │
|
|
420
|
-
▼ ▼ ▼ ▼
|
|
421
|
-
┌──────────────────────────────────────────────────────────────────────────────────────────────┐
|
|
422
|
-
│ WORKER LAYER (Per-Entity) │
|
|
423
|
-
│ │
|
|
424
|
-
│ ┌───────────────┐ ┌───────────────┐ ┌───────────────┐ ┌───────────────┐ │
|
|
425
|
-
│ │ Worker A │ │ Worker B │ │ Worker C │ │ Worker N │ │
|
|
426
|
-
│ │ concurrency=1 │ │ concurrency=1 │ │ concurrency=1 │ │ concurrency=1 │ │
|
|
427
|
-
│ │ │ │ │ │ │ │ │ │
|
|
428
|
-
│ │ ┌─────────┐ │ │ ┌─────────┐ │ │ ┌─────────┐ │ │ ┌─────────┐ │ │
|
|
429
|
-
│ │ │Heartbeat│ │ │ │Heartbeat│ │ │ │Heartbeat│ │ │ │Heartbeat│ │ │
|
|
430
|
-
│ │ │ TTL=3s │ │ │ │ TTL=3s │ │ │ │ TTL=3s │ │ │ │ TTL=3s │ │ │
|
|
431
|
-
│ │ └─────────┘ │ │ └─────────┘ │ │ └─────────┘ │ │ └─────────┘ │ │
|
|
432
|
-
│ └───────┬───────┘ └───────┬───────┘ └───────┬───────┘ └───────┬───────┘ │
|
|
433
|
-
│ │ │ │ │ │
|
|
434
|
-
│ │ WorkerManagerService (Lifecycle, Heartbeats, Shutdown Signals) │
|
|
435
|
-
│ └────────────────────┴────────────────────┴────────────────────┘ │
|
|
436
|
-
│ │ │
|
|
437
|
-
└──────────────────────────────────────────┼───────────────────────────────────────────────────┘
|
|
438
|
-
│
|
|
439
|
-
▼
|
|
440
|
-
┌──────────────────────────────────────────────────────────────────────────────────────────────┐
|
|
441
|
-
│ JOB PROCESSOR SERVICE │
|
|
442
|
-
│ │
|
|
443
|
-
│ ┌───────────────────────────────────────────────────────────────────────────────────────┐ │
|
|
444
|
-
│ │ JobProcessorRegistry │ │
|
|
445
|
-
│ │ │ │
|
|
446
|
-
│ │ @JobProcessor('validate') @JobProcessor('pay') @JobProcessor('ship') │ │
|
|
447
|
-
│ │ class ValidateProcessor {} class PayProcessor {} class ShipProcessor {} │ │
|
|
448
|
-
│ │ │ │
|
|
449
|
-
│ └───────────────────────────────────────────────────────────────────────────────────────┘ │
|
|
450
|
-
│ │ │
|
|
451
|
-
│ ▼ │
|
|
452
|
-
│ ┌───────────────────────────────────────────────────────────────────────────────────────┐ │
|
|
453
|
-
│ │ CQRS CommandBus / QueryBus │ │
|
|
454
|
-
│ │ │ │
|
|
455
|
-
│ │ commandBus.execute(new ValidateOrderCommand(...)) │ │
|
|
456
|
-
│ │ commandBus.execute(new ProcessPaymentCommand(...)) │ │
|
|
457
|
-
│ │ │ │
|
|
458
|
-
│ └───────────────────────────────────────────────────────────────────────────────────────┘ │
|
|
459
|
-
│ │
|
|
460
|
-
└──────────────────────────────────────────────────────────────────────────────────────────────┘
|
|
461
|
-
|
|
462
|
-
|
|
463
|
-
┌──────────────────────────────────────────────────────────────────────────────────────────────┐
|
|
464
|
-
│ SUPPORTING SERVICES │
|
|
465
|
-
│ │
|
|
466
|
-
│ ┌─────────────────────────┐ ┌─────────────────────────┐ ┌─────────────────────────┐ │
|
|
467
|
-
│ │ CronManagerService │ │ IndexManagerService │ │ ResourceLockService │ │
|
|
468
|
-
│ │ │ │ │ │ │ │
|
|
469
|
-
│ │ • Poll for entities │ │ • Track jobs per │ │ • Lua-based atomic │ │
|
|
470
|
-
│ │ needing workers │ │ entity │ │ locks │ │
|
|
471
|
-
│ │ • Spawn workers on │ │ • Track worker states │ │ • Lock pooling │ │
|
|
472
|
-
│ │ demand │ │ • Track queue states │ │ • TTL-based expiry │ │
|
|
473
|
-
│ │ • Terminate idle │ │ • Cleanup on entity │ │ • Owner tracking │ │
|
|
474
|
-
│ │ workers │ │ completion │ │ │ │
|
|
475
|
-
│ │ │ │ │ │ │ │
|
|
476
|
-
│ └─────────────────────────┘ └─────────────────────────┘ └─────────────────────────┘ │
|
|
477
|
-
│ │
|
|
478
|
-
└──────────────────────────────────────────────────────────────────────────────────────────────┘
|
|
479
|
-
```
|
|
480
|
-
|
|
481
|
-
### Detailed Component Interaction
|
|
482
|
-
|
|
483
|
-
```
|
|
484
|
-
┌─────────────────────────────────────────────────────────────────────────────────────────────────┐
|
|
485
|
-
│ COMPLETE JOB LIFECYCLE │
|
|
486
|
-
└─────────────────────────────────────────────────────────────────────────────────────────────────┘
|
|
487
|
-
|
|
488
|
-
1. JOB CREATION 2. WORKER SPAWNING 3. JOB PROCESSING
|
|
489
|
-
───────────────── ────────────────── ─────────────────
|
|
490
|
-
|
|
491
|
-
┌─────────────┐ ┌─────────────────┐ ┌─────────────────┐
|
|
492
|
-
│ Service │ │ CronManager │ │ Worker │
|
|
493
|
-
│ (HTTP/WS) │ │ Service │ │ (BullMQ) │
|
|
494
|
-
└──────┬──────┘ └────────┬────────┘ └────────┬────────┘
|
|
495
|
-
│ │ │
|
|
496
|
-
│ 1. Receive request │ 1. Every N seconds │ 1. Poll queue
|
|
497
|
-
│ (create order, etc) │ check entities │ for jobs
|
|
498
|
-
▼ │ with pending jobs │
|
|
499
|
-
┌─────────────┐ ▼ ▼
|
|
500
|
-
│ Queue │ ┌─────────────────┐ ┌─────────────────┐
|
|
501
|
-
│ Manager │ │ Index │ │ Job │
|
|
502
|
-
│ Service │ │ Manager │ │ Processor │
|
|
503
|
-
└──────┬──────┘ └────────┬────────┘ │ Registry │
|
|
504
|
-
│ │ └────────┬────────┘
|
|
505
|
-
│ 2. Get/create queue │ 2. Return entities │
|
|
506
|
-
│ for entity │ with job counts │ 2. Lookup processor
|
|
507
|
-
▼ │ │ by job name
|
|
508
|
-
┌─────────────┐ ▼ ▼
|
|
509
|
-
│ Redis │ ┌─────────────────┐ ┌─────────────────┐
|
|
510
|
-
│ Queue │◄────────────────── │ Worker │ │ @JobProcessor │
|
|
511
|
-
│ (entity-X) │ │ Manager │ │ Handler Class │
|
|
512
|
-
└──────┬──────┘ └────────┬────────┘ └────────┬────────┘
|
|
513
|
-
│ │ │
|
|
514
|
-
│ 3. Add job to queue │ 3. Spawn worker │ 3. Execute
|
|
515
|
-
│ (FIFO ordered) │ for entity │ command/query
|
|
516
|
-
▼ │ ▼
|
|
517
|
-
┌─────────────┐ ▼ ┌─────────────────┐
|
|
518
|
-
│ Index │ ┌─────────────────┐ │ CommandBus │
|
|
519
|
-
│ Manager │ │ New Worker │ │ / QueryBus │
|
|
520
|
-
└─────────────┘ │ (concurrency=1)│ └────────┬────────┘
|
|
521
|
-
│ └─────────────────┘ │
|
|
522
|
-
│ 4. Track job in index │ 4. Domain
|
|
523
|
-
│ for entity │ logic
|
|
524
|
-
▼ ▼
|
|
525
|
-
┌─────────────────────────────────────────────────────────────────────────────────────┐
|
|
526
|
-
│ REDIS │
|
|
527
|
-
│ ┌─────────────┐ ┌─────────────┐ ┌─────────────┐ ┌─────────────┐ │
|
|
528
|
-
│ │ Queues │ │ Workers │ │ Indices │ │ Locks │ │
|
|
529
|
-
│ │ (BullMQ) │ │ (Heartbeat) │ │ (Jobs/Qs) │ │ (Lua Atom) │ │
|
|
530
|
-
│ └─────────────┘ └─────────────┘ └─────────────┘ └─────────────┘ │
|
|
531
|
-
└─────────────────────────────────────────────────────────────────────────────────────┘
|
|
532
|
-
|
|
533
|
-
|
|
534
|
-
4. JOB COMPLETION 5. WORKER TERMINATION 6. GRACEFUL SHUTDOWN
|
|
535
|
-
───────────────── ───────────────────── ────────────────────
|
|
536
|
-
|
|
537
|
-
┌─────────────────┐ ┌─────────────────┐ ┌─────────────────┐
|
|
538
|
-
│ Worker │ │ CronManager │ │ SIGTERM/INT │
|
|
539
|
-
│ completes │ │ Service │ │ Signal │
|
|
540
|
-
└────────┬────────┘ └────────┬────────┘ └────────┬────────┘
|
|
541
|
-
│ │ │
|
|
542
|
-
│ 1. Job finished │ 1. Check worker │ 1. Caught by
|
|
543
|
-
│ │ idle time │ process handler
|
|
544
|
-
▼ │ ▼
|
|
545
|
-
┌─────────────────┐ ▼ ┌─────────────────┐
|
|
546
|
-
│ Index │ ┌─────────────────┐ │ Worker │
|
|
547
|
-
│ Manager │ │ No pending │ │ Manager │
|
|
548
|
-
└────────┬────────┘ │ jobs for │ └────────┬────────┘
|
|
549
|
-
│ │ entity? │ │
|
|
550
|
-
│ 2. Remove job from └────────┬────────┘ │ 2. Signal all
|
|
551
|
-
│ entity index │ │ workers to close
|
|
552
|
-
▼ │ YES ▼
|
|
553
|
-
┌─────────────────┐ ▼ ┌─────────────────┐
|
|
554
|
-
│ Check pending │ ┌─────────────────┐ │ Redis │
|
|
555
|
-
│ jobs for │ │ Worker │ │ Pub/Sub │
|
|
556
|
-
│ entity │ │ Manager │ │ (shutdown │
|
|
557
|
-
└────────┬────────┘ └────────┬────────┘ │ channel) │
|
|
558
|
-
│ │ └────────┬────────┘
|
|
559
|
-
│ 3. If no pending │ 2. Signal worker │
|
|
560
|
-
│ jobs, cleanup │ to close │ 3. Workers receive
|
|
561
|
-
▼ ▼ │ shutdown signal
|
|
562
|
-
┌─────────────────┐ ┌─────────────────┐ ▼
|
|
563
|
-
│ Entity indices │ │ Worker │ ┌─────────────────┐
|
|
564
|
-
│ cleaned up │ │ gracefully │ │ Workers │
|
|
565
|
-
│ │ │ closes │ │ finish │
|
|
566
|
-
└─────────────────┘ └─────────────────┘ │ current job │
|
|
567
|
-
│ then exit │
|
|
568
|
-
└─────────────────┘
|
|
569
|
-
```
|
|
570
|
-
|
|
571
|
-
### Multi-Node Cluster Architecture
|
|
572
|
-
|
|
573
|
-
```
|
|
574
|
-
┌─────────────────────────────────────────────────────────────────────────────────────────────────┐
|
|
575
|
-
│ MULTI-NODE CLUSTER DEPLOYMENT │
|
|
576
|
-
└─────────────────────────────────────────────────────────────────────────────────────────────────┘
|
|
577
|
-
|
|
578
|
-
┌─────────────────┐
|
|
579
|
-
│ Load Balancer │
|
|
580
|
-
└────────┬────────┘
|
|
581
|
-
│
|
|
582
|
-
┌──────────────────────────────┼──────────────────────────────┐
|
|
583
|
-
│ │ │
|
|
584
|
-
▼ ▼ ▼
|
|
585
|
-
┌─────────────────┐ ┌─────────────────┐ ┌─────────────────┐
|
|
586
|
-
│ Node 1 │ │ Node 2 │ │ Node 3 │
|
|
587
|
-
│ (PM2 Cluster) │ │ (PM2 Cluster) │ │ (K8s Pod) │
|
|
588
|
-
├─────────────────┤ ├─────────────────┤ ├─────────────────┤
|
|
589
|
-
│ │ │ │ │ │
|
|
590
|
-
│ ┌───────────┐ │ │ ┌───────────┐ │ │ ┌───────────┐ │
|
|
591
|
-
│ │ Worker A │ │ │ │ Worker C │ │ │ │ Worker E │ │
|
|
592
|
-
│ │(Entity 1) │ │ │ │(Entity 3) │ │ │ │(Entity 5) │ │
|
|
593
|
-
│ └───────────┘ │ │ └───────────┘ │ │ └───────────┘ │
|
|
594
|
-
│ │ │ │ │ │
|
|
595
|
-
│ ┌───────────┐ │ │ ┌───────────┐ │ │ ┌───────────┐ │
|
|
596
|
-
│ │ Worker B │ │ │ │ Worker D │ │ │ │ Worker F │ │
|
|
597
|
-
│ │(Entity 2) │ │ │ │(Entity 4) │ │ │ │(Entity 6) │ │
|
|
598
|
-
│ └───────────┘ │ │ └───────────┘ │ │ └───────────┘ │
|
|
599
|
-
│ │ │ │ │ │
|
|
600
|
-
└────────┬────────┘ └────────┬────────┘ └────────┬────────┘
|
|
601
|
-
│ │ │
|
|
602
|
-
└──────────────────────────────┼──────────────────────────────┘
|
|
603
|
-
│
|
|
604
|
-
▼
|
|
605
|
-
┌─────────────────────────────────────────────────────────────────────────────────┐
|
|
606
|
-
│ REDIS CLUSTER │
|
|
607
|
-
│ │
|
|
608
|
-
│ ┌─────────────────────────────────────────────────────────────────────────┐ │
|
|
609
|
-
│ │ BullMQ Queues │ │
|
|
610
|
-
│ │ entity-1-queue │ entity-2-queue │ entity-3-queue │ ... │ entity-N-q │ │
|
|
611
|
-
│ └─────────────────────────────────────────────────────────────────────────┘ │
|
|
612
|
-
│ │
|
|
613
|
-
│ ┌─────────────────────────────────────────────────────────────────────────┐ │
|
|
614
|
-
│ │ Worker Heartbeats (TTL) │ │
|
|
615
|
-
│ │ aq:workers:entity-1-worker │ aq:workers:entity-2-worker │ ... │ │
|
|
616
|
-
│ └─────────────────────────────────────────────────────────────────────────┘ │
|
|
617
|
-
│ │
|
|
618
|
-
│ ┌─────────────────────────────────────────────────────────────────────────┐ │
|
|
619
|
-
│ │ Job/Entity Indices │ │
|
|
620
|
-
│ │ aq:idx:entity:jobs │ aq:idx:entity:queues │ aq:idx:entity:workers │ │
|
|
621
|
-
│ └─────────────────────────────────────────────────────────────────────────┘ │
|
|
622
|
-
│ │
|
|
623
|
-
│ ┌─────────────────────────────────────────────────────────────────────────┐ │
|
|
624
|
-
│ │ Pub/Sub Shutdown Channels │ │
|
|
625
|
-
│ │ aq:worker:entity-1-worker:shutdown │ aq:worker:entity-2-worker:shut │ │
|
|
626
|
-
│ └─────────────────────────────────────────────────────────────────────────┘ │
|
|
627
|
-
│ │
|
|
628
|
-
└─────────────────────────────────────────────────────────────────────────────────┘
|
|
629
|
-
|
|
630
|
-
|
|
631
|
-
KEY GUARANTEES:
|
|
632
|
-
───────────────
|
|
633
|
-
✓ Only ONE worker processes jobs for each entity (concurrency=1)
|
|
634
|
-
✓ Jobs for same entity are processed in FIFO order
|
|
635
|
-
✓ Worker heartbeats detected across all nodes
|
|
636
|
-
✓ Graceful shutdown via Redis pub/sub (not local signals)
|
|
637
|
-
✓ Any node can spawn workers for any entity
|
|
638
|
-
✓ Dead workers detected via TTL expiration
|
|
639
297
|
```
|
|
640
298
|
|
|
641
299
|
---
|
|
642
300
|
|
|
643
|
-
##
|
|
644
|
-
|
|
645
|
-
The manual API is still available for advanced use cases or gradual migration. **For most use cases, prefer the decorator-based API above.**
|
|
301
|
+
## Complete Example
|
|
646
302
|
|
|
647
|
-
|
|
303
|
+
A document processing service where multiple users can edit the same document:
|
|
648
304
|
|
|
649
305
|
```typescript
|
|
650
|
-
|
|
651
|
-
|
|
652
|
-
|
|
653
|
-
|
|
654
|
-
|
|
655
|
-
|
|
656
|
-
|
|
657
|
-
|
|
658
|
-
|
|
659
|
-
|
|
660
|
-
|
|
661
|
-
cronInterval: 5000,
|
|
662
|
-
keyPrefix: 'myapp',
|
|
663
|
-
}),
|
|
664
|
-
],
|
|
665
|
-
})
|
|
666
|
-
export class AppModule {}
|
|
667
|
-
```
|
|
306
|
+
// ─────────────────────────────────────────────────────────────────
|
|
307
|
+
// commands/update-document.command.ts
|
|
308
|
+
// ─────────────────────────────────────────────────────────────────
|
|
309
|
+
export class UpdateDocumentCommand {
|
|
310
|
+
constructor(
|
|
311
|
+
public readonly documentId: string,
|
|
312
|
+
public readonly userId: string,
|
|
313
|
+
public readonly content: string,
|
|
314
|
+
public readonly version: number,
|
|
315
|
+
) {}
|
|
316
|
+
}
|
|
668
317
|
|
|
669
|
-
|
|
318
|
+
// ─────────────────────────────────────────────────────────────────
|
|
319
|
+
// commands/publish-document.command.ts
|
|
320
|
+
// ─────────────────────────────────────────────────────────────────
|
|
321
|
+
export class PublishDocumentCommand {
|
|
322
|
+
constructor(
|
|
323
|
+
public readonly documentId: string,
|
|
324
|
+
public readonly publishedBy: string,
|
|
325
|
+
) {}
|
|
326
|
+
}
|
|
670
327
|
|
|
671
|
-
|
|
672
|
-
|
|
673
|
-
|
|
674
|
-
import {
|
|
328
|
+
// ─────────────────────────────────────────────────────────────────
|
|
329
|
+
// handlers/update-document.handler.ts
|
|
330
|
+
// ─────────────────────────────────────────────────────────────────
|
|
331
|
+
import { CommandHandler, ICommandHandler } from '@nestjs/cqrs';
|
|
332
|
+
import { UpdateDocumentCommand } from '../commands';
|
|
675
333
|
|
|
676
|
-
@
|
|
677
|
-
|
|
678
|
-
|
|
679
|
-
constructor(private readonly commandBus: CommandBus) {}
|
|
334
|
+
@CommandHandler(UpdateDocumentCommand)
|
|
335
|
+
export class UpdateDocumentHandler implements ICommandHandler<UpdateDocumentCommand> {
|
|
336
|
+
constructor(private readonly documentRepo: DocumentRepository) {}
|
|
680
337
|
|
|
681
|
-
async
|
|
682
|
-
const {
|
|
683
|
-
|
|
338
|
+
async execute(command: UpdateDocumentCommand) {
|
|
339
|
+
const { documentId, userId, content, version } = command;
|
|
340
|
+
|
|
341
|
+
// Safe! No race conditions - one update at a time per document
|
|
342
|
+
await this.documentRepo.update(documentId, { content, version, lastEditedBy: userId });
|
|
343
|
+
|
|
344
|
+
return { success: true, documentId, version };
|
|
684
345
|
}
|
|
685
346
|
}
|
|
686
|
-
```
|
|
687
347
|
|
|
688
|
-
|
|
689
|
-
|
|
690
|
-
|
|
691
|
-
import {
|
|
692
|
-
import {
|
|
348
|
+
// ─────────────────────────────────────────────────────────────────
|
|
349
|
+
// handlers/publish-document.handler.ts
|
|
350
|
+
// ─────────────────────────────────────────────────────────────────
|
|
351
|
+
import { CommandHandler, ICommandHandler } from '@nestjs/cqrs';
|
|
352
|
+
import { PublishDocumentCommand } from '../commands';
|
|
693
353
|
|
|
694
|
-
@
|
|
695
|
-
export class
|
|
696
|
-
constructor(
|
|
697
|
-
private readonly queueManager: QueueManagerService,
|
|
698
|
-
private readonly indexManager: IndexManagerService,
|
|
699
|
-
) {}
|
|
354
|
+
@CommandHandler(PublishDocumentCommand)
|
|
355
|
+
export class PublishDocumentHandler implements ICommandHandler<PublishDocumentCommand> {
|
|
356
|
+
constructor(private readonly documentRepo: DocumentRepository) {}
|
|
700
357
|
|
|
701
|
-
async
|
|
702
|
-
const
|
|
703
|
-
|
|
704
|
-
// Queue validation job
|
|
705
|
-
const job = await this.queueManager.addJob(queue.name, 'validate-order', { orderId, items });
|
|
358
|
+
async execute(command: PublishDocumentCommand) {
|
|
359
|
+
const { documentId, publishedBy } = command;
|
|
706
360
|
|
|
707
|
-
|
|
708
|
-
await this.queueManager.addJob(queue.name, 'process-payment', { orderId, amount });
|
|
361
|
+
await this.documentRepo.publish(documentId, publishedBy);
|
|
709
362
|
|
|
710
|
-
|
|
711
|
-
await this.indexManager.indexJob('order', orderId, job.id!);
|
|
712
|
-
|
|
713
|
-
return orderId;
|
|
363
|
+
return { success: true, documentId, publishedAt: new Date() };
|
|
714
364
|
}
|
|
715
365
|
}
|
|
716
|
-
```
|
|
717
366
|
|
|
718
|
-
|
|
719
|
-
|
|
720
|
-
|
|
367
|
+
// ─────────────────────────────────────────────────────────────────
|
|
368
|
+
// document.processor.ts
|
|
369
|
+
// ─────────────────────────────────────────────────────────────────
|
|
721
370
|
import { Injectable } from '@nestjs/common';
|
|
722
|
-
import {
|
|
371
|
+
import { WorkerProcessor } from 'atomic-queues';
|
|
723
372
|
|
|
373
|
+
@WorkerProcessor({
|
|
374
|
+
entityType: 'document',
|
|
375
|
+
queueName: (documentId) => `doc-${documentId}-queue`,
|
|
376
|
+
workerName: (documentId) => `doc-${documentId}-worker`,
|
|
377
|
+
})
|
|
724
378
|
@Injectable()
|
|
725
|
-
export class
|
|
726
|
-
constructor(
|
|
727
|
-
private readonly workerManager: WorkerManagerService,
|
|
728
|
-
private readonly jobRegistry: JobProcessorRegistry,
|
|
729
|
-
) {}
|
|
730
|
-
|
|
731
|
-
async createOrderWorker(orderId: string) {
|
|
732
|
-
const queueName = `order-${orderId}-queue`;
|
|
733
|
-
|
|
734
|
-
await this.workerManager.createWorker({
|
|
735
|
-
workerName: `${orderId}-worker`,
|
|
736
|
-
queueName,
|
|
737
|
-
processor: async (job) => {
|
|
738
|
-
const processor = this.jobRegistry.getProcessor(job.name);
|
|
739
|
-
if (!processor) {
|
|
740
|
-
throw new Error(`No processor for job: ${job.name}`);
|
|
741
|
-
}
|
|
742
|
-
await processor.process(job);
|
|
743
|
-
},
|
|
744
|
-
events: {
|
|
745
|
-
onReady: async (worker, name) => {
|
|
746
|
-
console.log(`Worker ${name} is ready`);
|
|
747
|
-
},
|
|
748
|
-
onCompleted: async (job, name) => {
|
|
749
|
-
console.log(`Job ${job.id} completed by ${name}`);
|
|
750
|
-
},
|
|
751
|
-
onFailed: async (job, error, name) => {
|
|
752
|
-
console.error(`Job ${job?.id} failed in ${name}:`, error.message);
|
|
753
|
-
},
|
|
754
|
-
},
|
|
755
|
-
});
|
|
756
|
-
}
|
|
757
|
-
}
|
|
758
|
-
```
|
|
759
|
-
|
|
760
|
-
---
|
|
379
|
+
export class DocumentProcessor {}
|
|
761
380
|
|
|
762
|
-
|
|
763
|
-
|
|
764
|
-
|
|
765
|
-
|
|
766
|
-
|
|
767
|
-
|
|
768
|
-
|
|
769
|
-
|
|
770
|
-
const queue = queueManager.getOrCreateEntityQueue('order', '123');
|
|
771
|
-
|
|
772
|
-
// Add a job to a queue
|
|
773
|
-
await queueManager.addJob(queue.name, 'process', { data: 'hello' });
|
|
774
|
-
|
|
775
|
-
// Get job counts
|
|
776
|
-
const counts = await queueManager.getJobCounts(queue.name);
|
|
777
|
-
|
|
778
|
-
// Close a queue
|
|
779
|
-
await queueManager.closeQueue(queue.name);
|
|
780
|
-
```
|
|
781
|
-
|
|
782
|
-
### WorkerManagerService
|
|
783
|
-
|
|
784
|
-
Manages worker lifecycle with heartbeat-based liveness tracking.
|
|
785
|
-
|
|
786
|
-
```typescript
|
|
787
|
-
// Create a worker
|
|
788
|
-
await workerManager.createWorker({
|
|
789
|
-
workerName: 'my-worker',
|
|
790
|
-
queueName: 'my-queue',
|
|
791
|
-
processor: async (job) => { /* process job */ },
|
|
792
|
-
config: {
|
|
793
|
-
concurrency: 1,
|
|
794
|
-
heartbeatTTL: 3,
|
|
795
|
-
},
|
|
796
|
-
});
|
|
797
|
-
|
|
798
|
-
// Check if worker exists
|
|
799
|
-
const exists = await workerManager.workerExists('my-worker');
|
|
800
|
-
|
|
801
|
-
// Signal worker to close via Redis pub/sub
|
|
802
|
-
await workerManager.signalWorkerClose('my-worker');
|
|
803
|
-
|
|
804
|
-
// Get all workers for an entity
|
|
805
|
-
const workers = await workerManager.getEntityWorkers('order', '123');
|
|
806
|
-
```
|
|
381
|
+
// ─────────────────────────────────────────────────────────────────
|
|
382
|
+
// document.module.ts
|
|
383
|
+
// ─────────────────────────────────────────────────────────────────
|
|
384
|
+
import { Module } from '@nestjs/common';
|
|
385
|
+
import { CqrsModule } from '@nestjs/cqrs';
|
|
386
|
+
import { DocumentProcessor } from './document.processor';
|
|
387
|
+
import { DocumentController } from './document.controller';
|
|
388
|
+
import { UpdateDocumentHandler, PublishDocumentHandler } from './handlers';
|
|
807
389
|
|
|
808
|
-
|
|
390
|
+
@Module({
|
|
391
|
+
imports: [CqrsModule],
|
|
392
|
+
providers: [
|
|
393
|
+
DocumentProcessor,
|
|
394
|
+
UpdateDocumentHandler, // Commands auto-discovered!
|
|
395
|
+
PublishDocumentHandler,
|
|
396
|
+
],
|
|
397
|
+
controllers: [DocumentController],
|
|
398
|
+
})
|
|
399
|
+
export class DocumentModule {}
|
|
400
|
+
|
|
401
|
+
// ─────────────────────────────────────────────────────────────────
|
|
402
|
+
// document.controller.ts
|
|
403
|
+
// ─────────────────────────────────────────────────────────────────
|
|
404
|
+
import { Controller, Post, Body, Param } from '@nestjs/common';
|
|
405
|
+
import { QueueBus } from 'atomic-queues';
|
|
406
|
+
import { DocumentProcessor } from './document.processor';
|
|
407
|
+
import { UpdateDocumentCommand, PublishDocumentCommand } from './commands';
|
|
408
|
+
|
|
409
|
+
@Controller('documents')
|
|
410
|
+
export class DocumentController {
|
|
411
|
+
constructor(private readonly queueBus: QueueBus) {}
|
|
412
|
+
|
|
413
|
+
@Post(':id/update')
|
|
414
|
+
async updateDocument(
|
|
415
|
+
@Param('id') documentId: string,
|
|
416
|
+
@Body() body: { userId: string; content: string; version: number },
|
|
417
|
+
) {
|
|
418
|
+
// Multiple users editing same doc? No problem!
|
|
419
|
+
// Updates are queued and processed one at a time
|
|
420
|
+
await this.queueBus
|
|
421
|
+
.forProcessor(DocumentProcessor)
|
|
422
|
+
.enqueue(new UpdateDocumentCommand(
|
|
423
|
+
documentId,
|
|
424
|
+
body.userId,
|
|
425
|
+
body.content,
|
|
426
|
+
body.version,
|
|
427
|
+
));
|
|
428
|
+
|
|
429
|
+
return { queued: true, documentId };
|
|
430
|
+
}
|
|
809
431
|
|
|
810
|
-
|
|
432
|
+
@Post(':id/publish')
|
|
433
|
+
async publishDocument(
|
|
434
|
+
@Param('id') documentId: string,
|
|
435
|
+
@Body() body: { publishedBy: string },
|
|
436
|
+
) {
|
|
437
|
+
await this.queueBus
|
|
438
|
+
.forProcessor(DocumentProcessor)
|
|
439
|
+
.enqueue(new PublishDocumentCommand(documentId, body.publishedBy));
|
|
811
440
|
|
|
812
|
-
|
|
813
|
-
// Acquire a lock
|
|
814
|
-
const result = await lockService.acquireLock(
|
|
815
|
-
'resource', // resourceType
|
|
816
|
-
'resource-123', // resourceId
|
|
817
|
-
'owner-456', // ownerId
|
|
818
|
-
'service', // ownerType
|
|
819
|
-
60, // TTL in seconds
|
|
820
|
-
);
|
|
821
|
-
|
|
822
|
-
if (result.acquired) {
|
|
823
|
-
try {
|
|
824
|
-
// Do work with the locked resource
|
|
825
|
-
} finally {
|
|
826
|
-
await lockService.releaseLock('resource', 'resource-123');
|
|
441
|
+
return { queued: true, documentId };
|
|
827
442
|
}
|
|
828
443
|
}
|
|
829
|
-
|
|
830
|
-
// Get first available resource from a pool
|
|
831
|
-
const available = await lockService.getAvailableResource(
|
|
832
|
-
'resource',
|
|
833
|
-
['res-1', 'res-2', 'res-3'],
|
|
834
|
-
'owner-456',
|
|
835
|
-
'service',
|
|
836
|
-
);
|
|
837
444
|
```
|
|
838
445
|
|
|
839
|
-
|
|
840
|
-
|
|
841
|
-
Automatic worker scaling based on demand.
|
|
842
|
-
|
|
843
|
-
**Recommended: Use `@EntityScaler` decorator (see Quick Start section above)**
|
|
844
|
-
|
|
845
|
-
The decorator-based approach is preferred as it's cleaner and auto-discovered:
|
|
846
|
-
|
|
847
|
-
```typescript
|
|
848
|
-
@EntityScaler({ entityType: 'order', maxWorkersPerEntity: 1 })
|
|
849
|
-
@Injectable()
|
|
850
|
-
export class OrderEntityScaler {
|
|
851
|
-
@GetActiveEntities()
|
|
852
|
-
async getActiveOrders(): Promise<string[]> { ... }
|
|
853
|
-
|
|
854
|
-
@GetDesiredWorkerCount()
|
|
855
|
-
async getWorkerCount(orderId: string): Promise<number> { return 1; }
|
|
856
|
-
}
|
|
857
|
-
```
|
|
446
|
+
---
|
|
858
447
|
|
|
859
|
-
|
|
448
|
+
## Configuration
|
|
860
449
|
|
|
861
450
|
```typescript
|
|
862
|
-
|
|
863
|
-
|
|
864
|
-
|
|
865
|
-
|
|
866
|
-
|
|
867
|
-
return 1;
|
|
868
|
-
},
|
|
869
|
-
getActiveEntityIds: async () => {
|
|
870
|
-
return Object.keys(await indexManager.getEntitiesWithJobs('order'));
|
|
451
|
+
AtomicQueuesModule.forRoot({
|
|
452
|
+
redis: {
|
|
453
|
+
host: 'localhost',
|
|
454
|
+
port: 6379,
|
|
455
|
+
password: 'secret',
|
|
871
456
|
},
|
|
872
|
-
|
|
873
|
-
|
|
874
|
-
|
|
457
|
+
|
|
458
|
+
keyPrefix: 'myapp', // Redis key prefix (default: 'aq')
|
|
459
|
+
|
|
460
|
+
enableCronManager: true, // Enable auto-scaling (default: false)
|
|
461
|
+
cronInterval: 5000, // Scaling check interval (default: 5000ms)
|
|
462
|
+
|
|
463
|
+
verbose: false, // Enable verbose logging (default: false)
|
|
464
|
+
// When true, logs service job processing details
|
|
465
|
+
|
|
466
|
+
workerDefaults: {
|
|
467
|
+
concurrency: 1, // Jobs processed simultaneously
|
|
468
|
+
stalledInterval: 1000, // Stalled job check interval
|
|
469
|
+
lockDuration: 30000, // Job lock duration
|
|
470
|
+
heartbeatTTL: 3, // Worker heartbeat TTL (seconds)
|
|
875
471
|
},
|
|
876
472
|
});
|
|
877
|
-
|
|
878
|
-
// Start the cron manager
|
|
879
|
-
cronManager.start(5000);
|
|
880
473
|
```
|
|
881
474
|
|
|
882
|
-
|
|
883
|
-
|
|
884
|
-
Track jobs, workers, and queue states.
|
|
885
|
-
|
|
886
|
-
```typescript
|
|
887
|
-
// Index a job
|
|
888
|
-
await indexManager.indexJob('order', '123', 'job-456');
|
|
889
|
-
|
|
890
|
-
// Get all entities with pending jobs
|
|
891
|
-
const entitiesWithJobs = await indexManager.getEntitiesWithJobs('order');
|
|
892
|
-
// Returns: { '123': 5, '456': 2 } (entityId: jobCount)
|
|
475
|
+
---
|
|
893
476
|
|
|
894
|
-
|
|
895
|
-
await indexManager.indexEntityQueue('order', '123');
|
|
477
|
+
## Command Registration
|
|
896
478
|
|
|
897
|
-
|
|
898
|
-
await indexManager.cleanupEntityIndices('order', '123');
|
|
899
|
-
```
|
|
479
|
+
By default, atomic-queues **auto-discovers** all commands from your `@CommandHandler` and `@QueryHandler` decorators. No manual registration needed!
|
|
900
480
|
|
|
901
|
-
|
|
481
|
+
### Auto-Discovery (Default)
|
|
902
482
|
|
|
903
|
-
|
|
483
|
+
Commands are automatically discovered when you have CQRS handlers:
|
|
904
484
|
|
|
905
485
|
```typescript
|
|
906
|
-
|
|
907
|
-
|
|
908
|
-
|
|
909
|
-
|
|
910
|
-
|
|
911
|
-
|
|
912
|
-
db?: number;
|
|
913
|
-
url?: string;
|
|
914
|
-
maxRetriesPerRequest?: number | null;
|
|
915
|
-
};
|
|
916
|
-
|
|
917
|
-
// Worker defaults
|
|
918
|
-
workerDefaults?: {
|
|
919
|
-
concurrency?: number; // Default: 1
|
|
920
|
-
stalledInterval?: number; // Default: 1000ms
|
|
921
|
-
lockDuration?: number; // Default: 30000ms
|
|
922
|
-
maxStalledCount?: number; // Default: MAX_SAFE_INTEGER
|
|
923
|
-
heartbeatTTL?: number; // Default: 3 seconds
|
|
924
|
-
heartbeatInterval?: number; // Default: 1000ms
|
|
925
|
-
};
|
|
926
|
-
|
|
927
|
-
// Queue defaults
|
|
928
|
-
queueDefaults?: {
|
|
929
|
-
defaultJobOptions?: {
|
|
930
|
-
removeOnComplete?: boolean;
|
|
931
|
-
removeOnFail?: boolean;
|
|
932
|
-
attempts?: number;
|
|
933
|
-
backoff?: { type: 'fixed' | 'exponential'; delay: number };
|
|
934
|
-
priority?: number;
|
|
935
|
-
};
|
|
936
|
-
};
|
|
937
|
-
|
|
938
|
-
// Cron manager
|
|
939
|
-
enableCronManager?: boolean; // Default: false
|
|
940
|
-
cronInterval?: number; // Default: 5000ms
|
|
941
|
-
|
|
942
|
-
// Key prefix for Redis keys
|
|
943
|
-
keyPrefix?: string; // Default: 'aq'
|
|
486
|
+
// Your handler - that's all you need!
|
|
487
|
+
@CommandHandler(ProcessOrderCommand)
|
|
488
|
+
export class ProcessOrderHandler implements ICommandHandler<ProcessOrderCommand> {
|
|
489
|
+
async execute(command: ProcessOrderCommand) {
|
|
490
|
+
// ProcessOrderCommand is auto-registered with QueueBus
|
|
491
|
+
}
|
|
944
492
|
}
|
|
945
493
|
```
|
|
946
494
|
|
|
947
|
-
|
|
948
|
-
|
|
949
|
-
## Graceful Shutdown
|
|
950
|
-
|
|
951
|
-
The library handles graceful shutdown automatically via Redis pub/sub:
|
|
495
|
+
### Manual Registration (Optional)
|
|
952
496
|
|
|
953
|
-
|
|
954
|
-
2. All workers (even on other nodes) subscribed to shutdown channels receive the signal
|
|
955
|
-
3. Workers finish their current job (with configurable timeout)
|
|
956
|
-
4. Heartbeat TTLs expire, marking workers as dead
|
|
957
|
-
5. Resources are cleaned up
|
|
497
|
+
If you need to register commands without handlers, or disable auto-discovery:
|
|
958
498
|
|
|
959
499
|
```typescript
|
|
960
|
-
//
|
|
961
|
-
|
|
962
|
-
|
|
500
|
+
// Disable auto-discovery in config
|
|
501
|
+
AtomicQueuesModule.forRoot({
|
|
502
|
+
redis: { host: 'localhost', port: 6379 },
|
|
503
|
+
autoRegisterCommands: false, // Disable auto-discovery
|
|
504
|
+
});
|
|
505
|
+
|
|
506
|
+
// Then manually register
|
|
507
|
+
QueueBus.registerCommands(ProcessOrderCommand, ShipOrderCommand);
|
|
963
508
|
```
|
|
964
509
|
|
|
965
510
|
---
|
|
966
511
|
|
|
967
|
-
## Use
|
|
968
|
-
|
|
969
|
-
### 1. Per-Order Processing (E-commerce)
|
|
970
|
-
Each order has its own queue ensuring stages (validate → pay → ship) happen sequentially.
|
|
971
|
-
|
|
972
|
-
### 2. Per-User Message Queues (Chat/Messaging)
|
|
973
|
-
Each user has their own queue for message delivery, ensuring order.
|
|
974
|
-
|
|
975
|
-
### 3. Per-Tenant Job Processing (SaaS)
|
|
976
|
-
Each tenant's jobs are isolated and processed sequentially.
|
|
977
|
-
|
|
978
|
-
### 4. Per-Document Processing (Document Management)
|
|
979
|
-
Each document goes through OCR → validation → storage in sequence.
|
|
512
|
+
## Why Use atomic-queues?
|
|
980
513
|
|
|
981
|
-
|
|
982
|
-
|
|
514
|
+
| Feature | Without | With atomic-queues |
|
|
515
|
+
|---------|---------|-------------------|
|
|
516
|
+
| Sequential per-entity | Manual locking | Automatic via queues |
|
|
517
|
+
| Race conditions | Possible | Prevented |
|
|
518
|
+
| Worker management | Manual | Automatic |
|
|
519
|
+
| Horizontal scaling | Complex | Built-in |
|
|
520
|
+
| Code organization | Scattered | Clean decorators |
|
|
983
521
|
|
|
984
522
|
---
|
|
985
523
|
|