atomic-queues 1.2.2 → 1.2.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +317 -834
- package/dist/domain/interfaces.d.ts +8 -0
- package/dist/domain/interfaces.d.ts.map +1 -1
- package/dist/services/command-discovery/command-discovery.service.d.ts.map +1 -1
- package/dist/services/command-discovery/command-discovery.service.js +4 -1
- package/dist/services/command-discovery/command-discovery.service.js.map +1 -1
- package/dist/services/processor-discovery/processor-discovery.service.d.ts +4 -0
- package/dist/services/processor-discovery/processor-discovery.service.d.ts.map +1 -1
- package/dist/services/processor-discovery/processor-discovery.service.js +13 -0
- package/dist/services/processor-discovery/processor-discovery.service.js.map +1 -1
- package/dist/services/queue-bus/queue-bus.service.d.ts +28 -0
- package/dist/services/queue-bus/queue-bus.service.d.ts.map +1 -1
- package/dist/services/queue-bus/queue-bus.service.js +54 -0
- package/dist/services/queue-bus/queue-bus.service.js.map +1 -1
- package/dist/services/service-queue/service-queue.service.js +3 -1
- package/dist/services/service-queue/service-queue.service.js.map +1 -1
- package/package.json +1 -1
package/README.md
CHANGED
|
@@ -1,34 +1,35 @@
|
|
|
1
1
|
# atomic-queues
|
|
2
2
|
|
|
3
|
-
A
|
|
4
|
-
|
|
5
|
-
##
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
3
|
+
A NestJS library for atomic, sequential job processing per entity with BullMQ and Redis.
|
|
4
|
+
|
|
5
|
+
## What It Does
|
|
6
|
+
|
|
7
|
+
```
|
|
8
|
+
┌─────────────────────────────────────────────────────────────────┐
|
|
9
|
+
│ THE PROBLEM │
|
|
10
|
+
├─────────────────────────────────────────────────────────────────┤
|
|
11
|
+
│ │
|
|
12
|
+
│ Multiple requests for the same entity arrive simultaneously: │
|
|
13
|
+
│ │
|
|
14
|
+
│ Request A ───┐ │
|
|
15
|
+
│ Request B ───┼──► Entity 123 ──► 💥 RACE CONDITION! │
|
|
16
|
+
│ Request C ───┘ │
|
|
17
|
+
│ │
|
|
18
|
+
└─────────────────────────────────────────────────────────────────┘
|
|
19
|
+
|
|
20
|
+
┌─────────────────────────────────────────────────────────────────┐
|
|
21
|
+
│ THE SOLUTION │
|
|
22
|
+
├─────────────────────────────────────────────────────────────────┤
|
|
23
|
+
│ │
|
|
24
|
+
│ atomic-queues ensures sequential processing per entity: │
|
|
25
|
+
│ │
|
|
26
|
+
│ Request A ───┐ ┌─────────┐ │
|
|
27
|
+
│ Request B ───┼──► │ Queue │ ──► Worker ──► Entity 123 │
|
|
28
|
+
│ Request C ───┘ │ A, B, C │ (1 at a time) │
|
|
29
|
+
│ └─────────┘ │
|
|
30
|
+
│ │
|
|
31
|
+
└─────────────────────────────────────────────────────────────────┘
|
|
32
|
+
```
|
|
32
33
|
|
|
33
34
|
## Installation
|
|
34
35
|
|
|
@@ -36,13 +37,9 @@ This library solves all of these with a single, cohesive module.
|
|
|
36
37
|
npm install atomic-queues bullmq ioredis
|
|
37
38
|
```
|
|
38
39
|
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
## Quick Start (Decorator-based API) ✨
|
|
40
|
+
## Quick Start
|
|
42
41
|
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
### 1. Import the Module
|
|
42
|
+
### 1. Configure the Module
|
|
46
43
|
|
|
47
44
|
```typescript
|
|
48
45
|
import { Module } from '@nestjs/common';
|
|
@@ -50,936 +47,422 @@ import { AtomicQueuesModule } from 'atomic-queues';
|
|
|
50
47
|
|
|
51
48
|
@Module({
|
|
52
49
|
imports: [
|
|
53
|
-
AtomicQueuesModule.
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
redis: {
|
|
57
|
-
url: configService.get('REDIS_URL'),
|
|
58
|
-
},
|
|
59
|
-
keyPrefix: 'myapp',
|
|
60
|
-
enableCronManager: true,
|
|
61
|
-
workerDefaults: {
|
|
62
|
-
concurrency: 1,
|
|
63
|
-
heartbeatTTL: 3,
|
|
64
|
-
},
|
|
65
|
-
}),
|
|
66
|
-
inject: [ConfigService],
|
|
50
|
+
AtomicQueuesModule.forRoot({
|
|
51
|
+
redis: { host: 'localhost', port: 6379 },
|
|
52
|
+
keyPrefix: 'myapp',
|
|
67
53
|
}),
|
|
68
54
|
],
|
|
69
55
|
})
|
|
70
56
|
export class AppModule {}
|
|
71
57
|
```
|
|
72
58
|
|
|
73
|
-
### 2. Create
|
|
59
|
+
### 2. Create Your Commands
|
|
60
|
+
|
|
61
|
+
Plain classes - no decorators needed:
|
|
62
|
+
|
|
63
|
+
```typescript
|
|
64
|
+
// commands/process-order.command.ts
|
|
65
|
+
export class ProcessOrderCommand {
|
|
66
|
+
constructor(
|
|
67
|
+
public readonly orderId: string,
|
|
68
|
+
public readonly items: string[],
|
|
69
|
+
public readonly amount: number,
|
|
70
|
+
) {}
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
// commands/ship-order.command.ts
|
|
74
|
+
export class ShipOrderCommand {
|
|
75
|
+
constructor(
|
|
76
|
+
public readonly orderId: string,
|
|
77
|
+
public readonly address: string,
|
|
78
|
+
) {}
|
|
79
|
+
}
|
|
80
|
+
```
|
|
74
81
|
|
|
75
|
-
|
|
82
|
+
### 3. Create a Worker Processor
|
|
76
83
|
|
|
77
84
|
```typescript
|
|
78
85
|
import { Injectable } from '@nestjs/common';
|
|
79
|
-
import {
|
|
80
|
-
import { Job } from 'bullmq';
|
|
81
|
-
import { WorkerProcessor, JobHandler } from 'atomic-queues';
|
|
86
|
+
import { WorkerProcessor } from 'atomic-queues';
|
|
82
87
|
|
|
83
88
|
@WorkerProcessor({
|
|
84
89
|
entityType: 'order',
|
|
85
90
|
queueName: (orderId) => `order-${orderId}-queue`,
|
|
86
91
|
workerName: (orderId) => `order-${orderId}-worker`,
|
|
87
|
-
workerConfig: {
|
|
88
|
-
concurrency: 1,
|
|
89
|
-
heartbeatTTL: 3,
|
|
90
|
-
},
|
|
91
92
|
})
|
|
92
93
|
@Injectable()
|
|
93
|
-
export class
|
|
94
|
-
constructor(private readonly commandBus: CommandBus) {}
|
|
95
|
-
|
|
96
|
-
@JobHandler('validate')
|
|
97
|
-
async handleValidate(job: Job, orderId: string) {
|
|
98
|
-
const { items } = job.data;
|
|
99
|
-
return this.commandBus.execute(new ValidateOrderCommand(orderId, items));
|
|
100
|
-
}
|
|
101
|
-
|
|
102
|
-
@JobHandler('process-payment')
|
|
103
|
-
async handlePayment(job: Job, orderId: string) {
|
|
104
|
-
const { amount } = job.data;
|
|
105
|
-
return this.commandBus.execute(new ProcessPaymentCommand(orderId, amount));
|
|
106
|
-
}
|
|
107
|
-
|
|
108
|
-
@JobHandler('ship')
|
|
109
|
-
async handleShip(job: Job, orderId: string) {
|
|
110
|
-
return this.commandBus.execute(new ShipOrderCommand(orderId));
|
|
111
|
-
}
|
|
112
|
-
|
|
113
|
-
// Wildcard handler for any unmatched job names
|
|
114
|
-
@JobHandler('*')
|
|
115
|
-
async handleOther(job: Job, orderId: string) {
|
|
116
|
-
console.log(`Unknown job type: ${job.name} for order ${orderId}`);
|
|
117
|
-
}
|
|
118
|
-
}
|
|
94
|
+
export class OrderProcessor {}
|
|
119
95
|
```
|
|
120
96
|
|
|
121
|
-
###
|
|
97
|
+
### 4. Queue Jobs with the Fluent API
|
|
122
98
|
|
|
123
|
-
|
|
99
|
+
Commands are **automatically registered** from your `@CommandHandler` classes - no manual registration needed!
|
|
124
100
|
|
|
125
101
|
```typescript
|
|
126
102
|
import { Injectable } from '@nestjs/common';
|
|
127
|
-
import {
|
|
103
|
+
import { QueueBus } from 'atomic-queues';
|
|
104
|
+
import { OrderProcessor } from './order.processor';
|
|
105
|
+
import { ProcessOrderCommand, ShipOrderCommand } from './commands';
|
|
128
106
|
|
|
129
|
-
@EntityScaler({
|
|
130
|
-
entityType: 'order',
|
|
131
|
-
maxWorkersPerEntity: 1,
|
|
132
|
-
})
|
|
133
107
|
@Injectable()
|
|
134
|
-
export class
|
|
135
|
-
constructor(private readonly
|
|
108
|
+
export class OrderService {
|
|
109
|
+
constructor(private readonly queueBus: QueueBus) {}
|
|
136
110
|
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
111
|
+
async createOrder(orderId: string, items: string[], amount: number) {
|
|
112
|
+
// Jobs are queued and processed sequentially per orderId
|
|
113
|
+
await this.queueBus
|
|
114
|
+
.forProcessor(OrderProcessor)
|
|
115
|
+
.enqueue(new ProcessOrderCommand(orderId, items, amount));
|
|
142
116
|
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
return 1;
|
|
117
|
+
await this.queueBus
|
|
118
|
+
.forProcessor(OrderProcessor)
|
|
119
|
+
.enqueue(new ShipOrderCommand(orderId, '123 Main St'));
|
|
147
120
|
}
|
|
148
121
|
}
|
|
149
122
|
```
|
|
150
123
|
|
|
151
|
-
|
|
124
|
+
That's it! The library automatically:
|
|
125
|
+
- Discovers commands from `@CommandHandler` decorators
|
|
126
|
+
- Creates a queue for each `orderId`
|
|
127
|
+
- Spawns a worker to process jobs sequentially
|
|
128
|
+
- Routes jobs to the correct command handlers
|
|
152
129
|
|
|
153
|
-
|
|
154
|
-
@Module({
|
|
155
|
-
imports: [AtomicQueuesModule.forRootAsync({ ... })],
|
|
156
|
-
providers: [
|
|
157
|
-
OrderWorkerProcessor, // Auto-discovered by @WorkerProcessor
|
|
158
|
-
OrderEntityScaler, // Auto-discovered by @EntityScaler
|
|
159
|
-
],
|
|
160
|
-
})
|
|
161
|
-
export class OrderModule {}
|
|
162
|
-
```
|
|
163
|
-
|
|
164
|
-
### 5. Queue Jobs
|
|
165
|
-
|
|
166
|
-
```typescript
|
|
167
|
-
import { Injectable } from '@nestjs/common';
|
|
168
|
-
import { QueueManagerService } from 'atomic-queues';
|
|
130
|
+
---
|
|
169
131
|
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
132
|
+
## How It Works
|
|
133
|
+
|
|
134
|
+
```
|
|
135
|
+
┌─────────────────────────────────────────────────────────────────┐
|
|
136
|
+
│ FLOW DIAGRAM │
|
|
137
|
+
└─────────────────────────────────────────────────────────────────┘
|
|
138
|
+
|
|
139
|
+
YOUR CODE ATOMIC-QUEUES WORKER
|
|
140
|
+
───────── ───────────── ──────
|
|
141
|
+
|
|
142
|
+
queueBus
|
|
143
|
+
.forProcessor(OrderProcessor)
|
|
144
|
+
.enqueue(new ProcessOrderCommand(...))
|
|
145
|
+
│
|
|
146
|
+
│ 1. Extract queue config from @WorkerProcessor
|
|
147
|
+
│ 2. Extract orderId from command.orderId
|
|
148
|
+
│ 3. Build queue name: order-{orderId}-queue
|
|
149
|
+
▼
|
|
150
|
+
┌─────────────┐
|
|
151
|
+
│ Redis │
|
|
152
|
+
│ Queue │ ◄─── Job: { name: "ProcessOrderCommand", data: {...} }
|
|
153
|
+
└──────┬──────┘
|
|
154
|
+
│
|
|
155
|
+
│ 4. Worker pulls job from queue
|
|
156
|
+
▼
|
|
157
|
+
┌─────────────┐
|
|
158
|
+
│ Worker │
|
|
159
|
+
│ (1 per ID) │
|
|
160
|
+
└──────┬──────┘
|
|
161
|
+
│
|
|
162
|
+
│ 5. Lookup ProcessOrderCommand in registry
|
|
163
|
+
│ 6. Instantiate command from job.data
|
|
164
|
+
│ 7. Execute via CommandBus
|
|
165
|
+
▼
|
|
166
|
+
┌─────────────┐
|
|
167
|
+
│ CommandBus │ ──► ProcessOrderCommandHandler.execute()
|
|
168
|
+
└─────────────┘
|
|
185
169
|
```
|
|
186
170
|
|
|
187
|
-
That's it! The library will:
|
|
188
|
-
1. **Auto-discover** your `OrderWorkerProcessor` and `OrderEntityScaler`
|
|
189
|
-
2. **Create workers** for active jobs via `CronManagerService`
|
|
190
|
-
3. **Route jobs** to the correct `@JobHandler` method
|
|
191
|
-
4. **Clean up** workers when jobs are complete
|
|
192
|
-
|
|
193
171
|
---
|
|
194
172
|
|
|
195
|
-
##
|
|
173
|
+
## API Reference
|
|
196
174
|
|
|
197
|
-
###
|
|
175
|
+
### QueueBus
|
|
198
176
|
|
|
199
|
-
|
|
177
|
+
The main way to add jobs to queues:
|
|
200
178
|
|
|
201
179
|
```typescript
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
workerConfig?: {
|
|
207
|
-
concurrency?: number; // Default: 1
|
|
208
|
-
stalledInterval?: number; // Default: 1000ms
|
|
209
|
-
lockDuration?: number; // Default: 30000ms
|
|
210
|
-
heartbeatTTL?: number; // Default: 3 seconds
|
|
211
|
-
heartbeatInterval?: number; // Default: 1000ms
|
|
212
|
-
};
|
|
213
|
-
})
|
|
214
|
-
```
|
|
180
|
+
// Enqueue a single command
|
|
181
|
+
await queueBus
|
|
182
|
+
.forProcessor(MyProcessor)
|
|
183
|
+
.enqueue(new MyCommand(entityId, data));
|
|
215
184
|
|
|
216
|
-
|
|
185
|
+
// Enqueue and wait for result
|
|
186
|
+
const result = await queueBus
|
|
187
|
+
.forProcessor(MyProcessor)
|
|
188
|
+
.enqueueAndWait(new MyQuery(entityId));
|
|
217
189
|
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
190
|
+
// Enqueue multiple commands
|
|
191
|
+
await queueBus
|
|
192
|
+
.forProcessor(MyProcessor)
|
|
193
|
+
.enqueueBulk([
|
|
194
|
+
new CommandA(entityId),
|
|
195
|
+
new CommandB(entityId),
|
|
196
|
+
]);
|
|
223
197
|
|
|
224
|
-
|
|
225
|
-
|
|
198
|
+
// With job options (delay, priority, etc.)
|
|
199
|
+
await queueBus
|
|
200
|
+
.forProcessor(MyProcessor)
|
|
201
|
+
.enqueue(new MyCommand(entityId), {
|
|
202
|
+
jobOptions: { delay: 5000, priority: 1 }
|
|
203
|
+
});
|
|
226
204
|
```
|
|
227
205
|
|
|
228
|
-
### @
|
|
206
|
+
### @WorkerProcessor
|
|
229
207
|
|
|
230
|
-
|
|
208
|
+
Defines how workers are created for an entity type:
|
|
231
209
|
|
|
232
210
|
```typescript
|
|
233
|
-
@
|
|
234
|
-
entityType:
|
|
235
|
-
|
|
211
|
+
@WorkerProcessor({
|
|
212
|
+
entityType: 'order', // Required
|
|
213
|
+
queueName: (id) => `order-${id}-queue`, // Optional
|
|
214
|
+
workerName: (id) => `order-${id}-worker`, // Optional
|
|
215
|
+
workerConfig: {
|
|
216
|
+
concurrency: 1, // Jobs per worker (default: 1)
|
|
217
|
+
stalledInterval: 1000, // Check stalled jobs (ms)
|
|
218
|
+
lockDuration: 30000, // Job lock duration (ms)
|
|
219
|
+
},
|
|
236
220
|
})
|
|
237
221
|
```
|
|
238
222
|
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
Method decorator marking the method that returns active entity IDs.
|
|
242
|
-
|
|
243
|
-
```typescript
|
|
244
|
-
@GetActiveEntities()
|
|
245
|
-
async getActiveOrders(): Promise<string[]> {
|
|
246
|
-
return ['order-1', 'order-2'];
|
|
247
|
-
}
|
|
248
|
-
```
|
|
223
|
+
---
|
|
249
224
|
|
|
250
|
-
|
|
225
|
+
## Entity ID Extraction
|
|
251
226
|
|
|
252
|
-
|
|
227
|
+
The `entityId` is automatically extracted from your command's properties:
|
|
253
228
|
|
|
254
229
|
```typescript
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
return 1;
|
|
258
|
-
}
|
|
259
|
-
```
|
|
230
|
+
// These property names are checked in order:
|
|
231
|
+
// entityId, tableId, userId, id, gameId, playerId
|
|
260
232
|
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
@OnSpawnWorker()
|
|
267
|
-
async customSpawn(entityId: string): Promise<void> {
|
|
268
|
-
console.log(`Spawning worker for ${entityId}`);
|
|
233
|
+
export class ProcessOrderCommand {
|
|
234
|
+
constructor(
|
|
235
|
+
public readonly orderId: string, // ✓ 'orderId' contains 'Id' → entityId
|
|
236
|
+
public readonly items: string[],
|
|
237
|
+
) {}
|
|
269
238
|
}
|
|
270
239
|
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
|
|
240
|
+
// Or use standard names
|
|
241
|
+
export class UpdateUserCommand {
|
|
242
|
+
constructor(
|
|
243
|
+
public readonly userId: string, // ✓ Matches 'userId' → entityId
|
|
244
|
+
public readonly name: string,
|
|
245
|
+
) {}
|
|
274
246
|
}
|
|
275
247
|
```
|
|
276
248
|
|
|
277
249
|
---
|
|
278
250
|
|
|
279
|
-
##
|
|
280
|
-
|
|
281
|
-
### Migrating from Manual Registration to Decorators
|
|
251
|
+
## Scaling with Entity Scalers
|
|
282
252
|
|
|
283
|
-
|
|
253
|
+
For dynamic worker management based on demand:
|
|
284
254
|
|
|
285
255
|
```typescript
|
|
286
|
-
|
|
287
|
-
|
|
288
|
-
@JobProcessor('validate-order')
|
|
289
|
-
export class ValidateOrderProcessor {
|
|
290
|
-
async process(job: Job) {
|
|
291
|
-
// validation logic
|
|
292
|
-
}
|
|
293
|
-
}
|
|
294
|
-
|
|
295
|
-
// order-worker.service.ts (manual worker creation)
|
|
296
|
-
@Injectable()
|
|
297
|
-
export class OrderWorkerService {
|
|
298
|
-
constructor(
|
|
299
|
-
private workerManager: WorkerManagerService,
|
|
300
|
-
private jobRegistry: JobProcessorRegistry,
|
|
301
|
-
) {}
|
|
302
|
-
|
|
303
|
-
async createOrderWorker(orderId: string) {
|
|
304
|
-
await this.workerManager.createWorker({
|
|
305
|
-
workerName: `order-${orderId}-worker`,
|
|
306
|
-
queueName: `order-${orderId}-queue`,
|
|
307
|
-
processor: async (job) => {
|
|
308
|
-
const processor = this.jobRegistry.getProcessor(job.name);
|
|
309
|
-
await processor.process(job);
|
|
310
|
-
},
|
|
311
|
-
});
|
|
312
|
-
}
|
|
313
|
-
}
|
|
314
|
-
|
|
315
|
-
// app.module.ts (manual entity type registration)
|
|
316
|
-
cronManager.registerEntityType({
|
|
317
|
-
entityType: 'order',
|
|
318
|
-
getActiveEntityIds: async () => [...],
|
|
319
|
-
getDesiredWorkerCount: async (id) => 1,
|
|
320
|
-
onSpawnWorker: async (id) => orderWorkerService.createOrderWorker(id),
|
|
321
|
-
});
|
|
322
|
-
```
|
|
323
|
-
|
|
324
|
-
**After (Decorator-based):**
|
|
256
|
+
import { Injectable } from '@nestjs/common';
|
|
257
|
+
import { EntityScaler, GetActiveEntities, GetDesiredWorkerCount } from 'atomic-queues';
|
|
325
258
|
|
|
326
|
-
|
|
327
|
-
// table-worker.processor.ts (single file with all handlers)
|
|
328
|
-
@WorkerProcessor({
|
|
259
|
+
@EntityScaler({
|
|
329
260
|
entityType: 'order',
|
|
330
|
-
|
|
331
|
-
workerName: (id) => `order-${id}-worker`,
|
|
261
|
+
maxWorkersPerEntity: 1,
|
|
332
262
|
})
|
|
333
263
|
@Injectable()
|
|
334
|
-
export class
|
|
335
|
-
|
|
336
|
-
async handleValidate(job: Job, orderId: string) {
|
|
337
|
-
// validation logic
|
|
338
|
-
}
|
|
339
|
-
|
|
340
|
-
@JobHandler('process-payment')
|
|
341
|
-
async handlePayment(job: Job, orderId: string) {
|
|
342
|
-
// payment logic
|
|
343
|
-
}
|
|
344
|
-
}
|
|
264
|
+
export class OrderScaler {
|
|
265
|
+
constructor(private readonly orderRepo: OrderRepository) {}
|
|
345
266
|
|
|
346
|
-
// table-entity.scaler.ts (scaling config in one place)
|
|
347
|
-
@EntityScaler({ entityType: 'order', maxWorkersPerEntity: 1 })
|
|
348
|
-
@Injectable()
|
|
349
|
-
export class OrderEntityScaler {
|
|
350
267
|
@GetActiveEntities()
|
|
351
|
-
async getActiveOrders(): Promise<string[]> {
|
|
352
|
-
|
|
268
|
+
async getActiveOrders(): Promise<string[]> {
|
|
269
|
+
// Return IDs that need workers
|
|
270
|
+
return this.orderRepo.findPendingOrderIds();
|
|
271
|
+
}
|
|
272
|
+
|
|
353
273
|
@GetDesiredWorkerCount()
|
|
354
|
-
async getWorkerCount(
|
|
274
|
+
async getWorkerCount(orderId: string): Promise<number> {
|
|
275
|
+
return 1; // One worker per order
|
|
276
|
+
}
|
|
355
277
|
}
|
|
356
|
-
|
|
357
|
-
// app.module.ts (just provide the classes, auto-discovery handles the rest)
|
|
358
|
-
@Module({
|
|
359
|
-
providers: [OrderWorkerProcessor, OrderEntityScaler],
|
|
360
|
-
})
|
|
361
|
-
export class OrderModule {}
|
|
362
278
|
```
|
|
363
279
|
|
|
364
|
-
### Key Benefits of Migration
|
|
365
|
-
|
|
366
|
-
| Aspect | Manual API | Decorator API |
|
|
367
|
-
|--------|-----------|---------------|
|
|
368
|
-
| **Job routing** | Manual switch/case or registry lookup | Automatic via `@JobHandler` |
|
|
369
|
-
| **Worker creation** | Explicit service method | Auto-generated by library |
|
|
370
|
-
| **Scaling config** | Imperative `registerEntityType()` call | Declarative `@EntityScaler` class |
|
|
371
|
-
| **Entity ID access** | Manual parsing from job data | Injected as method parameter |
|
|
372
|
-
| **Code organization** | Multiple files and services | Single processor class per entity type |
|
|
373
|
-
| **Registration** | Manual in `onModuleInit` | Auto-discovered at startup |
|
|
374
|
-
|
|
375
280
|
---
|
|
376
281
|
|
|
377
|
-
##
|
|
378
|
-
|
|
379
|
-
### High-Level Flow
|
|
282
|
+
## Complete Example
|
|
380
283
|
|
|
381
|
-
```
|
|
382
|
-
|
|
383
|
-
|
|
384
|
-
|
|
385
|
-
|
|
386
|
-
|
|
387
|
-
|
|
388
|
-
|
|
389
|
-
|
|
390
|
-
|
|
391
|
-
|
|
392
|
-
▼
|
|
393
|
-
┌──────────────────────────────────────────────────────────────────────────────────────────────┐
|
|
394
|
-
│ APPLICATION LAYER │
|
|
395
|
-
│ ┌────────────────────────────────────────────────────────────────────────────────────────┐ │
|
|
396
|
-
│ │ QueueManagerService │ │
|
|
397
|
-
│ │ │ │
|
|
398
|
-
│ │ queueManager.addJob(entityQueue, jobName, { entityId, action, payload }) │ │
|
|
399
|
-
│ │ │ │
|
|
400
|
-
│ └────────────────────────────────────────────────────────────────────────────────────────┘ │
|
|
401
|
-
└──────────────────────────────────────────────────────────────────────────────────────────────┘
|
|
402
|
-
│
|
|
403
|
-
▼
|
|
404
|
-
┌──────────────────────────────────────────────────────────────────────────────────────────────┐
|
|
405
|
-
│ REDIS (BullMQ) │
|
|
406
|
-
│ │
|
|
407
|
-
│ ┌───────────────┐ ┌───────────────┐ ┌───────────────┐ ┌───────────────┐ │
|
|
408
|
-
│ │ entity-A-q │ │ entity-B-q │ │ entity-C-q │ │ entity-N-q │ │
|
|
409
|
-
│ │ │ │ │ │ │ │ │ │
|
|
410
|
-
│ │ ┌─────────┐ │ │ ┌─────────┐ │ │ ┌─────────┐ │ │ ┌─────────┐ │ │
|
|
411
|
-
│ │ │ Job 1 │ │ │ │ Job 1 │ │ │ │ Job 1 │ │ │ │ Job 1 │ │ │
|
|
412
|
-
│ │ │ Job 2 │ │ │ │ Job 2 │ │ │ └─────────┘ │ │ │ Job 2 │ │ │
|
|
413
|
-
│ │ │ Job 3 │ │ │ └─────────┘ │ │ │ │ │ Job 3 │ │ │
|
|
414
|
-
│ │ │ ... │ │ │ │ │ │ │ │ ... │ │ │
|
|
415
|
-
│ │ └─────────┘ │ │ │ │ │ │ └─────────┘ │ │
|
|
416
|
-
│ └───────┬───────┘ └───────┬───────┘ └───────┬───────┘ └───────┬───────┘ │
|
|
417
|
-
│ │ │ │ │ │
|
|
418
|
-
└───────────┼────────────────────┼────────────────────┼────────────────────┼───────────────────┘
|
|
419
|
-
│ │ │ │
|
|
420
|
-
▼ ▼ ▼ ▼
|
|
421
|
-
┌──────────────────────────────────────────────────────────────────────────────────────────────┐
|
|
422
|
-
│ WORKER LAYER (Per-Entity) │
|
|
423
|
-
│ │
|
|
424
|
-
│ ┌───────────────┐ ┌───────────────┐ ┌───────────────┐ ┌───────────────┐ │
|
|
425
|
-
│ │ Worker A │ │ Worker B │ │ Worker C │ │ Worker N │ │
|
|
426
|
-
│ │ concurrency=1 │ │ concurrency=1 │ │ concurrency=1 │ │ concurrency=1 │ │
|
|
427
|
-
│ │ │ │ │ │ │ │ │ │
|
|
428
|
-
│ │ ┌─────────┐ │ │ ┌─────────┐ │ │ ┌─────────┐ │ │ ┌─────────┐ │ │
|
|
429
|
-
│ │ │Heartbeat│ │ │ │Heartbeat│ │ │ │Heartbeat│ │ │ │Heartbeat│ │ │
|
|
430
|
-
│ │ │ TTL=3s │ │ │ │ TTL=3s │ │ │ │ TTL=3s │ │ │ │ TTL=3s │ │ │
|
|
431
|
-
│ │ └─────────┘ │ │ └─────────┘ │ │ └─────────┘ │ │ └─────────┘ │ │
|
|
432
|
-
│ └───────┬───────┘ └───────┬───────┘ └───────┬───────┘ └───────┬───────┘ │
|
|
433
|
-
│ │ │ │ │ │
|
|
434
|
-
│ │ WorkerManagerService (Lifecycle, Heartbeats, Shutdown Signals) │
|
|
435
|
-
│ └────────────────────┴────────────────────┴────────────────────┘ │
|
|
436
|
-
│ │ │
|
|
437
|
-
└──────────────────────────────────────────┼───────────────────────────────────────────────────┘
|
|
438
|
-
│
|
|
439
|
-
▼
|
|
440
|
-
┌──────────────────────────────────────────────────────────────────────────────────────────────┐
|
|
441
|
-
│ JOB PROCESSOR SERVICE │
|
|
442
|
-
│ │
|
|
443
|
-
│ ┌───────────────────────────────────────────────────────────────────────────────────────┐ │
|
|
444
|
-
│ │ JobProcessorRegistry │ │
|
|
445
|
-
│ │ │ │
|
|
446
|
-
│ │ @JobProcessor('validate') @JobProcessor('pay') @JobProcessor('ship') │ │
|
|
447
|
-
│ │ class ValidateProcessor {} class PayProcessor {} class ShipProcessor {} │ │
|
|
448
|
-
│ │ │ │
|
|
449
|
-
│ └───────────────────────────────────────────────────────────────────────────────────────┘ │
|
|
450
|
-
│ │ │
|
|
451
|
-
│ ▼ │
|
|
452
|
-
│ ┌───────────────────────────────────────────────────────────────────────────────────────┐ │
|
|
453
|
-
│ │ CQRS CommandBus / QueryBus │ │
|
|
454
|
-
│ │ │ │
|
|
455
|
-
│ │ commandBus.execute(new ValidateOrderCommand(...)) │ │
|
|
456
|
-
│ │ commandBus.execute(new ProcessPaymentCommand(...)) │ │
|
|
457
|
-
│ │ │ │
|
|
458
|
-
│ └───────────────────────────────────────────────────────────────────────────────────────┘ │
|
|
459
|
-
│ │
|
|
460
|
-
└──────────────────────────────────────────────────────────────────────────────────────────────┘
|
|
461
|
-
|
|
462
|
-
|
|
463
|
-
┌──────────────────────────────────────────────────────────────────────────────────────────────┐
|
|
464
|
-
│ SUPPORTING SERVICES │
|
|
465
|
-
│ │
|
|
466
|
-
│ ┌─────────────────────────┐ ┌─────────────────────────┐ ┌─────────────────────────┐ │
|
|
467
|
-
│ │ CronManagerService │ │ IndexManagerService │ │ ResourceLockService │ │
|
|
468
|
-
│ │ │ │ │ │ │ │
|
|
469
|
-
│ │ • Poll for entities │ │ • Track jobs per │ │ • Lua-based atomic │ │
|
|
470
|
-
│ │ needing workers │ │ entity │ │ locks │ │
|
|
471
|
-
│ │ • Spawn workers on │ │ • Track worker states │ │ • Lock pooling │ │
|
|
472
|
-
│ │ demand │ │ • Track queue states │ │ • TTL-based expiry │ │
|
|
473
|
-
│ │ • Terminate idle │ │ • Cleanup on entity │ │ • Owner tracking │ │
|
|
474
|
-
│ │ workers │ │ completion │ │ │ │
|
|
475
|
-
│ │ │ │ │ │ │ │
|
|
476
|
-
│ └─────────────────────────┘ └─────────────────────────┘ └─────────────────────────┘ │
|
|
477
|
-
│ │
|
|
478
|
-
└──────────────────────────────────────────────────────────────────────────────────────────────┘
|
|
479
|
-
```
|
|
480
|
-
|
|
481
|
-
### Detailed Component Interaction
|
|
284
|
+
```typescript
|
|
285
|
+
// ─────────────────────────────────────────────────────────────────
|
|
286
|
+
// commands/place-bet.command.ts
|
|
287
|
+
// ─────────────────────────────────────────────────────────────────
|
|
288
|
+
export class PlaceBetCommand {
|
|
289
|
+
constructor(
|
|
290
|
+
public readonly tableId: string,
|
|
291
|
+
public readonly playerId: string,
|
|
292
|
+
public readonly amount: number,
|
|
293
|
+
) {}
|
|
294
|
+
}
|
|
482
295
|
|
|
483
|
-
|
|
484
|
-
|
|
485
|
-
|
|
486
|
-
|
|
487
|
-
|
|
488
|
-
|
|
489
|
-
|
|
490
|
-
|
|
491
|
-
┌─────────────┐ ┌─────────────────┐ ┌─────────────────┐
|
|
492
|
-
│ Service │ │ CronManager │ │ Worker │
|
|
493
|
-
│ (HTTP/WS) │ │ Service │ │ (BullMQ) │
|
|
494
|
-
└──────┬──────┘ └────────┬────────┘ └────────┬────────┘
|
|
495
|
-
│ │ │
|
|
496
|
-
│ 1. Receive request │ 1. Every N seconds │ 1. Poll queue
|
|
497
|
-
│ (create order, etc) │ check entities │ for jobs
|
|
498
|
-
▼ │ with pending jobs │
|
|
499
|
-
┌─────────────┐ ▼ ▼
|
|
500
|
-
│ Queue │ ┌─────────────────┐ ┌─────────────────┐
|
|
501
|
-
│ Manager │ │ Index │ │ Job │
|
|
502
|
-
│ Service │ │ Manager │ │ Processor │
|
|
503
|
-
└──────┬──────┘ └────────┬────────┘ │ Registry │
|
|
504
|
-
│ │ └────────┬────────┘
|
|
505
|
-
│ 2. Get/create queue │ 2. Return entities │
|
|
506
|
-
│ for entity │ with job counts │ 2. Lookup processor
|
|
507
|
-
▼ │ │ by job name
|
|
508
|
-
┌─────────────┐ ▼ ▼
|
|
509
|
-
│ Redis │ ┌─────────────────┐ ┌─────────────────┐
|
|
510
|
-
│ Queue │◄────────────────── │ Worker │ │ @JobProcessor │
|
|
511
|
-
│ (entity-X) │ │ Manager │ │ Handler Class │
|
|
512
|
-
└──────┬──────┘ └────────┬────────┘ └────────┬────────┘
|
|
513
|
-
│ │ │
|
|
514
|
-
│ 3. Add job to queue │ 3. Spawn worker │ 3. Execute
|
|
515
|
-
│ (FIFO ordered) │ for entity │ command/query
|
|
516
|
-
▼ │ ▼
|
|
517
|
-
┌─────────────┐ ▼ ┌─────────────────┐
|
|
518
|
-
│ Index │ ┌─────────────────┐ │ CommandBus │
|
|
519
|
-
│ Manager │ │ New Worker │ │ / QueryBus │
|
|
520
|
-
└─────────────┘ │ (concurrency=1)│ └────────┬────────┘
|
|
521
|
-
│ └─────────────────┘ │
|
|
522
|
-
│ 4. Track job in index │ 4. Domain
|
|
523
|
-
│ for entity │ logic
|
|
524
|
-
▼ ▼
|
|
525
|
-
┌─────────────────────────────────────────────────────────────────────────────────────┐
|
|
526
|
-
│ REDIS │
|
|
527
|
-
│ ┌─────────────┐ ┌─────────────┐ ┌─────────────┐ ┌─────────────┐ │
|
|
528
|
-
│ │ Queues │ │ Workers │ │ Indices │ │ Locks │ │
|
|
529
|
-
│ │ (BullMQ) │ │ (Heartbeat) │ │ (Jobs/Qs) │ │ (Lua Atom) │ │
|
|
530
|
-
│ └─────────────┘ └─────────────┘ └─────────────┘ └─────────────┘ │
|
|
531
|
-
└─────────────────────────────────────────────────────────────────────────────────────┘
|
|
532
|
-
|
|
533
|
-
|
|
534
|
-
4. JOB COMPLETION 5. WORKER TERMINATION 6. GRACEFUL SHUTDOWN
|
|
535
|
-
───────────────── ───────────────────── ────────────────────
|
|
536
|
-
|
|
537
|
-
┌─────────────────┐ ┌─────────────────┐ ┌─────────────────┐
|
|
538
|
-
│ Worker │ │ CronManager │ │ SIGTERM/INT │
|
|
539
|
-
│ completes │ │ Service │ │ Signal │
|
|
540
|
-
└────────┬────────┘ └────────┬────────┘ └────────┬────────┘
|
|
541
|
-
│ │ │
|
|
542
|
-
│ 1. Job finished │ 1. Check worker │ 1. Caught by
|
|
543
|
-
│ │ idle time │ process handler
|
|
544
|
-
▼ │ ▼
|
|
545
|
-
┌─────────────────┐ ▼ ┌─────────────────┐
|
|
546
|
-
│ Index │ ┌─────────────────┐ │ Worker │
|
|
547
|
-
│ Manager │ │ No pending │ │ Manager │
|
|
548
|
-
└────────┬────────┘ │ jobs for │ └────────┬────────┘
|
|
549
|
-
│ │ entity? │ │
|
|
550
|
-
│ 2. Remove job from └────────┬────────┘ │ 2. Signal all
|
|
551
|
-
│ entity index │ │ workers to close
|
|
552
|
-
▼ │ YES ▼
|
|
553
|
-
┌─────────────────┐ ▼ ┌─────────────────┐
|
|
554
|
-
│ Check pending │ ┌─────────────────┐ │ Redis │
|
|
555
|
-
│ jobs for │ │ Worker │ │ Pub/Sub │
|
|
556
|
-
│ entity │ │ Manager │ │ (shutdown │
|
|
557
|
-
└────────┬────────┘ └────────┬────────┘ │ channel) │
|
|
558
|
-
│ │ └────────┬────────┘
|
|
559
|
-
│ 3. If no pending │ 2. Signal worker │
|
|
560
|
-
│ jobs, cleanup │ to close │ 3. Workers receive
|
|
561
|
-
▼ ▼ │ shutdown signal
|
|
562
|
-
┌─────────────────┐ ┌─────────────────┐ ▼
|
|
563
|
-
│ Entity indices │ │ Worker │ ┌─────────────────┐
|
|
564
|
-
│ cleaned up │ │ gracefully │ │ Workers │
|
|
565
|
-
│ │ │ closes │ │ finish │
|
|
566
|
-
└─────────────────┘ └─────────────────┘ │ current job │
|
|
567
|
-
│ then exit │
|
|
568
|
-
└─────────────────┘
|
|
569
|
-
```
|
|
296
|
+
// ─────────────────────────────────────────────────────────────────
|
|
297
|
+
// commands/deal-cards.command.ts
|
|
298
|
+
// ─────────────────────────────────────────────────────────────────
|
|
299
|
+
export class DealCardsCommand {
|
|
300
|
+
constructor(
|
|
301
|
+
public readonly tableId: string,
|
|
302
|
+
) {}
|
|
303
|
+
}
|
|
570
304
|
|
|
571
|
-
|
|
305
|
+
// ─────────────────────────────────────────────────────────────────
|
|
306
|
+
// handlers/place-bet.handler.ts (auto-registers PlaceBetCommand)
|
|
307
|
+
// ─────────────────────────────────────────────────────────────────
|
|
308
|
+
import { CommandHandler, ICommandHandler } from '@nestjs/cqrs';
|
|
309
|
+
import { PlaceBetCommand } from '../commands/place-bet.command';
|
|
572
310
|
|
|
573
|
-
|
|
574
|
-
|
|
575
|
-
|
|
576
|
-
|
|
577
|
-
|
|
578
|
-
|
|
579
|
-
│ Load Balancer │
|
|
580
|
-
└────────┬────────┘
|
|
581
|
-
│
|
|
582
|
-
┌──────────────────────────────┼──────────────────────────────┐
|
|
583
|
-
│ │ │
|
|
584
|
-
▼ ▼ ▼
|
|
585
|
-
┌─────────────────┐ ┌─────────────────┐ ┌─────────────────┐
|
|
586
|
-
│ Node 1 │ │ Node 2 │ │ Node 3 │
|
|
587
|
-
│ (PM2 Cluster) │ │ (PM2 Cluster) │ │ (K8s Pod) │
|
|
588
|
-
├─────────────────┤ ├─────────────────┤ ├─────────────────┤
|
|
589
|
-
│ │ │ │ │ │
|
|
590
|
-
│ ┌───────────┐ │ │ ┌───────────┐ │ │ ┌───────────┐ │
|
|
591
|
-
│ │ Worker A │ │ │ │ Worker C │ │ │ │ Worker E │ │
|
|
592
|
-
│ │(Entity 1) │ │ │ │(Entity 3) │ │ │ │(Entity 5) │ │
|
|
593
|
-
│ └───────────┘ │ │ └───────────┘ │ │ └───────────┘ │
|
|
594
|
-
│ │ │ │ │ │
|
|
595
|
-
│ ┌───────────┐ │ │ ┌───────────┐ │ │ ┌───────────┐ │
|
|
596
|
-
│ │ Worker B │ │ │ │ Worker D │ │ │ │ Worker F │ │
|
|
597
|
-
│ │(Entity 2) │ │ │ │(Entity 4) │ │ │ │(Entity 6) │ │
|
|
598
|
-
│ └───────────┘ │ │ └───────────┘ │ │ └───────────┘ │
|
|
599
|
-
│ │ │ │ │ │
|
|
600
|
-
└────────┬────────┘ └────────┬────────┘ └────────┬────────┘
|
|
601
|
-
│ │ │
|
|
602
|
-
└──────────────────────────────┼──────────────────────────────┘
|
|
603
|
-
│
|
|
604
|
-
▼
|
|
605
|
-
┌─────────────────────────────────────────────────────────────────────────────────┐
|
|
606
|
-
│ REDIS CLUSTER │
|
|
607
|
-
│ │
|
|
608
|
-
│ ┌─────────────────────────────────────────────────────────────────────────┐ │
|
|
609
|
-
│ │ BullMQ Queues │ │
|
|
610
|
-
│ │ entity-1-queue │ entity-2-queue │ entity-3-queue │ ... │ entity-N-q │ │
|
|
611
|
-
│ └─────────────────────────────────────────────────────────────────────────┘ │
|
|
612
|
-
│ │
|
|
613
|
-
│ ┌─────────────────────────────────────────────────────────────────────────┐ │
|
|
614
|
-
│ │ Worker Heartbeats (TTL) │ │
|
|
615
|
-
│ │ aq:workers:entity-1-worker │ aq:workers:entity-2-worker │ ... │ │
|
|
616
|
-
│ └─────────────────────────────────────────────────────────────────────────┘ │
|
|
617
|
-
│ │
|
|
618
|
-
│ ┌─────────────────────────────────────────────────────────────────────────┐ │
|
|
619
|
-
│ │ Job/Entity Indices │ │
|
|
620
|
-
│ │ aq:idx:entity:jobs │ aq:idx:entity:queues │ aq:idx:entity:workers │ │
|
|
621
|
-
│ └─────────────────────────────────────────────────────────────────────────┘ │
|
|
622
|
-
│ │
|
|
623
|
-
│ ┌─────────────────────────────────────────────────────────────────────────┐ │
|
|
624
|
-
│ │ Pub/Sub Shutdown Channels │ │
|
|
625
|
-
│ │ aq:worker:entity-1-worker:shutdown │ aq:worker:entity-2-worker:shut │ │
|
|
626
|
-
│ └─────────────────────────────────────────────────────────────────────────┘ │
|
|
627
|
-
│ │
|
|
628
|
-
└─────────────────────────────────────────────────────────────────────────────────┘
|
|
629
|
-
|
|
630
|
-
|
|
631
|
-
KEY GUARANTEES:
|
|
632
|
-
───────────────
|
|
633
|
-
✓ Only ONE worker processes jobs for each entity (concurrency=1)
|
|
634
|
-
✓ Jobs for same entity are processed in FIFO order
|
|
635
|
-
✓ Worker heartbeats detected across all nodes
|
|
636
|
-
✓ Graceful shutdown via Redis pub/sub (not local signals)
|
|
637
|
-
✓ Any node can spawn workers for any entity
|
|
638
|
-
✓ Dead workers detected via TTL expiration
|
|
639
|
-
```
|
|
311
|
+
@CommandHandler(PlaceBetCommand)
|
|
312
|
+
export class PlaceBetHandler implements ICommandHandler<PlaceBetCommand> {
|
|
313
|
+
async execute(command: PlaceBetCommand) {
|
|
314
|
+
console.log(`Placing bet of ${command.amount} for player ${command.playerId}`);
|
|
315
|
+
}
|
|
316
|
+
}
|
|
640
317
|
|
|
641
|
-
|
|
318
|
+
// ─────────────────────────────────────────────────────────────────
|
|
319
|
+
// handlers/deal-cards.handler.ts (auto-registers DealCardsCommand)
|
|
320
|
+
// ─────────────────────────────────────────────────────────────────
|
|
321
|
+
import { CommandHandler, ICommandHandler } from '@nestjs/cqrs';
|
|
322
|
+
import { DealCardsCommand } from '../commands/deal-cards.command';
|
|
642
323
|
|
|
643
|
-
|
|
324
|
+
@CommandHandler(DealCardsCommand)
|
|
325
|
+
export class DealCardsHandler implements ICommandHandler<DealCardsCommand> {
|
|
326
|
+
async execute(command: DealCardsCommand) {
|
|
327
|
+
console.log(`Dealing cards for table ${command.tableId}`);
|
|
328
|
+
}
|
|
329
|
+
}
|
|
644
330
|
|
|
645
|
-
|
|
331
|
+
// ─────────────────────────────────────────────────────────────────
|
|
332
|
+
// table.processor.ts
|
|
333
|
+
// ─────────────────────────────────────────────────────────────────
|
|
334
|
+
import { Injectable } from '@nestjs/common';
|
|
335
|
+
import { WorkerProcessor } from 'atomic-queues';
|
|
646
336
|
|
|
647
|
-
|
|
337
|
+
@WorkerProcessor({
|
|
338
|
+
entityType: 'table',
|
|
339
|
+
queueName: (tableId) => `table-${tableId}-queue`,
|
|
340
|
+
workerName: (tableId) => `table-${tableId}-worker`,
|
|
341
|
+
})
|
|
342
|
+
@Injectable()
|
|
343
|
+
export class TableProcessor {}
|
|
648
344
|
|
|
649
|
-
|
|
345
|
+
// ─────────────────────────────────────────────────────────────────
|
|
346
|
+
// table.module.ts - No manual registration needed!
|
|
347
|
+
// ─────────────────────────────────────────────────────────────────
|
|
650
348
|
import { Module } from '@nestjs/common';
|
|
651
|
-
import {
|
|
349
|
+
import { CqrsModule } from '@nestjs/cqrs';
|
|
350
|
+
import { TableProcessor } from './table.processor';
|
|
351
|
+
import { TableGateway } from './table.gateway';
|
|
352
|
+
import { PlaceBetHandler, DealCardsHandler } from './handlers';
|
|
652
353
|
|
|
653
354
|
@Module({
|
|
654
|
-
imports: [
|
|
655
|
-
|
|
656
|
-
|
|
657
|
-
|
|
658
|
-
|
|
659
|
-
|
|
660
|
-
enableCronManager: true,
|
|
661
|
-
cronInterval: 5000,
|
|
662
|
-
keyPrefix: 'myapp',
|
|
663
|
-
}),
|
|
355
|
+
imports: [CqrsModule],
|
|
356
|
+
providers: [
|
|
357
|
+
TableProcessor,
|
|
358
|
+
TableGateway,
|
|
359
|
+
PlaceBetHandler, // Commands auto-discovered from handlers!
|
|
360
|
+
DealCardsHandler,
|
|
664
361
|
],
|
|
665
362
|
})
|
|
666
|
-
export class
|
|
667
|
-
```
|
|
668
|
-
|
|
669
|
-
### 2. Register Job Processors Manually
|
|
363
|
+
export class TableModule {}
|
|
670
364
|
|
|
671
|
-
|
|
365
|
+
// ─────────────────────────────────────────────────────────────────
|
|
366
|
+
// table.gateway.ts (WebSocket example)
|
|
367
|
+
// ─────────────────────────────────────────────────────────────────
|
|
672
368
|
import { Injectable } from '@nestjs/common';
|
|
673
|
-
import {
|
|
674
|
-
import {
|
|
369
|
+
import { QueueBus } from 'atomic-queues';
|
|
370
|
+
import { TableProcessor } from './table.processor';
|
|
371
|
+
import { PlaceBetCommand, DealCardsCommand } from './commands';
|
|
675
372
|
|
|
676
373
|
@Injectable()
|
|
677
|
-
|
|
678
|
-
|
|
679
|
-
constructor(private readonly commandBus: CommandBus) {}
|
|
374
|
+
export class TableGateway {
|
|
375
|
+
constructor(private readonly queueBus: QueueBus) {}
|
|
680
376
|
|
|
681
|
-
async
|
|
682
|
-
|
|
683
|
-
|
|
377
|
+
async onPlaceBet(tableId: string, playerId: string, amount: number) {
|
|
378
|
+
await this.queueBus
|
|
379
|
+
.forProcessor(TableProcessor)
|
|
380
|
+
.enqueue(new PlaceBetCommand(tableId, playerId, amount));
|
|
684
381
|
}
|
|
685
|
-
}
|
|
686
|
-
```
|
|
687
|
-
|
|
688
|
-
### 3. Queue Jobs Manually
|
|
689
|
-
|
|
690
|
-
```typescript
|
|
691
|
-
import { Injectable } from '@nestjs/common';
|
|
692
|
-
import { QueueManagerService, IndexManagerService } from 'atomic-queues';
|
|
693
382
|
|
|
694
|
-
|
|
695
|
-
|
|
696
|
-
|
|
697
|
-
|
|
698
|
-
private readonly indexManager: IndexManagerService,
|
|
699
|
-
) {}
|
|
700
|
-
|
|
701
|
-
async createOrder(orderId: string, items: any[], amount: number) {
|
|
702
|
-
const queue = this.queueManager.getOrCreateEntityQueue('order', orderId);
|
|
703
|
-
|
|
704
|
-
// Queue validation job
|
|
705
|
-
const job = await this.queueManager.addJob(queue.name, 'validate-order', { orderId, items });
|
|
706
|
-
|
|
707
|
-
// Queue payment job (will run after validation completes due to FIFO)
|
|
708
|
-
await this.queueManager.addJob(queue.name, 'process-payment', { orderId, amount });
|
|
709
|
-
|
|
710
|
-
// Track job for scaling decisions
|
|
711
|
-
await this.indexManager.indexJob('order', orderId, job.id!);
|
|
712
|
-
|
|
713
|
-
return orderId;
|
|
714
|
-
}
|
|
715
|
-
}
|
|
716
|
-
```
|
|
717
|
-
|
|
718
|
-
### 4. Create Workers Manually
|
|
719
|
-
|
|
720
|
-
```typescript
|
|
721
|
-
import { Injectable } from '@nestjs/common';
|
|
722
|
-
import { WorkerManagerService, JobProcessorRegistry } from 'atomic-queues';
|
|
723
|
-
|
|
724
|
-
@Injectable()
|
|
725
|
-
export class OrderWorkerService {
|
|
726
|
-
constructor(
|
|
727
|
-
private readonly workerManager: WorkerManagerService,
|
|
728
|
-
private readonly jobRegistry: JobProcessorRegistry,
|
|
729
|
-
) {}
|
|
730
|
-
|
|
731
|
-
async createOrderWorker(orderId: string) {
|
|
732
|
-
const queueName = `order-${orderId}-queue`;
|
|
733
|
-
|
|
734
|
-
await this.workerManager.createWorker({
|
|
735
|
-
workerName: `${orderId}-worker`,
|
|
736
|
-
queueName,
|
|
737
|
-
processor: async (job) => {
|
|
738
|
-
const processor = this.jobRegistry.getProcessor(job.name);
|
|
739
|
-
if (!processor) {
|
|
740
|
-
throw new Error(`No processor for job: ${job.name}`);
|
|
741
|
-
}
|
|
742
|
-
await processor.process(job);
|
|
743
|
-
},
|
|
744
|
-
events: {
|
|
745
|
-
onReady: async (worker, name) => {
|
|
746
|
-
console.log(`Worker ${name} is ready`);
|
|
747
|
-
},
|
|
748
|
-
onCompleted: async (job, name) => {
|
|
749
|
-
console.log(`Job ${job.id} completed by ${name}`);
|
|
750
|
-
},
|
|
751
|
-
onFailed: async (job, error, name) => {
|
|
752
|
-
console.error(`Job ${job?.id} failed in ${name}:`, error.message);
|
|
753
|
-
},
|
|
754
|
-
},
|
|
755
|
-
});
|
|
383
|
+
async onDealCards(tableId: string) {
|
|
384
|
+
await this.queueBus
|
|
385
|
+
.forProcessor(TableProcessor)
|
|
386
|
+
.enqueue(new DealCardsCommand(tableId));
|
|
756
387
|
}
|
|
757
388
|
}
|
|
758
389
|
```
|
|
759
390
|
|
|
760
391
|
---
|
|
761
392
|
|
|
762
|
-
##
|
|
763
|
-
|
|
764
|
-
### QueueManagerService
|
|
765
|
-
|
|
766
|
-
Manages dynamic queue creation and destruction per entity.
|
|
767
|
-
|
|
768
|
-
```typescript
|
|
769
|
-
// Get or create a queue for an entity
|
|
770
|
-
const queue = queueManager.getOrCreateEntityQueue('order', '123');
|
|
771
|
-
|
|
772
|
-
// Add a job to a queue
|
|
773
|
-
await queueManager.addJob(queue.name, 'process', { data: 'hello' });
|
|
774
|
-
|
|
775
|
-
// Get job counts
|
|
776
|
-
const counts = await queueManager.getJobCounts(queue.name);
|
|
777
|
-
|
|
778
|
-
// Close a queue
|
|
779
|
-
await queueManager.closeQueue(queue.name);
|
|
780
|
-
```
|
|
781
|
-
|
|
782
|
-
### WorkerManagerService
|
|
783
|
-
|
|
784
|
-
Manages worker lifecycle with heartbeat-based liveness tracking.
|
|
393
|
+
## Configuration
|
|
785
394
|
|
|
786
395
|
```typescript
|
|
787
|
-
|
|
788
|
-
|
|
789
|
-
|
|
790
|
-
|
|
791
|
-
|
|
792
|
-
config: {
|
|
793
|
-
concurrency: 1,
|
|
794
|
-
heartbeatTTL: 3,
|
|
396
|
+
AtomicQueuesModule.forRoot({
|
|
397
|
+
redis: {
|
|
398
|
+
host: 'localhost',
|
|
399
|
+
port: 6379,
|
|
400
|
+
password: 'secret',
|
|
795
401
|
},
|
|
796
|
-
});
|
|
797
|
-
|
|
798
|
-
// Check if worker exists
|
|
799
|
-
const exists = await workerManager.workerExists('my-worker');
|
|
800
|
-
|
|
801
|
-
// Signal worker to close via Redis pub/sub
|
|
802
|
-
await workerManager.signalWorkerClose('my-worker');
|
|
803
|
-
|
|
804
|
-
// Get all workers for an entity
|
|
805
|
-
const workers = await workerManager.getEntityWorkers('order', '123');
|
|
806
|
-
```
|
|
807
|
-
|
|
808
|
-
### ResourceLockService
|
|
809
|
-
|
|
810
|
-
Provides distributed resource locking using Redis Lua scripts.
|
|
811
|
-
|
|
812
|
-
```typescript
|
|
813
|
-
// Acquire a lock
|
|
814
|
-
const result = await lockService.acquireLock(
|
|
815
|
-
'resource', // resourceType
|
|
816
|
-
'resource-123', // resourceId
|
|
817
|
-
'owner-456', // ownerId
|
|
818
|
-
'service', // ownerType
|
|
819
|
-
60, // TTL in seconds
|
|
820
|
-
);
|
|
821
|
-
|
|
822
|
-
if (result.acquired) {
|
|
823
|
-
try {
|
|
824
|
-
// Do work with the locked resource
|
|
825
|
-
} finally {
|
|
826
|
-
await lockService.releaseLock('resource', 'resource-123');
|
|
827
|
-
}
|
|
828
|
-
}
|
|
829
|
-
|
|
830
|
-
// Get first available resource from a pool
|
|
831
|
-
const available = await lockService.getAvailableResource(
|
|
832
|
-
'resource',
|
|
833
|
-
['res-1', 'res-2', 'res-3'],
|
|
834
|
-
'owner-456',
|
|
835
|
-
'service',
|
|
836
|
-
);
|
|
837
|
-
```
|
|
838
|
-
|
|
839
|
-
### CronManagerService
|
|
840
|
-
|
|
841
|
-
Automatic worker scaling based on demand.
|
|
842
|
-
|
|
843
|
-
**Recommended: Use `@EntityScaler` decorator (see Quick Start section above)**
|
|
844
|
-
|
|
845
|
-
The decorator-based approach is preferred as it's cleaner and auto-discovered:
|
|
846
|
-
|
|
847
|
-
```typescript
|
|
848
|
-
@EntityScaler({ entityType: 'order', maxWorkersPerEntity: 1 })
|
|
849
|
-
@Injectable()
|
|
850
|
-
export class OrderEntityScaler {
|
|
851
|
-
@GetActiveEntities()
|
|
852
|
-
async getActiveOrders(): Promise<string[]> { ... }
|
|
853
402
|
|
|
854
|
-
|
|
855
|
-
|
|
856
|
-
|
|
857
|
-
|
|
858
|
-
|
|
859
|
-
|
|
860
|
-
|
|
861
|
-
|
|
862
|
-
|
|
863
|
-
|
|
864
|
-
|
|
865
|
-
|
|
866
|
-
//
|
|
867
|
-
return 1;
|
|
868
|
-
},
|
|
869
|
-
getActiveEntityIds: async () => {
|
|
870
|
-
return Object.keys(await indexManager.getEntitiesWithJobs('order'));
|
|
871
|
-
},
|
|
872
|
-
maxWorkersPerEntity: 5,
|
|
873
|
-
onSpawnWorker: async (orderId) => {
|
|
874
|
-
await orderWorkerService.createOrderWorker(orderId);
|
|
403
|
+
keyPrefix: 'myapp', // Redis key prefix (default: 'aq')
|
|
404
|
+
|
|
405
|
+
enableCronManager: true, // Enable auto-scaling (default: false)
|
|
406
|
+
cronInterval: 5000, // Scaling check interval (default: 5000ms)
|
|
407
|
+
|
|
408
|
+
verbose: false, // Enable verbose logging (default: false)
|
|
409
|
+
// When true, logs service job processing details
|
|
410
|
+
|
|
411
|
+
workerDefaults: {
|
|
412
|
+
concurrency: 1, // Jobs processed simultaneously
|
|
413
|
+
stalledInterval: 1000, // Stalled job check interval
|
|
414
|
+
lockDuration: 30000, // Job lock duration
|
|
415
|
+
heartbeatTTL: 3, // Worker heartbeat TTL (seconds)
|
|
875
416
|
},
|
|
876
417
|
});
|
|
877
|
-
|
|
878
|
-
// Start the cron manager
|
|
879
|
-
cronManager.start(5000);
|
|
880
418
|
```
|
|
881
419
|
|
|
882
|
-
|
|
883
|
-
|
|
884
|
-
Track jobs, workers, and queue states.
|
|
885
|
-
|
|
886
|
-
```typescript
|
|
887
|
-
// Index a job
|
|
888
|
-
await indexManager.indexJob('order', '123', 'job-456');
|
|
889
|
-
|
|
890
|
-
// Get all entities with pending jobs
|
|
891
|
-
const entitiesWithJobs = await indexManager.getEntitiesWithJobs('order');
|
|
892
|
-
// Returns: { '123': 5, '456': 2 } (entityId: jobCount)
|
|
420
|
+
---
|
|
893
421
|
|
|
894
|
-
|
|
895
|
-
await indexManager.indexEntityQueue('order', '123');
|
|
422
|
+
## Command Registration
|
|
896
423
|
|
|
897
|
-
|
|
898
|
-
await indexManager.cleanupEntityIndices('order', '123');
|
|
899
|
-
```
|
|
424
|
+
By default, atomic-queues **auto-discovers** all commands from your `@CommandHandler` and `@QueryHandler` decorators. No manual registration needed!
|
|
900
425
|
|
|
901
|
-
|
|
426
|
+
### Auto-Discovery (Default)
|
|
902
427
|
|
|
903
|
-
|
|
428
|
+
Commands are automatically discovered when you have CQRS handlers:
|
|
904
429
|
|
|
905
430
|
```typescript
|
|
906
|
-
|
|
907
|
-
|
|
908
|
-
|
|
909
|
-
|
|
910
|
-
|
|
911
|
-
|
|
912
|
-
db?: number;
|
|
913
|
-
url?: string;
|
|
914
|
-
maxRetriesPerRequest?: number | null;
|
|
915
|
-
};
|
|
916
|
-
|
|
917
|
-
// Worker defaults
|
|
918
|
-
workerDefaults?: {
|
|
919
|
-
concurrency?: number; // Default: 1
|
|
920
|
-
stalledInterval?: number; // Default: 1000ms
|
|
921
|
-
lockDuration?: number; // Default: 30000ms
|
|
922
|
-
maxStalledCount?: number; // Default: MAX_SAFE_INTEGER
|
|
923
|
-
heartbeatTTL?: number; // Default: 3 seconds
|
|
924
|
-
heartbeatInterval?: number; // Default: 1000ms
|
|
925
|
-
};
|
|
926
|
-
|
|
927
|
-
// Queue defaults
|
|
928
|
-
queueDefaults?: {
|
|
929
|
-
defaultJobOptions?: {
|
|
930
|
-
removeOnComplete?: boolean;
|
|
931
|
-
removeOnFail?: boolean;
|
|
932
|
-
attempts?: number;
|
|
933
|
-
backoff?: { type: 'fixed' | 'exponential'; delay: number };
|
|
934
|
-
priority?: number;
|
|
935
|
-
};
|
|
936
|
-
};
|
|
937
|
-
|
|
938
|
-
// Cron manager
|
|
939
|
-
enableCronManager?: boolean; // Default: false
|
|
940
|
-
cronInterval?: number; // Default: 5000ms
|
|
941
|
-
|
|
942
|
-
// Key prefix for Redis keys
|
|
943
|
-
keyPrefix?: string; // Default: 'aq'
|
|
431
|
+
// Your handler - that's all you need!
|
|
432
|
+
@CommandHandler(ProcessOrderCommand)
|
|
433
|
+
export class ProcessOrderHandler implements ICommandHandler<ProcessOrderCommand> {
|
|
434
|
+
async execute(command: ProcessOrderCommand) {
|
|
435
|
+
// ProcessOrderCommand is auto-registered with QueueBus
|
|
436
|
+
}
|
|
944
437
|
}
|
|
945
438
|
```
|
|
946
439
|
|
|
947
|
-
|
|
948
|
-
|
|
949
|
-
## Graceful Shutdown
|
|
440
|
+
### Manual Registration (Optional)
|
|
950
441
|
|
|
951
|
-
|
|
952
|
-
|
|
953
|
-
1. On `SIGTERM`/`SIGINT`, the node publishes shutdown signals to Redis
|
|
954
|
-
2. All workers (even on other nodes) subscribed to shutdown channels receive the signal
|
|
955
|
-
3. Workers finish their current job (with configurable timeout)
|
|
956
|
-
4. Heartbeat TTLs expire, marking workers as dead
|
|
957
|
-
5. Resources are cleaned up
|
|
442
|
+
If you need to register commands without handlers, or disable auto-discovery:
|
|
958
443
|
|
|
959
444
|
```typescript
|
|
960
|
-
//
|
|
961
|
-
|
|
962
|
-
|
|
445
|
+
// Disable auto-discovery in config
|
|
446
|
+
AtomicQueuesModule.forRoot({
|
|
447
|
+
redis: { host: 'localhost', port: 6379 },
|
|
448
|
+
autoRegisterCommands: false, // Disable auto-discovery
|
|
449
|
+
});
|
|
450
|
+
|
|
451
|
+
// Then manually register
|
|
452
|
+
QueueBus.registerCommands(ProcessOrderCommand, ShipOrderCommand);
|
|
963
453
|
```
|
|
964
454
|
|
|
965
455
|
---
|
|
966
456
|
|
|
967
|
-
## Use
|
|
968
|
-
|
|
969
|
-
### 1. Per-Order Processing (E-commerce)
|
|
970
|
-
Each order has its own queue ensuring stages (validate → pay → ship) happen sequentially.
|
|
971
|
-
|
|
972
|
-
### 2. Per-User Message Queues (Chat/Messaging)
|
|
973
|
-
Each user has their own queue for message delivery, ensuring order.
|
|
974
|
-
|
|
975
|
-
### 3. Per-Tenant Job Processing (SaaS)
|
|
976
|
-
Each tenant's jobs are isolated and processed sequentially.
|
|
977
|
-
|
|
978
|
-
### 4. Per-Document Processing (Document Management)
|
|
979
|
-
Each document goes through OCR → validation → storage in sequence.
|
|
457
|
+
## Why Use atomic-queues?
|
|
980
458
|
|
|
981
|
-
|
|
982
|
-
|
|
459
|
+
| Feature | Without | With atomic-queues |
|
|
460
|
+
|---------|---------|-------------------|
|
|
461
|
+
| Sequential per-entity | Manual locking | Automatic via queues |
|
|
462
|
+
| Race conditions | Possible | Prevented |
|
|
463
|
+
| Worker management | Manual | Automatic |
|
|
464
|
+
| Horizontal scaling | Complex | Built-in |
|
|
465
|
+
| Code organization | Scattered | Clean decorators |
|
|
983
466
|
|
|
984
467
|
---
|
|
985
468
|
|