atomic-queues 1.2.3 → 1.2.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +161 -106
- package/package.json +1 -1
package/README.md
CHANGED
|
@@ -5,30 +5,41 @@ A NestJS library for atomic, sequential job processing per entity with BullMQ an
|
|
|
5
5
|
## What It Does
|
|
6
6
|
|
|
7
7
|
```
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
│
|
|
16
|
-
│
|
|
17
|
-
│
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
│
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
8
|
+
╔═══════════════════════════════════════════════════════════════════════════════╗
|
|
9
|
+
║ THE PROBLEM ║
|
|
10
|
+
╠═══════════════════════════════════════════════════════════════════════════════╣
|
|
11
|
+
║ ║
|
|
12
|
+
║ Multiple requests for the same entity arrive simultaneously: ║
|
|
13
|
+
║ ║
|
|
14
|
+
║ ┌──────────┐ ║
|
|
15
|
+
║ │ Request A │──┐ ║
|
|
16
|
+
║ └──────────┘ │ ║
|
|
17
|
+
║ ┌──────────┐ │ ┌─────────────┐ ║
|
|
18
|
+
║ │ Request B │──┼───▶│ Entity 123 │───▶ 💥 RACE CONDITION! ║
|
|
19
|
+
║ └──────────┘ │ └─────────────┘ ║
|
|
20
|
+
║ ┌──────────┐ │ ║
|
|
21
|
+
║ │ Request C │──┘ ║
|
|
22
|
+
║ └──────────┘ ║
|
|
23
|
+
║ ║
|
|
24
|
+
╚═══════════════════════════════════════════════════════════════════════════════╝
|
|
25
|
+
|
|
26
|
+
╔═══════════════════════════════════════════════════════════════════════════════╗
|
|
27
|
+
║ THE SOLUTION ║
|
|
28
|
+
╠═══════════════════════════════════════════════════════════════════════════════╣
|
|
29
|
+
║ ║
|
|
30
|
+
║ atomic-queues ensures sequential processing per entity: ║
|
|
31
|
+
║ ║
|
|
32
|
+
║ ┌──────────┐ ┌─────────────────┐ ┌──────────┐ ║
|
|
33
|
+
║ │ Request A │──┐ │ │ │ │ ║
|
|
34
|
+
║ └──────────┘ │ │ Redis Queue │ │ Worker │ ┌───────────┐ ║
|
|
35
|
+
║ ┌──────────┐ │ │ ┌───┬───┬───┐ │ │ │ │ │ ║
|
|
36
|
+
║ │ Request B │──┼──▶│ │ A │ B │ C │ │─────▶│ (1 job │─▶│Entity 123 │ ║
|
|
37
|
+
║ └──────────┘ │ │ └───┴───┴───┘ │ │ at a time│ │ │ ║
|
|
38
|
+
║ ┌──────────┐ │ │ │ │ │ └───────────┘ ║
|
|
39
|
+
║ │ Request C │──┘ └─────────────────┘ └──────────┘ ║
|
|
40
|
+
║ └──────────┘ ║
|
|
41
|
+
║ ║
|
|
42
|
+
╚═══════════════════════════════════════════════════════════════════════════════╝
|
|
32
43
|
```
|
|
33
44
|
|
|
34
45
|
## Installation
|
|
@@ -132,40 +143,48 @@ That's it! The library automatically:
|
|
|
132
143
|
## How It Works
|
|
133
144
|
|
|
134
145
|
```
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
YOUR CODE
|
|
140
|
-
─────────
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
146
|
+
╔═══════════════════════════════════════════════════════════════════════════════╗
|
|
147
|
+
║ ARCHITECTURE ║
|
|
148
|
+
╚═══════════════════════════════════════════════════════════════════════════════╝
|
|
149
|
+
|
|
150
|
+
YOUR CODE ATOMIC-QUEUES EXECUTION
|
|
151
|
+
───────── ───────────── ─────────
|
|
152
|
+
|
|
153
|
+
┌─────────────────────────┐
|
|
154
|
+
│ queueBus │
|
|
155
|
+
│ .forProcessor(...) │
|
|
156
|
+
│ .enqueue(command) │
|
|
157
|
+
└───────────┬─────────────┘
|
|
158
|
+
│
|
|
159
|
+
│ ① Extract queue config from @WorkerProcessor
|
|
160
|
+
│ ② Extract entityId from command properties
|
|
161
|
+
│ ③ Build queue name: {prefix}-{entityId}-queue
|
|
162
|
+
▼
|
|
163
|
+
┌───────────────────┐
|
|
164
|
+
│ │
|
|
165
|
+
│ Redis Queue │◀─── Job { name: "MyCommand", data: {...} }
|
|
166
|
+
│ (per entity) │
|
|
167
|
+
│ │
|
|
168
|
+
└─────────┬─────────┘
|
|
169
|
+
│
|
|
170
|
+
│ ④ Worker pulls job (one at a time)
|
|
171
|
+
▼
|
|
172
|
+
┌───────────────────┐
|
|
173
|
+
│ │
|
|
174
|
+
│ BullMQ Worker │
|
|
175
|
+
│ (1 per entity) │
|
|
176
|
+
│ │
|
|
177
|
+
└─────────┬─────────┘
|
|
178
|
+
│
|
|
179
|
+
│ ⑤ Lookup command class in registry
|
|
180
|
+
│ ⑥ Instantiate from job.data
|
|
181
|
+
│ ⑦ Execute via CQRS CommandBus
|
|
182
|
+
▼
|
|
183
|
+
┌───────────────────┐ ┌─────────────────────────┐
|
|
184
|
+
│ │ │ │
|
|
185
|
+
│ CommandBus │─────▶│ MyCommandHandler │
|
|
186
|
+
│ │ │ .execute(command) │
|
|
187
|
+
└───────────────────┘ └─────────────────────────┘
|
|
169
188
|
```
|
|
170
189
|
|
|
171
190
|
---
|
|
@@ -281,109 +300,145 @@ export class OrderScaler {
|
|
|
281
300
|
|
|
282
301
|
## Complete Example
|
|
283
302
|
|
|
303
|
+
A document processing service where multiple users can edit the same document:
|
|
304
|
+
|
|
284
305
|
```typescript
|
|
285
306
|
// ─────────────────────────────────────────────────────────────────
|
|
286
|
-
// commands/
|
|
307
|
+
// commands/update-document.command.ts
|
|
287
308
|
// ─────────────────────────────────────────────────────────────────
|
|
288
|
-
export class
|
|
309
|
+
export class UpdateDocumentCommand {
|
|
289
310
|
constructor(
|
|
290
|
-
public readonly
|
|
291
|
-
public readonly
|
|
292
|
-
public readonly
|
|
311
|
+
public readonly documentId: string,
|
|
312
|
+
public readonly userId: string,
|
|
313
|
+
public readonly content: string,
|
|
314
|
+
public readonly version: number,
|
|
293
315
|
) {}
|
|
294
316
|
}
|
|
295
317
|
|
|
296
318
|
// ─────────────────────────────────────────────────────────────────
|
|
297
|
-
// commands/
|
|
319
|
+
// commands/publish-document.command.ts
|
|
298
320
|
// ─────────────────────────────────────────────────────────────────
|
|
299
|
-
export class
|
|
321
|
+
export class PublishDocumentCommand {
|
|
300
322
|
constructor(
|
|
301
|
-
public readonly
|
|
323
|
+
public readonly documentId: string,
|
|
324
|
+
public readonly publishedBy: string,
|
|
302
325
|
) {}
|
|
303
326
|
}
|
|
304
327
|
|
|
305
328
|
// ─────────────────────────────────────────────────────────────────
|
|
306
|
-
// handlers/
|
|
329
|
+
// handlers/update-document.handler.ts
|
|
307
330
|
// ─────────────────────────────────────────────────────────────────
|
|
308
331
|
import { CommandHandler, ICommandHandler } from '@nestjs/cqrs';
|
|
309
|
-
import {
|
|
310
|
-
|
|
311
|
-
@CommandHandler(
|
|
312
|
-
export class
|
|
313
|
-
|
|
314
|
-
|
|
332
|
+
import { UpdateDocumentCommand } from '../commands';
|
|
333
|
+
|
|
334
|
+
@CommandHandler(UpdateDocumentCommand)
|
|
335
|
+
export class UpdateDocumentHandler implements ICommandHandler<UpdateDocumentCommand> {
|
|
336
|
+
constructor(private readonly documentRepo: DocumentRepository) {}
|
|
337
|
+
|
|
338
|
+
async execute(command: UpdateDocumentCommand) {
|
|
339
|
+
const { documentId, userId, content, version } = command;
|
|
340
|
+
|
|
341
|
+
// Safe! No race conditions - one update at a time per document
|
|
342
|
+
await this.documentRepo.update(documentId, { content, version, lastEditedBy: userId });
|
|
343
|
+
|
|
344
|
+
return { success: true, documentId, version };
|
|
315
345
|
}
|
|
316
346
|
}
|
|
317
347
|
|
|
318
348
|
// ─────────────────────────────────────────────────────────────────
|
|
319
|
-
// handlers/
|
|
349
|
+
// handlers/publish-document.handler.ts
|
|
320
350
|
// ─────────────────────────────────────────────────────────────────
|
|
321
351
|
import { CommandHandler, ICommandHandler } from '@nestjs/cqrs';
|
|
322
|
-
import {
|
|
323
|
-
|
|
324
|
-
@CommandHandler(
|
|
325
|
-
export class
|
|
326
|
-
|
|
327
|
-
|
|
352
|
+
import { PublishDocumentCommand } from '../commands';
|
|
353
|
+
|
|
354
|
+
@CommandHandler(PublishDocumentCommand)
|
|
355
|
+
export class PublishDocumentHandler implements ICommandHandler<PublishDocumentCommand> {
|
|
356
|
+
constructor(private readonly documentRepo: DocumentRepository) {}
|
|
357
|
+
|
|
358
|
+
async execute(command: PublishDocumentCommand) {
|
|
359
|
+
const { documentId, publishedBy } = command;
|
|
360
|
+
|
|
361
|
+
await this.documentRepo.publish(documentId, publishedBy);
|
|
362
|
+
|
|
363
|
+
return { success: true, documentId, publishedAt: new Date() };
|
|
328
364
|
}
|
|
329
365
|
}
|
|
330
366
|
|
|
331
367
|
// ─────────────────────────────────────────────────────────────────
|
|
332
|
-
//
|
|
368
|
+
// document.processor.ts
|
|
333
369
|
// ─────────────────────────────────────────────────────────────────
|
|
334
370
|
import { Injectable } from '@nestjs/common';
|
|
335
371
|
import { WorkerProcessor } from 'atomic-queues';
|
|
336
372
|
|
|
337
373
|
@WorkerProcessor({
|
|
338
|
-
entityType: '
|
|
339
|
-
queueName: (
|
|
340
|
-
workerName: (
|
|
374
|
+
entityType: 'document',
|
|
375
|
+
queueName: (documentId) => `doc-${documentId}-queue`,
|
|
376
|
+
workerName: (documentId) => `doc-${documentId}-worker`,
|
|
341
377
|
})
|
|
342
378
|
@Injectable()
|
|
343
|
-
export class
|
|
379
|
+
export class DocumentProcessor {}
|
|
344
380
|
|
|
345
381
|
// ─────────────────────────────────────────────────────────────────
|
|
346
|
-
//
|
|
382
|
+
// document.module.ts
|
|
347
383
|
// ─────────────────────────────────────────────────────────────────
|
|
348
384
|
import { Module } from '@nestjs/common';
|
|
349
385
|
import { CqrsModule } from '@nestjs/cqrs';
|
|
350
|
-
import {
|
|
351
|
-
import {
|
|
352
|
-
import {
|
|
386
|
+
import { DocumentProcessor } from './document.processor';
|
|
387
|
+
import { DocumentController } from './document.controller';
|
|
388
|
+
import { UpdateDocumentHandler, PublishDocumentHandler } from './handlers';
|
|
353
389
|
|
|
354
390
|
@Module({
|
|
355
391
|
imports: [CqrsModule],
|
|
356
392
|
providers: [
|
|
357
|
-
|
|
358
|
-
|
|
359
|
-
|
|
360
|
-
DealCardsHandler,
|
|
393
|
+
DocumentProcessor,
|
|
394
|
+
UpdateDocumentHandler, // Commands auto-discovered!
|
|
395
|
+
PublishDocumentHandler,
|
|
361
396
|
],
|
|
397
|
+
controllers: [DocumentController],
|
|
362
398
|
})
|
|
363
|
-
export class
|
|
399
|
+
export class DocumentModule {}
|
|
364
400
|
|
|
365
401
|
// ─────────────────────────────────────────────────────────────────
|
|
366
|
-
//
|
|
402
|
+
// document.controller.ts
|
|
367
403
|
// ─────────────────────────────────────────────────────────────────
|
|
368
|
-
import {
|
|
404
|
+
import { Controller, Post, Body, Param } from '@nestjs/common';
|
|
369
405
|
import { QueueBus } from 'atomic-queues';
|
|
370
|
-
import {
|
|
371
|
-
import {
|
|
406
|
+
import { DocumentProcessor } from './document.processor';
|
|
407
|
+
import { UpdateDocumentCommand, PublishDocumentCommand } from './commands';
|
|
372
408
|
|
|
373
|
-
@
|
|
374
|
-
export class
|
|
409
|
+
@Controller('documents')
|
|
410
|
+
export class DocumentController {
|
|
375
411
|
constructor(private readonly queueBus: QueueBus) {}
|
|
376
412
|
|
|
377
|
-
|
|
413
|
+
@Post(':id/update')
|
|
414
|
+
async updateDocument(
|
|
415
|
+
@Param('id') documentId: string,
|
|
416
|
+
@Body() body: { userId: string; content: string; version: number },
|
|
417
|
+
) {
|
|
418
|
+
// Multiple users editing same doc? No problem!
|
|
419
|
+
// Updates are queued and processed one at a time
|
|
378
420
|
await this.queueBus
|
|
379
|
-
.forProcessor(
|
|
380
|
-
.enqueue(new
|
|
421
|
+
.forProcessor(DocumentProcessor)
|
|
422
|
+
.enqueue(new UpdateDocumentCommand(
|
|
423
|
+
documentId,
|
|
424
|
+
body.userId,
|
|
425
|
+
body.content,
|
|
426
|
+
body.version,
|
|
427
|
+
));
|
|
428
|
+
|
|
429
|
+
return { queued: true, documentId };
|
|
381
430
|
}
|
|
382
431
|
|
|
383
|
-
|
|
432
|
+
@Post(':id/publish')
|
|
433
|
+
async publishDocument(
|
|
434
|
+
@Param('id') documentId: string,
|
|
435
|
+
@Body() body: { publishedBy: string },
|
|
436
|
+
) {
|
|
384
437
|
await this.queueBus
|
|
385
|
-
.forProcessor(
|
|
386
|
-
.enqueue(new
|
|
438
|
+
.forProcessor(DocumentProcessor)
|
|
439
|
+
.enqueue(new PublishDocumentCommand(documentId, body.publishedBy));
|
|
440
|
+
|
|
441
|
+
return { queued: true, documentId };
|
|
387
442
|
}
|
|
388
443
|
}
|
|
389
444
|
```
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "atomic-queues",
|
|
3
|
-
"version": "1.2.
|
|
3
|
+
"version": "1.2.4",
|
|
4
4
|
"description": "A plug-and-play NestJS library for atomic process handling per entity with BullMQ, Redis distributed locking, and dynamic worker management",
|
|
5
5
|
"main": "dist/index.js",
|
|
6
6
|
"types": "dist/index.d.ts",
|