atomic-queues 1.4.1 → 1.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +300 -283
- package/dist/decorators/constants.d.ts +17 -0
- package/dist/decorators/constants.d.ts.map +1 -0
- package/dist/decorators/constants.js +23 -0
- package/dist/decorators/constants.js.map +1 -0
- package/dist/decorators/entity.decorators.d.ts +88 -0
- package/dist/decorators/entity.decorators.d.ts.map +1 -0
- package/dist/decorators/entity.decorators.js +150 -0
- package/dist/decorators/entity.decorators.js.map +1 -0
- package/dist/decorators/index.d.ts +9 -1
- package/dist/decorators/index.d.ts.map +1 -1
- package/dist/decorators/index.js +9 -1
- package/dist/decorators/index.js.map +1 -1
- package/dist/decorators/interfaces.d.ts +130 -0
- package/dist/decorators/interfaces.d.ts.map +1 -0
- package/dist/decorators/interfaces.js +3 -0
- package/dist/decorators/interfaces.js.map +1 -0
- package/dist/decorators/job.decorators.d.ts +60 -0
- package/dist/decorators/job.decorators.d.ts.map +1 -0
- package/dist/decorators/job.decorators.js +97 -0
- package/dist/decorators/job.decorators.js.map +1 -0
- package/dist/decorators/legacy.decorators.d.ts +36 -0
- package/dist/decorators/legacy.decorators.d.ts.map +1 -0
- package/dist/decorators/legacy.decorators.js +61 -0
- package/dist/decorators/legacy.decorators.js.map +1 -0
- package/dist/decorators/metadata-readers.d.ts +31 -0
- package/dist/decorators/metadata-readers.d.ts.map +1 -0
- package/dist/decorators/metadata-readers.js +53 -0
- package/dist/decorators/metadata-readers.js.map +1 -0
- package/dist/decorators/registry.d.ts +2 -0
- package/dist/decorators/registry.d.ts.map +1 -0
- package/dist/decorators/registry.js +6 -0
- package/dist/decorators/registry.js.map +1 -0
- package/dist/decorators/scaler.decorators.d.ts +65 -0
- package/dist/decorators/scaler.decorators.d.ts.map +1 -0
- package/dist/decorators/scaler.decorators.js +103 -0
- package/dist/decorators/scaler.decorators.js.map +1 -0
- package/dist/decorators/type-guards.d.ts +18 -0
- package/dist/decorators/type-guards.d.ts.map +1 -0
- package/dist/decorators/type-guards.js +32 -0
- package/dist/decorators/type-guards.js.map +1 -0
- package/dist/decorators/utils.d.ts +20 -0
- package/dist/decorators/utils.d.ts.map +1 -0
- package/dist/decorators/utils.js +98 -0
- package/dist/decorators/utils.js.map +1 -0
- package/dist/decorators/worker.decorators.d.ts +58 -0
- package/dist/decorators/worker.decorators.d.ts.map +1 -0
- package/dist/decorators/worker.decorators.js +92 -0
- package/dist/decorators/worker.decorators.js.map +1 -0
- package/dist/domain/interfaces/config.interfaces.d.ts +188 -0
- package/dist/domain/interfaces/config.interfaces.d.ts.map +1 -0
- package/dist/domain/interfaces/config.interfaces.js +3 -0
- package/dist/domain/interfaces/config.interfaces.js.map +1 -0
- package/dist/domain/interfaces/cqrs.interfaces.d.ts +7 -0
- package/dist/domain/interfaces/cqrs.interfaces.d.ts.map +1 -0
- package/dist/domain/interfaces/cqrs.interfaces.js +3 -0
- package/dist/domain/interfaces/cqrs.interfaces.js.map +1 -0
- package/dist/domain/interfaces/event.interfaces.d.ts +71 -0
- package/dist/domain/interfaces/event.interfaces.d.ts.map +1 -0
- package/dist/domain/interfaces/event.interfaces.js +3 -0
- package/dist/domain/interfaces/event.interfaces.js.map +1 -0
- package/dist/domain/interfaces/index-tracking.interfaces.d.ts +69 -0
- package/dist/domain/interfaces/index-tracking.interfaces.d.ts.map +1 -0
- package/dist/domain/interfaces/index-tracking.interfaces.js +3 -0
- package/dist/domain/interfaces/index-tracking.interfaces.js.map +1 -0
- package/dist/domain/interfaces/index.d.ts +12 -0
- package/dist/domain/interfaces/index.d.ts.map +1 -0
- package/dist/domain/interfaces/index.js +28 -0
- package/dist/domain/interfaces/index.js.map +1 -0
- package/dist/domain/interfaces/job.interfaces.d.ts +76 -0
- package/dist/domain/interfaces/job.interfaces.d.ts.map +1 -0
- package/dist/domain/interfaces/job.interfaces.js +3 -0
- package/dist/domain/interfaces/job.interfaces.js.map +1 -0
- package/dist/domain/interfaces/lock.interfaces.d.ts +54 -0
- package/dist/domain/interfaces/lock.interfaces.d.ts.map +1 -0
- package/dist/domain/interfaces/lock.interfaces.js +3 -0
- package/dist/domain/interfaces/lock.interfaces.js.map +1 -0
- package/dist/domain/interfaces/process.interfaces.d.ts +44 -0
- package/dist/domain/interfaces/process.interfaces.d.ts.map +1 -0
- package/dist/domain/interfaces/process.interfaces.js +3 -0
- package/dist/domain/interfaces/process.interfaces.js.map +1 -0
- package/dist/domain/interfaces/queue.interfaces.d.ts +46 -0
- package/dist/domain/interfaces/queue.interfaces.d.ts.map +1 -0
- package/dist/domain/interfaces/queue.interfaces.js +3 -0
- package/dist/domain/interfaces/queue.interfaces.js.map +1 -0
- package/dist/domain/interfaces/scaling.interfaces.d.ts +62 -0
- package/dist/domain/interfaces/scaling.interfaces.d.ts.map +1 -0
- package/dist/domain/interfaces/scaling.interfaces.js +3 -0
- package/dist/domain/interfaces/scaling.interfaces.js.map +1 -0
- package/dist/domain/interfaces/utility.types.d.ts +15 -0
- package/dist/domain/interfaces/utility.types.d.ts.map +1 -0
- package/dist/domain/interfaces/utility.types.js +3 -0
- package/dist/domain/interfaces/utility.types.js.map +1 -0
- package/dist/domain/interfaces/worker.interfaces.d.ts +120 -0
- package/dist/domain/interfaces/worker.interfaces.d.ts.map +1 -0
- package/dist/domain/interfaces/worker.interfaces.js +3 -0
- package/dist/domain/interfaces/worker.interfaces.js.map +1 -0
- package/dist/module/atomic-queues.module.d.ts.map +1 -1
- package/dist/module/atomic-queues.module.js +5 -0
- package/dist/module/atomic-queues.module.js.map +1 -1
- package/dist/services/cron-manager/cron-manager.service.d.ts +5 -4
- package/dist/services/cron-manager/cron-manager.service.d.ts.map +1 -1
- package/dist/services/cron-manager/cron-manager.service.js +26 -57
- package/dist/services/cron-manager/cron-manager.service.js.map +1 -1
- package/dist/services/index-manager/index-manager.service.d.ts +0 -4
- package/dist/services/index-manager/index-manager.service.d.ts.map +1 -1
- package/dist/services/index-manager/index-manager.service.js +4 -16
- package/dist/services/index-manager/index-manager.service.js.map +1 -1
- package/dist/services/index.d.ts +1 -0
- package/dist/services/index.d.ts.map +1 -1
- package/dist/services/index.js +1 -0
- package/dist/services/index.js.map +1 -1
- package/dist/services/processor-discovery/decorator-discovery.service.d.ts +40 -0
- package/dist/services/processor-discovery/decorator-discovery.service.d.ts.map +1 -0
- package/dist/services/processor-discovery/decorator-discovery.service.js +191 -0
- package/dist/services/processor-discovery/decorator-discovery.service.js.map +1 -0
- package/dist/services/processor-discovery/index.d.ts +4 -0
- package/dist/services/processor-discovery/index.d.ts.map +1 -1
- package/dist/services/processor-discovery/index.js +4 -0
- package/dist/services/processor-discovery/index.js.map +1 -1
- package/dist/services/processor-discovery/processor-discovery.service.d.ts +30 -138
- package/dist/services/processor-discovery/processor-discovery.service.d.ts.map +1 -1
- package/dist/services/processor-discovery/processor-discovery.service.js +125 -502
- package/dist/services/processor-discovery/processor-discovery.service.js.map +1 -1
- package/dist/services/processor-discovery/processor-registry.d.ts +58 -0
- package/dist/services/processor-discovery/processor-registry.d.ts.map +1 -0
- package/dist/services/processor-discovery/processor-registry.js +74 -0
- package/dist/services/processor-discovery/processor-registry.js.map +1 -0
- package/dist/services/processor-discovery/scaling-registration.service.d.ts +60 -0
- package/dist/services/processor-discovery/scaling-registration.service.d.ts.map +1 -0
- package/dist/services/processor-discovery/scaling-registration.service.js +261 -0
- package/dist/services/processor-discovery/scaling-registration.service.js.map +1 -0
- package/dist/services/processor-discovery/worker-factory.service.d.ts +54 -0
- package/dist/services/processor-discovery/worker-factory.service.d.ts.map +1 -0
- package/dist/services/processor-discovery/worker-factory.service.js +185 -0
- package/dist/services/processor-discovery/worker-factory.service.js.map +1 -0
- package/dist/services/queue-bus/entity-target.d.ts +58 -0
- package/dist/services/queue-bus/entity-target.d.ts.map +1 -0
- package/dist/services/queue-bus/entity-target.js +109 -0
- package/dist/services/queue-bus/entity-target.js.map +1 -0
- package/dist/services/queue-bus/index.d.ts +4 -0
- package/dist/services/queue-bus/index.d.ts.map +1 -1
- package/dist/services/queue-bus/index.js +4 -0
- package/dist/services/queue-bus/index.js.map +1 -1
- package/dist/services/queue-bus/queue-bus.service.d.ts +9 -145
- package/dist/services/queue-bus/queue-bus.service.d.ts.map +1 -1
- package/dist/services/queue-bus/queue-bus.service.js +23 -311
- package/dist/services/queue-bus/queue-bus.service.js.map +1 -1
- package/dist/services/queue-bus/queue-bus.types.d.ts +40 -0
- package/dist/services/queue-bus/queue-bus.types.d.ts.map +1 -0
- package/dist/services/queue-bus/queue-bus.types.js +3 -0
- package/dist/services/queue-bus/queue-bus.types.js.map +1 -0
- package/dist/services/queue-bus/queue-bus.utils.d.ts +34 -0
- package/dist/services/queue-bus/queue-bus.utils.d.ts.map +1 -0
- package/dist/services/queue-bus/queue-bus.utils.js +82 -0
- package/dist/services/queue-bus/queue-bus.utils.js.map +1 -0
- package/dist/services/queue-bus/queue-target.d.ts +61 -0
- package/dist/services/queue-bus/queue-target.d.ts.map +1 -0
- package/dist/services/queue-bus/queue-target.js +123 -0
- package/dist/services/queue-bus/queue-target.js.map +1 -0
- package/dist/services/queue-events-manager/queue-events-manager.service.d.ts +23 -6
- package/dist/services/queue-events-manager/queue-events-manager.service.d.ts.map +1 -1
- package/dist/services/queue-events-manager/queue-events-manager.service.js +69 -37
- package/dist/services/queue-events-manager/queue-events-manager.service.js.map +1 -1
- package/dist/services/resource-lock/resource-lock.service.d.ts +0 -4
- package/dist/services/resource-lock/resource-lock.service.d.ts.map +1 -1
- package/dist/services/resource-lock/resource-lock.service.js +4 -16
- package/dist/services/resource-lock/resource-lock.service.js.map +1 -1
- package/dist/services/service-queue/index.d.ts +1 -0
- package/dist/services/service-queue/index.d.ts.map +1 -1
- package/dist/services/service-queue/index.js +1 -0
- package/dist/services/service-queue/index.js.map +1 -1
- package/dist/services/service-queue/service-queue.service.d.ts +2 -35
- package/dist/services/service-queue/service-queue.service.d.ts.map +1 -1
- package/dist/services/service-queue/service-queue.service.js +17 -49
- package/dist/services/service-queue/service-queue.service.js.map +1 -1
- package/dist/services/service-queue/service-queue.types.d.ts +32 -0
- package/dist/services/service-queue/service-queue.types.d.ts.map +1 -0
- package/dist/services/service-queue/service-queue.types.js +27 -0
- package/dist/services/service-queue/service-queue.types.js.map +1 -0
- package/dist/services/spawn-queue/index.d.ts +2 -0
- package/dist/services/spawn-queue/index.d.ts.map +1 -0
- package/dist/services/spawn-queue/index.js +18 -0
- package/dist/services/spawn-queue/index.js.map +1 -0
- package/dist/services/spawn-queue/spawn-queue.service.d.ts +119 -0
- package/dist/services/spawn-queue/spawn-queue.service.d.ts.map +1 -0
- package/dist/services/spawn-queue/spawn-queue.service.js +273 -0
- package/dist/services/spawn-queue/spawn-queue.service.js.map +1 -0
- package/dist/services/worker-manager/worker-manager.service.d.ts +18 -3
- package/dist/services/worker-manager/worker-manager.service.d.ts.map +1 -1
- package/dist/services/worker-manager/worker-manager.service.js +46 -21
- package/dist/services/worker-manager/worker-manager.service.js.map +1 -1
- package/dist/utils/async.utils.d.ts +51 -0
- package/dist/utils/async.utils.d.ts.map +1 -0
- package/dist/utils/async.utils.js +87 -0
- package/dist/utils/async.utils.js.map +1 -0
- package/dist/utils/helpers.d.ts +4 -123
- package/dist/utils/helpers.d.ts.map +1 -1
- package/dist/utils/helpers.js +18 -226
- package/dist/utils/helpers.js.map +1 -1
- package/dist/utils/index.d.ts +1 -0
- package/dist/utils/index.d.ts.map +1 -1
- package/dist/utils/index.js +1 -0
- package/dist/utils/index.js.map +1 -1
- package/dist/utils/job.utils.d.ts +50 -0
- package/dist/utils/job.utils.d.ts.map +1 -0
- package/dist/utils/job.utils.js +89 -0
- package/dist/utils/job.utils.js.map +1 -0
- package/dist/utils/naming.utils.d.ts +21 -0
- package/dist/utils/naming.utils.d.ts.map +1 -0
- package/dist/utils/naming.utils.js +38 -0
- package/dist/utils/naming.utils.js.map +1 -0
- package/dist/utils/rate-limit.utils.d.ts +9 -0
- package/dist/utils/rate-limit.utils.d.ts.map +1 -0
- package/dist/utils/rate-limit.utils.js +30 -0
- package/dist/utils/rate-limit.utils.js.map +1 -0
- package/dist/utils/redis.utils.d.ts +3 -0
- package/dist/utils/redis.utils.d.ts.map +1 -0
- package/dist/utils/redis.utils.js +14 -0
- package/dist/utils/redis.utils.js.map +1 -0
- package/package.json +17 -17
- package/dist/decorators/decorators.d.ts +0 -489
- package/dist/decorators/decorators.d.ts.map +0 -1
- package/dist/decorators/decorators.js +0 -680
- package/dist/decorators/decorators.js.map +0 -1
- package/dist/domain/interfaces.d.ts +0 -748
- package/dist/domain/interfaces.d.ts.map +0 -1
- package/dist/domain/interfaces.js +0 -19
- package/dist/domain/interfaces.js.map +0 -1
package/README.md
CHANGED
|
@@ -1,132 +1,137 @@
|
|
|
1
|
-
|
|
1
|
+
<p align="center">
|
|
2
|
+
<img src="https://img.shields.io/npm/v/atomic-queues?style=flat-square&color=cb3837" alt="npm version" />
|
|
3
|
+
<img src="https://img.shields.io/badge/NestJS-11-ea2845?style=flat-square&logo=nestjs" alt="NestJS 11" />
|
|
4
|
+
<img src="https://img.shields.io/badge/BullMQ-5-3c873a?style=flat-square" alt="BullMQ 5" />
|
|
5
|
+
<img src="https://img.shields.io/badge/Redis-7-dc382d?style=flat-square&logo=redis&logoColor=white" alt="Redis 7" />
|
|
6
|
+
<img src="https://img.shields.io/badge/license-MIT-blue?style=flat-square" alt="MIT License" />
|
|
7
|
+
</p>
|
|
2
8
|
|
|
3
|
-
|
|
9
|
+
<h1 align="center">atomic-queues</h1>
|
|
10
|
+
|
|
11
|
+
<p align="center">
|
|
12
|
+
<strong>Zero-contention, per-entity sequential processing for NestJS.</strong><br/>
|
|
13
|
+
Distributed. Lock-free.
|
|
14
|
+
</p>
|
|
15
|
+
|
|
16
|
+
---
|
|
17
|
+
|
|
18
|
+
## Why atomic-queues?
|
|
19
|
+
|
|
20
|
+
Distributed locks (Redlock, advisory locks, optimistic locking) all share the same fundamental flaw: **contention collapse**. When multiple pods fight for the same lock simultaneously, they spend more time retrying failed acquisitions than doing actual work. The harder you push, the slower they go.
|
|
21
|
+
|
|
22
|
+
**atomic-queues** eliminates contention entirely. Instead of locking, each entity gets its own dedicated BullMQ queue. Operations execute sequentially — back-to-back with zero wasted cycles. There's nothing to contend over.
|
|
23
|
+
|
|
24
|
+
### atomic-queues vs Redlock
|
|
25
|
+
|
|
26
|
+
| | Redlock | atomic-queues |
|
|
27
|
+
|---|---|---|
|
|
28
|
+
| **Architecture** | Distributed mutex (quorum-based) | Per-entity queue (sequential) |
|
|
29
|
+
| **Under contention** | Degrades — retry storms, backoff delays | **Constant** — jobs queue up, execute instantly |
|
|
30
|
+
| **Failure mode** | Silent double-execution (clock drift) | Job stuck in queue (visible, retryable) |
|
|
31
|
+
| **Split-brain risk** | Yes (timing assumptions) | **Impossible** (serial queue) |
|
|
32
|
+
| **Warm-path overhead** | Acquire + release per op | **0 Redis calls** (in-memory hot cache) |
|
|
33
|
+
| **Cold-start** | None | One-time per entity |
|
|
34
|
+
| **Multi-pod scaling** | Contention increases with pods | **Throughput increases with pods** |
|
|
4
35
|
|
|
5
36
|
---
|
|
6
37
|
|
|
7
38
|
## Table of Contents
|
|
8
39
|
|
|
9
|
-
- [
|
|
10
|
-
- [
|
|
11
|
-
- [The Per-Entity Queue Architecture](#the-per-entity-queue-architecture)
|
|
40
|
+
- [Why atomic-queues?](#why-atomic-queues)
|
|
41
|
+
- [How It Works](#how-it-works)
|
|
12
42
|
- [Installation](#installation)
|
|
13
43
|
- [Quick Start](#quick-start)
|
|
14
|
-
- [Commands
|
|
44
|
+
- [Commands & Decorators](#commands--decorators)
|
|
15
45
|
- [Configuration](#configuration)
|
|
46
|
+
- [Distributed Worker Lifecycle](#distributed-worker-lifecycle)
|
|
16
47
|
- [Complete Example](#complete-example)
|
|
17
48
|
- [Advanced: Custom Worker Processors](#advanced-custom-worker-processors)
|
|
49
|
+
- [Performance](#performance)
|
|
18
50
|
- [License](#license)
|
|
19
51
|
|
|
20
52
|
---
|
|
21
53
|
|
|
22
|
-
##
|
|
54
|
+
## How It Works
|
|
23
55
|
|
|
24
|
-
|
|
56
|
+
### The Problem
|
|
25
57
|
|
|
26
|
-
|
|
58
|
+
Every distributed system eventually hits this:
|
|
27
59
|
|
|
28
|
-
---
|
|
29
|
-
|
|
30
|
-
## The Concurrency Problem
|
|
31
|
-
|
|
32
|
-
Consider a banking system where a user with a $100 balance submits two concurrent $80 withdrawal requests:
|
|
33
|
-
|
|
34
|
-
```
|
|
35
|
-
Time Request A Request B Database State
|
|
36
|
-
─────────────────────────────────────────────────────────────────────────────────
|
|
37
|
-
T₀ SELECT balance → $100 SELECT balance → $100 balance = $100
|
|
38
|
-
T₁ CHECK: $100 >= $80 ✓ CHECK: $100 >= $80 ✓
|
|
39
|
-
T₂ UPDATE: balance = $20 UPDATE: balance = $20 balance = $20
|
|
40
|
-
T₃ UPDATE: balance = -$60 balance = -$60
|
|
41
|
-
─────────────────────────────────────────────────────────────────────────────────
|
|
42
|
-
Result: Both withdrawals succeed. Balance becomes -$60. Integrity violated.
|
|
43
60
|
```
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
T₂ [] Process Op₂: $20 < $80 → REJECT balance = $20
|
|
53
|
-
───────────────────────────────────────────────────────────────────────────────────
|
|
54
|
-
Result: First withdrawal succeeds. Second is rejected. Integrity preserved.
|
|
61
|
+
Time Request A Request B Database
|
|
62
|
+
──────────────────────────────────────────────────────────────────────────
|
|
63
|
+
T₀ SELECT balance → $100 SELECT balance → $100 $100
|
|
64
|
+
T₁ CHECK: $100 ≥ $80 ✓ CHECK: $100 ≥ $80 ✓
|
|
65
|
+
T₂ UPDATE: $100 − $80 = $20 $20
|
|
66
|
+
T₃ UPDATE: $100 − $80 = $20 −$60
|
|
67
|
+
──────────────────────────────────────────────────────────────────────────
|
|
68
|
+
Result: Balance is −$60. Both withdrawals succeed. Integrity violated.
|
|
55
69
|
```
|
|
56
70
|
|
|
57
|
-
|
|
71
|
+
### The Solution
|
|
58
72
|
|
|
59
|
-
|
|
73
|
+
atomic-queues routes operations through per-entity queues. Same entity → same queue → sequential execution. Different entities → parallel queues → full throughput.
|
|
60
74
|
|
|
61
75
|
```
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
76
|
+
┌─────────────────────────────────────────────────┐
|
|
77
|
+
Request A ─┐ │ Entity: account-42 │
|
|
78
|
+
│ │ ┌──────┐ ┌──────┐ ┌──────┐ │
|
|
79
|
+
Request B ─┼─► Route ─┼─►│ Op 1 │─►│ Op 2 │─►│ Op 3 │─► [Worker] ──┐ │
|
|
80
|
+
│ │ └──────┘ └──────┘ └──────┘ │ │
|
|
81
|
+
Request C ─┘ │ Sequential ◄─────────────┘ │
|
|
82
|
+
└─────────────────────────────────────────────────┘
|
|
83
|
+
|
|
84
|
+
┌─────────────────────────────────────────────────┐
|
|
85
|
+
Request D ─┐ │ Entity: account-99 │
|
|
86
|
+
│ │ ┌──────┐ ┌──────┐ │
|
|
87
|
+
Request E ─┼─► Route ─┼─►│ Op 1 │─►│ Op 2 │─────────► [Worker] ──┐ │
|
|
88
|
+
│ │ └──────┘ └──────┘ │ │
|
|
89
|
+
Request F ─┘ │ Sequential ◄───────────┘ │
|
|
90
|
+
└─────────────────────────────────────────────────┘
|
|
91
|
+
|
|
92
|
+
▲ These two queues run in PARALLEL across pods ▲
|
|
70
93
|
```
|
|
71
94
|
|
|
72
|
-
**Key
|
|
73
|
-
-
|
|
74
|
-
-
|
|
75
|
-
-
|
|
76
|
-
-
|
|
95
|
+
**Key properties:**
|
|
96
|
+
- **One worker per entity** — enforced via Redis heartbeat TTL. No duplicates, ever.
|
|
97
|
+
- **Auto-spawn** — workers materialize when jobs arrive, on the pod that sees them first.
|
|
98
|
+
- **Auto-terminate** — idle workers shut down after a configurable timeout.
|
|
99
|
+
- **Self-healing** — node failure → heartbeat expires → worker respawns on a healthy pod.
|
|
100
|
+
- **Distributed** — workers spread across all pods via atomic `SET NX` claim. No leader election, no single point of failure.
|
|
77
101
|
|
|
78
102
|
---
|
|
79
103
|
|
|
80
104
|
## Installation
|
|
81
105
|
|
|
82
106
|
```bash
|
|
83
|
-
npm install atomic-queues
|
|
107
|
+
npm install atomic-queues
|
|
84
108
|
```
|
|
85
109
|
|
|
110
|
+
BullMQ, ioredis, and `@nestjs/bullmq` are bundled — no extra installs needed.
|
|
111
|
+
|
|
112
|
+
**Peer dependencies** (provided by your NestJS app): `@nestjs/common` 10+, `@nestjs/core` 10+, `reflect-metadata`, `rxjs` 7+. Optional: `@nestjs/cqrs` (for auto-routing commands/queries).
|
|
113
|
+
|
|
86
114
|
---
|
|
87
115
|
|
|
88
116
|
## Quick Start
|
|
89
117
|
|
|
90
118
|
### 1. Configure the Module
|
|
91
119
|
|
|
92
|
-
The `entities` configuration is **optional**. Choose the approach that fits your needs:
|
|
93
|
-
|
|
94
|
-
#### Option A: Minimal Setup (uses default naming)
|
|
95
|
-
|
|
96
120
|
```typescript
|
|
97
121
|
import { Module } from '@nestjs/common';
|
|
122
|
+
import { CqrsModule } from '@nestjs/cqrs';
|
|
98
123
|
import { AtomicQueuesModule } from 'atomic-queues';
|
|
99
124
|
|
|
100
125
|
@Module({
|
|
101
126
|
imports: [
|
|
127
|
+
CqrsModule,
|
|
102
128
|
AtomicQueuesModule.forRoot({
|
|
103
129
|
redis: { host: 'localhost', port: 6379 },
|
|
104
130
|
keyPrefix: 'myapp',
|
|
105
|
-
enableCronManager: true,
|
|
106
|
-
// No entities config needed! Uses default naming:
|
|
107
|
-
// Queue: {keyPrefix}:{entityType}:{entityId}:queue
|
|
108
|
-
// Worker: {keyPrefix}:{entityType}:{entityId}:worker
|
|
109
|
-
}),
|
|
110
|
-
],
|
|
111
|
-
})
|
|
112
|
-
export class AppModule {}
|
|
113
|
-
```
|
|
114
|
-
|
|
115
|
-
#### Option B: Custom Queue/Worker Naming (via entities config)
|
|
116
|
-
|
|
117
|
-
```typescript
|
|
118
|
-
@Module({
|
|
119
|
-
imports: [
|
|
120
|
-
AtomicQueuesModule.forRoot({
|
|
121
|
-
redis: { host: 'localhost', port: 6379 },
|
|
122
|
-
keyPrefix: 'myapp',
|
|
123
|
-
enableCronManager: true,
|
|
124
|
-
|
|
125
|
-
// Optional: Define custom naming and settings per entity type
|
|
126
131
|
entities: {
|
|
127
132
|
account: {
|
|
128
|
-
queueName: (id) =>
|
|
129
|
-
workerName: (id) =>
|
|
133
|
+
queueName: (id) => `account-${id}-queue`,
|
|
134
|
+
workerName: (id) => `account-${id}-worker`,
|
|
130
135
|
maxWorkersPerEntity: 1,
|
|
131
136
|
idleTimeoutSeconds: 15,
|
|
132
137
|
},
|
|
@@ -137,28 +142,31 @@ export class AppModule {}
|
|
|
137
142
|
export class AppModule {}
|
|
138
143
|
```
|
|
139
144
|
|
|
140
|
-
|
|
145
|
+
> **Tip:** The `entities` config is optional. Without it, default naming applies: `{keyPrefix}:{entityType}:{entityId}:queue`.
|
|
141
146
|
|
|
142
|
-
|
|
147
|
+
<details>
|
|
148
|
+
<summary><strong>Async configuration (ConfigService)</strong></summary>
|
|
143
149
|
|
|
144
150
|
```typescript
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
151
|
+
AtomicQueuesModule.forRootAsync({
|
|
152
|
+
imports: [ConfigModule],
|
|
153
|
+
useFactory: (config: ConfigService) => ({
|
|
154
|
+
redis: { url: config.get('REDIS_URL') },
|
|
155
|
+
keyPrefix: 'myapp',
|
|
156
|
+
entities: {
|
|
157
|
+
account: {
|
|
158
|
+
queueName: (id) => `account-${id}-queue`,
|
|
159
|
+
workerName: (id) => `account-${id}-worker`,
|
|
160
|
+
},
|
|
161
|
+
},
|
|
162
|
+
}),
|
|
163
|
+
inject: [ConfigService],
|
|
164
|
+
}),
|
|
154
165
|
```
|
|
155
166
|
|
|
156
|
-
>
|
|
157
|
-
> - **Option A**: Default naming works for you
|
|
158
|
-
> - **Option B**: Need custom naming but no custom job handling logic
|
|
159
|
-
> - **Option C**: Need custom naming AND custom `@JobHandler` methods
|
|
167
|
+
</details>
|
|
160
168
|
|
|
161
|
-
### 2.
|
|
169
|
+
### 2. Define Commands
|
|
162
170
|
|
|
163
171
|
```typescript
|
|
164
172
|
import { QueueEntity, QueueEntityId } from 'atomic-queues';
|
|
@@ -168,7 +176,6 @@ export class WithdrawCommand {
|
|
|
168
176
|
constructor(
|
|
169
177
|
@QueueEntityId() public readonly accountId: string,
|
|
170
178
|
public readonly amount: number,
|
|
171
|
-
public readonly transactionId: string,
|
|
172
179
|
) {}
|
|
173
180
|
}
|
|
174
181
|
|
|
@@ -177,35 +184,29 @@ export class DepositCommand {
|
|
|
177
184
|
constructor(
|
|
178
185
|
@QueueEntityId() public readonly accountId: string,
|
|
179
186
|
public readonly amount: number,
|
|
180
|
-
public readonly source: string,
|
|
181
187
|
) {}
|
|
182
188
|
}
|
|
183
189
|
```
|
|
184
190
|
|
|
185
|
-
### 3.
|
|
191
|
+
### 3. Write Handlers (standard @nestjs/cqrs)
|
|
186
192
|
|
|
187
193
|
```typescript
|
|
188
194
|
import { CommandHandler, ICommandHandler } from '@nestjs/cqrs';
|
|
189
|
-
import { WithdrawCommand } from './commands';
|
|
190
195
|
|
|
191
196
|
@CommandHandler(WithdrawCommand)
|
|
192
197
|
export class WithdrawHandler implements ICommandHandler<WithdrawCommand> {
|
|
193
|
-
constructor(private readonly
|
|
194
|
-
|
|
195
|
-
async execute(
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
const account = await this.accountRepo.findById(accountId);
|
|
200
|
-
|
|
198
|
+
constructor(private readonly repo: AccountRepository) {}
|
|
199
|
+
|
|
200
|
+
async execute({ accountId, amount }: WithdrawCommand) {
|
|
201
|
+
// SAFE: No race conditions. Sequential execution per account.
|
|
202
|
+
const account = await this.repo.findById(accountId);
|
|
203
|
+
|
|
201
204
|
if (account.balance < amount) {
|
|
202
205
|
throw new InsufficientFundsError(accountId, account.balance, amount);
|
|
203
206
|
}
|
|
204
|
-
|
|
207
|
+
|
|
205
208
|
account.balance -= amount;
|
|
206
|
-
await this.
|
|
207
|
-
|
|
208
|
-
return { success: true, newBalance: account.balance };
|
|
209
|
+
await this.repo.save(account);
|
|
209
210
|
}
|
|
210
211
|
}
|
|
211
212
|
```
|
|
@@ -215,137 +216,196 @@ export class WithdrawHandler implements ICommandHandler<WithdrawCommand> {
|
|
|
215
216
|
```typescript
|
|
216
217
|
import { Injectable } from '@nestjs/common';
|
|
217
218
|
import { QueueBus } from 'atomic-queues';
|
|
218
|
-
import { WithdrawCommand, DepositCommand } from './commands';
|
|
219
219
|
|
|
220
220
|
@Injectable()
|
|
221
221
|
export class AccountService {
|
|
222
222
|
constructor(private readonly queueBus: QueueBus) {}
|
|
223
223
|
|
|
224
|
-
async withdraw(accountId: string, amount: number
|
|
225
|
-
|
|
226
|
-
await this.queueBus.enqueue(new WithdrawCommand(accountId, amount, transactionId));
|
|
227
|
-
}
|
|
228
|
-
|
|
229
|
-
async deposit(accountId: string, amount: number, source: string) {
|
|
230
|
-
await this.queueBus.enqueue(new DepositCommand(accountId, amount, source));
|
|
224
|
+
async withdraw(accountId: string, amount: number) {
|
|
225
|
+
await this.queueBus.enqueue(new WithdrawCommand(accountId, amount));
|
|
231
226
|
}
|
|
232
227
|
}
|
|
233
228
|
```
|
|
234
229
|
|
|
235
|
-
**That's it
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
|
|
230
|
+
**That's it.** The library automatically:
|
|
231
|
+
1. Creates a queue for each `accountId` when jobs arrive
|
|
232
|
+
2. Spawns a worker (spread across pods) to process jobs sequentially
|
|
233
|
+
3. Routes jobs to the correct `@CommandHandler` via CQRS
|
|
234
|
+
4. Terminates idle workers after the configured timeout
|
|
235
|
+
5. Self-heals if a pod dies (heartbeat expires → respawn elsewhere)
|
|
240
236
|
|
|
241
237
|
---
|
|
242
238
|
|
|
243
|
-
## Commands
|
|
239
|
+
## Commands & Decorators
|
|
244
240
|
|
|
245
|
-
###
|
|
241
|
+
### `@QueueEntity(entityType, entityIdProperty?)`
|
|
246
242
|
|
|
247
|
-
Marks a command class for queue routing. The
|
|
243
|
+
Marks a command/query class for queue routing. The optional second argument specifies which property holds the entity ID — this is the simplest approach when you don't want to decorate individual properties.
|
|
248
244
|
|
|
249
245
|
```typescript
|
|
246
|
+
// Option 1: Explicit property name (no @QueueEntityId needed)
|
|
247
|
+
@QueueEntity('account', 'accountId')
|
|
248
|
+
export class TransferCommand {
|
|
249
|
+
constructor(
|
|
250
|
+
public readonly accountId: string,
|
|
251
|
+
public readonly toAccountId: string,
|
|
252
|
+
public readonly amount: number,
|
|
253
|
+
) {}
|
|
254
|
+
}
|
|
255
|
+
|
|
256
|
+
// Option 2: Rely on module-level defaultEntityId from entities config
|
|
250
257
|
@QueueEntity('account')
|
|
251
|
-
export class
|
|
258
|
+
export class DepositCommand {
|
|
259
|
+
constructor(
|
|
260
|
+
public readonly accountId: string, // Matched by entities.account.defaultEntityId
|
|
261
|
+
public readonly amount: number,
|
|
262
|
+
) {}
|
|
263
|
+
}
|
|
252
264
|
```
|
|
253
265
|
|
|
254
|
-
###
|
|
266
|
+
### `@QueueEntityId()`
|
|
255
267
|
|
|
256
|
-
Marks
|
|
268
|
+
Marks the property that contains the entity ID. One per class. Use this when you need per-command control over which property is the entity ID, or when you can't use the two-argument `@QueueEntity` shorthand.
|
|
257
269
|
|
|
258
270
|
```typescript
|
|
259
271
|
@QueueEntity('account')
|
|
260
272
|
export class TransferCommand {
|
|
261
273
|
constructor(
|
|
262
|
-
@QueueEntityId() public readonly
|
|
274
|
+
@QueueEntityId() public readonly accountId: string, // Routes to this account's queue
|
|
263
275
|
public readonly targetAccountId: string,
|
|
264
276
|
public readonly amount: number,
|
|
265
277
|
) {}
|
|
266
278
|
}
|
|
267
279
|
```
|
|
268
280
|
|
|
269
|
-
|
|
281
|
+
> **Entity ID resolution order:** `@QueueEntityId()` decorator > `@QueueEntity('type', 'prop')` second argument > `@WorkerProcessor({ defaultEntityId })` > `entities[type].defaultEntityId` in module config.
|
|
270
282
|
|
|
271
|
-
|
|
283
|
+
### `@WorkerProcessor(options)`
|
|
272
284
|
|
|
273
|
-
|
|
274
|
-
// In module config
|
|
275
|
-
entities: {
|
|
276
|
-
account: {
|
|
277
|
-
defaultEntityId: 'accountId', // Commands without @QueueEntityId use this
|
|
278
|
-
// ...
|
|
279
|
-
},
|
|
280
|
-
}
|
|
285
|
+
Optional. Define a processor class for custom job handling on top of CQRS auto-routing.
|
|
281
286
|
|
|
282
|
-
|
|
283
|
-
@
|
|
284
|
-
|
|
285
|
-
|
|
286
|
-
|
|
287
|
-
|
|
288
|
-
|
|
287
|
+
```typescript
|
|
288
|
+
@WorkerProcessor({
|
|
289
|
+
entityType: 'account',
|
|
290
|
+
queueName: (id) => `account-${id}-queue`,
|
|
291
|
+
workerName: (id) => `account-${id}-worker`,
|
|
292
|
+
maxWorkersPerEntity: 1,
|
|
293
|
+
idleTimeoutSeconds: 15,
|
|
294
|
+
})
|
|
295
|
+
@Injectable()
|
|
296
|
+
export class AccountProcessor {
|
|
297
|
+
@JobHandler('special-audit')
|
|
298
|
+
async handleAudit(job: Job, entityId: string) { ... }
|
|
289
299
|
}
|
|
290
300
|
```
|
|
291
301
|
|
|
302
|
+
### `@JobHandler(jobName)` / `@JobHandler('*')`
|
|
303
|
+
|
|
304
|
+
Custom job handlers on a `@WorkerProcessor`. The wildcard `'*'` catches anything not matched by a specific handler.
|
|
305
|
+
|
|
292
306
|
---
|
|
293
307
|
|
|
294
308
|
## Configuration
|
|
295
309
|
|
|
296
310
|
```typescript
|
|
297
311
|
AtomicQueuesModule.forRoot({
|
|
312
|
+
// ── Redis connection ──────────────────────────────────────
|
|
298
313
|
redis: {
|
|
299
|
-
host: '
|
|
314
|
+
host: 'redis',
|
|
300
315
|
port: 6379,
|
|
301
|
-
password: 'secret',
|
|
316
|
+
password: 'secret', // optional
|
|
302
317
|
},
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
|
|
306
|
-
|
|
307
|
-
|
|
318
|
+
|
|
319
|
+
// ── Global settings ───────────────────────────────────────
|
|
320
|
+
keyPrefix: 'myapp', // Redis key namespace (default: 'aq')
|
|
321
|
+
enableCronManager: true, // Legacy cron-based scaling (optional)
|
|
322
|
+
cronInterval: 5000, // Cron tick interval in ms
|
|
323
|
+
|
|
324
|
+
// ── Worker defaults ───────────────────────────────────────
|
|
308
325
|
workerDefaults: {
|
|
309
|
-
concurrency: 1,
|
|
310
|
-
stalledInterval: 1000,
|
|
311
|
-
lockDuration: 30000,
|
|
312
|
-
heartbeatTTL: 3,
|
|
326
|
+
concurrency: 1, // Jobs processed concurrently per worker
|
|
327
|
+
stalledInterval: 1000, // ms between stalled-job checks
|
|
328
|
+
lockDuration: 30000, // ms a job is locked during processing
|
|
329
|
+
heartbeatTTL: 3, // Heartbeat key TTL in seconds
|
|
313
330
|
},
|
|
314
|
-
|
|
315
|
-
//
|
|
316
|
-
// If omitted, uses default naming: {keyPrefix}:{entityType}:{entityId}:queue/worker
|
|
331
|
+
|
|
332
|
+
// ── Per-entity configuration (optional) ───────────────────
|
|
317
333
|
entities: {
|
|
318
334
|
account: {
|
|
319
|
-
|
|
320
|
-
|
|
321
|
-
workerName: (id) => `${id}-worker`,
|
|
335
|
+
queueName: (id) => `account-${id}-queue`,
|
|
336
|
+
workerName: (id) => `account-${id}-worker`,
|
|
322
337
|
maxWorkersPerEntity: 1,
|
|
323
338
|
idleTimeoutSeconds: 15,
|
|
324
|
-
|
|
325
|
-
|
|
339
|
+
|
|
340
|
+
// Fallback property name for entity ID extraction.
|
|
341
|
+
// Used when a command has no @QueueEntityId() decorator
|
|
342
|
+
// and no second argument to @QueueEntity().
|
|
343
|
+
defaultEntityId: 'accountId',
|
|
344
|
+
|
|
345
|
+
workerConfig: { // Override workerDefaults per entity
|
|
326
346
|
concurrency: 1,
|
|
327
347
|
lockDuration: 60000,
|
|
328
348
|
},
|
|
329
349
|
},
|
|
330
|
-
order: {
|
|
331
|
-
defaultEntityId: 'orderId',
|
|
332
|
-
queueName: (id) => `order-${id}-queue`,
|
|
333
|
-
idleTimeoutSeconds: 30,
|
|
334
|
-
},
|
|
335
350
|
},
|
|
336
351
|
});
|
|
337
352
|
```
|
|
338
353
|
|
|
339
354
|
---
|
|
340
355
|
|
|
356
|
+
## Distributed Worker Lifecycle
|
|
357
|
+
|
|
358
|
+
Workers in atomic-queues have a fully automated lifecycle, distributed across all pods with no leader election:
|
|
359
|
+
|
|
360
|
+
```
|
|
361
|
+
Job arrives SET NX claim
|
|
362
|
+
on any pod ──────► ┌──────────────────────┐
|
|
363
|
+
│ Pod claims worker? │
|
|
364
|
+
└──────┬───────┬───────┘
|
|
365
|
+
YES │ │ NO (another pod won)
|
|
366
|
+
▼ ▼
|
|
367
|
+
┌────────┐ ┌──────────────┐
|
|
368
|
+
│ Spawn │ │ Wait — other │
|
|
369
|
+
│ worker │ │ pod handles │
|
|
370
|
+
│ locally│ └──────────────┘
|
|
371
|
+
└───┬────┘
|
|
372
|
+
▼
|
|
373
|
+
┌──────────────┐
|
|
374
|
+
│ Processing │◄──── Heartbeat refresh (pipeline)
|
|
375
|
+
│ jobs back- │ every 1s (1 Redis round-trip)
|
|
376
|
+
│ to-back │
|
|
377
|
+
└──────┬───────┘
|
|
378
|
+
│ No jobs for idleTimeoutSeconds
|
|
379
|
+
▼
|
|
380
|
+
┌──────────────┐
|
|
381
|
+
│ Idle sweep │──── Hot cache eviction
|
|
382
|
+
│ closes │ Heartbeat keys cleaned up
|
|
383
|
+
│ worker │
|
|
384
|
+
└──────────────┘
|
|
385
|
+
```
|
|
386
|
+
|
|
387
|
+
### Hot Cache
|
|
388
|
+
|
|
389
|
+
After a worker is confirmed alive, subsequent job arrivals for that entity hit an **in-memory cache** — zero Redis calls on the warm path. This eliminates the per-job Redis overhead that plagues lock-based approaches.
|
|
390
|
+
|
|
391
|
+
| Path | Redis calls | When |
|
|
392
|
+
|---|---|---|
|
|
393
|
+
| **Hot** (cache hit) | 0 | Worker known alive |
|
|
394
|
+
| **Warm** (cache miss) | 1 (`EXISTS`) | First time seeing entity |
|
|
395
|
+
| **Cold** (no worker) | 1 (`SET NX`) | Worker needs creation |
|
|
396
|
+
|
|
397
|
+
### SpawnQueueService
|
|
398
|
+
|
|
399
|
+
For multi-pod deployments, the `SpawnQueueService` distributes worker creation across all pods via a shared BullMQ spawn queue. The **direct local spawn** path bypasses this queue entirely — the pod that first sees a job for a new entity claims it with an atomic `SET NX` and spawns the worker locally.
|
|
400
|
+
|
|
401
|
+
---
|
|
402
|
+
|
|
341
403
|
## Complete Example
|
|
342
404
|
|
|
343
|
-
A banking service
|
|
405
|
+
A banking service with withdrawals, deposits, and cross-account transfers:
|
|
344
406
|
|
|
345
407
|
```typescript
|
|
346
|
-
//
|
|
347
|
-
// app.module.ts
|
|
348
|
-
// ─────────────────────────────────────────────────────────────────
|
|
408
|
+
// ── Module ──────────────────────────────────────────────
|
|
349
409
|
import { Module } from '@nestjs/common';
|
|
350
410
|
import { CqrsModule } from '@nestjs/cqrs';
|
|
351
411
|
import { AtomicQueuesModule } from 'atomic-queues';
|
|
@@ -354,19 +414,14 @@ import { AtomicQueuesModule } from 'atomic-queues';
|
|
|
354
414
|
imports: [
|
|
355
415
|
CqrsModule,
|
|
356
416
|
AtomicQueuesModule.forRoot({
|
|
357
|
-
redis: { host: '
|
|
417
|
+
redis: { host: 'redis', port: 6379 },
|
|
358
418
|
keyPrefix: 'banking',
|
|
359
|
-
enableCronManager: true,
|
|
360
419
|
entities: {
|
|
361
420
|
account: {
|
|
362
|
-
queueName: (id) =>
|
|
363
|
-
workerName: (id) =>
|
|
421
|
+
queueName: (id) => `account-${id}-queue`,
|
|
422
|
+
workerName: (id) => `account-${id}-worker`,
|
|
364
423
|
maxWorkersPerEntity: 1,
|
|
365
424
|
idleTimeoutSeconds: 15,
|
|
366
|
-
workerConfig: {
|
|
367
|
-
concurrency: 1,
|
|
368
|
-
lockDuration: 60000,
|
|
369
|
-
},
|
|
370
425
|
},
|
|
371
426
|
},
|
|
372
427
|
}),
|
|
@@ -377,13 +432,10 @@ import { AtomicQueuesModule } from 'atomic-queues';
|
|
|
377
432
|
DepositHandler,
|
|
378
433
|
TransferHandler,
|
|
379
434
|
],
|
|
380
|
-
controllers: [AccountController],
|
|
381
435
|
})
|
|
382
|
-
export class
|
|
436
|
+
export class BankingModule {}
|
|
383
437
|
|
|
384
|
-
//
|
|
385
|
-
// commands/withdraw.command.ts
|
|
386
|
-
// ─────────────────────────────────────────────────────────────────
|
|
438
|
+
// ── Commands ────────────────────────────────────────────
|
|
387
439
|
import { QueueEntity, QueueEntityId } from 'atomic-queues';
|
|
388
440
|
|
|
389
441
|
@QueueEntity('account')
|
|
@@ -395,11 +447,6 @@ export class WithdrawCommand {
|
|
|
395
447
|
) {}
|
|
396
448
|
}
|
|
397
449
|
|
|
398
|
-
// ─────────────────────────────────────────────────────────────────
|
|
399
|
-
// commands/deposit.command.ts
|
|
400
|
-
// ─────────────────────────────────────────────────────────────────
|
|
401
|
-
import { QueueEntity, QueueEntityId } from 'atomic-queues';
|
|
402
|
-
|
|
403
450
|
@QueueEntity('account')
|
|
404
451
|
export class DepositCommand {
|
|
405
452
|
constructor(
|
|
@@ -409,123 +456,72 @@ export class DepositCommand {
|
|
|
409
456
|
) {}
|
|
410
457
|
}
|
|
411
458
|
|
|
412
|
-
// ─────────────────────────────────────────────────────────────────
|
|
413
|
-
// commands/transfer.command.ts
|
|
414
|
-
// ─────────────────────────────────────────────────────────────────
|
|
415
|
-
import { QueueEntity, QueueEntityId } from 'atomic-queues';
|
|
416
|
-
|
|
417
459
|
@QueueEntity('account')
|
|
418
460
|
export class TransferCommand {
|
|
419
461
|
constructor(
|
|
420
|
-
@QueueEntityId() public readonly accountId: string,
|
|
462
|
+
@QueueEntityId() public readonly accountId: string,
|
|
421
463
|
public readonly toAccountId: string,
|
|
422
464
|
public readonly amount: number,
|
|
423
|
-
public readonly transactionId: string,
|
|
424
465
|
) {}
|
|
425
466
|
}
|
|
426
467
|
|
|
427
|
-
//
|
|
428
|
-
// handlers/withdraw.handler.ts
|
|
429
|
-
// ─────────────────────────────────────────────────────────────────
|
|
468
|
+
// ── Handlers ────────────────────────────────────────────
|
|
430
469
|
import { CommandHandler, ICommandHandler } from '@nestjs/cqrs';
|
|
431
|
-
import { WithdrawCommand } from '../commands';
|
|
432
470
|
|
|
433
471
|
@CommandHandler(WithdrawCommand)
|
|
434
472
|
export class WithdrawHandler implements ICommandHandler<WithdrawCommand> {
|
|
435
|
-
constructor(private readonly
|
|
436
|
-
|
|
437
|
-
async execute(
|
|
438
|
-
const
|
|
439
|
-
|
|
440
|
-
// SAFE: Sequential execution per account
|
|
441
|
-
const account = await this.accountRepo.findById(accountId);
|
|
442
|
-
|
|
443
|
-
if (account.balance < amount) {
|
|
444
|
-
throw new InsufficientFundsError(accountId, account.balance, amount);
|
|
445
|
-
}
|
|
446
|
-
|
|
473
|
+
constructor(private readonly repo: AccountRepository) {}
|
|
474
|
+
|
|
475
|
+
async execute({ accountId, amount }: WithdrawCommand) {
|
|
476
|
+
const account = await this.repo.findById(accountId);
|
|
477
|
+
if (account.balance < amount) throw new InsufficientFundsError();
|
|
447
478
|
account.balance -= amount;
|
|
448
|
-
await this.
|
|
449
|
-
|
|
450
|
-
return { success: true, newBalance: account.balance };
|
|
479
|
+
await this.repo.save(account);
|
|
451
480
|
}
|
|
452
481
|
}
|
|
453
482
|
|
|
454
|
-
// ─────────────────────────────────────────────────────────────────
|
|
455
|
-
// handlers/transfer.handler.ts
|
|
456
|
-
// ─────────────────────────────────────────────────────────────────
|
|
457
|
-
import { CommandHandler, ICommandHandler } from '@nestjs/cqrs';
|
|
458
|
-
import { TransferCommand, DepositCommand } from '../commands';
|
|
459
|
-
import { QueueBus } from 'atomic-queues';
|
|
460
|
-
|
|
461
483
|
@CommandHandler(TransferCommand)
|
|
462
484
|
export class TransferHandler implements ICommandHandler<TransferCommand> {
|
|
463
485
|
constructor(
|
|
464
|
-
private readonly
|
|
486
|
+
private readonly repo: AccountRepository,
|
|
465
487
|
private readonly queueBus: QueueBus,
|
|
466
488
|
) {}
|
|
467
489
|
|
|
468
|
-
async execute(
|
|
469
|
-
|
|
470
|
-
|
|
471
|
-
|
|
472
|
-
const source = await this.accountRepo.findById(accountId);
|
|
473
|
-
if (source.balance < amount) {
|
|
474
|
-
throw new InsufficientFundsError(accountId, source.balance, amount);
|
|
475
|
-
}
|
|
476
|
-
|
|
490
|
+
async execute({ accountId, toAccountId, amount }: TransferCommand) {
|
|
491
|
+
// Debit source (we're in source account's queue — safe)
|
|
492
|
+
const source = await this.repo.findById(accountId);
|
|
493
|
+
if (source.balance < amount) throw new InsufficientFundsError();
|
|
477
494
|
source.balance -= amount;
|
|
478
|
-
await this.
|
|
479
|
-
|
|
480
|
-
// Credit destination (enqueued to destination's queue)
|
|
481
|
-
await this.queueBus.enqueue(
|
|
482
|
-
toAccountId,
|
|
483
|
-
|
|
484
|
-
`transfer:${accountId}`,
|
|
485
|
-
));
|
|
486
|
-
|
|
487
|
-
return { success: true };
|
|
495
|
+
await this.repo.save(source);
|
|
496
|
+
|
|
497
|
+
// Credit destination (enqueued to destination's queue — also safe)
|
|
498
|
+
await this.queueBus.enqueue(
|
|
499
|
+
new DepositCommand(toAccountId, amount, `transfer:${accountId}`),
|
|
500
|
+
);
|
|
488
501
|
}
|
|
489
502
|
}
|
|
490
503
|
|
|
491
|
-
//
|
|
492
|
-
// account.controller.ts
|
|
493
|
-
// ─────────────────────────────────────────────────────────────────
|
|
504
|
+
// ── Controller ──────────────────────────────────────────
|
|
494
505
|
import { Controller, Post, Body, Param } from '@nestjs/common';
|
|
495
506
|
import { QueueBus } from 'atomic-queues';
|
|
496
|
-
import { WithdrawCommand, TransferCommand } from './commands';
|
|
497
|
-
import { v4 as uuid } from 'uuid';
|
|
498
507
|
|
|
499
508
|
@Controller('accounts')
|
|
500
509
|
export class AccountController {
|
|
501
510
|
constructor(private readonly queueBus: QueueBus) {}
|
|
502
511
|
|
|
503
|
-
@Post(':
|
|
504
|
-
async withdraw(
|
|
505
|
-
|
|
506
|
-
|
|
507
|
-
) {
|
|
508
|
-
const transactionId = uuid();
|
|
509
|
-
|
|
510
|
-
await this.queueBus.enqueue(
|
|
511
|
-
new WithdrawCommand(accountId, body.amount, transactionId)
|
|
512
|
-
);
|
|
513
|
-
|
|
514
|
-
return { queued: true, transactionId };
|
|
512
|
+
@Post(':id/withdraw')
|
|
513
|
+
async withdraw(@Param('id') id: string, @Body() body: { amount: number }) {
|
|
514
|
+
await this.queueBus.enqueue(new WithdrawCommand(id, body.amount, uuid()));
|
|
515
|
+
return { queued: true };
|
|
515
516
|
}
|
|
516
517
|
|
|
517
|
-
@Post(':
|
|
518
|
+
@Post(':id/transfer')
|
|
518
519
|
async transfer(
|
|
519
|
-
@Param('
|
|
520
|
-
@Body() body: {
|
|
520
|
+
@Param('id') id: string,
|
|
521
|
+
@Body() body: { to: string; amount: number },
|
|
521
522
|
) {
|
|
522
|
-
|
|
523
|
-
|
|
524
|
-
await this.queueBus.enqueue(
|
|
525
|
-
new TransferCommand(accountId, body.toAccountId, body.amount, transactionId)
|
|
526
|
-
);
|
|
527
|
-
|
|
528
|
-
return { queued: true, transactionId };
|
|
523
|
+
await this.queueBus.enqueue(new TransferCommand(id, body.to, body.amount));
|
|
524
|
+
return { queued: true };
|
|
529
525
|
}
|
|
530
526
|
}
|
|
531
527
|
```
|
|
@@ -534,7 +530,7 @@ export class AccountController {
|
|
|
534
530
|
|
|
535
531
|
## Advanced: Custom Worker Processors
|
|
536
532
|
|
|
537
|
-
For
|
|
533
|
+
For cases where CQRS auto-routing isn't enough, define a `@WorkerProcessor` with explicit `@JobHandler` methods:
|
|
538
534
|
|
|
539
535
|
```typescript
|
|
540
536
|
import { Injectable } from '@nestjs/common';
|
|
@@ -543,28 +539,49 @@ import { Job } from 'bullmq';
|
|
|
543
539
|
|
|
544
540
|
@WorkerProcessor({
|
|
545
541
|
entityType: 'account',
|
|
546
|
-
queueName: (id) =>
|
|
547
|
-
workerName: (id) =>
|
|
542
|
+
queueName: (id) => `account-${id}-queue`,
|
|
543
|
+
workerName: (id) => `account-${id}-worker`,
|
|
548
544
|
maxWorkersPerEntity: 1,
|
|
549
545
|
idleTimeoutSeconds: 15,
|
|
550
546
|
})
|
|
551
547
|
@Injectable()
|
|
552
548
|
export class AccountProcessor {
|
|
553
|
-
|
|
554
|
-
|
|
555
|
-
|
|
556
|
-
// Custom logic here
|
|
549
|
+
@JobHandler('high-priority-audit')
|
|
550
|
+
async handleAudit(job: Job, entityId: string) {
|
|
551
|
+
// Specific handler for this job type
|
|
557
552
|
}
|
|
558
553
|
|
|
559
|
-
// Wildcard handler for everything else
|
|
560
554
|
@JobHandler('*')
|
|
561
555
|
async handleAll(job: Job, entityId: string) {
|
|
562
|
-
//
|
|
556
|
+
// Wildcard — catches everything not explicitly handled
|
|
557
|
+
// Falls back to CQRS routing automatically when not defined
|
|
563
558
|
}
|
|
564
559
|
}
|
|
565
560
|
```
|
|
566
561
|
|
|
567
|
-
**
|
|
562
|
+
> **Priority order:** Explicit `@JobHandler` → CQRS auto-routing (`@JobCommand`/`@JobQuery`) → Wildcard handler
|
|
563
|
+
|
|
564
|
+
---
|
|
565
|
+
|
|
566
|
+
## Performance
|
|
567
|
+
|
|
568
|
+
### Why it's fast
|
|
569
|
+
|
|
570
|
+
1. **Zero contention** — no locks, no retries, no backoff. Jobs queue and execute.
|
|
571
|
+
2. **Hot cache** — after first check, subsequent job arrivals for an entity incur 0 Redis calls.
|
|
572
|
+
3. **Direct local spawn** — atomic `SET NX` claim, local worker creation. No queue round-trip.
|
|
573
|
+
4. **Pipelined heartbeats** — heartbeat refresh uses a single Redis pipeline (1 round-trip for 2 keys).
|
|
574
|
+
5. **O(1) worker existence check** — global alive key replaces `KEYS` pattern scan.
|
|
575
|
+
|
|
576
|
+
### When to use what
|
|
577
|
+
|
|
578
|
+
| Use case | Recommendation |
|
|
579
|
+
|---|---|
|
|
580
|
+
| Per-entity operations that must be serialized (payments, inventory, game state) | **atomic-queues** |
|
|
581
|
+
| Rare, low-frequency mutual exclusion (config updates, migrations) | Redlock / advisory locks |
|
|
582
|
+
| Exactly-once semantics with audit trail | **atomic-queues** (BullMQ job IDs) |
|
|
583
|
+
| Sub-millisecond synchronous response required | Redlock (synchronous acquire) |
|
|
584
|
+
| Multi-pod, many entities, sustained load | **atomic-queues** (contention-free scaling) |
|
|
568
585
|
|
|
569
586
|
---
|
|
570
587
|
|