cosmonats 0.3.0 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +208 -156
- data/lib/cosmo/active_job/adapter.rb +46 -0
- data/lib/cosmo/active_job/executor.rb +16 -0
- data/lib/cosmo/active_job/options.rb +50 -0
- data/lib/cosmo/active_job.rb +29 -0
- data/lib/cosmo/api/busy.rb +2 -2
- data/lib/cosmo/api/counter.rb +2 -2
- data/lib/cosmo/api/cron/entry.rb +99 -0
- data/lib/cosmo/api/cron.rb +118 -0
- data/lib/cosmo/api/kv.rb +35 -13
- data/lib/cosmo/api/stream.rb +10 -5
- data/lib/cosmo/api.rb +1 -0
- data/lib/cosmo/cli.rb +27 -10
- data/lib/cosmo/client.rb +48 -2
- data/lib/cosmo/config.rb +9 -0
- data/lib/cosmo/job/data.rb +1 -1
- data/lib/cosmo/job/limit.rb +51 -0
- data/lib/cosmo/job/processor.rb +49 -5
- data/lib/cosmo/job.rb +51 -2
- data/lib/cosmo/processor.rb +1 -1
- data/lib/cosmo/railtie.rb +21 -0
- data/lib/cosmo/stream/processor.rb +2 -2
- data/lib/cosmo/stream.rb +2 -1
- data/lib/cosmo/utils/hash.rb +13 -0
- data/lib/cosmo/utils/overrides.rb +1 -1
- data/lib/cosmo/version.rb +1 -1
- data/lib/cosmo/web/assets/app.css +42 -0
- data/lib/cosmo/web/controllers/crons.rb +41 -0
- data/lib/cosmo/web/controllers/jobs.rb +7 -3
- data/lib/cosmo/web/controllers/streams.rb +1 -1
- data/lib/cosmo/web/helpers/application.rb +4 -0
- data/lib/cosmo/web/views/actions/index.erb +1 -1
- data/lib/cosmo/web/views/crons/_table.erb +58 -0
- data/lib/cosmo/web/views/crons/index.erb +10 -0
- data/lib/cosmo/web/views/jobs/_busy.erb +54 -49
- data/lib/cosmo/web/views/jobs/_dead.erb +70 -65
- data/lib/cosmo/web/views/jobs/_enqueued.erb +82 -56
- data/lib/cosmo/web/views/jobs/_scheduled.erb +53 -48
- data/lib/cosmo/web/views/jobs/_tabs.erb +6 -0
- data/lib/cosmo/web/views/jobs/busy.erb +8 -6
- data/lib/cosmo/web/views/jobs/dead.erb +6 -5
- data/lib/cosmo/web/views/jobs/enqueued.erb +8 -6
- data/lib/cosmo/web/views/jobs/index.erb +1 -1
- data/lib/cosmo/web/views/jobs/scheduled.erb +6 -5
- data/lib/cosmo/web/views/layout.erb +1 -1
- data/lib/cosmo/web.rb +5 -0
- data/lib/cosmo.rb +1 -0
- data/sig/cosmo/active_job/adapter.rbs +13 -0
- data/sig/cosmo/active_job/executor.rbs +9 -0
- data/sig/cosmo/active_job/options.rbs +14 -0
- data/sig/cosmo/api/cron/entry.rbs +30 -0
- data/sig/cosmo/api/cron.rbs +25 -0
- data/sig/cosmo/api/kv.rbs +4 -6
- data/sig/cosmo/client.rbs +9 -1
- data/sig/cosmo/job/data.rbs +1 -1
- data/sig/cosmo/job/limit.rbs +18 -0
- data/sig/cosmo/job/processor.rbs +3 -1
- data/sig/cosmo/job.rbs +9 -4
- data/sig/cosmo/railtie.rbs +4 -0
- data/sig/cosmo/utils/hash.rbs +4 -0
- metadata +20 -1
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 6f94d9b1e192b098e1bcb3ce77998d7b0ed9aca359af41fefe2a572624fa6b91
|
|
4
|
+
data.tar.gz: 12bba3015a4beb335efae8dfe5fc178a6c467d203eaf61f922bf1b3091b67ce6
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: e24c844b4f17aa7eaffceebdd6b847858855fc0b5526b5537832893fd144e71f427ae352700adff3c5b39ed5cd04c89bc84007d01548ff97ba701a45e8c836b9
|
|
7
|
+
data.tar.gz: d1a7d5e0d45c5f9b03388259b45a5a6ce659dd39378a0f13e8f31cf36d7f969a332076cada74b564f2fd7d504c646df0715778679d0e67743a5f0ef08398e7c9
|
data/README.md
CHANGED
|
@@ -1,12 +1,65 @@
|
|
|
1
|
-
# 🚀 Cosmonats
|
|
1
|
+
# 🚀 Cosmonats
|
|
2
2
|
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
3
|
+
Background jobs + real-time event streaming for Ruby — unified, in one gem, backed by NATS.
|
|
4
|
+
**No Redis. No DB polling. Disk-backed, horizontally scalable — no message is ever silently dropped.**
|
|
5
|
+
|
|
6
|
+
<div align="center">
|
|
7
7
|
|
|
8
8
|

|
|
9
9
|
|
|
10
|
+
[](https://rubygems.org/gems/cosmonats)
|
|
11
|
+
[](https://rubygems.org/gems/cosmonats)
|
|
12
|
+
[](https://www.ruby-lang.org)
|
|
13
|
+
[](LICENSE.txt)
|
|
14
|
+
[](https://github.com/bitsbeam/cosmonats/actions)
|
|
15
|
+
|
|
16
|
+
*Battle-tested in production. Tens of millions of jobs processed and counting.*
|
|
17
|
+
|
|
18
|
+
</div>
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
## ⚡ Taste it
|
|
22
|
+
|
|
23
|
+
```ruby
|
|
24
|
+
# Define a job with a familiar look
|
|
25
|
+
class SendEmailJob
|
|
26
|
+
include Cosmo::Job
|
|
27
|
+
options stream: :default, retry: 3, dead: true
|
|
28
|
+
|
|
29
|
+
def perform(user_id, template)
|
|
30
|
+
EmailService.send(user_id, template)
|
|
31
|
+
end
|
|
32
|
+
end
|
|
33
|
+
|
|
34
|
+
# Enqueue it
|
|
35
|
+
SendEmailJob.perform_async(123, "welcome")
|
|
36
|
+
SendEmailJob.perform_in(1.day, 123, "followup")
|
|
37
|
+
```
|
|
38
|
+
|
|
39
|
+
```ruby
|
|
40
|
+
# Process a continuous real-time event stream
|
|
41
|
+
class ClicksProcessor
|
|
42
|
+
include Cosmo::Stream
|
|
43
|
+
options stream: :clickstream, batch_size: 100,
|
|
44
|
+
consumer: { subjects: ["events.clicks.>"] }
|
|
45
|
+
|
|
46
|
+
def process_one
|
|
47
|
+
Analytics.track(message.data)
|
|
48
|
+
message.ack
|
|
49
|
+
end
|
|
50
|
+
end
|
|
51
|
+
|
|
52
|
+
ClicksProcessor.publish({ user_id: 123, page: "/home" }, subject: "events.clicks.homepage")
|
|
53
|
+
```
|
|
54
|
+
|
|
55
|
+
```bash
|
|
56
|
+
bundle exec cosmo -C config/cosmo.yml -c 20 # Run jobs + streams with 20 threads
|
|
57
|
+
bundle exec cosmo -C config/cosmo.yml -c 20 jobs # Jobs only
|
|
58
|
+
bundle exec cosmo -C config/cosmo.yml -c 20 streams # Streams only
|
|
59
|
+
```
|
|
60
|
+
|
|
61
|
+

|
|
62
|
+
|
|
10
63
|
## 📖 Index
|
|
11
64
|
|
|
12
65
|
- [Why?](#-why)
|
|
@@ -25,65 +78,73 @@ disk-backed persistence.
|
|
|
25
78
|
|
|
26
79
|
|
|
27
80
|
## 🎯 Why?
|
|
28
|
-
Among many others, why creating another? Cosmonats is a background processing framework for Ruby, powered by **[NATS](https://nats.io/)**.
|
|
29
|
-
It's designed to solve the fundamental scaling problems that plague Redis/DB-based job queues and at the same time to provide both job and stream
|
|
30
|
-
processing capabilities.
|
|
31
81
|
|
|
32
|
-
|
|
82
|
+
Most background job libraries use Redis or Postgres — tools that were never designed for this. Think of NATS as Redis — but Redis is KV first then messaging;
|
|
83
|
+
NATS is messaging first, then KV. What NATS is:
|
|
84
|
+
|
|
85
|
+
- **~20 MB binary, ~10 MB RAM at idle** Trivial to run anywhere.
|
|
86
|
+
- **Disk-backed persistent streams** Messages survive restarts, don't require RAM to fit.
|
|
87
|
+
- **True horizontal clustering** Lose a node — other nodes take over, zero message loss.
|
|
88
|
+
- **Multilingual** Official clients for Ruby, Go, Python, Rust, Java, .NET, and more. Any service can publish or consume.
|
|
89
|
+
|
|
90
|
+
One NATS server replaces your message broker, job queue, and KV store — with lower operational overhead.
|
|
91
|
+
|
|
92
|
+
| | Redis/DB-backed | NATS/Cosmonats |
|
|
93
|
+
|-------------------|-------------------------------|----------------------------|
|
|
94
|
+
| Persistence | In-memory / DB bloat | Disk-backed, TB-scale |
|
|
95
|
+
| Scaling | Sentinel only / Vertical only | True horizontal clustering |
|
|
96
|
+
| Background jobs | Yes | Yes |
|
|
97
|
+
| Real-time stream | No | Yes |
|
|
98
|
+
| Zero message loss | No | Yes |
|
|
99
|
+
| Message replay | No | Yes |
|
|
100
|
+
| Backpressure | No, grow unbounded | Yes |
|
|
101
|
+
| Multi-DC | Complex setup | Native geo-distribution |
|
|
102
|
+
|
|
103
|
+
|
|
104
|
+
### Killer Features:
|
|
33
105
|
|
|
34
|
-
|
|
35
|
-
- **Memory-only persistence** - Everything must fit in RAM, expensive to scale
|
|
36
|
-
- **Vertical scaling only** - Can't truly distribute a single queue across nodes
|
|
37
|
-
- **Polling overhead** - Thousands of blocked connections
|
|
38
|
-
- **No native backpressure** - Queues can grow unbounded
|
|
39
|
-
- **Weak durability** - Async replication can lose jobs during failures
|
|
106
|
+
#### — Jobs + Streams, unified in one gem.
|
|
40
107
|
|
|
41
|
-
|
|
108
|
+
Most Ruby gems handle exactly that — background jobs. If you also need to consume a continuous event feed, that's a second system, second config, second set of
|
|
109
|
+
worker processes, second Dockerfile entry. Cosmonats is the only Ruby gem with a first-class `Job` primitive *and* a first-class `Stream` primitive, sharing
|
|
110
|
+
one server, one config, one CLI, one monitoring endpoint.
|
|
42
111
|
|
|
43
|
-
|
|
112
|
+
#### — Message replay and time-travel debugging.
|
|
44
113
|
|
|
45
|
-
|
|
46
|
-
- **
|
|
47
|
-
- **
|
|
48
|
-
- **
|
|
49
|
-
- **Vertical scaling only** - Limited by single database instance capabilities
|
|
50
|
-
- **Index bloat** - High UPDATE/DELETE volume causes index degradation over time
|
|
51
|
-
- **Table bloat** - Constant row updates fragment tables, requiring maintenance
|
|
52
|
-
- **`LISTEN/NOTIFY` limitations** - 8KB payload limit, no persistence, breaks down at high volumes (10K+ notifications/sec)
|
|
53
|
-
- **No native horizontal scaling** - Cannot distribute a single job queue across multiple database nodes
|
|
114
|
+
NATS persists messages to disk and lets any consumer rewind to any point — beginning of time, a specific timestamp, or only new messages.
|
|
115
|
+
- **Incident recovery** — your pipeline crashed for 3 hours. Replay from the crash timestamp.
|
|
116
|
+
- **New consumer bootstrap** — a new service needs historical events. Start it from the beginning.
|
|
117
|
+
- **Bug reproduction** — replay the exact sequence of messages that caused a production issue.
|
|
54
118
|
|
|
55
|
-
|
|
119
|
+
#### — Multi-datacenter queues, natively.
|
|
56
120
|
|
|
57
|
-
|
|
121
|
+
NATS has a first-class cluster + leaf-node architecture for geo-distribution. Spanning multiple regions or datacenters is a config block — not a separate
|
|
122
|
+
product or a third-party replication tool. NATS was built for edge computing, IoT, and satellite communication — multi-DC is a first-class concern, not an
|
|
123
|
+
afterthought.
|
|
58
124
|
|
|
59
|
-
|
|
125
|
+
#### — Transport-level deduplication + built-in KV. No extra infrastructure.
|
|
60
126
|
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
✅ **Built-in flow control** - Automatic backpressure
|
|
65
|
-
✅ **Multi-DC support** - Native geo-distribution, and super clusters
|
|
66
|
-
✅ **High throughput & low latency** - Millions of messages per second
|
|
67
|
-
✅ **Stream processing** - Beyond simple job queues
|
|
127
|
+
NATS deduplicates messages at the **broker** — same-ID messages within the configured window are dropped before they ever reach a worker. No uniqueness gems,
|
|
128
|
+
no advisory locks, no extra round-trips. It also ships a built-in Key/Value store usable for distributed locks and rate limiting — no Redis, no Memcached,
|
|
129
|
+
nothing else to run.
|
|
68
130
|
|
|
69
131
|
|
|
70
132
|
## ✨ Features
|
|
71
133
|
|
|
72
134
|
### 🎪 Job Processing
|
|
73
|
-
- **Familiar
|
|
74
|
-
- **Priority queues**
|
|
75
|
-
- **Scheduled jobs**
|
|
76
|
-
- **Automatic retries**
|
|
77
|
-
- **Dead letter queue**
|
|
78
|
-
- **Job uniqueness**
|
|
135
|
+
- **Familiar API** — `perform_async`, `perform_in`, `perform_at`
|
|
136
|
+
- **Priority queues** — critical, high, default, low with weighted round-robin
|
|
137
|
+
- **Scheduled jobs** — execute at a specific time or after a delay
|
|
138
|
+
- **Automatic retries** — exponential backoff, configurable attempts
|
|
139
|
+
- **Dead letter queue** — capture permanently failed jobs
|
|
140
|
+
- **Job uniqueness** — prevent duplicate execution
|
|
79
141
|
|
|
80
142
|
### 🌊 Stream Processing
|
|
81
|
-
- **Real-time
|
|
82
|
-
- **Batch processing**
|
|
83
|
-
- **Message replay**
|
|
84
|
-
- **Consumer groups** -
|
|
85
|
-
- **
|
|
86
|
-
- **Custom serialization** - JSON, MessagePack, Protobuf support
|
|
143
|
+
- **Real-time event streams** — process continuous data feeds
|
|
144
|
+
- **Batch processing** — handle multiple messages in one go
|
|
145
|
+
- **Message replay** — reprocess from any point in time
|
|
146
|
+
- **Consumer groups** — load-balanced across workers
|
|
147
|
+
- **Custom serialization** — JSON, MessagePack, Protobuf
|
|
87
148
|
|
|
88
149
|
|
|
89
150
|
## 📦 Installation
|
|
@@ -93,71 +154,88 @@ Built on **NATS**, `cosmonats` provides:
|
|
|
93
154
|
gem "cosmonats"
|
|
94
155
|
```
|
|
95
156
|
|
|
96
|
-
**Requirements:** Ruby 3.1
|
|
157
|
+
**Requirements:** Ruby ≥ 3.1, NATS Server ([install guide](https://docs.nats.io/running-a-nats-service/introduction/installation))
|
|
97
158
|
|
|
98
|
-
|
|
159
|
+
Spin up NATS instantly with Docker — one command, that's it:
|
|
160
|
+
```bash
|
|
161
|
+
docker run -p 4222:4222 -p 8222:8222 nats:alpine -js
|
|
162
|
+
```
|
|
163
|
+
|
|
164
|
+
Or add it to your existing `docker-compose.yml`:
|
|
165
|
+
```yaml
|
|
166
|
+
services:
|
|
167
|
+
nats:
|
|
168
|
+
image: nats:alpine
|
|
169
|
+
command: -js
|
|
170
|
+
ports:
|
|
171
|
+
- "4222:4222"
|
|
172
|
+
- "8222:8222"
|
|
173
|
+
```
|
|
174
|
+
|
|
175
|
+
Mount the monitoring UI in your Rack app:
|
|
99
176
|
```ruby
|
|
100
177
|
require "cosmo/web"
|
|
101
178
|
|
|
102
|
-
Rails
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
179
|
+
# Rails
|
|
180
|
+
mount Cosmo::Web => "/cosmo"
|
|
181
|
+
|
|
182
|
+
# Any Rack app (config.ru)
|
|
183
|
+
map "/cosmo" { run Cosmo::Web }
|
|
106
184
|
```
|
|
107
185
|
|
|
108
186
|
|
|
109
187
|
## 🚀 Quick Start
|
|
110
188
|
|
|
111
|
-
### 1. Create `config/cosmo.yml`
|
|
189
|
+
### 1. Create `config/cosmo.yml`
|
|
112
190
|
|
|
113
191
|
```yaml
|
|
114
|
-
concurrency: 5
|
|
115
|
-
|
|
192
|
+
concurrency: 5 # Number of worker threads
|
|
193
|
+
|
|
194
|
+
consumers: # Declare consumer groups for streams, things that pull messages and process them
|
|
195
|
+
jobs: # Consumer configs for jobs (or streams)
|
|
196
|
+
default: # Stream name
|
|
197
|
+
ack_policy: explicit # Acknowledgment required for each message, can be explicit, none, or all
|
|
198
|
+
max_deliver: 10 # Max retry attempts before sending to a dead stream
|
|
199
|
+
max_ack_pending: 10 # Max messages waiting for ack, if exceeded, the server will stop delivering new messages until some are acked
|
|
200
|
+
ack_wait: 15 # Seconds to wait for ack before redelivering
|
|
201
|
+
subject: jobs.%{name}.> # Subject pattern for this consumer, %{name} replaced with stream name, becomes `jobs.default.>`
|
|
202
|
+
|
|
203
|
+
setup: # Initial stream creation only `cosmo -S`
|
|
204
|
+
jobs: # Stream configs for jobs (or streams)
|
|
205
|
+
default: # Stream name
|
|
206
|
+
storage: file # Storage type (file or memory)
|
|
207
|
+
retention: workqueue # Retention policy (limits, interest, workqueue). workqueue - deletes acked/nacked, limits - append only
|
|
208
|
+
subjects: ["jobs.%{name}.>"] # Subject pattern for this stream, %{name} replaced with stream name
|
|
209
|
+
allow_direct: true # Allow direct messages to stream (required for web UI)
|
|
210
|
+
```
|
|
116
211
|
|
|
117
|
-
|
|
118
|
-
jobs:
|
|
119
|
-
default:
|
|
120
|
-
ack_policy: explicit
|
|
121
|
-
max_deliver: 10
|
|
122
|
-
max_ack_pending: 10
|
|
123
|
-
ack_wait: 15
|
|
124
|
-
subject: jobs.%{name}.>
|
|
212
|
+
### 2. Create streams in NATS (one-time), grabs config from setup section of `config/cosmo.yml`
|
|
125
213
|
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
storage: file
|
|
130
|
-
retention: workqueue
|
|
131
|
-
subjects: ["jobs.%{name}.>"]
|
|
132
|
-
allow_direct: true
|
|
133
|
-
```
|
|
214
|
+
```bash
|
|
215
|
+
bundle exec cosmo -S
|
|
216
|
+
```
|
|
134
217
|
|
|
135
|
-
###
|
|
218
|
+
### 3. Define a job in `app/jobs/`
|
|
136
219
|
|
|
137
220
|
```ruby
|
|
138
221
|
class SendEmailJob
|
|
139
222
|
include Cosmo::Job
|
|
140
|
-
|
|
141
223
|
options stream: :default, retry: 3, dead: true
|
|
142
224
|
|
|
143
225
|
def perform(user_id, email_type)
|
|
144
|
-
|
|
145
|
-
sleep 0.5 # Simulate work
|
|
146
|
-
puts "#{user_id}, #{email_type}"
|
|
226
|
+
UserMailer.send(email_type, user_id).deliver_now
|
|
147
227
|
end
|
|
148
228
|
end
|
|
149
229
|
```
|
|
150
230
|
|
|
151
|
-
###
|
|
231
|
+
### 4. Enqueue & run
|
|
152
232
|
|
|
153
233
|
```ruby
|
|
154
|
-
|
|
234
|
+
SendEmailJob.perform_async(42, "welcome")
|
|
155
235
|
```
|
|
156
236
|
|
|
157
|
-
### 4. Run
|
|
158
|
-
|
|
159
237
|
```bash
|
|
160
|
-
bundle exec cosmo jobs
|
|
238
|
+
bundle exec cosmo -C config/cosmo.yml -c 10 -r ./app/jobs jobs
|
|
161
239
|
```
|
|
162
240
|
|
|
163
241
|
|
|
@@ -165,16 +243,14 @@ bundle exec cosmo jobs
|
|
|
165
243
|
|
|
166
244
|
### Jobs
|
|
167
245
|
|
|
168
|
-
Simple background tasks with a familiar API:
|
|
169
|
-
|
|
170
246
|
```ruby
|
|
171
247
|
class ReportJob
|
|
172
248
|
include Cosmo::Job
|
|
173
|
-
|
|
249
|
+
|
|
174
250
|
options(
|
|
175
251
|
stream: :critical, # Stream name
|
|
176
252
|
retry: 5, # Retry attempts
|
|
177
|
-
dead: true #
|
|
253
|
+
dead: true # Send to dead letter queue on final failure
|
|
178
254
|
)
|
|
179
255
|
|
|
180
256
|
def perform(report_id)
|
|
@@ -182,20 +258,18 @@ class ReportJob
|
|
|
182
258
|
Report.find(report_id).generate!
|
|
183
259
|
rescue StandardError => e
|
|
184
260
|
logger.error "Failed: #{e.message}"
|
|
185
|
-
raise # Triggers retry
|
|
261
|
+
raise # Triggers retry with exponential backoff
|
|
186
262
|
end
|
|
187
263
|
end
|
|
188
264
|
|
|
189
|
-
# Usage
|
|
190
265
|
ReportJob.perform_async(42) # Enqueue now
|
|
191
266
|
ReportJob.perform_in(30.minutes, 42) # Delayed
|
|
192
267
|
ReportJob.perform_at(Time.parse("2026-01-25 10:00"), 42) # Scheduled
|
|
268
|
+
ReportJob.perform_sync(42) # Inline, no NATS (great for tests)
|
|
193
269
|
```
|
|
194
270
|
|
|
195
271
|
### Streams
|
|
196
272
|
|
|
197
|
-
Real-time event processing with powerful features:
|
|
198
|
-
|
|
199
273
|
```ruby
|
|
200
274
|
class ClicksProcessor
|
|
201
275
|
include Cosmo::Stream
|
|
@@ -212,35 +286,34 @@ class ClicksProcessor
|
|
|
212
286
|
}
|
|
213
287
|
)
|
|
214
288
|
|
|
215
|
-
# Process one message
|
|
289
|
+
# Process one message at a time
|
|
216
290
|
def process_one
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
message.ack # Success
|
|
291
|
+
Analytics.track_click(message.data)
|
|
292
|
+
message.ack
|
|
220
293
|
end
|
|
221
|
-
|
|
222
|
-
# OR process batch
|
|
294
|
+
|
|
295
|
+
# OR process a batch
|
|
223
296
|
def process(messages)
|
|
224
|
-
Analytics.
|
|
297
|
+
Analytics.bulk_track(messages.map(&:data))
|
|
225
298
|
messages.each(&:ack)
|
|
226
299
|
end
|
|
227
300
|
end
|
|
228
301
|
|
|
229
302
|
# Publishing
|
|
230
|
-
ClicksProcessor.publish(
|
|
231
|
-
{ user_id: 123, page: "/home" },
|
|
232
|
-
subject: "events.clicks.homepage"
|
|
233
|
-
)
|
|
303
|
+
ClicksProcessor.publish({ user_id: 123, page: "/home" }, subject: "events.clicks.homepage")
|
|
234
304
|
|
|
235
|
-
#
|
|
305
|
+
# Acknowledgment strategies
|
|
236
306
|
message.ack # Success
|
|
237
|
-
message.nack(delay: 5_000_000_000) # Retry
|
|
307
|
+
message.nack(delay: 5_000_000_000) # Retry in 5 seconds (nanoseconds)
|
|
238
308
|
message.term # Permanent failure, no retry
|
|
239
309
|
```
|
|
240
310
|
|
|
241
311
|
### Configuration
|
|
242
312
|
|
|
243
|
-
**
|
|
313
|
+
**NATS subjects** follow a dot-separated hierarchy (`events.clicks.homepage`).
|
|
314
|
+
The `>` wildcard matches everything after that prefix. Think of subjects as topic names — flexible routing with no extra configuration.
|
|
315
|
+
|
|
316
|
+
**Full `config/cosmo.yml` example:**
|
|
244
317
|
```yaml
|
|
245
318
|
timeout: 25 # Shutdown timeout in seconds
|
|
246
319
|
concurrency: &concurrency 1 # Number of worker threads
|
|
@@ -338,28 +411,15 @@ export COSMO_STREAMS_FETCH_TIMEOUT=0.1
|
|
|
338
411
|
```ruby
|
|
339
412
|
class UrgentJob
|
|
340
413
|
include Cosmo::Job
|
|
341
|
-
options stream: :critical # priority: 50 in config
|
|
414
|
+
options stream: :critical # priority: 50 in config — polled most frequently
|
|
342
415
|
end
|
|
343
|
-
|
|
344
|
-
# config/cosmo.yml
|
|
345
|
-
consumers:
|
|
346
|
-
jobs:
|
|
347
|
-
critical:
|
|
348
|
-
priority: 50 # Polled more frequently
|
|
349
|
-
default:
|
|
350
|
-
priority: 15
|
|
351
416
|
```
|
|
352
417
|
|
|
353
418
|
**Custom Serializers:**
|
|
354
419
|
```ruby
|
|
355
420
|
module MessagePackSerializer
|
|
356
|
-
def self.serialize(data)
|
|
357
|
-
|
|
358
|
-
end
|
|
359
|
-
|
|
360
|
-
def self.deserialize(payload)
|
|
361
|
-
MessagePack.unpack(payload)
|
|
362
|
-
end
|
|
421
|
+
def self.serialize(data) = MessagePack.pack(data)
|
|
422
|
+
def self.deserialize(payload) = MessagePack.unpack(payload)
|
|
363
423
|
end
|
|
364
424
|
|
|
365
425
|
class FastStream
|
|
@@ -378,20 +438,20 @@ class ResilientJob
|
|
|
378
438
|
process_data(data)
|
|
379
439
|
rescue RetryableError => e
|
|
380
440
|
logger.warn "Retryable: #{e.message}"
|
|
381
|
-
raise # Will retry
|
|
441
|
+
raise # Will retry with exponential backoff
|
|
382
442
|
rescue FatalError => e
|
|
383
443
|
logger.error "Fatal: #{e.message}"
|
|
384
|
-
# Don't raise
|
|
444
|
+
# Don't raise — won't retry, won't go to DLQ
|
|
385
445
|
end
|
|
386
446
|
end
|
|
387
447
|
```
|
|
388
448
|
|
|
389
449
|
**Testing:**
|
|
390
450
|
```ruby
|
|
391
|
-
# Synchronous
|
|
451
|
+
# Synchronous — no NATS needed
|
|
392
452
|
SendEmailJob.perform_sync(123, "test")
|
|
393
453
|
|
|
394
|
-
#
|
|
454
|
+
# Async — returns a job ID
|
|
395
455
|
jid = SendEmailJob.perform_async(123, "welcome")
|
|
396
456
|
assert_kind_of String, jid
|
|
397
457
|
```
|
|
@@ -400,29 +460,24 @@ assert_kind_of String, jid
|
|
|
400
460
|
## 🖥️ CLI Reference
|
|
401
461
|
|
|
402
462
|
```bash
|
|
403
|
-
#
|
|
404
|
-
cosmo -C config/cosmo.yml
|
|
405
|
-
|
|
406
|
-
#
|
|
407
|
-
cosmo -C config/cosmo.yml -c 20 -r ./app/jobs jobs # Jobs only
|
|
408
|
-
cosmo -C config/cosmo.yml -c 20 streams # Streams only
|
|
409
|
-
cosmo -C config/cosmo.yml -c 20 # Both
|
|
463
|
+
cosmo -C config/cosmo.yml --setup # Create streams in NATS (idempotent)
|
|
464
|
+
cosmo -C config/cosmo.yml -c 20 -r ./app/jobs jobs # Jobs only
|
|
465
|
+
cosmo -C config/cosmo.yml -c 20 streams # Streams only
|
|
466
|
+
cosmo -C config/cosmo.yml -c 20 # Both
|
|
410
467
|
```
|
|
411
468
|
|
|
412
|
-
|
|
413
|
-
|
|
414
|
-
|
|
|
415
|
-
|
|
416
|
-
| `-
|
|
417
|
-
| `-
|
|
418
|
-
| `-
|
|
419
|
-
| `-t, --timeout NUM` | Shutdown timeout (sec) | `-t 60` |
|
|
420
|
-
| `-S, --setup` | Setup streams & exit | `--setup` |
|
|
469
|
+
| Flag | Description | Example |
|
|
470
|
+
|-------------------------|------------------------|-----------------------|
|
|
471
|
+
| `-C, --config PATH` | Config file path | `-C config/cosmo.yml` |
|
|
472
|
+
| `-c, --concurrency INT` | Worker threads | `-c 20` |
|
|
473
|
+
| `-r, --require PATH` | Auto-require directory | `-r ./app/jobs` |
|
|
474
|
+
| `-t, --timeout NUM` | Shutdown timeout (sec) | `-t 60` |
|
|
475
|
+
| `-S, --setup` | Setup streams & exit | `--setup` |
|
|
421
476
|
|
|
422
477
|
|
|
423
478
|
## 🚢 Deployment
|
|
424
479
|
|
|
425
|
-
**NATS Cluster:**
|
|
480
|
+
**NATS Cluster config:**
|
|
426
481
|
```bash
|
|
427
482
|
# nats-server.conf
|
|
428
483
|
port: 4222
|
|
@@ -446,7 +501,7 @@ services:
|
|
|
446
501
|
volumes:
|
|
447
502
|
- ./nats.conf:/etc/nats/nats-server.conf
|
|
448
503
|
- nats-data:/var/lib/nats
|
|
449
|
-
|
|
504
|
+
|
|
450
505
|
worker:
|
|
451
506
|
build: .
|
|
452
507
|
environment:
|
|
@@ -480,17 +535,14 @@ SyslogIdentifier=cosmo
|
|
|
480
535
|
WantedBy=multi-user.target
|
|
481
536
|
```
|
|
482
537
|
|
|
483
|
-
Enable and start:
|
|
484
538
|
```bash
|
|
485
|
-
sudo systemctl enable cosmo
|
|
486
|
-
sudo systemctl start cosmo
|
|
487
|
-
sudo systemctl status cosmo
|
|
539
|
+
sudo systemctl enable cosmo && sudo systemctl start cosmo
|
|
488
540
|
```
|
|
489
541
|
|
|
490
542
|
|
|
491
543
|
## 📊 Monitoring
|
|
492
544
|
|
|
493
|
-
**Structured
|
|
545
|
+
**Structured logs:**
|
|
494
546
|
```
|
|
495
547
|
2026-01-23T10:15:30.123Z INFO pid=12345 tid=abc jid=def: start
|
|
496
548
|
2026-01-23T10:15:32.456Z INFO pid=12345 tid=abc jid=def elapsed=2.333: done
|
|
@@ -506,15 +558,15 @@ info.state.bytes # Total bytes
|
|
|
506
558
|
info.state.consumer_count # Number of consumers
|
|
507
559
|
```
|
|
508
560
|
|
|
509
|
-
**Prometheus
|
|
510
|
-
- `jetstream_server_store_msgs`
|
|
511
|
-
- `jetstream_consumer_delivered_msgs`
|
|
512
|
-
- `jetstream_consumer_ack_pending`
|
|
561
|
+
**Prometheus** — NATS exposes metrics at `:8222/metrics`:
|
|
562
|
+
- `jetstream_server_store_msgs` — Messages in stream
|
|
563
|
+
- `jetstream_consumer_delivered_msgs` — Delivered messages
|
|
564
|
+
- `jetstream_consumer_ack_pending` — Pending acknowledgments
|
|
513
565
|
|
|
514
566
|
|
|
515
567
|
## 💼 Examples
|
|
516
568
|
|
|
517
|
-
**Email
|
|
569
|
+
**Email queue with scheduling:**
|
|
518
570
|
```ruby
|
|
519
571
|
class EmailJob
|
|
520
572
|
include Cosmo::Job
|
|
@@ -545,7 +597,7 @@ class ImageProcessor
|
|
|
545
597
|
message.ack
|
|
546
598
|
rescue => e
|
|
547
599
|
logger.error "Processing failed: #{e.message}"
|
|
548
|
-
message.nack(delay: 30_000_000_000)
|
|
600
|
+
message.nack(delay: 30_000_000_000) # retry in 30s
|
|
549
601
|
end
|
|
550
602
|
end
|
|
551
603
|
|
|
@@ -559,14 +611,14 @@ class AnalyticsAggregator
|
|
|
559
611
|
options batch_size: 1000, consumer: { subjects: ["events.*.>"] }
|
|
560
612
|
|
|
561
613
|
def process(messages)
|
|
562
|
-
|
|
563
|
-
aggregates = events.group_by { |e| e["type"] }.transform_values(&:count)
|
|
614
|
+
aggregates = messages.map(&:data).group_by { |e| e["type"] }.transform_values(&:count)
|
|
564
615
|
Analytics.bulk_insert(aggregates)
|
|
565
616
|
messages.each(&:ack)
|
|
566
617
|
end
|
|
567
618
|
end
|
|
568
619
|
```
|
|
569
620
|
|
|
621
|
+
---
|
|
570
622
|
|
|
571
623
|
<div align="center">
|
|
572
624
|
|
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Cosmo
|
|
4
|
+
module ActiveJobAdapter
|
|
5
|
+
# ActiveJob queue adapter that enqueues jobs via NATS JetStream.
|
|
6
|
+
#
|
|
7
|
+
# Usage:
|
|
8
|
+
# config.active_job.queue_adapter = :cosmonats
|
|
9
|
+
# # or explicitly:
|
|
10
|
+
# config.active_job.queue_adapter = Cosmo::ActiveJobAdapter::Adapter.new
|
|
11
|
+
#
|
|
12
|
+
# The ActiveJob queue name maps directly to the Cosmo stream name.
|
|
13
|
+
class Adapter
|
|
14
|
+
# Enqueue a job to be run as soon as possible.
|
|
15
|
+
# @param job [ActiveJob::Base]
|
|
16
|
+
def enqueue(job)
|
|
17
|
+
publish(job, nil)
|
|
18
|
+
end
|
|
19
|
+
|
|
20
|
+
# Enqueue a job to be run at (or after) a given time.
|
|
21
|
+
# @param job [ActiveJob::Base]
|
|
22
|
+
# @param timestamp [Numeric] Unix timestamp (seconds, float)
|
|
23
|
+
def enqueue_at(job, timestamp)
|
|
24
|
+
publish(job, timestamp)
|
|
25
|
+
end
|
|
26
|
+
|
|
27
|
+
private
|
|
28
|
+
|
|
29
|
+
def publish(job, timestamp)
|
|
30
|
+
cosmo_opts = job_cosmo_options(job)
|
|
31
|
+
stream = cosmo_opts.delete(:stream) || job.queue_name.to_sym
|
|
32
|
+
options = { stream: stream }.merge(cosmo_opts)
|
|
33
|
+
options[:at] = timestamp if timestamp
|
|
34
|
+
|
|
35
|
+
data = Job::Data.new(Executor.name, [job.serialize], options)
|
|
36
|
+
Publisher.publish_job(data)
|
|
37
|
+
end
|
|
38
|
+
|
|
39
|
+
# Returns Cosmo-specific options declared on the job class via
|
|
40
|
+
# +cosmo_options+, falling back to an empty hash.
|
|
41
|
+
def job_cosmo_options(job)
|
|
42
|
+
job.class.respond_to?(:get_cosmo_options) ? job.class.get_cosmo_options : {}
|
|
43
|
+
end
|
|
44
|
+
end
|
|
45
|
+
end
|
|
46
|
+
end
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Cosmo
|
|
4
|
+
module ActiveJobAdapter
|
|
5
|
+
# Cosmo::Job that deserializes and executes an ActiveJob payload
|
|
6
|
+
class Executor
|
|
7
|
+
include Cosmo::Job
|
|
8
|
+
|
|
9
|
+
options stream: :default
|
|
10
|
+
|
|
11
|
+
def perform(job_data)
|
|
12
|
+
::ActiveJob::Base.execute(Utils::Hash.stringify_keys(job_data))
|
|
13
|
+
end
|
|
14
|
+
end
|
|
15
|
+
end
|
|
16
|
+
end
|