cosmonats 0.3.0 → 0.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (62) hide show
  1. checksums.yaml +4 -4
  2. data/README.md +208 -156
  3. data/lib/cosmo/active_job/adapter.rb +46 -0
  4. data/lib/cosmo/active_job/executor.rb +16 -0
  5. data/lib/cosmo/active_job/options.rb +50 -0
  6. data/lib/cosmo/active_job.rb +29 -0
  7. data/lib/cosmo/api/busy.rb +2 -2
  8. data/lib/cosmo/api/counter.rb +2 -2
  9. data/lib/cosmo/api/cron/entry.rb +99 -0
  10. data/lib/cosmo/api/cron.rb +118 -0
  11. data/lib/cosmo/api/kv.rb +35 -13
  12. data/lib/cosmo/api/stream.rb +10 -5
  13. data/lib/cosmo/api.rb +1 -0
  14. data/lib/cosmo/cli.rb +27 -10
  15. data/lib/cosmo/client.rb +48 -2
  16. data/lib/cosmo/config.rb +9 -0
  17. data/lib/cosmo/job/data.rb +1 -1
  18. data/lib/cosmo/job/limit.rb +51 -0
  19. data/lib/cosmo/job/processor.rb +49 -5
  20. data/lib/cosmo/job.rb +51 -2
  21. data/lib/cosmo/processor.rb +1 -1
  22. data/lib/cosmo/railtie.rb +21 -0
  23. data/lib/cosmo/stream/processor.rb +2 -2
  24. data/lib/cosmo/stream.rb +2 -1
  25. data/lib/cosmo/utils/hash.rb +13 -0
  26. data/lib/cosmo/utils/overrides.rb +1 -1
  27. data/lib/cosmo/version.rb +1 -1
  28. data/lib/cosmo/web/assets/app.css +42 -0
  29. data/lib/cosmo/web/controllers/crons.rb +41 -0
  30. data/lib/cosmo/web/controllers/jobs.rb +7 -3
  31. data/lib/cosmo/web/controllers/streams.rb +1 -1
  32. data/lib/cosmo/web/helpers/application.rb +4 -0
  33. data/lib/cosmo/web/views/actions/index.erb +1 -1
  34. data/lib/cosmo/web/views/crons/_table.erb +58 -0
  35. data/lib/cosmo/web/views/crons/index.erb +10 -0
  36. data/lib/cosmo/web/views/jobs/_busy.erb +54 -49
  37. data/lib/cosmo/web/views/jobs/_dead.erb +70 -65
  38. data/lib/cosmo/web/views/jobs/_enqueued.erb +82 -56
  39. data/lib/cosmo/web/views/jobs/_scheduled.erb +53 -48
  40. data/lib/cosmo/web/views/jobs/_tabs.erb +6 -0
  41. data/lib/cosmo/web/views/jobs/busy.erb +8 -6
  42. data/lib/cosmo/web/views/jobs/dead.erb +6 -5
  43. data/lib/cosmo/web/views/jobs/enqueued.erb +8 -6
  44. data/lib/cosmo/web/views/jobs/index.erb +1 -1
  45. data/lib/cosmo/web/views/jobs/scheduled.erb +6 -5
  46. data/lib/cosmo/web/views/layout.erb +1 -1
  47. data/lib/cosmo/web.rb +5 -0
  48. data/lib/cosmo.rb +1 -0
  49. data/sig/cosmo/active_job/adapter.rbs +13 -0
  50. data/sig/cosmo/active_job/executor.rbs +9 -0
  51. data/sig/cosmo/active_job/options.rbs +14 -0
  52. data/sig/cosmo/api/cron/entry.rbs +30 -0
  53. data/sig/cosmo/api/cron.rbs +25 -0
  54. data/sig/cosmo/api/kv.rbs +4 -6
  55. data/sig/cosmo/client.rbs +9 -1
  56. data/sig/cosmo/job/data.rbs +1 -1
  57. data/sig/cosmo/job/limit.rbs +18 -0
  58. data/sig/cosmo/job/processor.rbs +3 -1
  59. data/sig/cosmo/job.rbs +9 -4
  60. data/sig/cosmo/railtie.rbs +4 -0
  61. data/sig/cosmo/utils/hash.rbs +4 -0
  62. metadata +20 -1
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: f850638b66506538b131180c03b64a13e31cd86f924503703812da49b6ba3d20
4
- data.tar.gz: 0dfdab08193600f3bca2c32558ffb016a38657f9488789770eea5e60612281e9
3
+ metadata.gz: 6f94d9b1e192b098e1bcb3ce77998d7b0ed9aca359af41fefe2a572624fa6b91
4
+ data.tar.gz: 12bba3015a4beb335efae8dfe5fc178a6c467d203eaf61f922bf1b3091b67ce6
5
5
  SHA512:
6
- metadata.gz: 56f46ac8a8c1409b1583033ad8b5cf9e513652fae2643b07344aa8f8bf1cca80f948463af92e2a989ec9679cc830ac05a6b873b1f0c9b5e10a1e6ef5e8399ad0
7
- data.tar.gz: '086ae5c5b26471417e54c37d2f782c0c103340953e9a27ca8c02131478122f01ede30f2f8ab5a0b29c81ce7b4ea07a5c8de8e7f51957e4d9d5dfd5d670a0ff08'
6
+ metadata.gz: e24c844b4f17aa7eaffceebdd6b847858855fc0b5526b5537832893fd144e71f427ae352700adff3c5b39ed5cd04c89bc84007d01548ff97ba701a45e8c836b9
7
+ data.tar.gz: d1a7d5e0d45c5f9b03388259b45a5a6ce659dd39378a0f13e8f31cf36d7f969a332076cada74b564f2fd7d504c646df0715778679d0e67743a5f0ef08398e7c9
data/README.md CHANGED
@@ -1,12 +1,65 @@
1
- # 🚀 Cosmonats - lightweight background and stream processing
1
+ # 🚀 Cosmonats
2
2
 
3
- It is a Ruby background job and stream processing framework powered by NATS JetStream.
4
- It provides a familiar API for job queues while adding powerful stream processing capabilities,
5
- solving the scalability limitations of Redis and database-backed queues through true horizontal scaling and
6
- disk-backed persistence.
3
+ Background jobs + real-time event streaming for Ruby unified, in one gem, backed by NATS.
4
+ **No Redis. No DB polling. Disk-backed, horizontally scalable no message is ever silently dropped.**
5
+
6
+ <div align="center">
7
7
 
8
8
  ![logo.svg](logo.svg)
9
9
 
10
+ [![Gem Version](https://badge.fury.io/rb/cosmonats.svg)](https://rubygems.org/gems/cosmonats)
11
+ [![Downloads](https://img.shields.io/gem/dt/cosmonats.svg)](https://rubygems.org/gems/cosmonats)
12
+ [![Ruby](https://img.shields.io/badge/ruby-%3E%3D%203.1-red)](https://www.ruby-lang.org)
13
+ [![License: LGPL v3](https://img.shields.io/badge/License-LGPL%20v3-blue.svg)](LICENSE.txt)
14
+ [![Build Status](https://github.com/bitsbeam/cosmonats/actions/workflows/ci.yml/badge.svg)](https://github.com/bitsbeam/cosmonats/actions)
15
+
16
+ *Battle-tested in production. Tens of millions of jobs processed and counting.*
17
+
18
+ </div>
19
+
20
+
21
+ ## ⚡ Taste it
22
+
23
+ ```ruby
24
+ # Define a job with a familiar look
25
+ class SendEmailJob
26
+ include Cosmo::Job
27
+ options stream: :default, retry: 3, dead: true
28
+
29
+ def perform(user_id, template)
30
+ EmailService.send(user_id, template)
31
+ end
32
+ end
33
+
34
+ # Enqueue it
35
+ SendEmailJob.perform_async(123, "welcome")
36
+ SendEmailJob.perform_in(1.day, 123, "followup")
37
+ ```
38
+
39
+ ```ruby
40
+ # Process a continuous real-time event stream
41
+ class ClicksProcessor
42
+ include Cosmo::Stream
43
+ options stream: :clickstream, batch_size: 100,
44
+ consumer: { subjects: ["events.clicks.>"] }
45
+
46
+ def process_one
47
+ Analytics.track(message.data)
48
+ message.ack
49
+ end
50
+ end
51
+
52
+ ClicksProcessor.publish({ user_id: 123, page: "/home" }, subject: "events.clicks.homepage")
53
+ ```
54
+
55
+ ```bash
56
+ bundle exec cosmo -C config/cosmo.yml -c 20 # Run jobs + streams with 20 threads
57
+ bundle exec cosmo -C config/cosmo.yml -c 20 jobs # Jobs only
58
+ bundle exec cosmo -C config/cosmo.yml -c 20 streams # Streams only
59
+ ```
60
+
61
+ ![webui.gif](webui.gif)
62
+
10
63
  ## 📖 Index
11
64
 
12
65
  - [Why?](#-why)
@@ -25,65 +78,73 @@ disk-backed persistence.
25
78
 
26
79
 
27
80
  ## 🎯 Why?
28
- Among many others, why creating another? Cosmonats is a background processing framework for Ruby, powered by **[NATS](https://nats.io/)**.
29
- It's designed to solve the fundamental scaling problems that plague Redis/DB-based job queues and at the same time to provide both job and stream
30
- processing capabilities.
31
81
 
32
- ### The Problem with Redis at Scale
82
+ Most background job libraries use Redis or Postgres — tools that were never designed for this. Think of NATS as Redis — but Redis is KV first then messaging;
83
+ NATS is messaging first, then KV. What NATS is:
84
+
85
+ - **~20 MB binary, ~10 MB RAM at idle** Trivial to run anywhere.
86
+ - **Disk-backed persistent streams** Messages survive restarts, don't require RAM to fit.
87
+ - **True horizontal clustering** Lose a node — other nodes take over, zero message loss.
88
+ - **Multilingual** Official clients for Ruby, Go, Python, Rust, Java, .NET, and more. Any service can publish or consume.
89
+
90
+ One NATS server replaces your message broker, job queue, and KV store — with lower operational overhead.
91
+
92
+ | | Redis/DB-backed | NATS/Cosmonats |
93
+ |-------------------|-------------------------------|----------------------------|
94
+ | Persistence | In-memory / DB bloat | Disk-backed, TB-scale |
95
+ | Scaling | Sentinel only / Vertical only | True horizontal clustering |
96
+ | Background jobs | Yes | Yes |
97
+ | Real-time stream | No | Yes |
98
+ | Zero message loss | No | Yes |
99
+ | Message replay | No | Yes |
100
+ | Backpressure | No, grow unbounded | Yes |
101
+ | Multi-DC | Complex setup | Native geo-distribution |
102
+
103
+
104
+ ### Killer Features:
33
105
 
34
- - **Single-threaded command processing** - All operations serialized, creating contention with many workers
35
- - **Memory-only persistence** - Everything must fit in RAM, expensive to scale
36
- - **Vertical scaling only** - Can't truly distribute a single queue across nodes
37
- - **Polling overhead** - Thousands of blocked connections
38
- - **No native backpressure** - Queues can grow unbounded
39
- - **Weak durability** - Async replication can lose jobs during failures
106
+ #### Jobs + Streams, unified in one gem.
40
107
 
41
- **Note:** Alternatives like Dragonfly solve the threading bottleneck but still face memory/scaling limitations.
108
+ Most Ruby gems handle exactly that background jobs. If you also need to consume a continuous event feed, that's a second system, second config, second set of
109
+ worker processes, second Dockerfile entry. Cosmonats is the only Ruby gem with a first-class `Job` primitive *and* a first-class `Stream` primitive, sharing
110
+ one server, one config, one CLI, one monitoring endpoint.
42
111
 
43
- ### The Problem with RDBMS at Scale
112
+ #### Message replay and time-travel debugging.
44
113
 
45
- - **Database contention** - Polling queries compete with application queries for resources
46
- - **Connection pool pressure** - Workers consume database connections, starving the application
47
- - **Row-level locking overhead** - `SELECT FOR UPDATE SKIP LOCKED` still scans rows under high concurrency
48
- - **Vacuum/autovacuum impact** - High-churn job tables degrade database performance
49
- - **Vertical scaling only** - Limited by single database instance capabilities
50
- - **Index bloat** - High UPDATE/DELETE volume causes index degradation over time
51
- - **Table bloat** - Constant row updates fragment tables, requiring maintenance
52
- - **`LISTEN/NOTIFY` limitations** - 8KB payload limit, no persistence, breaks down at high volumes (10K+ notifications/sec)
53
- - **No native horizontal scaling** - Cannot distribute a single job queue across multiple database nodes
114
+ NATS persists messages to disk and lets any consumer rewind to any point — beginning of time, a specific timestamp, or only new messages.
115
+ - **Incident recovery** your pipeline crashed for 3 hours. Replay from the crash timestamp.
116
+ - **New consumer bootstrap** a new service needs historical events. Start it from the beginning.
117
+ - **Bug reproduction** replay the exact sequence of messages that caused a production issue.
54
118
 
55
- **Note:** Solutions using DB might be ok for moderate workloads but face these fundamental limitations at higher scales.
119
+ #### Multi-datacenter queues, natively.
56
120
 
57
- ### The Solution
121
+ NATS has a first-class cluster + leaf-node architecture for geo-distribution. Spanning multiple regions or datacenters is a config block — not a separate
122
+ product or a third-party replication tool. NATS was built for edge computing, IoT, and satellite communication — multi-DC is a first-class concern, not an
123
+ afterthought.
58
124
 
59
- Built on **NATS**, `cosmonats` provides:
125
+ #### Transport-level deduplication + built-in KV. No extra infrastructure.
60
126
 
61
- **True horizontal scaling** - Distribute streams across cluster nodes
62
- **Disk-backed persistence** - TB-scale queues with memory cache
63
- **Replicated acknowledgments** - Survive multi-node failures
64
- ✅ **Built-in flow control** - Automatic backpressure
65
- ✅ **Multi-DC support** - Native geo-distribution, and super clusters
66
- ✅ **High throughput & low latency** - Millions of messages per second
67
- ✅ **Stream processing** - Beyond simple job queues
127
+ NATS deduplicates messages at the **broker** — same-ID messages within the configured window are dropped before they ever reach a worker. No uniqueness gems,
128
+ no advisory locks, no extra round-trips. It also ships a built-in Key/Value store usable for distributed locks and rate limiting — no Redis, no Memcached,
129
+ nothing else to run.
68
130
 
69
131
 
70
132
  ## ✨ Features
71
133
 
72
134
  ### 🎪 Job Processing
73
- - **Familiar compatible API** - Easy migration from existing codebases
74
- - **Priority queues** - Multiple priority levels (critical, high, default, low)
75
- - **Scheduled jobs** - Execute jobs at specific times or after delays
76
- - **Automatic retries** - Configurable retry strategies with exponential backoff
77
- - **Dead letter queue** - Capture permanently failed jobs
78
- - **Job uniqueness** - Prevent duplicate job execution
135
+ - **Familiar API** `perform_async`, `perform_in`, `perform_at`
136
+ - **Priority queues** critical, high, default, low with weighted round-robin
137
+ - **Scheduled jobs** execute at a specific time or after a delay
138
+ - **Automatic retries** exponential backoff, configurable attempts
139
+ - **Dead letter queue** capture permanently failed jobs
140
+ - **Job uniqueness** prevent duplicate execution
79
141
 
80
142
  ### 🌊 Stream Processing
81
- - **Real-time data streams** - Process continuous event streams
82
- - **Batch processing** - Handle multiple messages efficiently
83
- - **Message replay** - Reprocess messages from any point in time
84
- - **Consumer groups** - Multiple consumers with load balancing
85
- - **Exactly-once semantics** - With proper configuration
86
- - **Custom serialization** - JSON, MessagePack, Protobuf support
143
+ - **Real-time event streams** process continuous data feeds
144
+ - **Batch processing** handle multiple messages in one go
145
+ - **Message replay** reprocess from any point in time
146
+ - **Consumer groups** — load-balanced across workers
147
+ - **Custom serialization** JSON, MessagePack, Protobuf
87
148
 
88
149
 
89
150
  ## 📦 Installation
@@ -93,71 +154,88 @@ Built on **NATS**, `cosmonats` provides:
93
154
  gem "cosmonats"
94
155
  ```
95
156
 
96
- **Requirements:** Ruby 3.1.0+, NATS Server ([installation guide](https://docs.nats.io/running-a-nats-service/introduction/installation))
157
+ **Requirements:** Ruby 3.1, NATS Server ([install guide](https://docs.nats.io/running-a-nats-service/introduction/installation))
97
158
 
98
- Add these lines to config/routes.rb:
159
+ Spin up NATS instantly with Docker — one command, that's it:
160
+ ```bash
161
+ docker run -p 4222:4222 -p 8222:8222 nats:alpine -js
162
+ ```
163
+
164
+ Or add it to your existing `docker-compose.yml`:
165
+ ```yaml
166
+ services:
167
+ nats:
168
+ image: nats:alpine
169
+ command: -js
170
+ ports:
171
+ - "4222:4222"
172
+ - "8222:8222"
173
+ ```
174
+
175
+ Mount the monitoring UI in your Rack app:
99
176
  ```ruby
100
177
  require "cosmo/web"
101
178
 
102
- Rails.application.routes.draw do
103
- mount Cosmo::Web => "/cosmo" # access web UI at http://localhost:3000/cosmo
104
- ...
105
- end
179
+ # Rails
180
+ mount Cosmo::Web => "/cosmo"
181
+
182
+ # Any Rack app (config.ru)
183
+ map "/cosmo" { run Cosmo::Web }
106
184
  ```
107
185
 
108
186
 
109
187
  ## 🚀 Quick Start
110
188
 
111
- ### 1. Create `config/cosmo.yml` and run `bundle exec cosmo -S` to create streams in NATS:
189
+ ### 1. Create `config/cosmo.yml`
112
190
 
113
191
  ```yaml
114
- concurrency: 5
115
- max_retries: 3
192
+ concurrency: 5 # Number of worker threads
193
+
194
+ consumers: # Declare consumer groups for streams, things that pull messages and process them
195
+ jobs: # Consumer configs for jobs (or streams)
196
+ default: # Stream name
197
+ ack_policy: explicit # Acknowledgment required for each message, can be explicit, none, or all
198
+ max_deliver: 10 # Max retry attempts before sending to a dead stream
199
+ max_ack_pending: 10 # Max messages waiting for ack, if exceeded, the server will stop delivering new messages until some are acked
200
+ ack_wait: 15 # Seconds to wait for ack before redelivering
201
+ subject: jobs.%{name}.> # Subject pattern for this consumer, %{name} replaced with stream name, becomes `jobs.default.>`
202
+
203
+ setup: # Initial stream creation only `cosmo -S`
204
+ jobs: # Stream configs for jobs (or streams)
205
+ default: # Stream name
206
+ storage: file # Storage type (file or memory)
207
+ retention: workqueue # Retention policy (limits, interest, workqueue). workqueue - deletes acked/nacked, limits - append only
208
+ subjects: ["jobs.%{name}.>"] # Subject pattern for this stream, %{name} replaced with stream name
209
+ allow_direct: true # Allow direct messages to stream (required for web UI)
210
+ ```
116
211
 
117
- consumers:
118
- jobs:
119
- default:
120
- ack_policy: explicit
121
- max_deliver: 10
122
- max_ack_pending: 10
123
- ack_wait: 15
124
- subject: jobs.%{name}.>
212
+ ### 2. Create streams in NATS (one-time), grabs config from setup section of `config/cosmo.yml`
125
213
 
126
- setup:
127
- jobs:
128
- default:
129
- storage: file
130
- retention: workqueue
131
- subjects: ["jobs.%{name}.>"]
132
- allow_direct: true
133
- ```
214
+ ```bash
215
+ bundle exec cosmo -S
216
+ ```
134
217
 
135
- ### 2. Create a Job in app/workers
218
+ ### 3. Define a job in `app/jobs/`
136
219
 
137
220
  ```ruby
138
221
  class SendEmailJob
139
222
  include Cosmo::Job
140
-
141
223
  options stream: :default, retry: 3, dead: true
142
224
 
143
225
  def perform(user_id, email_type)
144
- # Pretend to send email to user: UserMailer.send(email_type, user_id).deliver_now
145
- sleep 0.5 # Simulate work
146
- puts "#{user_id}, #{email_type}"
226
+ UserMailer.send(email_type, user_id).deliver_now
147
227
  end
148
228
  end
149
229
  ```
150
230
 
151
- ### 3. Enqueue Jobs
231
+ ### 4. Enqueue & run
152
232
 
153
233
  ```ruby
154
- 10.times { |i| SendEmailJob.perform_async(i, "welcome") }
234
+ SendEmailJob.perform_async(42, "welcome")
155
235
  ```
156
236
 
157
- ### 4. Run
158
-
159
237
  ```bash
160
- bundle exec cosmo jobs
238
+ bundle exec cosmo -C config/cosmo.yml -c 10 -r ./app/jobs jobs
161
239
  ```
162
240
 
163
241
 
@@ -165,16 +243,14 @@ bundle exec cosmo jobs
165
243
 
166
244
  ### Jobs
167
245
 
168
- Simple background tasks with a familiar API:
169
-
170
246
  ```ruby
171
247
  class ReportJob
172
248
  include Cosmo::Job
173
-
249
+
174
250
  options(
175
251
  stream: :critical, # Stream name
176
252
  retry: 5, # Retry attempts
177
- dead: true # Use dead letter queue
253
+ dead: true # Send to dead letter queue on final failure
178
254
  )
179
255
 
180
256
  def perform(report_id)
@@ -182,20 +258,18 @@ class ReportJob
182
258
  Report.find(report_id).generate!
183
259
  rescue StandardError => e
184
260
  logger.error "Failed: #{e.message}"
185
- raise # Triggers retry
261
+ raise # Triggers retry with exponential backoff
186
262
  end
187
263
  end
188
264
 
189
- # Usage
190
265
  ReportJob.perform_async(42) # Enqueue now
191
266
  ReportJob.perform_in(30.minutes, 42) # Delayed
192
267
  ReportJob.perform_at(Time.parse("2026-01-25 10:00"), 42) # Scheduled
268
+ ReportJob.perform_sync(42) # Inline, no NATS (great for tests)
193
269
  ```
194
270
 
195
271
  ### Streams
196
272
 
197
- Real-time event processing with powerful features:
198
-
199
273
  ```ruby
200
274
  class ClicksProcessor
201
275
  include Cosmo::Stream
@@ -212,35 +286,34 @@ class ClicksProcessor
212
286
  }
213
287
  )
214
288
 
215
- # Process one message
289
+ # Process one message at a time
216
290
  def process_one
217
- data = message.data
218
- Analytics.track_click(data)
219
- message.ack # Success
291
+ Analytics.track_click(message.data)
292
+ message.ack
220
293
  end
221
-
222
- # OR process batch
294
+
295
+ # OR process a batch
223
296
  def process(messages)
224
- Analytics.track_click(messages.map(&:data))
297
+ Analytics.bulk_track(messages.map(&:data))
225
298
  messages.each(&:ack)
226
299
  end
227
300
  end
228
301
 
229
302
  # Publishing
230
- ClicksProcessor.publish(
231
- { user_id: 123, page: "/home" },
232
- subject: "events.clicks.homepage"
233
- )
303
+ ClicksProcessor.publish({ user_id: 123, page: "/home" }, subject: "events.clicks.homepage")
234
304
 
235
- # Message acknowledgment strategies
305
+ # Acknowledgment strategies
236
306
  message.ack # Success
237
- message.nack(delay: 5_000_000_000) # Retry (5 seconds in nanoseconds)
307
+ message.nack(delay: 5_000_000_000) # Retry in 5 seconds (nanoseconds)
238
308
  message.term # Permanent failure, no retry
239
309
  ```
240
310
 
241
311
  ### Configuration
242
312
 
243
- **File-based (config/cosmo.yml):**
313
+ **NATS subjects** follow a dot-separated hierarchy (`events.clicks.homepage`).
314
+ The `>` wildcard matches everything after that prefix. Think of subjects as topic names — flexible routing with no extra configuration.
315
+
316
+ **Full `config/cosmo.yml` example:**
244
317
  ```yaml
245
318
  timeout: 25 # Shutdown timeout in seconds
246
319
  concurrency: &concurrency 1 # Number of worker threads
@@ -338,28 +411,15 @@ export COSMO_STREAMS_FETCH_TIMEOUT=0.1
338
411
  ```ruby
339
412
  class UrgentJob
340
413
  include Cosmo::Job
341
- options stream: :critical # priority: 50 in config
414
+ options stream: :critical # priority: 50 in config — polled most frequently
342
415
  end
343
-
344
- # config/cosmo.yml
345
- consumers:
346
- jobs:
347
- critical:
348
- priority: 50 # Polled more frequently
349
- default:
350
- priority: 15
351
416
  ```
352
417
 
353
418
  **Custom Serializers:**
354
419
  ```ruby
355
420
  module MessagePackSerializer
356
- def self.serialize(data)
357
- MessagePack.pack(data)
358
- end
359
-
360
- def self.deserialize(payload)
361
- MessagePack.unpack(payload)
362
- end
421
+ def self.serialize(data) = MessagePack.pack(data)
422
+ def self.deserialize(payload) = MessagePack.unpack(payload)
363
423
  end
364
424
 
365
425
  class FastStream
@@ -378,20 +438,20 @@ class ResilientJob
378
438
  process_data(data)
379
439
  rescue RetryableError => e
380
440
  logger.warn "Retryable: #{e.message}"
381
- raise # Will retry
441
+ raise # Will retry with exponential backoff
382
442
  rescue FatalError => e
383
443
  logger.error "Fatal: #{e.message}"
384
- # Don't raise - won't retry
444
+ # Don't raise won't retry, won't go to DLQ
385
445
  end
386
446
  end
387
447
  ```
388
448
 
389
449
  **Testing:**
390
450
  ```ruby
391
- # Synchronous execution
451
+ # Synchronous — no NATS needed
392
452
  SendEmailJob.perform_sync(123, "test")
393
453
 
394
- # Test job creation
454
+ # Async — returns a job ID
395
455
  jid = SendEmailJob.perform_async(123, "welcome")
396
456
  assert_kind_of String, jid
397
457
  ```
@@ -400,29 +460,24 @@ assert_kind_of String, jid
400
460
  ## 🖥️ CLI Reference
401
461
 
402
462
  ```bash
403
- # Setup streams
404
- cosmo -C config/cosmo.yml --setup
405
-
406
- # Run processors
407
- cosmo -C config/cosmo.yml -c 20 -r ./app/jobs jobs # Jobs only
408
- cosmo -C config/cosmo.yml -c 20 streams # Streams only
409
- cosmo -C config/cosmo.yml -c 20 # Both
463
+ cosmo -C config/cosmo.yml --setup # Create streams in NATS (idempotent)
464
+ cosmo -C config/cosmo.yml -c 20 -r ./app/jobs jobs # Jobs only
465
+ cosmo -C config/cosmo.yml -c 20 streams # Streams only
466
+ cosmo -C config/cosmo.yml -c 20 # Both
410
467
  ```
411
468
 
412
- **Common Flags:**
413
-
414
- | Flag | Description | Example |
415
- |------|-------------|---------|
416
- | `-C, --config PATH` | Config file path | `-C config/cosmo.yml` |
417
- | `-c, --concurrency INT` | Worker threads | `-c 20` |
418
- | `-r, --require PATH` | Auto-require directory | `-r ./app/jobs` |
419
- | `-t, --timeout NUM` | Shutdown timeout (sec) | `-t 60` |
420
- | `-S, --setup` | Setup streams & exit | `--setup` |
469
+ | Flag | Description | Example |
470
+ |-------------------------|------------------------|-----------------------|
471
+ | `-C, --config PATH` | Config file path | `-C config/cosmo.yml` |
472
+ | `-c, --concurrency INT` | Worker threads | `-c 20` |
473
+ | `-r, --require PATH` | Auto-require directory | `-r ./app/jobs` |
474
+ | `-t, --timeout NUM` | Shutdown timeout (sec) | `-t 60` |
475
+ | `-S, --setup` | Setup streams & exit | `--setup` |
421
476
 
422
477
 
423
478
  ## 🚢 Deployment
424
479
 
425
- **NATS Cluster:**
480
+ **NATS Cluster config:**
426
481
  ```bash
427
482
  # nats-server.conf
428
483
  port: 4222
@@ -446,7 +501,7 @@ services:
446
501
  volumes:
447
502
  - ./nats.conf:/etc/nats/nats-server.conf
448
503
  - nats-data:/var/lib/nats
449
-
504
+
450
505
  worker:
451
506
  build: .
452
507
  environment:
@@ -480,17 +535,14 @@ SyslogIdentifier=cosmo
480
535
  WantedBy=multi-user.target
481
536
  ```
482
537
 
483
- Enable and start:
484
538
  ```bash
485
- sudo systemctl enable cosmo
486
- sudo systemctl start cosmo
487
- sudo systemctl status cosmo
539
+ sudo systemctl enable cosmo && sudo systemctl start cosmo
488
540
  ```
489
541
 
490
542
 
491
543
  ## 📊 Monitoring
492
544
 
493
- **Structured Logging:**
545
+ **Structured logs:**
494
546
  ```
495
547
  2026-01-23T10:15:30.123Z INFO pid=12345 tid=abc jid=def: start
496
548
  2026-01-23T10:15:32.456Z INFO pid=12345 tid=abc jid=def elapsed=2.333: done
@@ -506,15 +558,15 @@ info.state.bytes # Total bytes
506
558
  info.state.consumer_count # Number of consumers
507
559
  ```
508
560
 
509
- **Prometheus:** NATS exposes metrics at `:8222/metrics`
510
- - `jetstream_server_store_msgs` - Messages in stream
511
- - `jetstream_consumer_delivered_msgs` - Delivered messages
512
- - `jetstream_consumer_ack_pending` - Pending acknowledgments
561
+ **Prometheus** NATS exposes metrics at `:8222/metrics`:
562
+ - `jetstream_server_store_msgs` Messages in stream
563
+ - `jetstream_consumer_delivered_msgs` Delivered messages
564
+ - `jetstream_consumer_ack_pending` Pending acknowledgments
513
565
 
514
566
 
515
567
  ## 💼 Examples
516
568
 
517
- **Email Queue:**
569
+ **Email queue with scheduling:**
518
570
  ```ruby
519
571
  class EmailJob
520
572
  include Cosmo::Job
@@ -545,7 +597,7 @@ class ImageProcessor
545
597
  message.ack
546
598
  rescue => e
547
599
  logger.error "Processing failed: #{e.message}"
548
- message.nack(delay: 30_000_000_000)
600
+ message.nack(delay: 30_000_000_000) # retry in 30s
549
601
  end
550
602
  end
551
603
 
@@ -559,14 +611,14 @@ class AnalyticsAggregator
559
611
  options batch_size: 1000, consumer: { subjects: ["events.*.>"] }
560
612
 
561
613
  def process(messages)
562
- events = messages.map(&:data)
563
- aggregates = events.group_by { |e| e["type"] }.transform_values(&:count)
614
+ aggregates = messages.map(&:data).group_by { |e| e["type"] }.transform_values(&:count)
564
615
  Analytics.bulk_insert(aggregates)
565
616
  messages.each(&:ack)
566
617
  end
567
618
  end
568
619
  ```
569
620
 
621
+ ---
570
622
 
571
623
  <div align="center">
572
624
 
@@ -0,0 +1,46 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Cosmo
4
+ module ActiveJobAdapter
5
+ # ActiveJob queue adapter that enqueues jobs via NATS JetStream.
6
+ #
7
+ # Usage:
8
+ # config.active_job.queue_adapter = :cosmonats
9
+ # # or explicitly:
10
+ # config.active_job.queue_adapter = Cosmo::ActiveJobAdapter::Adapter.new
11
+ #
12
+ # The ActiveJob queue name maps directly to the Cosmo stream name.
13
+ class Adapter
14
+ # Enqueue a job to be run as soon as possible.
15
+ # @param job [ActiveJob::Base]
16
+ def enqueue(job)
17
+ publish(job, nil)
18
+ end
19
+
20
+ # Enqueue a job to be run at (or after) a given time.
21
+ # @param job [ActiveJob::Base]
22
+ # @param timestamp [Numeric] Unix timestamp (seconds, float)
23
+ def enqueue_at(job, timestamp)
24
+ publish(job, timestamp)
25
+ end
26
+
27
+ private
28
+
29
+ def publish(job, timestamp)
30
+ cosmo_opts = job_cosmo_options(job)
31
+ stream = cosmo_opts.delete(:stream) || job.queue_name.to_sym
32
+ options = { stream: stream }.merge(cosmo_opts)
33
+ options[:at] = timestamp if timestamp
34
+
35
+ data = Job::Data.new(Executor.name, [job.serialize], options)
36
+ Publisher.publish_job(data)
37
+ end
38
+
39
+ # Returns Cosmo-specific options declared on the job class via
40
+ # +cosmo_options+, falling back to an empty hash.
41
+ def job_cosmo_options(job)
42
+ job.class.respond_to?(:get_cosmo_options) ? job.class.get_cosmo_options : {}
43
+ end
44
+ end
45
+ end
46
+ end
@@ -0,0 +1,16 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Cosmo
4
+ module ActiveJobAdapter
5
+ # Cosmo::Job that deserializes and executes an ActiveJob payload
6
+ class Executor
7
+ include Cosmo::Job
8
+
9
+ options stream: :default
10
+
11
+ def perform(job_data)
12
+ ::ActiveJob::Base.execute(Utils::Hash.stringify_keys(job_data))
13
+ end
14
+ end
15
+ end
16
+ end