lepus 0.0.1.beta2 → 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.github/workflows/linter.yml +21 -0
- data/.github/workflows/specs.yml +93 -13
- data/.gitignore +2 -0
- data/.rubocop.yml +10 -0
- data/.tool-versions +1 -1
- data/Gemfile +7 -0
- data/Gemfile.lock +36 -9
- data/Makefile +19 -0
- data/README.md +562 -7
- data/bin/setup +5 -2
- data/config.ru +14 -0
- data/docker-compose.yml +5 -3
- data/docs/README.md +80 -0
- data/docs/cli.md +108 -0
- data/docs/configuration.md +171 -0
- data/docs/consumers.md +168 -0
- data/docs/getting-started.md +136 -0
- data/docs/images/lepus-web.png +0 -0
- data/docs/middleware.md +240 -0
- data/docs/producers.md +173 -0
- data/docs/prometheus.md +112 -0
- data/docs/rails.md +161 -0
- data/docs/supervisor.md +112 -0
- data/docs/testing.md +141 -0
- data/docs/web.md +85 -0
- data/examples/grafana-dashboard.json +450 -0
- data/gemfiles/Gemfile.rails-5.2 +7 -0
- data/gemfiles/{rails52.gemfile.lock → Gemfile.rails-5.2.lock} +102 -69
- data/gemfiles/Gemfile.rails-6.1 +7 -0
- data/gemfiles/{rails61.gemfile.lock → Gemfile.rails-6.1.lock} +113 -79
- data/gemfiles/{rails52.gemfile → Gemfile.rails-7.2} +1 -1
- data/gemfiles/Gemfile.rails-7.2.lock +321 -0
- data/gemfiles/{rails61.gemfile → Gemfile.rails-8.0} +1 -1
- data/gemfiles/Gemfile.rails-8.0.lock +322 -0
- data/lepus.gemspec +7 -1
- data/lib/lepus/cli.rb +35 -4
- data/lib/lepus/configuration.rb +107 -0
- data/lib/lepus/connection_pool.rb +135 -0
- data/lib/lepus/consumer.rb +59 -41
- data/lib/lepus/consumers/config.rb +183 -0
- data/lib/lepus/consumers/handler.rb +56 -0
- data/lib/lepus/consumers/middleware_chain.rb +22 -0
- data/lib/lepus/consumers/middlewares/exception_logger.rb +27 -0
- data/lib/lepus/consumers/middlewares/honeybadger.rb +33 -0
- data/lib/lepus/consumers/middlewares/json.rb +37 -0
- data/lib/lepus/consumers/middlewares/max_retry.rb +83 -0
- data/lib/lepus/consumers/middlewares/unique.rb +65 -0
- data/lib/lepus/consumers/stats.rb +70 -0
- data/lib/lepus/consumers/stats_registry.rb +29 -0
- data/lib/lepus/consumers/worker.rb +141 -0
- data/lib/lepus/consumers/worker_factory.rb +124 -0
- data/lib/lepus/consumers.rb +6 -0
- data/lib/lepus/message/delivery_info.rb +72 -0
- data/lib/lepus/message/metadata.rb +99 -0
- data/lib/lepus/message.rb +88 -5
- data/lib/lepus/middleware_chain.rb +83 -0
- data/lib/lepus/primitive/hash.rb +29 -0
- data/lib/lepus/process.rb +24 -24
- data/lib/lepus/process_registry/backend.rb +49 -0
- data/lib/lepus/process_registry/file_backend.rb +108 -0
- data/lib/lepus/process_registry/message_builder.rb +72 -0
- data/lib/lepus/process_registry/rabbitmq_backend.rb +153 -0
- data/lib/lepus/process_registry.rb +56 -23
- data/lib/lepus/processes/base.rb +0 -5
- data/lib/lepus/processes/callbacks.rb +3 -0
- data/lib/lepus/processes/interruptible.rb +4 -8
- data/lib/lepus/processes/procline.rb +1 -1
- data/lib/lepus/processes/registrable.rb +1 -1
- data/lib/lepus/processes/runnable.rb +1 -1
- data/lib/lepus/processes.rb +15 -0
- data/lib/lepus/producer.rb +141 -30
- data/lib/lepus/producers/config.rb +46 -0
- data/lib/lepus/producers/definition.rb +48 -0
- data/lib/lepus/producers/hooks.rb +170 -0
- data/lib/lepus/producers/middleware_chain.rb +22 -0
- data/lib/lepus/producers/middlewares/correlation_id.rb +37 -0
- data/lib/lepus/producers/middlewares/header.rb +47 -0
- data/lib/lepus/producers/middlewares/instrumentation.rb +30 -0
- data/lib/lepus/producers/middlewares/json.rb +47 -0
- data/lib/lepus/producers/middlewares/unique.rb +67 -0
- data/lib/lepus/producers.rb +7 -0
- data/lib/lepus/prometheus/collector.rb +149 -0
- data/lib/lepus/prometheus/instrumentation.rb +168 -0
- data/lib/lepus/prometheus.rb +48 -0
- data/lib/lepus/publisher.rb +67 -0
- data/lib/lepus/supervisor/children_pipes.rb +25 -0
- data/lib/lepus/supervisor/lifecycle_hooks.rb +50 -0
- data/lib/lepus/supervisor/pidfiled.rb +1 -1
- data/lib/lepus/supervisor/registry_cleaner.rb +22 -0
- data/lib/lepus/supervisor.rb +129 -25
- data/lib/lepus/testing/exchange.rb +95 -0
- data/lib/lepus/testing/message_builder.rb +177 -0
- data/lib/lepus/testing/rspec_matchers.rb +258 -0
- data/lib/lepus/testing.rb +210 -0
- data/lib/lepus/unique.rb +18 -0
- data/lib/lepus/version.rb +1 -1
- data/lib/lepus/web/aggregator.rb +154 -0
- data/lib/lepus/web/api.rb +132 -0
- data/lib/lepus/web/app.rb +37 -0
- data/lib/lepus/web/management_api.rb +192 -0
- data/lib/lepus/web/respond_with.rb +28 -0
- data/lib/lepus/web.rb +238 -0
- data/lib/lepus.rb +39 -28
- data/test_offline.html +189 -0
- data/web/assets/css/styles.css +635 -0
- data/web/assets/js/app.js +6 -0
- data/web/assets/js/bootstrap.js +20 -0
- data/web/assets/js/controllers/connection_controller.js +44 -0
- data/web/assets/js/controllers/dashboard_controller.js +499 -0
- data/web/assets/js/controllers/queue_controller.js +17 -0
- data/web/assets/js/controllers/theme_controller.js +31 -0
- data/web/assets/js/offline-manager.js +233 -0
- data/web/assets/js/service-worker-manager.js +65 -0
- data/web/index.html +159 -0
- data/web/sw.js +144 -0
- metadata +177 -18
- data/lib/lepus/consumer_config.rb +0 -149
- data/lib/lepus/consumer_wrapper.rb +0 -46
- data/lib/lepus/lifecycle_hooks.rb +0 -49
- data/lib/lepus/middlewares/honeybadger.rb +0 -23
- data/lib/lepus/middlewares/json.rb +0 -35
- data/lib/lepus/middlewares/max_retry.rb +0 -57
- data/lib/lepus/processes/consumer.rb +0 -113
- data/lib/lepus/supervisor/config.rb +0 -45
data/README.md
CHANGED
|
@@ -1,7 +1,12 @@
|
|
|
1
1
|
# Lepus
|
|
2
2
|
|
|
3
|
+

|
|
3
4
|
|
|
4
|
-
Lepus is a
|
|
5
|
+
Lepus is a lightweight but powerful Ruby library to help you to consume and produce messages to [RabbitMQ](https://www.rabbitmq.com/) using the [Bunny](https://github.com/ruby-amqp/bunny) gem. It's similar to the Sidekiq, Faktory, ActiveJob, SolidQueue, and other libraries, but using RabbitMQ as the message broker.
|
|
6
|
+
|
|
7
|
+
## Documentation
|
|
8
|
+
|
|
9
|
+
Full guides, consumer/producer recipes, middleware reference, and the web dashboard walkthrough are published at **[gems.marcosz.com.br/lepus](https://gems.marcosz.com.br/lepus/)** — part of the [marcosgz Ruby gem catalogue](https://gems.marcosz.com.br).
|
|
5
10
|
|
|
6
11
|
## Installation
|
|
7
12
|
|
|
@@ -28,7 +33,16 @@ gem install lepus
|
|
|
28
33
|
|
|
29
34
|
## Configuration
|
|
30
35
|
|
|
31
|
-
You can configure the Lepus using the `Lepus.configure` method.
|
|
36
|
+
You can configure the Lepus using the `Lepus.configure` method.
|
|
37
|
+
|
|
38
|
+
```ruby
|
|
39
|
+
Lepus.configure do |config|
|
|
40
|
+
config.connection_name = 'MyApp'
|
|
41
|
+
config.rabbitmq_url = ENV.fetch('RABBITMQ_URL', 'amqp://guest:guest@localhost:5672')
|
|
42
|
+
end
|
|
43
|
+
```
|
|
44
|
+
|
|
45
|
+
The configuration options are:
|
|
32
46
|
|
|
33
47
|
- `rabbitmq_url`: The RabbitMQ host. Default: to `RABBITMQ_URL` environment variable or `amqp://guest:guest@localhost:5672`.
|
|
34
48
|
- `connection_name`: The connection name. Default: `Lepus`.
|
|
@@ -37,17 +51,265 @@ You can configure the Lepus using the `Lepus.configure` method. The configuratio
|
|
|
37
51
|
- `app_executor`: The [Rails executor](https://guides.rubyonrails.org/threading_and_code_execution.html#executor) used to wrap asynchronous operations. Only available if you are using Rails. Default: `nil`.
|
|
38
52
|
- `on_thread_error`: The block to be executed when an error occurs on the thread. Default: `nil`.
|
|
39
53
|
- `process_heartbeat_interval`: The interval in seconds between heartbeats. Default is `60 seconds`.
|
|
40
|
-
- `
|
|
54
|
+
- `process_heartbeat_timeout`: The timeout in seconds to wait for a heartbeat. Default is `10 seconds`.
|
|
55
|
+
- `worker`: A block to configure the worker process that will run the consumers. You can set the `pool_size`, `pool_timeout`, and before/after fork callbacks inline options or using a block. Main worker is `:default`, but you can define more workers with different names for different consumers.
|
|
56
|
+
- `logger`: The logger instance. Default: `Logger.new($stdout)`.
|
|
57
|
+
|
|
58
|
+
### Configuration > Producer
|
|
59
|
+
|
|
60
|
+
Lepus can be used to both **produce** and **consume** RabbitMQ events. Producers and consumers use separate connection pools, allowing for efficient and isolated message publishing and processing.
|
|
41
61
|
|
|
62
|
+
You can configure the producer connection pool using the `producer` method inside the configuration block:
|
|
42
63
|
|
|
43
64
|
```ruby
|
|
44
65
|
Lepus.configure do |config|
|
|
45
|
-
|
|
46
|
-
config.
|
|
66
|
+
# Block
|
|
67
|
+
config.producer do |c|
|
|
68
|
+
c.pool_size = 2
|
|
69
|
+
c.pool_timeout = 10.0
|
|
70
|
+
end
|
|
71
|
+
# Inline
|
|
72
|
+
config.producer(pool_size: 1, pool_timeout: 5.0)
|
|
73
|
+
end
|
|
74
|
+
```
|
|
75
|
+
|
|
76
|
+
Once configured, you can use `Lepus::Publisher` to publish messages to RabbitMQ exchanges:
|
|
77
|
+
|
|
78
|
+
```ruby
|
|
79
|
+
# Create a publisher for a specific exchange
|
|
80
|
+
publisher = Lepus::Publisher.new("my_exchange", type: :topic, durable: true)
|
|
81
|
+
|
|
82
|
+
# Publish a string message
|
|
83
|
+
publisher.publish("Hello, RabbitMQ!")
|
|
84
|
+
|
|
85
|
+
# Publish a JSON message (automatically serialized)
|
|
86
|
+
publisher.publish({user_id: 123, action: "login"}, routing_key: "user.login")
|
|
87
|
+
|
|
88
|
+
# Publish with custom options
|
|
89
|
+
publisher.publish("Important message",
|
|
90
|
+
routing_key: "notifications.urgent",
|
|
91
|
+
expiration: 30000,
|
|
92
|
+
priority: 10
|
|
93
|
+
)
|
|
94
|
+
```
|
|
95
|
+
|
|
96
|
+
### Using Lepus::Producer
|
|
97
|
+
|
|
98
|
+
For a more structured approach, you can use `Lepus::Producer` to define reusable producer classes with pre-configured exchange settings:
|
|
99
|
+
|
|
100
|
+
```ruby
|
|
101
|
+
# Define a producer with exchange configuration
|
|
102
|
+
class UserEventsProducer < Lepus::Producer
|
|
103
|
+
configure(exchange: "user_events")
|
|
104
|
+
end
|
|
105
|
+
|
|
106
|
+
# Define a producer with detailed exchange and publish options
|
|
107
|
+
class OrderEventsProducer < Lepus::Producer
|
|
108
|
+
configure(
|
|
109
|
+
exchange: {
|
|
110
|
+
name: "order_events",
|
|
111
|
+
type: :direct,
|
|
112
|
+
durable: true
|
|
113
|
+
},
|
|
114
|
+
publish: {
|
|
115
|
+
persistent: true,
|
|
116
|
+
mandatory: false
|
|
117
|
+
}
|
|
118
|
+
)
|
|
47
119
|
end
|
|
120
|
+
|
|
121
|
+
# Define a producer with block configuration
|
|
122
|
+
class NotificationProducer < Lepus::Producer
|
|
123
|
+
configure(exchange: "notifications") do |definition|
|
|
124
|
+
definition.publish_options[:persistent] = true
|
|
125
|
+
end
|
|
126
|
+
end
|
|
127
|
+
|
|
128
|
+
# Usage examples:
|
|
129
|
+
|
|
130
|
+
# Publish using class methods
|
|
131
|
+
UserEventsProducer.publish("User created: 123")
|
|
132
|
+
OrderEventsProducer.publish(
|
|
133
|
+
{ order_id: 456, status: "created" },
|
|
134
|
+
routing_key: "order.created"
|
|
135
|
+
)
|
|
136
|
+
|
|
137
|
+
# Publish using instance methods
|
|
138
|
+
producer = NotificationProducer.new
|
|
139
|
+
producer.publish(
|
|
140
|
+
{ message: "Welcome!", user_id: 789 },
|
|
141
|
+
routing_key: "user.welcome"
|
|
142
|
+
)
|
|
48
143
|
```
|
|
49
144
|
|
|
50
|
-
|
|
145
|
+
The `Lepus::Producer` class provides:
|
|
146
|
+
- **Pre-configured exchanges**: Define exchange settings once in your producer class
|
|
147
|
+
- **Default publish options**: Set default publish behavior (persistent, mandatory, etc.)
|
|
148
|
+
- **Class and instance methods**: Use either `ProducerClass.publish()` or `producer_instance.publish()`
|
|
149
|
+
- **Block configuration**: Fine-tune settings using configuration blocks
|
|
150
|
+
|
|
151
|
+
### Producer Hooks
|
|
152
|
+
|
|
153
|
+
Lepus provides a powerful hooks system that allows you to control when producers can publish messages. This is particularly useful for testing, debugging, or temporarily disabling message publishing in specific environments.
|
|
154
|
+
|
|
155
|
+
#### Basic Usage
|
|
156
|
+
|
|
157
|
+
```ruby
|
|
158
|
+
# Disable all producers
|
|
159
|
+
Lepus::Producers.disable!
|
|
160
|
+
|
|
161
|
+
# Enable all producers
|
|
162
|
+
Lepus::Producers.enable!
|
|
163
|
+
|
|
164
|
+
# Disable specific producers
|
|
165
|
+
Lepus::Producers.disable!(UserEventsProducer, OrderEventsProducer)
|
|
166
|
+
|
|
167
|
+
# Enable specific producers
|
|
168
|
+
Lepus::Producers.enable!(UserEventsProducer)
|
|
169
|
+
|
|
170
|
+
# Disable by exchange name (affects all producers using that exchange)
|
|
171
|
+
Lepus::Producers.disable!("user_events", "order_events")
|
|
172
|
+
|
|
173
|
+
# Enable by exchange name
|
|
174
|
+
Lepus::Producers.enable!("notifications")
|
|
175
|
+
|
|
176
|
+
# Check if producers are enabled/disabled
|
|
177
|
+
Lepus::Producers.enabled?(UserEventsProducer) # => true/false
|
|
178
|
+
Lepus::Producers.disabled?(UserEventsProducer) # => true/false
|
|
179
|
+
```
|
|
180
|
+
|
|
181
|
+
#### Block-based Control
|
|
182
|
+
|
|
183
|
+
The hooks system provides block-based methods for temporary control:
|
|
184
|
+
|
|
185
|
+
```ruby
|
|
186
|
+
# Temporarily disable publishing for a block
|
|
187
|
+
Lepus::Producers.without_publishing do
|
|
188
|
+
# All producer.publish() calls will be ignored
|
|
189
|
+
UserEventsProducer.publish("This won't be sent")
|
|
190
|
+
OrderEventsProducer.publish("This won't be sent either")
|
|
191
|
+
end
|
|
192
|
+
# Publishing is automatically restored after the block
|
|
193
|
+
|
|
194
|
+
# Temporarily disable specific producers
|
|
195
|
+
Lepus::Producers.without_publishing(UserEventsProducer) do
|
|
196
|
+
UserEventsProducer.publish("This won't be sent") # Disabled
|
|
197
|
+
OrderEventsProducer.publish("This will be sent") # Still enabled
|
|
198
|
+
end
|
|
199
|
+
|
|
200
|
+
# Temporarily enable publishing for a block
|
|
201
|
+
Lepus::Producers.disable!
|
|
202
|
+
Lepus::Producers.with_publishing do
|
|
203
|
+
# Publishing is temporarily enabled
|
|
204
|
+
UserEventsProducer.publish("This will be sent")
|
|
205
|
+
end
|
|
206
|
+
# Publishing is automatically restored to disabled state
|
|
207
|
+
```
|
|
208
|
+
|
|
209
|
+
### Producer Middlewares
|
|
210
|
+
|
|
211
|
+
Producers support middlewares that can modify the message payload, headers, routing key, and other publish options before messages are sent to RabbitMQ. Middlewares are executed in the order they are registered.
|
|
212
|
+
|
|
213
|
+
#### Built-in Middlewares
|
|
214
|
+
|
|
215
|
+
* `:json`: Serializes Hash payloads to JSON and sets `content_type` to `application/json`.
|
|
216
|
+
* `:header`: Adds default headers to messages (static values or dynamic procs).
|
|
217
|
+
* `:correlation_id`: Auto-generates a `correlation_id` (UUID) if not already set.
|
|
218
|
+
* `:instrumentation`: Emits instrumentation events via `Lepus.instrument` for monitoring.
|
|
219
|
+
|
|
220
|
+
#### Per-Producer Middlewares
|
|
221
|
+
|
|
222
|
+
Use the `use` method to add middlewares to a specific producer:
|
|
223
|
+
|
|
224
|
+
```ruby
|
|
225
|
+
class OrderEventsProducer < Lepus::Producer
|
|
226
|
+
configure(exchange: "order_events")
|
|
227
|
+
|
|
228
|
+
use :json
|
|
229
|
+
use :correlation_id
|
|
230
|
+
use :header, defaults: {
|
|
231
|
+
"app" => "my-service",
|
|
232
|
+
"published_at" => -> { Time.now.iso8601 }
|
|
233
|
+
}
|
|
234
|
+
end
|
|
235
|
+
|
|
236
|
+
# Messages will be serialized to JSON, have a correlation_id,
|
|
237
|
+
# and include the default headers
|
|
238
|
+
OrderEventsProducer.publish({ order_id: 123, status: "created" })
|
|
239
|
+
```
|
|
240
|
+
|
|
241
|
+
#### Global Producer Middlewares
|
|
242
|
+
|
|
243
|
+
You can configure middlewares that apply to all producers:
|
|
244
|
+
|
|
245
|
+
```ruby
|
|
246
|
+
Lepus.configure do |config|
|
|
247
|
+
config.producer_middlewares do |chain|
|
|
248
|
+
chain.use :instrumentation
|
|
249
|
+
chain.use :correlation_id
|
|
250
|
+
end
|
|
251
|
+
end
|
|
252
|
+
```
|
|
253
|
+
|
|
254
|
+
Global middlewares are executed before per-producer middlewares.
|
|
255
|
+
|
|
256
|
+
#### Custom Producer Middlewares
|
|
257
|
+
|
|
258
|
+
Create custom middlewares by extending `Lepus::Middleware` (same interface as consumer middlewares):
|
|
259
|
+
|
|
260
|
+
```ruby
|
|
261
|
+
class TimestampMiddleware < Lepus::Middleware
|
|
262
|
+
def call(message, app)
|
|
263
|
+
# Add a timestamp header
|
|
264
|
+
current_headers = message.metadata.headers || {}
|
|
265
|
+
new_headers = current_headers.merge("published_at" => Time.now.iso8601)
|
|
266
|
+
|
|
267
|
+
new_metadata = Lepus::Message::Metadata.new(
|
|
268
|
+
**message.metadata.to_h,
|
|
269
|
+
headers: new_headers
|
|
270
|
+
)
|
|
271
|
+
|
|
272
|
+
# Pass the modified message to the next middleware
|
|
273
|
+
app.call(message.mutate(metadata: new_metadata))
|
|
274
|
+
end
|
|
275
|
+
end
|
|
276
|
+
|
|
277
|
+
class MyProducer < Lepus::Producer
|
|
278
|
+
configure(exchange: "my_exchange")
|
|
279
|
+
|
|
280
|
+
use TimestampMiddleware
|
|
281
|
+
end
|
|
282
|
+
```
|
|
283
|
+
|
|
284
|
+
### Configuration > Consumer Worker
|
|
285
|
+
|
|
286
|
+
You can configure the consumer process using the `worker` method. The default worker is named `:default`, but you can define more workers with different names for different consumers.
|
|
287
|
+
|
|
288
|
+
Configuration can be done inline or using a block:
|
|
289
|
+
|
|
290
|
+
```ruby
|
|
291
|
+
Lepus.configure do |config|
|
|
292
|
+
# Block
|
|
293
|
+
config.worker(:default) do |c|
|
|
294
|
+
c.pool_size = 2
|
|
295
|
+
c.pool_timeout = 10.0
|
|
296
|
+
c.before_fork do
|
|
297
|
+
ActiveRecord::Base.clear_all_connections!
|
|
298
|
+
end
|
|
299
|
+
c.after_fork do
|
|
300
|
+
ActiveRecord::Base.establish_connection
|
|
301
|
+
end
|
|
302
|
+
end
|
|
303
|
+
# Inline
|
|
304
|
+
config.worker(:datasync, pool_size: 1, pool_timeout: 5.0)
|
|
305
|
+
end
|
|
306
|
+
```
|
|
307
|
+
|
|
308
|
+
The options are:
|
|
309
|
+
- `pool_size`: The number of threads in the pool. Default: `1`.
|
|
310
|
+
- `pool_timeout`: The timeout in seconds to wait for a thread to be available. Default: `5.0`.
|
|
311
|
+
- `before_fork`: A block to be executed before forking the process. Default: `nil`.
|
|
312
|
+
- `after_fork`: A block to be executed after forking the process. Default: `nil`.
|
|
51
313
|
|
|
52
314
|
To define a consumer, you need to create a class inheriting from `Lepus::Consumer` and implement the `perform` method. The `perform` method will be called when a message is received. Use the `configure` method to set the queue name, exchange name, and other options.
|
|
53
315
|
|
|
@@ -132,11 +394,21 @@ Consumers can use middlewares for recurring tasks like logging, error handling,
|
|
|
132
394
|
|
|
133
395
|
* `:max_retry`: Rejects the message and routes it to the error queue after a number of attempts.
|
|
134
396
|
* `:json`: Parses the message payload as JSON.
|
|
397
|
+
* `:honeybadger`: Reports exceptions to Honeybadger.
|
|
398
|
+
* `:exception_logger`: Logs unhandled exceptions to `Lepus.logger` (or a custom logger) and re-raises.
|
|
135
399
|
|
|
136
400
|
You can use the `use` method to add middlewares to the consumer:
|
|
137
401
|
|
|
138
402
|
```ruby
|
|
139
403
|
class MyConsumer < Lepus::Consumer
|
|
404
|
+
configure(...)
|
|
405
|
+
|
|
406
|
+
# If you don't have an error-reporting middleware (e.g. Honeybadger, Airbrake),
|
|
407
|
+
# add :exception_logger to ensure errors are actually logged.
|
|
408
|
+
use(
|
|
409
|
+
:exception_logger
|
|
410
|
+
)
|
|
411
|
+
|
|
140
412
|
use(
|
|
141
413
|
:max_retry,
|
|
142
414
|
retries: 6,
|
|
@@ -146,7 +418,7 @@ class MyConsumer < Lepus::Consumer
|
|
|
146
418
|
use(
|
|
147
419
|
:json,
|
|
148
420
|
symbolize_keys: true,
|
|
149
|
-
on_error: proc { :
|
|
421
|
+
on_error: proc { :reject } # You can omit since the default value is :reject
|
|
150
422
|
)
|
|
151
423
|
|
|
152
424
|
def perform(message)
|
|
@@ -156,6 +428,23 @@ class MyConsumer < Lepus::Consumer
|
|
|
156
428
|
end
|
|
157
429
|
```
|
|
158
430
|
|
|
431
|
+
> Important: If you are not using an external error-reporting middleware like Honeybadger or Airbrake, make sure to add `:exception_logger` to all consumers. The worker execution flow rescues exceptions to keep the process alive; without a logging middleware, exceptions may be swallowed and go unnoticed. `:exception_logger` ensures the error message is written to your logs.
|
|
432
|
+
|
|
433
|
+
#### Global Consumer Middlewares
|
|
434
|
+
|
|
435
|
+
You can configure middlewares that apply to all consumers:
|
|
436
|
+
|
|
437
|
+
```ruby
|
|
438
|
+
Lepus.configure do |config|
|
|
439
|
+
config.consumer_middlewares do |chain|
|
|
440
|
+
chain.use :exception_logger
|
|
441
|
+
chain.use :json, symbolize_keys: true
|
|
442
|
+
end
|
|
443
|
+
end
|
|
444
|
+
```
|
|
445
|
+
|
|
446
|
+
Global middlewares are executed before per-consumer middlewares. This is useful for common cross-cutting concerns like logging or JSON parsing that you want applied consistently across all consumers.
|
|
447
|
+
|
|
159
448
|
You can also create your own middlewares, just create subclasses of `Lepus::Middleware` and implement the `call` method:
|
|
160
449
|
|
|
161
450
|
```ruby
|
|
@@ -171,6 +460,97 @@ class MyMiddleware < Lepus::Middleware
|
|
|
171
460
|
end
|
|
172
461
|
```
|
|
173
462
|
|
|
463
|
+
### Unique Middleware (Experimental)
|
|
464
|
+
|
|
465
|
+
> **Note:** This feature is experimental and may change in future versions.
|
|
466
|
+
|
|
467
|
+
The unique middleware prevents duplicate messages from being published using Redis-based distributed locking via the [de-dupe](https://github.com/marcosgz/de-dupe) gem. It works as a pair: the **producer middleware acquires a lock** before publishing, and the **consumer middleware releases the lock** after successful processing (`:ack`).
|
|
468
|
+
|
|
469
|
+
Multiple producers can share the same lock namespace. For example, `StoryCreatedProducer` and `StoryUpdatedProducer` can both use `lock_key: "story"` to prevent duplicate processing of the same story.
|
|
470
|
+
|
|
471
|
+
#### Setup
|
|
472
|
+
|
|
473
|
+
Add the `de-dupe` gem to your Gemfile:
|
|
474
|
+
|
|
475
|
+
```ruby
|
|
476
|
+
gem 'de-dupe'
|
|
477
|
+
```
|
|
478
|
+
|
|
479
|
+
Configure DeDupe with Redis, then require the middleware:
|
|
480
|
+
|
|
481
|
+
```ruby
|
|
482
|
+
# In an initializer or application setup:
|
|
483
|
+
DeDupe.configure do |config|
|
|
484
|
+
config.redis = Redis.new(url: ENV["REDIS_URL"])
|
|
485
|
+
end
|
|
486
|
+
|
|
487
|
+
require "lepus/unique"
|
|
488
|
+
```
|
|
489
|
+
|
|
490
|
+
The `require "lepus/unique"` call will raise an error if `de-dupe` is not installed or DeDupe is not configured with Redis.
|
|
491
|
+
|
|
492
|
+
#### Producer Usage
|
|
493
|
+
|
|
494
|
+
```ruby
|
|
495
|
+
class StoryCreatedProducer < Lepus::Producer
|
|
496
|
+
configure(exchange: "story_created")
|
|
497
|
+
use :json
|
|
498
|
+
use :unique, lock_key: "story", lock_id: ->(msg) { msg.payload[:story_id].to_s }, ttl: 3600
|
|
499
|
+
end
|
|
500
|
+
|
|
501
|
+
class StoryUpdatedProducer < Lepus::Producer
|
|
502
|
+
configure(exchange: "story_updated")
|
|
503
|
+
use :json
|
|
504
|
+
use :unique, lock_key: "story", lock_id: ->(msg) { msg.payload[:story_id].to_s }
|
|
505
|
+
end
|
|
506
|
+
```
|
|
507
|
+
|
|
508
|
+
Options:
|
|
509
|
+
- `lock_key` (required): Shared lock namespace (e.g., `"story"`).
|
|
510
|
+
- `lock_id` (required): A `Proc` that extracts a unique identifier from the message. If it returns `nil`, deduplication is skipped.
|
|
511
|
+
- `ttl` (optional): Lock TTL in seconds. Defaults to the DeDupe global configuration. The TTL is passed to the consumer via message headers (`x-dedupe-lock-ttl`), so the consumer uses the same TTL when releasing the lock.
|
|
512
|
+
|
|
513
|
+
When a duplicate is detected (lock already held), the publish is **silently skipped**.
|
|
514
|
+
|
|
515
|
+
#### Consumer Usage
|
|
516
|
+
|
|
517
|
+
Register the `:unique` middleware on your consumer. It reads lock information from message headers set by the producer (`x-dedupe-lock-key`, `x-dedupe-lock-id`, and optionally `x-dedupe-lock-ttl`):
|
|
518
|
+
|
|
519
|
+
```ruby
|
|
520
|
+
class StoryConsumer < Lepus::Consumer
|
|
521
|
+
configure(
|
|
522
|
+
queue: "stories",
|
|
523
|
+
exchange: "story_created",
|
|
524
|
+
routing_key: %w[story.created story.updated]
|
|
525
|
+
)
|
|
526
|
+
use :json, symbolize_keys: true
|
|
527
|
+
use :unique
|
|
528
|
+
|
|
529
|
+
def perform(message)
|
|
530
|
+
story_id = message.payload[:story_id]
|
|
531
|
+
Story.find(story_id).process!
|
|
532
|
+
|
|
533
|
+
ack!
|
|
534
|
+
rescue ActiveRecord::RecordNotFound
|
|
535
|
+
reject!
|
|
536
|
+
end
|
|
537
|
+
end
|
|
538
|
+
```
|
|
539
|
+
|
|
540
|
+
By default, the lock is **only released when the consumer returns `:ack`**. On `:reject`, `:requeue`, or `:nack`, the lock remains held so that retries are still deduplicated.
|
|
541
|
+
|
|
542
|
+
You can customize this behavior with the `release_on` option:
|
|
543
|
+
|
|
544
|
+
```ruby
|
|
545
|
+
# Release on ack or reject (e.g., message is permanently handled either way)
|
|
546
|
+
use :unique, release_on: [:ack, :reject]
|
|
547
|
+
|
|
548
|
+
# Release on error too (e.g., dead-letter scenarios where you don't want the lock held)
|
|
549
|
+
use :unique, release_on: [:ack, :error]
|
|
550
|
+
```
|
|
551
|
+
|
|
552
|
+
Valid values for `release_on`: `:ack`, `:reject`, `:requeue`, `:nack`, `:error`. When `:error` is included, the lock is released if the downstream raises an exception, and the exception is re-raised after release.
|
|
553
|
+
|
|
174
554
|
## Starting the Consumer Process
|
|
175
555
|
|
|
176
556
|
To start the consumer, can use the `lepus` CLI:
|
|
@@ -197,6 +577,181 @@ plugin :lepus
|
|
|
197
577
|
|
|
198
578
|
**Note**: The Puma plugin is only available if you are using Puma 6.x or higher.
|
|
199
579
|
|
|
580
|
+
## Web UI Dashboard
|
|
581
|
+
|
|
582
|
+
Lepus includes a built-in web dashboard that provides a real-time view of your message processing infrastructure. The dashboard allows you to monitor processes, queues, connections, and consumer performance.
|
|
583
|
+
|
|
584
|
+

|
|
585
|
+
|
|
586
|
+
### Starting the Web Dashboard
|
|
587
|
+
|
|
588
|
+
You can start the web dashboard using the `lepus web` command:
|
|
589
|
+
|
|
590
|
+
```bash
|
|
591
|
+
bundle exec lepus web
|
|
592
|
+
```
|
|
593
|
+
|
|
594
|
+
The dashboard will be available at `http://localhost:9292` by default. You can customize the host and port:
|
|
595
|
+
|
|
596
|
+
```bash
|
|
597
|
+
bundle exec lepus web --port 3000 --host 127.0.0.1
|
|
598
|
+
```
|
|
599
|
+
|
|
600
|
+
### Web Dashboard Features
|
|
601
|
+
|
|
602
|
+
The Lepus web dashboard provides:
|
|
603
|
+
|
|
604
|
+
- **Process Monitoring**: View all running supervisors and workers with their PIDs, memory usage, and heartbeat status
|
|
605
|
+
- **Queue Management**: Monitor queue statistics including message counts, consumer connections, and memory usage
|
|
606
|
+
- **Connection Tracking**: View active RabbitMQ connections and their states
|
|
607
|
+
- **Consumer Performance**: Track processed, rejected, and errored messages per consumer
|
|
608
|
+
- **Real-time Updates**: Dashboard automatically refreshes to show current system state
|
|
609
|
+
|
|
610
|
+
### Integrating with Rails
|
|
611
|
+
|
|
612
|
+
To integrate the Lepus web dashboard into your Rails application, you can mount it as a Rack application in your routes:
|
|
613
|
+
|
|
614
|
+
```ruby
|
|
615
|
+
# config/routes.rb
|
|
616
|
+
Rails.application.routes.draw do
|
|
617
|
+
# Your existing routes...
|
|
618
|
+
|
|
619
|
+
# Mount Lepus web dashboard (simple way)
|
|
620
|
+
mount Lepus::Web => "/lepus"
|
|
621
|
+
end
|
|
622
|
+
```
|
|
623
|
+
|
|
624
|
+
You can also use the more explicit syntax:
|
|
625
|
+
|
|
626
|
+
```ruby
|
|
627
|
+
# config/routes.rb
|
|
628
|
+
Rails.application.routes.draw do
|
|
629
|
+
# Your existing routes...
|
|
630
|
+
|
|
631
|
+
# Mount Lepus web dashboard (explicit way)
|
|
632
|
+
mount Lepus::Web::App.build => "/lepus"
|
|
633
|
+
end
|
|
634
|
+
```
|
|
635
|
+
|
|
636
|
+
This will make the dashboard available at `http://your-app.com/lepus` in your Rails application.
|
|
637
|
+
|
|
638
|
+
#### Process registry backend
|
|
639
|
+
|
|
640
|
+
Lepus tracks running supervisors and workers in a **process registry**. Two
|
|
641
|
+
backends are available:
|
|
642
|
+
|
|
643
|
+
- `:file` (default for a core `require "lepus"`) — stores process data in a
|
|
644
|
+
local file under `/tmp`. Fast and dependency-free, but the file is only
|
|
645
|
+
visible to processes that share the same filesystem.
|
|
646
|
+
- `:rabbitmq` — stores the same data in a dedicated RabbitMQ queue, so every
|
|
647
|
+
process connected to the same broker sees the same registry.
|
|
648
|
+
|
|
649
|
+
**Requiring `lepus/web` automatically switches the default to `:rabbitmq`.**
|
|
650
|
+
This is because the dashboard is almost always run in a separate process (and
|
|
651
|
+
often a separate container) from the workers, and the `:file` backend cannot
|
|
652
|
+
bridge that gap — you'd see an empty dashboard even with workers running. The
|
|
653
|
+
dashboard still needs the RabbitMQ Management API for queue/connection data,
|
|
654
|
+
but the registry is what lets it discover your workers.
|
|
655
|
+
|
|
656
|
+
If you really want the file backend even with the dashboard loaded, set it
|
|
657
|
+
explicitly after your `require`:
|
|
658
|
+
|
|
659
|
+
```ruby
|
|
660
|
+
# config/initializers/lepus.rb
|
|
661
|
+
Lepus.configure do |config|
|
|
662
|
+
config.process_registry_backend = :file
|
|
663
|
+
end
|
|
664
|
+
```
|
|
665
|
+
|
|
666
|
+
`Lepus::Web` is a plain Rack app, so authentication is applied by wrapping it
|
|
667
|
+
in standard Rack middleware or by gating the mount with a real auth helper.
|
|
668
|
+
Rails routing `constraints:` is **not** an authentication mechanism — a falsy
|
|
669
|
+
constraint returns 404 and never prompts for credentials.
|
|
670
|
+
|
|
671
|
+
HTTP Basic Auth (wrap the Rack app):
|
|
672
|
+
|
|
673
|
+
```ruby
|
|
674
|
+
# config/routes.rb
|
|
675
|
+
require "rack/auth/basic"
|
|
676
|
+
|
|
677
|
+
lepus_web = Rack::Builder.new do
|
|
678
|
+
use Rack::Auth::Basic, "Lepus Dashboard" do |username, password|
|
|
679
|
+
ActiveSupport::SecurityUtils.secure_compare(username, ENV.fetch("LEPUS_USER")) &
|
|
680
|
+
ActiveSupport::SecurityUtils.secure_compare(password, ENV.fetch("LEPUS_PASSWORD"))
|
|
681
|
+
end
|
|
682
|
+
run Lepus::Web
|
|
683
|
+
end
|
|
684
|
+
|
|
685
|
+
Rails.application.routes.draw do
|
|
686
|
+
mount lepus_web => "/lepus"
|
|
687
|
+
end
|
|
688
|
+
```
|
|
689
|
+
|
|
690
|
+
Devise (only admins can see the dashboard):
|
|
691
|
+
|
|
692
|
+
```ruby
|
|
693
|
+
# config/routes.rb
|
|
694
|
+
Rails.application.routes.draw do
|
|
695
|
+
authenticate :user, ->(u) { u.admin? } do
|
|
696
|
+
mount Lepus::Web => "/lepus"
|
|
697
|
+
end
|
|
698
|
+
end
|
|
699
|
+
```
|
|
700
|
+
|
|
701
|
+
## Prometheus metrics (optional)
|
|
702
|
+
|
|
703
|
+
Lepus ships an optional integration with
|
|
704
|
+
[`prometheus_exporter`](https://github.com/discourse/prometheus_exporter). It is
|
|
705
|
+
not a required dependency and is not auto-loaded — add the gem to your `Gemfile`
|
|
706
|
+
and require `lepus/prometheus` explicitly from the Lepus process you want to
|
|
707
|
+
instrument.
|
|
708
|
+
|
|
709
|
+
```ruby
|
|
710
|
+
# Gemfile
|
|
711
|
+
gem "prometheus_exporter"
|
|
712
|
+
```
|
|
713
|
+
|
|
714
|
+
```ruby
|
|
715
|
+
# e.g. config/initializers/lepus.rb, or at the top of your consumer boot script
|
|
716
|
+
require "lepus/prometheus"
|
|
717
|
+
|
|
718
|
+
# Optional: poll the RabbitMQ Management API for queue-level gauges
|
|
719
|
+
# from a single process (typically the supervisor).
|
|
720
|
+
Lepus::Prometheus.watch_queues(interval: 30)
|
|
721
|
+
```
|
|
722
|
+
|
|
723
|
+
Requiring `lepus/prometheus` installs the necessary hooks into
|
|
724
|
+
`Lepus::Consumers::Handler` (delivery counters and latency) and
|
|
725
|
+
`Lepus::Consumers::Worker` (process RSS gauge), and subscribes to
|
|
726
|
+
`publish.lepus` notifications (publish counters). Metrics are sent over TCP to
|
|
727
|
+
the `PrometheusExporter::Client.default` client.
|
|
728
|
+
|
|
729
|
+
On the exporter side, load the bundled type collector so the server knows how
|
|
730
|
+
to turn Lepus payloads into Prometheus metrics:
|
|
731
|
+
|
|
732
|
+
```bash
|
|
733
|
+
bundle exec prometheus_exporter -a lepus/prometheus/collector
|
|
734
|
+
```
|
|
735
|
+
|
|
736
|
+
Point Prometheus at the exporter (default port `9394`) and import
|
|
737
|
+
[`examples/grafana-dashboard.json`](examples/grafana-dashboard.json) into
|
|
738
|
+
Grafana. The dashboard covers every metric exposed by the collector.
|
|
739
|
+
|
|
740
|
+
### Exposed metrics
|
|
741
|
+
|
|
742
|
+
| Metric | Type | Labels | Source |
|
|
743
|
+
|-------------------------------------------|-----------|-----------------------------------|--------------------------------------------|
|
|
744
|
+
| `lepus_messages_processed_total` | counter | `consumer`, `queue`, `result` | `Handler#process_delivery` |
|
|
745
|
+
| `lepus_delivery_duration_seconds` | histogram | `consumer`, `queue` | `Handler#process_delivery` |
|
|
746
|
+
| `lepus_messages_published_total` | counter | `exchange`, `routing_key` | `publish.lepus` notification |
|
|
747
|
+
| `lepus_publish_duration_seconds` | histogram | `exchange`, `routing_key` | `publish.lepus` notification |
|
|
748
|
+
| `lepus_process_rss_memory_bytes` | gauge | `kind`, `name`, `pid` | `Worker#heartbeat` |
|
|
749
|
+
| `lepus_queue_messages` | gauge | `name` | `watch_queues` via management API |
|
|
750
|
+
| `lepus_queue_messages_ready` | gauge | `name` | `watch_queues` via management API |
|
|
751
|
+
| `lepus_queue_messages_unacknowledged` | gauge | `name` | `watch_queues` via management API |
|
|
752
|
+
| `lepus_queue_consumers` | gauge | `name` | `watch_queues` via management API |
|
|
753
|
+
| `lepus_queue_memory_bytes` | gauge | `name` | `watch_queues` via management API |
|
|
754
|
+
|
|
200
755
|
## Development
|
|
201
756
|
|
|
202
757
|
After checking out the repo, run `bin/setup` to install dependencies. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
|
data/bin/setup
CHANGED
|
@@ -3,5 +3,8 @@ set -euo pipefail
|
|
|
3
3
|
IFS=$'\n\t'
|
|
4
4
|
set -vx
|
|
5
5
|
|
|
6
|
-
bundle install
|
|
7
|
-
|
|
6
|
+
mise exec ruby@2.7.8 -- bundle install
|
|
7
|
+
mise exec ruby@2.7.8 -- bundle install --gemfile gemfiles/Gemfile.rails-5.2
|
|
8
|
+
mise exec ruby@2.7.8 -- bundle install --gemfile gemfiles/Gemfile.rails-6.1
|
|
9
|
+
mise exec ruby@3.2.2 -- bundle install --gemfile gemfiles/Gemfile.rails-7.2
|
|
10
|
+
mise exec ruby@3.2.2 -- bundle install --gemfile gemfiles/Gemfile.rails-8.0
|
data/config.ru
ADDED
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
$LOAD_PATH.unshift File.expand_path("lib", __dir__)
|
|
4
|
+
|
|
5
|
+
require "lepus"
|
|
6
|
+
require "lepus/web"
|
|
7
|
+
|
|
8
|
+
# Start web services for real data
|
|
9
|
+
Lepus::Web.start
|
|
10
|
+
|
|
11
|
+
# Graceful shutdown
|
|
12
|
+
at_exit { Lepus::Web.stop }
|
|
13
|
+
|
|
14
|
+
run Lepus::Web::App.build
|
data/docker-compose.yml
CHANGED