leopard 0.2.4 → 0.2.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.release-please-manifest.json +1 -1
- data/.version.txt +1 -1
- data/.yardopts +5 -0
- data/CHANGELOG.md +14 -0
- data/Rakefile +110 -1
- data/Readme.adoc +61 -1
- data/ci/nats/start.sh +26 -1
- data/examples/echo_endpoint.rb +6 -0
- data/examples/jetstream_endpoint.rb +44 -0
- data/lib/leopard/errors.rb +10 -0
- data/lib/leopard/message_processor.rb +86 -0
- data/lib/leopard/message_wrapper.rb +4 -0
- data/lib/leopard/metrics_server.rb +139 -0
- data/lib/leopard/nats_api_server.rb +218 -57
- data/lib/leopard/nats_jetstream_callbacks.rb +76 -0
- data/lib/leopard/nats_jetstream_consumer.rb +186 -0
- data/lib/leopard/nats_jetstream_endpoint.rb +19 -0
- data/lib/leopard/nats_request_reply_callbacks.rb +70 -0
- data/lib/leopard/templates/prometheus_metrics.erb +17 -0
- data/lib/leopard/version.rb +1 -1
- data/lib/leopard.rb +17 -0
- data/mise.toml +2 -0
- metadata +13 -3
|
@@ -6,26 +6,55 @@ require 'dry/configurable'
|
|
|
6
6
|
require 'concurrent'
|
|
7
7
|
require_relative '../leopard'
|
|
8
8
|
require_relative 'message_wrapper'
|
|
9
|
+
require_relative 'message_processor'
|
|
10
|
+
require_relative 'metrics_server'
|
|
11
|
+
require_relative 'nats_jetstream_endpoint'
|
|
12
|
+
require_relative 'nats_jetstream_consumer'
|
|
13
|
+
require_relative 'nats_request_reply_callbacks'
|
|
9
14
|
|
|
10
15
|
module Rubyists
|
|
11
16
|
module Leopard
|
|
17
|
+
# DSL and runtime integration for Leopard request/reply and JetStream workers.
|
|
12
18
|
module NatsApiServer
|
|
13
19
|
include Dry::Monads[:result]
|
|
14
20
|
extend Dry::Monads[:result]
|
|
15
21
|
|
|
22
|
+
# Extends an including class with Leopard's DSL and worker lifecycle behavior.
|
|
23
|
+
#
|
|
24
|
+
# @param base [Class] The class including this module.
|
|
25
|
+
#
|
|
26
|
+
# @return [void]
|
|
16
27
|
def self.included(base)
|
|
17
28
|
base.extend(ClassMethods)
|
|
18
|
-
base.include(
|
|
29
|
+
base.include(WorkerLifecycle)
|
|
30
|
+
base.include(MessageHandling)
|
|
19
31
|
base.extend(Dry::Monads[:result])
|
|
20
32
|
base.extend(Dry::Configurable)
|
|
21
33
|
base.setting :logger, default: Rubyists::Leopard.logger, reader: true
|
|
22
34
|
end
|
|
23
35
|
|
|
24
|
-
|
|
36
|
+
# Configuration for a request/reply endpoint declared with {.endpoint}.
|
|
37
|
+
Endpoint = Struct.new(:name, :subject, :queue, :group, :handler, keyword_init: true)
|
|
25
38
|
|
|
39
|
+
# Class-level DSL for defining Leopard endpoints, middleware, and worker startup.
|
|
26
40
|
module ClassMethods
|
|
41
|
+
include MetricsServer
|
|
42
|
+
|
|
43
|
+
# Returns the configured request/reply endpoints for the service class.
|
|
44
|
+
#
|
|
45
|
+
# @return [Array<Endpoint>] Declared request/reply endpoints.
|
|
27
46
|
def endpoints = @endpoints ||= []
|
|
47
|
+
# Returns the configured JetStream endpoints for the service class.
|
|
48
|
+
#
|
|
49
|
+
# @return [Array<NatsJetstreamEndpoint>] Declared JetStream pull-consumer endpoints.
|
|
50
|
+
def jetstream_endpoints = @jetstream_endpoints ||= []
|
|
51
|
+
# Returns the configured endpoint groups for the service class.
|
|
52
|
+
#
|
|
53
|
+
# @return [Hash{Symbol,String => Hash}] Declared group definitions.
|
|
28
54
|
def groups = @groups ||= {}
|
|
55
|
+
# Returns the configured middleware stack for the service class.
|
|
56
|
+
#
|
|
57
|
+
# @return [Array<Array>] Middleware declarations in registration order.
|
|
29
58
|
def middleware = @middleware ||= []
|
|
30
59
|
|
|
31
60
|
# Define an endpoint for the NATS API server.
|
|
@@ -35,12 +64,36 @@ module Rubyists
|
|
|
35
64
|
# @param queue [String, nil] The NATS queue group to use. Defaults to nil.
|
|
36
65
|
# @param group [String, nil] The group this endpoint belongs to. Defaults to nil.
|
|
37
66
|
# @param handler [Proc] The block that will handle incoming messages.
|
|
67
|
+
# @yield [wrapper] Handles the wrapped request message.
|
|
68
|
+
# @yieldparam wrapper [MessageWrapper] The wrapped inbound NATS message.
|
|
69
|
+
# @yieldreturn [Dry::Monads::Result] The handler result.
|
|
38
70
|
#
|
|
39
71
|
# @return [void]
|
|
40
72
|
def endpoint(name, subject: nil, queue: nil, group: nil, &handler)
|
|
41
73
|
endpoints << Endpoint.new(name:, subject: subject || name, queue:, group:, handler:)
|
|
42
74
|
end
|
|
43
75
|
|
|
76
|
+
# Define a JetStream pull consumer endpoint.
|
|
77
|
+
#
|
|
78
|
+
# @param name [String] The name of the endpoint.
|
|
79
|
+
# @param options [Hash] JetStream endpoint configuration.
|
|
80
|
+
# @option options [String] :stream The JetStream stream name.
|
|
81
|
+
# @option options [String] :subject The JetStream subject filter.
|
|
82
|
+
# @option options [String] :durable The durable consumer name.
|
|
83
|
+
# @option options [Hash, NATS::JetStream::API::ConsumerConfig, nil] :consumer Optional consumer config.
|
|
84
|
+
# @option options [Integer] :batch (1) Number of messages to fetch per pull request.
|
|
85
|
+
# @option options [Numeric] :fetch_timeout (5) Maximum time to wait for fetched messages.
|
|
86
|
+
# @option options [Numeric, nil] :nak_delay Optional delayed redelivery value for `nak`.
|
|
87
|
+
# @param handler [Proc] The block that will handle incoming messages.
|
|
88
|
+
# @yield [wrapper] Handles the wrapped JetStream message.
|
|
89
|
+
# @yieldparam wrapper [MessageWrapper] The wrapped inbound JetStream message.
|
|
90
|
+
# @yieldreturn [Dry::Monads::Result] The handler result.
|
|
91
|
+
#
|
|
92
|
+
# @return [void]
|
|
93
|
+
def jetstream_endpoint(name, **options, &handler)
|
|
94
|
+
jetstream_endpoints << build_jetstream_endpoint(name, options, handler)
|
|
95
|
+
end
|
|
96
|
+
|
|
44
97
|
# Define a group for organizing endpoints.
|
|
45
98
|
#
|
|
46
99
|
# @param name [String] The name of the group.
|
|
@@ -71,13 +124,14 @@ module Rubyists
|
|
|
71
124
|
# @param instances [Integer] The number of instances to spawn. Defaults to 1.
|
|
72
125
|
# @param blocking [Boolean] If false, does not block current thread after starting the server. Defaults to true.
|
|
73
126
|
#
|
|
74
|
-
# @return [void]
|
|
127
|
+
# @return [Concurrent::FixedThreadPool, void] The worker pool for non-blocking runs, otherwise blocks forever.
|
|
75
128
|
def run(nats_url:, service_opts:, instances: 1, blocking: true)
|
|
76
129
|
logger.info 'Booting NATS API server...'
|
|
77
130
|
workers = Concurrent::Array.new
|
|
78
131
|
pool = spawn_instances(nats_url, service_opts, instances, workers, blocking)
|
|
79
132
|
logger.info 'Setting up signal trap...'
|
|
80
133
|
trap_signals(workers, pool)
|
|
134
|
+
start_metrics_server(workers) if ENV['LEOPARD_METRICS_PORT']
|
|
81
135
|
return pool unless blocking
|
|
82
136
|
|
|
83
137
|
sleep
|
|
@@ -94,6 +148,7 @@ module Rubyists
|
|
|
94
148
|
# @param blocking [Boolean] If false, does not block current thread after starting the server.
|
|
95
149
|
#
|
|
96
150
|
# @return [Concurrent::FixedThreadPool] The thread pool managing the worker threads.
|
|
151
|
+
# @raise [ArgumentError] If `instance_args` was provided but is not a hash.
|
|
97
152
|
def spawn_instances(url, opts, count, workers, blocking)
|
|
98
153
|
pool = Concurrent::FixedThreadPool.new(count)
|
|
99
154
|
@instance_args = opts.delete(:instance_args) || nil
|
|
@@ -173,50 +228,154 @@ module Rubyists
|
|
|
173
228
|
rescue StandardError
|
|
174
229
|
exit 1
|
|
175
230
|
end
|
|
231
|
+
|
|
232
|
+
# Builds a JetStream endpoint struct with Leopard defaults applied.
|
|
233
|
+
#
|
|
234
|
+
# @param name [String, Symbol] Endpoint name.
|
|
235
|
+
# @param options [Hash] JetStream endpoint options.
|
|
236
|
+
# @param handler [Proc] Endpoint handler block.
|
|
237
|
+
#
|
|
238
|
+
# @return [NatsJetstreamEndpoint] The configured JetStream endpoint definition.
|
|
239
|
+
def build_jetstream_endpoint(name, options, handler)
|
|
240
|
+
NatsJetstreamEndpoint.new(
|
|
241
|
+
name:,
|
|
242
|
+
handler:,
|
|
243
|
+
consumer: nil,
|
|
244
|
+
batch: 1,
|
|
245
|
+
fetch_timeout: 5,
|
|
246
|
+
nak_delay: nil,
|
|
247
|
+
**options,
|
|
248
|
+
)
|
|
249
|
+
end
|
|
176
250
|
end
|
|
177
251
|
|
|
178
|
-
|
|
252
|
+
# Instance-side worker boot and shutdown helpers.
|
|
253
|
+
module WorkerLifecycle
|
|
179
254
|
# Returns the logger configured for the NATS API server.
|
|
255
|
+
#
|
|
256
|
+
# @return [Object] The configured logger.
|
|
180
257
|
def logger = self.class.logger
|
|
181
258
|
|
|
182
259
|
# Sets up a worker thread for the NATS API server.
|
|
183
260
|
# This method connects to the NATS server, adds the service, groups, and endpoints,
|
|
184
261
|
#
|
|
185
|
-
# @param
|
|
186
|
-
# @param
|
|
187
|
-
# @param eps [Array<Hash>] The list of endpoints to add.
|
|
188
|
-
# @param gps [Hash] The groups to add.
|
|
262
|
+
# @param nats_url [String] The URL of the NATS server.
|
|
263
|
+
# @param service_opts [Hash] Options for the NATS service.
|
|
189
264
|
#
|
|
190
265
|
# @return [void]
|
|
191
266
|
def setup_worker(nats_url: 'nats://localhost:4222', service_opts: {})
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
group_map = add_groups(gps)
|
|
198
|
-
add_endpoints eps, group_map
|
|
267
|
+
initialize_worker_state
|
|
268
|
+
connect_client(nats_url)
|
|
269
|
+
initialize_service(service_opts)
|
|
270
|
+
add_endpoints(self.class.endpoints.dup, add_groups(self.class.groups.dup))
|
|
271
|
+
start_jetstream_consumer(self.class.jetstream_endpoints.dup)
|
|
199
272
|
end
|
|
200
273
|
|
|
201
274
|
# Sets up a worker thread for the NATS API server and blocks the current thread.
|
|
202
275
|
#
|
|
203
276
|
# @see #setup_worker
|
|
277
|
+
# @param nats_url [String] The URL of the NATS server.
|
|
278
|
+
# @param service_opts [Hash] Options for the NATS service.
|
|
279
|
+
#
|
|
280
|
+
# @return [void]
|
|
204
281
|
def setup_worker!(nats_url: 'nats://localhost:4222', service_opts: {})
|
|
205
282
|
setup_worker(nats_url:, service_opts:)
|
|
206
283
|
sleep
|
|
207
284
|
end
|
|
208
285
|
|
|
209
286
|
# Stops the NATS API server worker.
|
|
287
|
+
#
|
|
288
|
+
# @return [void]
|
|
210
289
|
def stop
|
|
211
|
-
@
|
|
212
|
-
|
|
213
|
-
|
|
290
|
+
@running = false
|
|
291
|
+
stop_jetstream
|
|
292
|
+
stop_service
|
|
293
|
+
wake_worker
|
|
214
294
|
rescue ThreadError
|
|
215
295
|
nil
|
|
216
296
|
end
|
|
217
297
|
|
|
218
298
|
private
|
|
219
299
|
|
|
300
|
+
# Captures the current thread for later wakeup during shutdown.
|
|
301
|
+
#
|
|
302
|
+
# @return [Thread] The current worker thread.
|
|
303
|
+
def initialize_worker_state
|
|
304
|
+
@thread = Thread.current
|
|
305
|
+
end
|
|
306
|
+
|
|
307
|
+
# Opens the NATS client connection for this worker.
|
|
308
|
+
#
|
|
309
|
+
# @param nats_url [String] The URL of the NATS server.
|
|
310
|
+
#
|
|
311
|
+
# @return [Object] The connected NATS client.
|
|
312
|
+
def connect_client(nats_url)
|
|
313
|
+
@client = NATS.connect(nats_url)
|
|
314
|
+
end
|
|
315
|
+
|
|
316
|
+
# Registers the NATS service for this worker.
|
|
317
|
+
#
|
|
318
|
+
# @param service_opts [Hash] Options for the NATS service.
|
|
319
|
+
#
|
|
320
|
+
# @return [Object] The created NATS service.
|
|
321
|
+
def initialize_service(service_opts)
|
|
322
|
+
@service = @client.services.add(build_service_opts(service_opts:))
|
|
323
|
+
end
|
|
324
|
+
|
|
325
|
+
# Starts the JetStream consumer coordinator when JetStream endpoints are present.
|
|
326
|
+
#
|
|
327
|
+
# @param endpoints [Array<NatsJetstreamEndpoint>] JetStream endpoints for this worker.
|
|
328
|
+
#
|
|
329
|
+
# @return [void]
|
|
330
|
+
def start_jetstream_consumer(endpoints)
|
|
331
|
+
return if endpoints.empty?
|
|
332
|
+
|
|
333
|
+
@jetstream_consumer = jetstream_consumer_class.new(
|
|
334
|
+
jetstream: @client.jetstream,
|
|
335
|
+
endpoints:,
|
|
336
|
+
logger:,
|
|
337
|
+
process_message: method(:process_transport_message),
|
|
338
|
+
thread_factory:,
|
|
339
|
+
)
|
|
340
|
+
@jetstream_consumer.start
|
|
341
|
+
end
|
|
342
|
+
|
|
343
|
+
# Stops the JetStream consumer coordinator if one was started.
|
|
344
|
+
#
|
|
345
|
+
# @return [void]
|
|
346
|
+
def stop_jetstream
|
|
347
|
+
@jetstream_consumer&.stop
|
|
348
|
+
end
|
|
349
|
+
|
|
350
|
+
# Stops the registered NATS service and closes the client connection.
|
|
351
|
+
#
|
|
352
|
+
# @return [void]
|
|
353
|
+
def stop_service
|
|
354
|
+
@service&.stop
|
|
355
|
+
@client&.close
|
|
356
|
+
end
|
|
357
|
+
|
|
358
|
+
# Wakes the worker thread if it is blocked.
|
|
359
|
+
#
|
|
360
|
+
# @return [Thread, nil] The awakened worker thread, if present.
|
|
361
|
+
def wake_worker
|
|
362
|
+
@thread&.wakeup
|
|
363
|
+
end
|
|
364
|
+
|
|
365
|
+
# Returns the JetStream consumer coordinator class for this worker.
|
|
366
|
+
#
|
|
367
|
+
# @return [Class] The JetStream consumer implementation class.
|
|
368
|
+
def jetstream_consumer_class
|
|
369
|
+
NatsJetstreamConsumer
|
|
370
|
+
end
|
|
371
|
+
|
|
372
|
+
# Returns the thread factory used for JetStream consumer loops.
|
|
373
|
+
#
|
|
374
|
+
# @return [Class] The thread factory class.
|
|
375
|
+
def thread_factory
|
|
376
|
+
Thread
|
|
377
|
+
end
|
|
378
|
+
|
|
220
379
|
# Builds the service options for the NATS service.
|
|
221
380
|
#
|
|
222
381
|
# @param service_opts [Hash] Options for the NATS service.
|
|
@@ -247,6 +406,7 @@ module Rubyists
|
|
|
247
406
|
# @param name [String] The name of the group to build.
|
|
248
407
|
#
|
|
249
408
|
# @return [NATS::Group] The created group object.
|
|
409
|
+
# @raise [ArgumentError] If the requested group was never defined.
|
|
250
410
|
def build_group(defs, cache, name)
|
|
251
411
|
return cache[name] if cache.key?(name)
|
|
252
412
|
|
|
@@ -263,6 +423,7 @@ module Rubyists
|
|
|
263
423
|
# @param group_map [Hash] A map of group names to their created group objects.
|
|
264
424
|
#
|
|
265
425
|
# @return [void]
|
|
426
|
+
# @raise [ArgumentError] If an endpoint references an undefined group.
|
|
266
427
|
def add_endpoints(endpoints, group_map)
|
|
267
428
|
endpoints.each do |ep|
|
|
268
429
|
grp = ep.group
|
|
@@ -272,6 +433,16 @@ module Rubyists
|
|
|
272
433
|
build_endpoint(parent, ep)
|
|
273
434
|
end
|
|
274
435
|
end
|
|
436
|
+
end
|
|
437
|
+
|
|
438
|
+
# Message execution helpers shared by request/reply and JetStream transports.
|
|
439
|
+
module MessageHandling
|
|
440
|
+
# Returns the logger configured for the NATS API server.
|
|
441
|
+
#
|
|
442
|
+
# @return [Object] The configured logger.
|
|
443
|
+
def logger = self.class.logger
|
|
444
|
+
|
|
445
|
+
private
|
|
275
446
|
|
|
276
447
|
# Builds an endpoint in the NATS service.
|
|
277
448
|
#
|
|
@@ -282,58 +453,48 @@ module Rubyists
|
|
|
282
453
|
# @return [void]
|
|
283
454
|
def build_endpoint(parent, ept)
|
|
284
455
|
parent.endpoints.add(ept.name, subject: ept.subject, queue: ept.queue) do |raw_msg|
|
|
285
|
-
|
|
286
|
-
dispatch_with_middleware(wrapper, ept.handler)
|
|
456
|
+
process_transport_message(raw_msg, ept.handler, request_reply_callbacks.callbacks)
|
|
287
457
|
end
|
|
288
458
|
end
|
|
289
459
|
|
|
290
|
-
#
|
|
460
|
+
# Processes a raw transport message through Leopard's middleware and callback pipeline.
|
|
291
461
|
#
|
|
292
|
-
# @param
|
|
293
|
-
# @param handler [Proc] The handler
|
|
462
|
+
# @param raw_msg [Object] The raw NATS transport message.
|
|
463
|
+
# @param handler [Proc] The endpoint handler block.
|
|
464
|
+
# @param callbacks [Hash{Symbol => #call}] Transport callbacks keyed by outcome.
|
|
294
465
|
#
|
|
295
|
-
# @return [
|
|
296
|
-
def
|
|
297
|
-
|
|
298
|
-
self.class.middleware.reverse_each do |(klass, args, blk)|
|
|
299
|
-
app = klass.new(app, *args, &blk)
|
|
300
|
-
end
|
|
301
|
-
app.call(wrapper)
|
|
466
|
+
# @return [Object] The transport-specific callback result.
|
|
467
|
+
def process_transport_message(raw_msg, handler, callbacks)
|
|
468
|
+
message_processor.process(raw_msg, handler, callbacks)
|
|
302
469
|
end
|
|
303
470
|
|
|
304
|
-
#
|
|
471
|
+
# Returns the callback helper for request/reply endpoints.
|
|
305
472
|
#
|
|
306
|
-
# @
|
|
307
|
-
|
|
473
|
+
# @return [NatsRequestReplyCallbacks] The request/reply callback helper.
|
|
474
|
+
def request_reply_callbacks
|
|
475
|
+
@request_reply_callbacks ||= NatsRequestReplyCallbacks.new(logger:)
|
|
476
|
+
end
|
|
477
|
+
|
|
478
|
+
# Returns the memoized message processor for this worker instance.
|
|
308
479
|
#
|
|
309
|
-
# @return [
|
|
310
|
-
def
|
|
311
|
-
|
|
312
|
-
|
|
313
|
-
|
|
314
|
-
|
|
315
|
-
|
|
316
|
-
|
|
480
|
+
# @return [MessageProcessor] The shared message processor.
|
|
481
|
+
def message_processor
|
|
482
|
+
@message_processor ||= MessageProcessor.new(
|
|
483
|
+
wrapper_factory: MessageWrapper.method(:new),
|
|
484
|
+
middleware: -> { self.class.middleware },
|
|
485
|
+
execute_handler: method(:execute_handler),
|
|
486
|
+
logger:,
|
|
487
|
+
)
|
|
317
488
|
end
|
|
318
489
|
|
|
319
|
-
#
|
|
490
|
+
# Executes an endpoint handler within the worker instance context.
|
|
320
491
|
#
|
|
321
|
-
# @param wrapper [MessageWrapper] The
|
|
322
|
-
# @param
|
|
492
|
+
# @param wrapper [MessageWrapper] The wrapped transport message.
|
|
493
|
+
# @param handler [Proc] The endpoint handler block.
|
|
323
494
|
#
|
|
324
|
-
# @return [
|
|
325
|
-
|
|
326
|
-
|
|
327
|
-
case result
|
|
328
|
-
in Dry::Monads::Success
|
|
329
|
-
wrapper.respond(result.value!)
|
|
330
|
-
in Dry::Monads::Failure
|
|
331
|
-
logger.error 'Error processing message: ', result.failure
|
|
332
|
-
wrapper.respond_with_error(result.failure)
|
|
333
|
-
else
|
|
334
|
-
logger.error('Unexpected result: ', result:)
|
|
335
|
-
raise ResultError, "Unexpected Response from Handler, must respond with a Success or Failure monad: #{result}"
|
|
336
|
-
end
|
|
495
|
+
# @return [Dry::Monads::Result] The handler result.
|
|
496
|
+
def execute_handler(wrapper, handler)
|
|
497
|
+
instance_exec(wrapper, &handler)
|
|
337
498
|
end
|
|
338
499
|
end
|
|
339
500
|
end
|
|
@@ -0,0 +1,76 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Rubyists
|
|
4
|
+
module Leopard
|
|
5
|
+
# Maps Leopard handler outcomes to JetStream ack, nak, and term operations.
|
|
6
|
+
class NatsJetstreamCallbacks
|
|
7
|
+
# Builds a callback set for JetStream message outcomes.
|
|
8
|
+
#
|
|
9
|
+
# @param logger [#error] Logger used for failures and unhandled exceptions.
|
|
10
|
+
#
|
|
11
|
+
# @return [void]
|
|
12
|
+
def initialize(logger:)
|
|
13
|
+
@logger = logger
|
|
14
|
+
end
|
|
15
|
+
|
|
16
|
+
# Returns transport callbacks for a JetStream endpoint.
|
|
17
|
+
#
|
|
18
|
+
# @param endpoint [NatsJetstreamEndpoint] The endpoint configuration being consumed.
|
|
19
|
+
#
|
|
20
|
+
# @return [Hash{Symbol => #call}] Outcome callbacks keyed by `:on_success`, `:on_failure`, and `:on_error`.
|
|
21
|
+
def callbacks_for(endpoint)
|
|
22
|
+
{
|
|
23
|
+
on_success: method(:ack_message),
|
|
24
|
+
on_failure: ->(wrapper, result) { nak_message(wrapper, result, endpoint) },
|
|
25
|
+
on_error: method(:term_message),
|
|
26
|
+
}
|
|
27
|
+
end
|
|
28
|
+
|
|
29
|
+
private
|
|
30
|
+
|
|
31
|
+
# Acknowledges a successfully processed JetStream message.
|
|
32
|
+
#
|
|
33
|
+
# @param wrapper [MessageWrapper] Wrapped JetStream message.
|
|
34
|
+
# @param _result [Dry::Monads::Success] Successful handler result.
|
|
35
|
+
#
|
|
36
|
+
# @return [void]
|
|
37
|
+
def ack_message(wrapper, _result)
|
|
38
|
+
wrapper.raw.ack
|
|
39
|
+
end
|
|
40
|
+
|
|
41
|
+
# Negatively acknowledges a failed JetStream message, optionally delaying redelivery.
|
|
42
|
+
#
|
|
43
|
+
# @param wrapper [MessageWrapper] Wrapped JetStream message.
|
|
44
|
+
# @param result [Dry::Monads::Failure] Failed handler result.
|
|
45
|
+
# @param endpoint [NatsJetstreamEndpoint] Endpoint configuration for the message.
|
|
46
|
+
#
|
|
47
|
+
# @return [void]
|
|
48
|
+
def nak_message(wrapper, result, endpoint)
|
|
49
|
+
log_failure(result.failure)
|
|
50
|
+
return wrapper.raw.nak unless endpoint.nak_delay
|
|
51
|
+
|
|
52
|
+
wrapper.raw.nak(delay: endpoint.nak_delay)
|
|
53
|
+
end
|
|
54
|
+
|
|
55
|
+
# Terminates a JetStream message after an unhandled exception.
|
|
56
|
+
#
|
|
57
|
+
# @param wrapper [MessageWrapper] Wrapped JetStream message.
|
|
58
|
+
# @param error [StandardError] The unhandled exception.
|
|
59
|
+
#
|
|
60
|
+
# @return [void]
|
|
61
|
+
def term_message(wrapper, error)
|
|
62
|
+
@logger.error 'Unhandled JetStream error: ', error
|
|
63
|
+
wrapper.raw.term
|
|
64
|
+
end
|
|
65
|
+
|
|
66
|
+
# Logs the failure payload returned by a handler.
|
|
67
|
+
#
|
|
68
|
+
# @param failure [Object] The failure payload from the handler.
|
|
69
|
+
#
|
|
70
|
+
# @return [void]
|
|
71
|
+
def log_failure(failure)
|
|
72
|
+
@logger.error 'Error processing message: ', failure
|
|
73
|
+
end
|
|
74
|
+
end
|
|
75
|
+
end
|
|
76
|
+
end
|
|
@@ -0,0 +1,186 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require_relative 'nats_jetstream_callbacks'
|
|
4
|
+
require_relative 'nats_jetstream_endpoint'
|
|
5
|
+
|
|
6
|
+
module Rubyists
|
|
7
|
+
module Leopard
|
|
8
|
+
# Coordinates JetStream pull subscriptions and dispatches fetched messages through Leopard.
|
|
9
|
+
class NatsJetstreamConsumer
|
|
10
|
+
# Consumer configuration keys Leopard owns and will not allow endpoint overrides to replace.
|
|
11
|
+
PROTECTED_CONSUMER_KEYS = %i[durable_name filter_subject ack_policy].freeze
|
|
12
|
+
|
|
13
|
+
# @!attribute [r] subscriptions
|
|
14
|
+
#
|
|
15
|
+
# @return [Array<Object>] Active JetStream pull subscriptions.
|
|
16
|
+
# @!attribute [r] threads
|
|
17
|
+
#
|
|
18
|
+
# @return [Array<Thread>] Consumer loop threads for each endpoint.
|
|
19
|
+
attr_reader :subscriptions, :threads
|
|
20
|
+
|
|
21
|
+
# Builds a pull-consumer coordinator for one Leopard worker.
|
|
22
|
+
#
|
|
23
|
+
# @param jetstream [Object] JetStream client used to manage consumers and subscriptions.
|
|
24
|
+
# @param endpoints [Array<NatsJetstreamEndpoint>] JetStream endpoint definitions for this worker.
|
|
25
|
+
# @param logger [#error] Logger used for loop failures.
|
|
26
|
+
# @param process_message [#call] Callable that processes a raw JetStream message through Leopard.
|
|
27
|
+
# @param dependencies [Hash{Symbol => Object}] Optional collaborators for callback and thread creation.
|
|
28
|
+
# @option dependencies [Class] :callback_builder (NatsJetstreamCallbacks) Builder for transport callbacks.
|
|
29
|
+
# @option dependencies [Class] :thread_factory (Thread) Thread-like factory used to spawn consumer loops.
|
|
30
|
+
#
|
|
31
|
+
# @return [void]
|
|
32
|
+
def initialize(jetstream:, endpoints:, logger:, process_message:, **dependencies)
|
|
33
|
+
@jetstream = jetstream
|
|
34
|
+
@endpoints = endpoints
|
|
35
|
+
@logger = logger
|
|
36
|
+
@process_message = process_message
|
|
37
|
+
@callbacks = dependencies.fetch(:callback_builder, NatsJetstreamCallbacks).new(logger:)
|
|
38
|
+
@thread_factory = dependencies.fetch(:thread_factory, Thread)
|
|
39
|
+
@subscriptions = []
|
|
40
|
+
@threads = []
|
|
41
|
+
@running = false
|
|
42
|
+
end
|
|
43
|
+
|
|
44
|
+
# Starts one pull-consumer loop per configured endpoint.
|
|
45
|
+
#
|
|
46
|
+
# @return [void]
|
|
47
|
+
def start
|
|
48
|
+
@running = true
|
|
49
|
+
@endpoints.each { |endpoint| start_endpoint(endpoint) }
|
|
50
|
+
end
|
|
51
|
+
|
|
52
|
+
# Stops all pull-consumer loops and waits for them to exit.
|
|
53
|
+
#
|
|
54
|
+
# @return [void]
|
|
55
|
+
def stop
|
|
56
|
+
@running = false
|
|
57
|
+
subscriptions.each(&:unsubscribe)
|
|
58
|
+
threads.each(&:join)
|
|
59
|
+
end
|
|
60
|
+
|
|
61
|
+
private
|
|
62
|
+
|
|
63
|
+
# Starts a consumer loop for one endpoint.
|
|
64
|
+
#
|
|
65
|
+
# @param endpoint [NatsJetstreamEndpoint] The endpoint configuration to consume.
|
|
66
|
+
#
|
|
67
|
+
# @return [void]
|
|
68
|
+
def start_endpoint(endpoint)
|
|
69
|
+
subscription = build_subscription(endpoint)
|
|
70
|
+
subscriptions << subscription
|
|
71
|
+
threads << @thread_factory.new { consume_endpoint(subscription, endpoint) }
|
|
72
|
+
end
|
|
73
|
+
|
|
74
|
+
# Ensures the durable consumer exists and creates a pull subscription for it.
|
|
75
|
+
#
|
|
76
|
+
# @param endpoint [NatsJetstreamEndpoint] The endpoint configuration to subscribe to.
|
|
77
|
+
#
|
|
78
|
+
# @return [Object] The JetStream pull subscription.
|
|
79
|
+
def build_subscription(endpoint)
|
|
80
|
+
ensure_consumer(endpoint)
|
|
81
|
+
@jetstream.pull_subscribe(
|
|
82
|
+
endpoint.subject,
|
|
83
|
+
endpoint.durable,
|
|
84
|
+
stream: endpoint.stream,
|
|
85
|
+
)
|
|
86
|
+
end
|
|
87
|
+
|
|
88
|
+
# Verifies that the durable consumer exists, creating it when missing.
|
|
89
|
+
#
|
|
90
|
+
# @param endpoint [NatsJetstreamEndpoint] The endpoint configuration to ensure.
|
|
91
|
+
#
|
|
92
|
+
# @return [Object] Consumer metadata from `consumer_info` or `add_consumer`.
|
|
93
|
+
def ensure_consumer(endpoint)
|
|
94
|
+
@jetstream.consumer_info(endpoint.stream, endpoint.durable)
|
|
95
|
+
rescue NATS::JetStream::Error::NotFound
|
|
96
|
+
@jetstream.add_consumer(endpoint.stream, consumer_config(endpoint))
|
|
97
|
+
end
|
|
98
|
+
|
|
99
|
+
# Builds the JetStream consumer configuration for an endpoint.
|
|
100
|
+
#
|
|
101
|
+
# @param endpoint [NatsJetstreamEndpoint] The endpoint configuration to translate.
|
|
102
|
+
#
|
|
103
|
+
# @return [Hash] Consumer configuration accepted by `add_consumer`.
|
|
104
|
+
def consumer_config(endpoint)
|
|
105
|
+
base = {
|
|
106
|
+
durable_name: endpoint.durable,
|
|
107
|
+
filter_subject: endpoint.subject,
|
|
108
|
+
ack_policy: 'explicit',
|
|
109
|
+
}
|
|
110
|
+
base.merge(safe_consumer_options(endpoint))
|
|
111
|
+
end
|
|
112
|
+
|
|
113
|
+
# Normalizes optional consumer overrides into a hash.
|
|
114
|
+
#
|
|
115
|
+
# @param endpoint [NatsJetstreamEndpoint] The endpoint configuration to inspect.
|
|
116
|
+
#
|
|
117
|
+
# @return [Hash] Consumer overrides, or an empty hash when none were provided.
|
|
118
|
+
def normalized_consumer_options(endpoint)
|
|
119
|
+
return {} unless endpoint.consumer
|
|
120
|
+
return endpoint.consumer.to_h if endpoint.consumer.respond_to?(:to_h)
|
|
121
|
+
|
|
122
|
+
endpoint.consumer
|
|
123
|
+
end
|
|
124
|
+
|
|
125
|
+
# Removes Leopard-managed consumer keys from user overrides.
|
|
126
|
+
#
|
|
127
|
+
# @param endpoint [NatsJetstreamEndpoint] The endpoint configuration to inspect.
|
|
128
|
+
#
|
|
129
|
+
# @return [Hash] Consumer overrides excluding protected keys required by Leopard.
|
|
130
|
+
def safe_consumer_options(endpoint)
|
|
131
|
+
normalized_consumer_options(endpoint).reject { |key, _value| PROTECTED_CONSUMER_KEYS.include?(key.to_sym) }
|
|
132
|
+
end
|
|
133
|
+
|
|
134
|
+
# Repeatedly fetches and processes batches for one endpoint while the consumer is running.
|
|
135
|
+
#
|
|
136
|
+
# @param subscription [Object] Pull subscription for the endpoint.
|
|
137
|
+
# @param endpoint [NatsJetstreamEndpoint] The endpoint configuration being consumed.
|
|
138
|
+
#
|
|
139
|
+
# @return [void]
|
|
140
|
+
def consume_endpoint(subscription, endpoint)
|
|
141
|
+
while @running
|
|
142
|
+
begin
|
|
143
|
+
consume_batch(subscription, endpoint)
|
|
144
|
+
rescue NATS::Timeout
|
|
145
|
+
next if @running
|
|
146
|
+
rescue StandardError => e
|
|
147
|
+
log_loop_error(endpoint, e)
|
|
148
|
+
break unless @running
|
|
149
|
+
end
|
|
150
|
+
end
|
|
151
|
+
end
|
|
152
|
+
|
|
153
|
+
# Fetches one batch from JetStream and processes each message through Leopard.
|
|
154
|
+
#
|
|
155
|
+
# @param subscription [Object] Pull subscription for the endpoint.
|
|
156
|
+
# @param endpoint [NatsJetstreamEndpoint] The endpoint configuration being consumed.
|
|
157
|
+
#
|
|
158
|
+
# @return [void]
|
|
159
|
+
def consume_batch(subscription, endpoint)
|
|
160
|
+
fetch_messages(subscription, endpoint).each do |raw_msg|
|
|
161
|
+
@process_message.call(raw_msg, endpoint.handler, @callbacks.callbacks_for(endpoint))
|
|
162
|
+
end
|
|
163
|
+
end
|
|
164
|
+
|
|
165
|
+
# Fetches a batch of messages for one endpoint.
|
|
166
|
+
#
|
|
167
|
+
# @param subscription [Object] Pull subscription for the endpoint.
|
|
168
|
+
# @param endpoint [NatsJetstreamEndpoint] The endpoint configuration being consumed.
|
|
169
|
+
#
|
|
170
|
+
# @return [Array<Object>] Raw JetStream messages returned by the subscription.
|
|
171
|
+
def fetch_messages(subscription, endpoint)
|
|
172
|
+
subscription.fetch(endpoint.batch, timeout: endpoint.fetch_timeout)
|
|
173
|
+
end
|
|
174
|
+
|
|
175
|
+
# Logs an endpoint-level loop failure.
|
|
176
|
+
#
|
|
177
|
+
# @param endpoint [NatsJetstreamEndpoint] The endpoint whose loop failed.
|
|
178
|
+
# @param error [StandardError] The raised exception.
|
|
179
|
+
#
|
|
180
|
+
# @return [void]
|
|
181
|
+
def log_loop_error(endpoint, error)
|
|
182
|
+
@logger.error "JetStream endpoint #{endpoint.name} loop error: ", error
|
|
183
|
+
end
|
|
184
|
+
end
|
|
185
|
+
end
|
|
186
|
+
end
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Rubyists
|
|
4
|
+
module Leopard
|
|
5
|
+
# Configuration for a Leopard JetStream pull-consumer endpoint.
|
|
6
|
+
NatsJetstreamEndpoint = Struct.new(
|
|
7
|
+
:name,
|
|
8
|
+
:stream,
|
|
9
|
+
:subject,
|
|
10
|
+
:durable,
|
|
11
|
+
:consumer,
|
|
12
|
+
:batch,
|
|
13
|
+
:fetch_timeout,
|
|
14
|
+
:nak_delay,
|
|
15
|
+
:handler,
|
|
16
|
+
keyword_init: true,
|
|
17
|
+
)
|
|
18
|
+
end
|
|
19
|
+
end
|