leopard 0.2.5 → 0.2.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.release-please-manifest.json +1 -1
- data/.version.txt +1 -1
- data/.yardopts +5 -0
- data/CHANGELOG.md +14 -0
- data/Rakefile +110 -1
- data/Readme.adoc +61 -1
- data/ci/nats/start.sh +26 -1
- data/examples/echo_endpoint.rb +6 -0
- data/examples/jetstream_endpoint.rb +44 -0
- data/lib/leopard/errors.rb +10 -0
- data/lib/leopard/message_processor.rb +90 -0
- data/lib/leopard/message_wrapper.rb +4 -0
- data/lib/leopard/metrics_server.rb +49 -0
- data/lib/leopard/nats_api_server.rb +214 -57
- data/lib/leopard/nats_jetstream_callbacks.rb +76 -0
- data/lib/leopard/nats_jetstream_consumer.rb +186 -0
- data/lib/leopard/nats_jetstream_endpoint.rb +19 -0
- data/lib/leopard/nats_request_reply_callbacks.rb +70 -0
- data/lib/leopard/version.rb +1 -1
- data/lib/leopard.rb +17 -0
- data/mise.toml +2 -0
- metadata +11 -3
|
@@ -6,29 +6,55 @@ require 'dry/configurable'
|
|
|
6
6
|
require 'concurrent'
|
|
7
7
|
require_relative '../leopard'
|
|
8
8
|
require_relative 'message_wrapper'
|
|
9
|
+
require_relative 'message_processor'
|
|
9
10
|
require_relative 'metrics_server'
|
|
11
|
+
require_relative 'nats_jetstream_endpoint'
|
|
12
|
+
require_relative 'nats_jetstream_consumer'
|
|
13
|
+
require_relative 'nats_request_reply_callbacks'
|
|
10
14
|
|
|
11
15
|
module Rubyists
|
|
12
16
|
module Leopard
|
|
17
|
+
# DSL and runtime integration for Leopard request/reply and JetStream workers.
|
|
13
18
|
module NatsApiServer
|
|
14
19
|
include Dry::Monads[:result]
|
|
15
20
|
extend Dry::Monads[:result]
|
|
16
21
|
|
|
22
|
+
# Extends an including class with Leopard's DSL and worker lifecycle behavior.
|
|
23
|
+
#
|
|
24
|
+
# @param base [Class] The class including this module.
|
|
25
|
+
#
|
|
26
|
+
# @return [void]
|
|
17
27
|
def self.included(base)
|
|
18
28
|
base.extend(ClassMethods)
|
|
19
|
-
base.include(
|
|
29
|
+
base.include(WorkerLifecycle)
|
|
30
|
+
base.include(MessageHandling)
|
|
20
31
|
base.extend(Dry::Monads[:result])
|
|
21
32
|
base.extend(Dry::Configurable)
|
|
22
33
|
base.setting :logger, default: Rubyists::Leopard.logger, reader: true
|
|
23
34
|
end
|
|
24
35
|
|
|
25
|
-
|
|
36
|
+
# Configuration for a request/reply endpoint declared with {.endpoint}.
|
|
37
|
+
Endpoint = Struct.new(:name, :subject, :queue, :group, :handler, keyword_init: true)
|
|
26
38
|
|
|
39
|
+
# Class-level DSL for defining Leopard endpoints, middleware, and worker startup.
|
|
27
40
|
module ClassMethods
|
|
28
41
|
include MetricsServer
|
|
29
42
|
|
|
43
|
+
# Returns the configured request/reply endpoints for the service class.
|
|
44
|
+
#
|
|
45
|
+
# @return [Array<Endpoint>] Declared request/reply endpoints.
|
|
30
46
|
def endpoints = @endpoints ||= []
|
|
47
|
+
# Returns the configured JetStream endpoints for the service class.
|
|
48
|
+
#
|
|
49
|
+
# @return [Array<NatsJetstreamEndpoint>] Declared JetStream pull-consumer endpoints.
|
|
50
|
+
def jetstream_endpoints = @jetstream_endpoints ||= []
|
|
51
|
+
# Returns the configured endpoint groups for the service class.
|
|
52
|
+
#
|
|
53
|
+
# @return [Hash{Symbol,String => Hash}] Declared group definitions.
|
|
31
54
|
def groups = @groups ||= {}
|
|
55
|
+
# Returns the configured middleware stack for the service class.
|
|
56
|
+
#
|
|
57
|
+
# @return [Array<Array>] Middleware declarations in registration order.
|
|
32
58
|
def middleware = @middleware ||= []
|
|
33
59
|
|
|
34
60
|
# Define an endpoint for the NATS API server.
|
|
@@ -38,12 +64,36 @@ module Rubyists
|
|
|
38
64
|
# @param queue [String, nil] The NATS queue group to use. Defaults to nil.
|
|
39
65
|
# @param group [String, nil] The group this endpoint belongs to. Defaults to nil.
|
|
40
66
|
# @param handler [Proc] The block that will handle incoming messages.
|
|
67
|
+
# @yield [wrapper] Handles the wrapped request message.
|
|
68
|
+
# @yieldparam wrapper [MessageWrapper] The wrapped inbound NATS message.
|
|
69
|
+
# @yieldreturn [Dry::Monads::Result] The handler result.
|
|
41
70
|
#
|
|
42
71
|
# @return [void]
|
|
43
72
|
def endpoint(name, subject: nil, queue: nil, group: nil, &handler)
|
|
44
73
|
endpoints << Endpoint.new(name:, subject: subject || name, queue:, group:, handler:)
|
|
45
74
|
end
|
|
46
75
|
|
|
76
|
+
# Define a JetStream pull consumer endpoint.
|
|
77
|
+
#
|
|
78
|
+
# @param name [String] The name of the endpoint.
|
|
79
|
+
# @param options [Hash] JetStream endpoint configuration.
|
|
80
|
+
# @option options [String] :stream The JetStream stream name.
|
|
81
|
+
# @option options [String] :subject The JetStream subject filter.
|
|
82
|
+
# @option options [String] :durable The durable consumer name.
|
|
83
|
+
# @option options [Hash, NATS::JetStream::API::ConsumerConfig, nil] :consumer Optional consumer config.
|
|
84
|
+
# @option options [Integer] :batch (1) Number of messages to fetch per pull request.
|
|
85
|
+
# @option options [Numeric] :fetch_timeout (5) Maximum time to wait for fetched messages.
|
|
86
|
+
# @option options [Numeric, nil] :nak_delay Optional delayed redelivery value for `nak`.
|
|
87
|
+
# @param handler [Proc] The block that will handle incoming messages.
|
|
88
|
+
# @yield [wrapper] Handles the wrapped JetStream message.
|
|
89
|
+
# @yieldparam wrapper [MessageWrapper] The wrapped inbound JetStream message.
|
|
90
|
+
# @yieldreturn [Dry::Monads::Result] The handler result.
|
|
91
|
+
#
|
|
92
|
+
# @return [void]
|
|
93
|
+
def jetstream_endpoint(name, **options, &handler)
|
|
94
|
+
jetstream_endpoints << build_jetstream_endpoint(name, options, handler)
|
|
95
|
+
end
|
|
96
|
+
|
|
47
97
|
# Define a group for organizing endpoints.
|
|
48
98
|
#
|
|
49
99
|
# @param name [String] The name of the group.
|
|
@@ -74,7 +124,7 @@ module Rubyists
|
|
|
74
124
|
# @param instances [Integer] The number of instances to spawn. Defaults to 1.
|
|
75
125
|
# @param blocking [Boolean] If false, does not block current thread after starting the server. Defaults to true.
|
|
76
126
|
#
|
|
77
|
-
# @return [void]
|
|
127
|
+
# @return [Concurrent::FixedThreadPool, void] The worker pool for non-blocking runs, otherwise blocks forever.
|
|
78
128
|
def run(nats_url:, service_opts:, instances: 1, blocking: true)
|
|
79
129
|
logger.info 'Booting NATS API server...'
|
|
80
130
|
workers = Concurrent::Array.new
|
|
@@ -98,6 +148,7 @@ module Rubyists
|
|
|
98
148
|
# @param blocking [Boolean] If false, does not block current thread after starting the server.
|
|
99
149
|
#
|
|
100
150
|
# @return [Concurrent::FixedThreadPool] The thread pool managing the worker threads.
|
|
151
|
+
# @raise [ArgumentError] If `instance_args` was provided but is not a hash.
|
|
101
152
|
def spawn_instances(url, opts, count, workers, blocking)
|
|
102
153
|
pool = Concurrent::FixedThreadPool.new(count)
|
|
103
154
|
@instance_args = opts.delete(:instance_args) || nil
|
|
@@ -177,50 +228,154 @@ module Rubyists
|
|
|
177
228
|
rescue StandardError
|
|
178
229
|
exit 1
|
|
179
230
|
end
|
|
231
|
+
|
|
232
|
+
# Builds a JetStream endpoint struct with Leopard defaults applied.
|
|
233
|
+
#
|
|
234
|
+
# @param name [String, Symbol] Endpoint name.
|
|
235
|
+
# @param options [Hash] JetStream endpoint options.
|
|
236
|
+
# @param handler [Proc] Endpoint handler block.
|
|
237
|
+
#
|
|
238
|
+
# @return [NatsJetstreamEndpoint] The configured JetStream endpoint definition.
|
|
239
|
+
def build_jetstream_endpoint(name, options, handler)
|
|
240
|
+
NatsJetstreamEndpoint.new(
|
|
241
|
+
name:,
|
|
242
|
+
handler:,
|
|
243
|
+
consumer: nil,
|
|
244
|
+
batch: 1,
|
|
245
|
+
fetch_timeout: 5,
|
|
246
|
+
nak_delay: nil,
|
|
247
|
+
**options,
|
|
248
|
+
)
|
|
249
|
+
end
|
|
180
250
|
end
|
|
181
251
|
|
|
182
|
-
|
|
252
|
+
# Instance-side worker boot and shutdown helpers.
|
|
253
|
+
module WorkerLifecycle
|
|
183
254
|
# Returns the logger configured for the NATS API server.
|
|
255
|
+
#
|
|
256
|
+
# @return [Object] The configured logger.
|
|
184
257
|
def logger = self.class.logger
|
|
185
258
|
|
|
186
259
|
# Sets up a worker thread for the NATS API server.
|
|
187
260
|
# This method connects to the NATS server, adds the service, groups, and endpoints,
|
|
188
261
|
#
|
|
189
|
-
# @param
|
|
190
|
-
# @param
|
|
191
|
-
# @param eps [Array<Hash>] The list of endpoints to add.
|
|
192
|
-
# @param gps [Hash] The groups to add.
|
|
262
|
+
# @param nats_url [String] The URL of the NATS server.
|
|
263
|
+
# @param service_opts [Hash] Options for the NATS service.
|
|
193
264
|
#
|
|
194
265
|
# @return [void]
|
|
195
266
|
def setup_worker(nats_url: 'nats://localhost:4222', service_opts: {})
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
group_map = add_groups(gps)
|
|
202
|
-
add_endpoints eps, group_map
|
|
267
|
+
initialize_worker_state
|
|
268
|
+
connect_client(nats_url)
|
|
269
|
+
initialize_service(service_opts)
|
|
270
|
+
add_endpoints(self.class.endpoints.dup, add_groups(self.class.groups.dup))
|
|
271
|
+
start_jetstream_consumer(self.class.jetstream_endpoints.dup)
|
|
203
272
|
end
|
|
204
273
|
|
|
205
274
|
# Sets up a worker thread for the NATS API server and blocks the current thread.
|
|
206
275
|
#
|
|
207
276
|
# @see #setup_worker
|
|
277
|
+
# @param nats_url [String] The URL of the NATS server.
|
|
278
|
+
# @param service_opts [Hash] Options for the NATS service.
|
|
279
|
+
#
|
|
280
|
+
# @return [void]
|
|
208
281
|
def setup_worker!(nats_url: 'nats://localhost:4222', service_opts: {})
|
|
209
282
|
setup_worker(nats_url:, service_opts:)
|
|
210
283
|
sleep
|
|
211
284
|
end
|
|
212
285
|
|
|
213
286
|
# Stops the NATS API server worker.
|
|
287
|
+
#
|
|
288
|
+
# @return [void]
|
|
214
289
|
def stop
|
|
215
|
-
@
|
|
216
|
-
|
|
217
|
-
|
|
290
|
+
@running = false
|
|
291
|
+
stop_jetstream
|
|
292
|
+
stop_service
|
|
293
|
+
wake_worker
|
|
218
294
|
rescue ThreadError
|
|
219
295
|
nil
|
|
220
296
|
end
|
|
221
297
|
|
|
222
298
|
private
|
|
223
299
|
|
|
300
|
+
# Captures the current thread for later wakeup during shutdown.
|
|
301
|
+
#
|
|
302
|
+
# @return [Thread] The current worker thread.
|
|
303
|
+
def initialize_worker_state
|
|
304
|
+
@thread = Thread.current
|
|
305
|
+
end
|
|
306
|
+
|
|
307
|
+
# Opens the NATS client connection for this worker.
|
|
308
|
+
#
|
|
309
|
+
# @param nats_url [String] The URL of the NATS server.
|
|
310
|
+
#
|
|
311
|
+
# @return [Object] The connected NATS client.
|
|
312
|
+
def connect_client(nats_url)
|
|
313
|
+
@client = NATS.connect(nats_url)
|
|
314
|
+
end
|
|
315
|
+
|
|
316
|
+
# Registers the NATS service for this worker.
|
|
317
|
+
#
|
|
318
|
+
# @param service_opts [Hash] Options for the NATS service.
|
|
319
|
+
#
|
|
320
|
+
# @return [Object] The created NATS service.
|
|
321
|
+
def initialize_service(service_opts)
|
|
322
|
+
@service = @client.services.add(build_service_opts(service_opts:))
|
|
323
|
+
end
|
|
324
|
+
|
|
325
|
+
# Starts the JetStream consumer coordinator when JetStream endpoints are present.
|
|
326
|
+
#
|
|
327
|
+
# @param endpoints [Array<NatsJetstreamEndpoint>] JetStream endpoints for this worker.
|
|
328
|
+
#
|
|
329
|
+
# @return [void]
|
|
330
|
+
def start_jetstream_consumer(endpoints)
|
|
331
|
+
return if endpoints.empty?
|
|
332
|
+
|
|
333
|
+
@jetstream_consumer = jetstream_consumer_class.new(
|
|
334
|
+
jetstream: @client.jetstream,
|
|
335
|
+
endpoints:,
|
|
336
|
+
logger:,
|
|
337
|
+
process_message: method(:process_transport_message),
|
|
338
|
+
thread_factory:,
|
|
339
|
+
)
|
|
340
|
+
@jetstream_consumer.start
|
|
341
|
+
end
|
|
342
|
+
|
|
343
|
+
# Stops the JetStream consumer coordinator if one was started.
|
|
344
|
+
#
|
|
345
|
+
# @return [void]
|
|
346
|
+
def stop_jetstream
|
|
347
|
+
@jetstream_consumer&.stop
|
|
348
|
+
end
|
|
349
|
+
|
|
350
|
+
# Stops the registered NATS service and closes the client connection.
|
|
351
|
+
#
|
|
352
|
+
# @return [void]
|
|
353
|
+
def stop_service
|
|
354
|
+
@service&.stop
|
|
355
|
+
@client&.close
|
|
356
|
+
end
|
|
357
|
+
|
|
358
|
+
# Wakes the worker thread if it is blocked.
|
|
359
|
+
#
|
|
360
|
+
# @return [Thread, nil] The awakened worker thread, if present.
|
|
361
|
+
def wake_worker
|
|
362
|
+
@thread&.wakeup
|
|
363
|
+
end
|
|
364
|
+
|
|
365
|
+
# Returns the JetStream consumer coordinator class for this worker.
|
|
366
|
+
#
|
|
367
|
+
# @return [Class] The JetStream consumer implementation class.
|
|
368
|
+
def jetstream_consumer_class
|
|
369
|
+
NatsJetstreamConsumer
|
|
370
|
+
end
|
|
371
|
+
|
|
372
|
+
# Returns the thread factory used for JetStream consumer loops.
|
|
373
|
+
#
|
|
374
|
+
# @return [Class] The thread factory class.
|
|
375
|
+
def thread_factory
|
|
376
|
+
Thread
|
|
377
|
+
end
|
|
378
|
+
|
|
224
379
|
# Builds the service options for the NATS service.
|
|
225
380
|
#
|
|
226
381
|
# @param service_opts [Hash] Options for the NATS service.
|
|
@@ -251,6 +406,7 @@ module Rubyists
|
|
|
251
406
|
# @param name [String] The name of the group to build.
|
|
252
407
|
#
|
|
253
408
|
# @return [NATS::Group] The created group object.
|
|
409
|
+
# @raise [ArgumentError] If the requested group was never defined.
|
|
254
410
|
def build_group(defs, cache, name)
|
|
255
411
|
return cache[name] if cache.key?(name)
|
|
256
412
|
|
|
@@ -267,6 +423,7 @@ module Rubyists
|
|
|
267
423
|
# @param group_map [Hash] A map of group names to their created group objects.
|
|
268
424
|
#
|
|
269
425
|
# @return [void]
|
|
426
|
+
# @raise [ArgumentError] If an endpoint references an undefined group.
|
|
270
427
|
def add_endpoints(endpoints, group_map)
|
|
271
428
|
endpoints.each do |ep|
|
|
272
429
|
grp = ep.group
|
|
@@ -276,6 +433,16 @@ module Rubyists
|
|
|
276
433
|
build_endpoint(parent, ep)
|
|
277
434
|
end
|
|
278
435
|
end
|
|
436
|
+
end
|
|
437
|
+
|
|
438
|
+
# Message execution helpers shared by request/reply and JetStream transports.
|
|
439
|
+
module MessageHandling
|
|
440
|
+
# Returns the logger configured for the NATS API server.
|
|
441
|
+
#
|
|
442
|
+
# @return [Object] The configured logger.
|
|
443
|
+
def logger = self.class.logger
|
|
444
|
+
|
|
445
|
+
private
|
|
279
446
|
|
|
280
447
|
# Builds an endpoint in the NATS service.
|
|
281
448
|
#
|
|
@@ -286,58 +453,48 @@ module Rubyists
|
|
|
286
453
|
# @return [void]
|
|
287
454
|
def build_endpoint(parent, ept)
|
|
288
455
|
parent.endpoints.add(ept.name, subject: ept.subject, queue: ept.queue) do |raw_msg|
|
|
289
|
-
|
|
290
|
-
dispatch_with_middleware(wrapper, ept.handler)
|
|
456
|
+
process_transport_message(raw_msg, ept.handler, request_reply_callbacks.callbacks)
|
|
291
457
|
end
|
|
292
458
|
end
|
|
293
459
|
|
|
294
|
-
#
|
|
460
|
+
# Processes a raw transport message through Leopard's middleware and callback pipeline.
|
|
295
461
|
#
|
|
296
|
-
# @param
|
|
297
|
-
# @param handler [Proc] The handler
|
|
462
|
+
# @param raw_msg [Object] The raw NATS transport message.
|
|
463
|
+
# @param handler [Proc] The endpoint handler block.
|
|
464
|
+
# @param callbacks [Hash{Symbol => #call}] Transport callbacks keyed by outcome.
|
|
298
465
|
#
|
|
299
|
-
# @return [
|
|
300
|
-
def
|
|
301
|
-
|
|
302
|
-
self.class.middleware.reverse_each do |(klass, args, blk)|
|
|
303
|
-
app = klass.new(app, *args, &blk)
|
|
304
|
-
end
|
|
305
|
-
app.call(wrapper)
|
|
466
|
+
# @return [Object] The transport-specific callback result.
|
|
467
|
+
def process_transport_message(raw_msg, handler, callbacks)
|
|
468
|
+
message_processor.process(raw_msg, handler, callbacks)
|
|
306
469
|
end
|
|
307
470
|
|
|
308
|
-
#
|
|
471
|
+
# Returns the callback helper for request/reply endpoints.
|
|
309
472
|
#
|
|
310
|
-
# @
|
|
311
|
-
|
|
473
|
+
# @return [NatsRequestReplyCallbacks] The request/reply callback helper.
|
|
474
|
+
def request_reply_callbacks
|
|
475
|
+
@request_reply_callbacks ||= NatsRequestReplyCallbacks.new(logger:)
|
|
476
|
+
end
|
|
477
|
+
|
|
478
|
+
# Returns the memoized message processor for this worker instance.
|
|
312
479
|
#
|
|
313
|
-
# @return [
|
|
314
|
-
def
|
|
315
|
-
|
|
316
|
-
|
|
317
|
-
|
|
318
|
-
|
|
319
|
-
|
|
320
|
-
|
|
480
|
+
# @return [MessageProcessor] The shared message processor.
|
|
481
|
+
def message_processor
|
|
482
|
+
@message_processor ||= MessageProcessor.new(
|
|
483
|
+
wrapper_factory: MessageWrapper.method(:new),
|
|
484
|
+
middleware: -> { self.class.middleware },
|
|
485
|
+
execute_handler: method(:execute_handler),
|
|
486
|
+
logger:,
|
|
487
|
+
)
|
|
321
488
|
end
|
|
322
489
|
|
|
323
|
-
#
|
|
490
|
+
# Executes an endpoint handler within the worker instance context.
|
|
324
491
|
#
|
|
325
|
-
# @param wrapper [MessageWrapper] The
|
|
326
|
-
# @param
|
|
492
|
+
# @param wrapper [MessageWrapper] The wrapped transport message.
|
|
493
|
+
# @param handler [Proc] The endpoint handler block.
|
|
327
494
|
#
|
|
328
|
-
# @return [
|
|
329
|
-
|
|
330
|
-
|
|
331
|
-
case result
|
|
332
|
-
in Dry::Monads::Success
|
|
333
|
-
wrapper.respond(result.value!)
|
|
334
|
-
in Dry::Monads::Failure
|
|
335
|
-
logger.error 'Error processing message: ', result.failure
|
|
336
|
-
wrapper.respond_with_error(result.failure)
|
|
337
|
-
else
|
|
338
|
-
logger.error('Unexpected result: ', result:)
|
|
339
|
-
raise ResultError, "Unexpected Response from Handler, must respond with a Success or Failure monad: #{result}"
|
|
340
|
-
end
|
|
495
|
+
# @return [Dry::Monads::Result] The handler result.
|
|
496
|
+
def execute_handler(wrapper, handler)
|
|
497
|
+
instance_exec(wrapper, &handler)
|
|
341
498
|
end
|
|
342
499
|
end
|
|
343
500
|
end
|
|
@@ -0,0 +1,76 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Rubyists
|
|
4
|
+
module Leopard
|
|
5
|
+
# Maps Leopard handler outcomes to JetStream ack, nak, and term operations.
|
|
6
|
+
class NatsJetstreamCallbacks
|
|
7
|
+
# Builds a callback set for JetStream message outcomes.
|
|
8
|
+
#
|
|
9
|
+
# @param logger [#error] Logger used for failures and unhandled exceptions.
|
|
10
|
+
#
|
|
11
|
+
# @return [void]
|
|
12
|
+
def initialize(logger:)
|
|
13
|
+
@logger = logger
|
|
14
|
+
end
|
|
15
|
+
|
|
16
|
+
# Returns transport callbacks for a JetStream endpoint.
|
|
17
|
+
#
|
|
18
|
+
# @param endpoint [NatsJetstreamEndpoint] The endpoint configuration being consumed.
|
|
19
|
+
#
|
|
20
|
+
# @return [Hash{Symbol => #call}] Outcome callbacks keyed by `:on_success`, `:on_failure`, and `:on_error`.
|
|
21
|
+
def callbacks_for(endpoint)
|
|
22
|
+
{
|
|
23
|
+
on_success: method(:ack_message),
|
|
24
|
+
on_failure: ->(wrapper, result) { nak_message(wrapper, result, endpoint) },
|
|
25
|
+
on_error: method(:term_message),
|
|
26
|
+
}
|
|
27
|
+
end
|
|
28
|
+
|
|
29
|
+
private
|
|
30
|
+
|
|
31
|
+
# Acknowledges a successfully processed JetStream message.
|
|
32
|
+
#
|
|
33
|
+
# @param wrapper [MessageWrapper] Wrapped JetStream message.
|
|
34
|
+
# @param _result [Dry::Monads::Success] Successful handler result.
|
|
35
|
+
#
|
|
36
|
+
# @return [void]
|
|
37
|
+
def ack_message(wrapper, _result)
|
|
38
|
+
wrapper.raw.ack
|
|
39
|
+
end
|
|
40
|
+
|
|
41
|
+
# Negatively acknowledges a failed JetStream message, optionally delaying redelivery.
|
|
42
|
+
#
|
|
43
|
+
# @param wrapper [MessageWrapper] Wrapped JetStream message.
|
|
44
|
+
# @param result [Dry::Monads::Failure] Failed handler result.
|
|
45
|
+
# @param endpoint [NatsJetstreamEndpoint] Endpoint configuration for the message.
|
|
46
|
+
#
|
|
47
|
+
# @return [void]
|
|
48
|
+
def nak_message(wrapper, result, endpoint)
|
|
49
|
+
log_failure(result.failure)
|
|
50
|
+
return wrapper.raw.nak unless endpoint.nak_delay
|
|
51
|
+
|
|
52
|
+
wrapper.raw.nak(delay: endpoint.nak_delay)
|
|
53
|
+
end
|
|
54
|
+
|
|
55
|
+
# Terminates a JetStream message after an unhandled exception.
|
|
56
|
+
#
|
|
57
|
+
# @param wrapper [MessageWrapper] Wrapped JetStream message.
|
|
58
|
+
# @param error [StandardError] The unhandled exception.
|
|
59
|
+
#
|
|
60
|
+
# @return [void]
|
|
61
|
+
def term_message(wrapper, error)
|
|
62
|
+
@logger.error 'Unhandled JetStream error: ', error
|
|
63
|
+
wrapper.raw.term
|
|
64
|
+
end
|
|
65
|
+
|
|
66
|
+
# Logs the failure payload returned by a handler.
|
|
67
|
+
#
|
|
68
|
+
# @param failure [Object] The failure payload from the handler.
|
|
69
|
+
#
|
|
70
|
+
# @return [void]
|
|
71
|
+
def log_failure(failure)
|
|
72
|
+
@logger.error 'Error processing message: ', failure
|
|
73
|
+
end
|
|
74
|
+
end
|
|
75
|
+
end
|
|
76
|
+
end
|
|
@@ -0,0 +1,186 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require_relative 'nats_jetstream_callbacks'
|
|
4
|
+
require_relative 'nats_jetstream_endpoint'
|
|
5
|
+
|
|
6
|
+
module Rubyists
|
|
7
|
+
module Leopard
|
|
8
|
+
# Coordinates JetStream pull subscriptions and dispatches fetched messages through Leopard.
|
|
9
|
+
class NatsJetstreamConsumer
|
|
10
|
+
# Consumer configuration keys Leopard owns and will not allow endpoint overrides to replace.
|
|
11
|
+
PROTECTED_CONSUMER_KEYS = %i[durable_name filter_subject ack_policy].freeze
|
|
12
|
+
|
|
13
|
+
# @!attribute [r] subscriptions
|
|
14
|
+
#
|
|
15
|
+
# @return [Array<Object>] Active JetStream pull subscriptions.
|
|
16
|
+
# @!attribute [r] threads
|
|
17
|
+
#
|
|
18
|
+
# @return [Array<Thread>] Consumer loop threads for each endpoint.
|
|
19
|
+
attr_reader :subscriptions, :threads
|
|
20
|
+
|
|
21
|
+
# Builds a pull-consumer coordinator for one Leopard worker.
|
|
22
|
+
#
|
|
23
|
+
# @param jetstream [Object] JetStream client used to manage consumers and subscriptions.
|
|
24
|
+
# @param endpoints [Array<NatsJetstreamEndpoint>] JetStream endpoint definitions for this worker.
|
|
25
|
+
# @param logger [#error] Logger used for loop failures.
|
|
26
|
+
# @param process_message [#call] Callable that processes a raw JetStream message through Leopard.
|
|
27
|
+
# @param dependencies [Hash{Symbol => Object}] Optional collaborators for callback and thread creation.
|
|
28
|
+
# @option dependencies [Class] :callback_builder (NatsJetstreamCallbacks) Builder for transport callbacks.
|
|
29
|
+
# @option dependencies [Class] :thread_factory (Thread) Thread-like factory used to spawn consumer loops.
|
|
30
|
+
#
|
|
31
|
+
# @return [void]
|
|
32
|
+
def initialize(jetstream:, endpoints:, logger:, process_message:, **dependencies)
|
|
33
|
+
@jetstream = jetstream
|
|
34
|
+
@endpoints = endpoints
|
|
35
|
+
@logger = logger
|
|
36
|
+
@process_message = process_message
|
|
37
|
+
@callbacks = dependencies.fetch(:callback_builder, NatsJetstreamCallbacks).new(logger:)
|
|
38
|
+
@thread_factory = dependencies.fetch(:thread_factory, Thread)
|
|
39
|
+
@subscriptions = []
|
|
40
|
+
@threads = []
|
|
41
|
+
@running = false
|
|
42
|
+
end
|
|
43
|
+
|
|
44
|
+
# Starts one pull-consumer loop per configured endpoint.
|
|
45
|
+
#
|
|
46
|
+
# @return [void]
|
|
47
|
+
def start
|
|
48
|
+
@running = true
|
|
49
|
+
@endpoints.each { |endpoint| start_endpoint(endpoint) }
|
|
50
|
+
end
|
|
51
|
+
|
|
52
|
+
# Stops all pull-consumer loops and waits for them to exit.
|
|
53
|
+
#
|
|
54
|
+
# @return [void]
|
|
55
|
+
def stop
|
|
56
|
+
@running = false
|
|
57
|
+
subscriptions.each(&:unsubscribe)
|
|
58
|
+
threads.each(&:join)
|
|
59
|
+
end
|
|
60
|
+
|
|
61
|
+
private
|
|
62
|
+
|
|
63
|
+
# Starts a consumer loop for one endpoint.
|
|
64
|
+
#
|
|
65
|
+
# @param endpoint [NatsJetstreamEndpoint] The endpoint configuration to consume.
|
|
66
|
+
#
|
|
67
|
+
# @return [void]
|
|
68
|
+
def start_endpoint(endpoint)
|
|
69
|
+
subscription = build_subscription(endpoint)
|
|
70
|
+
subscriptions << subscription
|
|
71
|
+
threads << @thread_factory.new { consume_endpoint(subscription, endpoint) }
|
|
72
|
+
end
|
|
73
|
+
|
|
74
|
+
# Ensures the durable consumer exists and creates a pull subscription for it.
|
|
75
|
+
#
|
|
76
|
+
# @param endpoint [NatsJetstreamEndpoint] The endpoint configuration to subscribe to.
|
|
77
|
+
#
|
|
78
|
+
# @return [Object] The JetStream pull subscription.
|
|
79
|
+
def build_subscription(endpoint)
|
|
80
|
+
ensure_consumer(endpoint)
|
|
81
|
+
@jetstream.pull_subscribe(
|
|
82
|
+
endpoint.subject,
|
|
83
|
+
endpoint.durable,
|
|
84
|
+
stream: endpoint.stream,
|
|
85
|
+
)
|
|
86
|
+
end
|
|
87
|
+
|
|
88
|
+
# Verifies that the durable consumer exists, creating it when missing.
|
|
89
|
+
#
|
|
90
|
+
# @param endpoint [NatsJetstreamEndpoint] The endpoint configuration to ensure.
|
|
91
|
+
#
|
|
92
|
+
# @return [Object] Consumer metadata from `consumer_info` or `add_consumer`.
|
|
93
|
+
def ensure_consumer(endpoint)
|
|
94
|
+
@jetstream.consumer_info(endpoint.stream, endpoint.durable)
|
|
95
|
+
rescue NATS::JetStream::Error::NotFound
|
|
96
|
+
@jetstream.add_consumer(endpoint.stream, consumer_config(endpoint))
|
|
97
|
+
end
|
|
98
|
+
|
|
99
|
+
# Builds the JetStream consumer configuration for an endpoint.
|
|
100
|
+
#
|
|
101
|
+
# @param endpoint [NatsJetstreamEndpoint] The endpoint configuration to translate.
|
|
102
|
+
#
|
|
103
|
+
# @return [Hash] Consumer configuration accepted by `add_consumer`.
|
|
104
|
+
def consumer_config(endpoint)
|
|
105
|
+
base = {
|
|
106
|
+
durable_name: endpoint.durable,
|
|
107
|
+
filter_subject: endpoint.subject,
|
|
108
|
+
ack_policy: 'explicit',
|
|
109
|
+
}
|
|
110
|
+
base.merge(safe_consumer_options(endpoint))
|
|
111
|
+
end
|
|
112
|
+
|
|
113
|
+
# Normalizes optional consumer overrides into a hash.
|
|
114
|
+
#
|
|
115
|
+
# @param endpoint [NatsJetstreamEndpoint] The endpoint configuration to inspect.
|
|
116
|
+
#
|
|
117
|
+
# @return [Hash] Consumer overrides, or an empty hash when none were provided.
|
|
118
|
+
def normalized_consumer_options(endpoint)
|
|
119
|
+
return {} unless endpoint.consumer
|
|
120
|
+
return endpoint.consumer.to_h if endpoint.consumer.respond_to?(:to_h)
|
|
121
|
+
|
|
122
|
+
endpoint.consumer
|
|
123
|
+
end
|
|
124
|
+
|
|
125
|
+
# Removes Leopard-managed consumer keys from user overrides.
|
|
126
|
+
#
|
|
127
|
+
# @param endpoint [NatsJetstreamEndpoint] The endpoint configuration to inspect.
|
|
128
|
+
#
|
|
129
|
+
# @return [Hash] Consumer overrides excluding protected keys required by Leopard.
|
|
130
|
+
def safe_consumer_options(endpoint)
|
|
131
|
+
normalized_consumer_options(endpoint).reject { |key, _value| PROTECTED_CONSUMER_KEYS.include?(key.to_sym) }
|
|
132
|
+
end
|
|
133
|
+
|
|
134
|
+
# Repeatedly fetches and processes batches for one endpoint while the consumer is running.
|
|
135
|
+
#
|
|
136
|
+
# @param subscription [Object] Pull subscription for the endpoint.
|
|
137
|
+
# @param endpoint [NatsJetstreamEndpoint] The endpoint configuration being consumed.
|
|
138
|
+
#
|
|
139
|
+
# @return [void]
|
|
140
|
+
def consume_endpoint(subscription, endpoint)
|
|
141
|
+
while @running
|
|
142
|
+
begin
|
|
143
|
+
consume_batch(subscription, endpoint)
|
|
144
|
+
rescue NATS::Timeout
|
|
145
|
+
next if @running
|
|
146
|
+
rescue StandardError => e
|
|
147
|
+
log_loop_error(endpoint, e)
|
|
148
|
+
break unless @running
|
|
149
|
+
end
|
|
150
|
+
end
|
|
151
|
+
end
|
|
152
|
+
|
|
153
|
+
# Fetches one batch from JetStream and processes each message through Leopard.
|
|
154
|
+
#
|
|
155
|
+
# @param subscription [Object] Pull subscription for the endpoint.
|
|
156
|
+
# @param endpoint [NatsJetstreamEndpoint] The endpoint configuration being consumed.
|
|
157
|
+
#
|
|
158
|
+
# @return [void]
|
|
159
|
+
def consume_batch(subscription, endpoint)
|
|
160
|
+
fetch_messages(subscription, endpoint).each do |raw_msg|
|
|
161
|
+
@process_message.call(raw_msg, endpoint.handler, @callbacks.callbacks_for(endpoint))
|
|
162
|
+
end
|
|
163
|
+
end
|
|
164
|
+
|
|
165
|
+
# Fetches a batch of messages for one endpoint.
|
|
166
|
+
#
|
|
167
|
+
# @param subscription [Object] Pull subscription for the endpoint.
|
|
168
|
+
# @param endpoint [NatsJetstreamEndpoint] The endpoint configuration being consumed.
|
|
169
|
+
#
|
|
170
|
+
# @return [Array<Object>] Raw JetStream messages returned by the subscription.
|
|
171
|
+
def fetch_messages(subscription, endpoint)
|
|
172
|
+
subscription.fetch(endpoint.batch, timeout: endpoint.fetch_timeout)
|
|
173
|
+
end
|
|
174
|
+
|
|
175
|
+
# Logs an endpoint-level loop failure.
|
|
176
|
+
#
|
|
177
|
+
# @param endpoint [NatsJetstreamEndpoint] The endpoint whose loop failed.
|
|
178
|
+
# @param error [StandardError] The raised exception.
|
|
179
|
+
#
|
|
180
|
+
# @return [void]
|
|
181
|
+
def log_loop_error(endpoint, error)
|
|
182
|
+
@logger.error "JetStream endpoint #{endpoint.name} loop error: ", error
|
|
183
|
+
end
|
|
184
|
+
end
|
|
185
|
+
end
|
|
186
|
+
end
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Rubyists
|
|
4
|
+
module Leopard
|
|
5
|
+
# Configuration for a Leopard JetStream pull-consumer endpoint.
|
|
6
|
+
NatsJetstreamEndpoint = Struct.new(
|
|
7
|
+
:name,
|
|
8
|
+
:stream,
|
|
9
|
+
:subject,
|
|
10
|
+
:durable,
|
|
11
|
+
:consumer,
|
|
12
|
+
:batch,
|
|
13
|
+
:fetch_timeout,
|
|
14
|
+
:nak_delay,
|
|
15
|
+
:handler,
|
|
16
|
+
keyword_init: true,
|
|
17
|
+
)
|
|
18
|
+
end
|
|
19
|
+
end
|