nats_wave 1.1.7 → 1.1.9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/Gemfile.lock +1 -1
- data/lib/nats_wave/active_record_extension.rb +93 -0
- data/lib/nats_wave/adapters/active_record.rb +207 -0
- data/lib/nats_wave/adapters/datadog_metrics.rb +1 -1
- data/lib/nats_wave/client.rb +426 -158
- data/lib/nats_wave/concerns/mappable.rb +481 -117
- data/lib/nats_wave/configuration.rb +1 -1
- data/lib/nats_wave/database_connector.rb +51 -0
- data/lib/nats_wave/publisher.rb +142 -39
- data/lib/nats_wave/railtie.rb +126 -6
- data/lib/nats_wave/subscriber.rb +588 -50
- data/lib/nats_wave/version.rb +1 -1
- data/lib/nats_wave.rb +99 -0
- metadata +3 -3
- data/lib/nats_wave/concerns/publishable.rb +0 -216
data/lib/nats_wave/subscriber.rb
CHANGED
@@ -1,3 +1,394 @@
|
|
1
|
+
# # frozen_string_literal: true
|
2
|
+
#
|
3
|
+
# module NatsWave
|
4
|
+
# class Subscriber
|
5
|
+
# attr_reader :config, :client
|
6
|
+
#
|
7
|
+
# def initialize(config, client, middleware_stack = [])
|
8
|
+
# @config = config
|
9
|
+
# @client = client
|
10
|
+
# @original_client_factory = nil # Store how to recreate the client
|
11
|
+
# @database_connector = DatabaseConnector.new(config)
|
12
|
+
# @model_mapper = ModelMapper.new(config)
|
13
|
+
# @message_transformer = MessageTransformer.new(config)
|
14
|
+
# @dead_letter_queue = DeadLetterQueue.new(config)
|
15
|
+
#
|
16
|
+
# @registry_subscriptions = ModelRegistry.subscriptions
|
17
|
+
# @nats_subscriptions = [] # NATS::Subscription objects
|
18
|
+
# @running = false
|
19
|
+
# @shutdown = false
|
20
|
+
# @reconnecting = false
|
21
|
+
# @connection_stats = {
|
22
|
+
# reconnect_count: 0,
|
23
|
+
# last_successful_connection: Time.current,
|
24
|
+
# consecutive_failures: 0
|
25
|
+
# }
|
26
|
+
# end
|
27
|
+
#
|
28
|
+
# # Add method to store client factory for reconnection
|
29
|
+
# def set_client_factory(factory_proc)
|
30
|
+
# @original_client_factory = factory_proc
|
31
|
+
# end
|
32
|
+
#
|
33
|
+
# def begin
|
34
|
+
# return if @running || @shutdown
|
35
|
+
# return unless @config.subscription_enabled
|
36
|
+
#
|
37
|
+
# @running = true
|
38
|
+
# @shutdown = false
|
39
|
+
#
|
40
|
+
# NatsWave.logger.info "Starting NATS subscriber for #{@config.service_name}"
|
41
|
+
#
|
42
|
+
# setup_subscriptions
|
43
|
+
#
|
44
|
+
# NatsWave.logger.info "Started #{@registry_subscriptions.size} subscriptions from Model Registry"
|
45
|
+
#
|
46
|
+
# # Keep the subscriber alive
|
47
|
+
# keep_alive
|
48
|
+
# end
|
49
|
+
#
|
50
|
+
# def listen(subjects:, model_mappings: {}, handler: nil)
|
51
|
+
# subjects = Array(subjects)
|
52
|
+
#
|
53
|
+
# subjects.each do |subject|
|
54
|
+
# subscribe_to_subject(subject, handler, model_mappings)
|
55
|
+
# NatsWave.logger.info "Subscribed to #{subject}"
|
56
|
+
# end
|
57
|
+
# end
|
58
|
+
#
|
59
|
+
# def reset
|
60
|
+
# @shutdown = true
|
61
|
+
# @running = false
|
62
|
+
#
|
63
|
+
# # Stop keep alive thread
|
64
|
+
# if @keep_alive_thread&.alive?
|
65
|
+
# @keep_alive_thread.kill
|
66
|
+
# @keep_alive_thread.join(5) # Wait up to 5 seconds for graceful shutdown
|
67
|
+
# @keep_alive_thread = nil
|
68
|
+
# end
|
69
|
+
#
|
70
|
+
# # Unsubscribe from all subscriptions
|
71
|
+
# cleanup_subscriptions
|
72
|
+
#
|
73
|
+
# NatsWave.logger.info "🛑 Subscriber shutdown complete"
|
74
|
+
# end
|
75
|
+
#
|
76
|
+
# def database_connected?
|
77
|
+
# @database_connector.connected?
|
78
|
+
# end
|
79
|
+
#
|
80
|
+
# def disconnect
|
81
|
+
# reset
|
82
|
+
# end
|
83
|
+
#
|
84
|
+
# private
|
85
|
+
#
|
86
|
+
# def setup_subscriptions
|
87
|
+
# return unless @client&.connected?
|
88
|
+
#
|
89
|
+
# # Clear existing subscriptions first
|
90
|
+
# cleanup_subscriptions
|
91
|
+
#
|
92
|
+
# # Use ModelRegistry subscriptions
|
93
|
+
# @registry_subscriptions.each do |subscription|
|
94
|
+
# subscription[:subjects].each do |subject|
|
95
|
+
# subscribe_to_subject(subject, subscription[:handler])
|
96
|
+
# end
|
97
|
+
# end
|
98
|
+
# end
|
99
|
+
#
|
100
|
+
# def cleanup_subscriptions
|
101
|
+
# @nats_subscriptions.each do |subscription|
|
102
|
+
# begin
|
103
|
+
# subscription.unsubscribe if subscription.respond_to?(:unsubscribe)
|
104
|
+
# rescue => e
|
105
|
+
# NatsWave.logger.error "Error unsubscribing: #{e.message}"
|
106
|
+
# end
|
107
|
+
# end
|
108
|
+
# @nats_subscriptions.clear
|
109
|
+
# end
|
110
|
+
#
|
111
|
+
# def subscribe_to_subject(subject_pattern, custom_handler = nil, model_mappings = {})
|
112
|
+
# return unless @client&.connected?
|
113
|
+
#
|
114
|
+
# NatsWave.logger.info "🔍 Attempting to subscribe to: #{subject_pattern}"
|
115
|
+
#
|
116
|
+
# begin
|
117
|
+
# # Create the NATS subscription
|
118
|
+
# nats_subscription = @client.subscribe(
|
119
|
+
# subject_pattern,
|
120
|
+
# queue: @config.queue_group
|
121
|
+
# ) do |msg|
|
122
|
+
# begin
|
123
|
+
# NatsWave.logger.debug "📨 Received message on #{msg.subject}"
|
124
|
+
# process_message(msg.data, custom_handler, model_mappings)
|
125
|
+
# rescue => e
|
126
|
+
# NatsWave.logger.error "Error in subscription handler: #{e.message}"
|
127
|
+
# NatsWave.logger.error e.backtrace.join("\n")
|
128
|
+
# # Don't re-raise - this would kill the subscription
|
129
|
+
# end
|
130
|
+
# end
|
131
|
+
#
|
132
|
+
# # Add to NATS subscriptions array
|
133
|
+
# @nats_subscriptions << nats_subscription
|
134
|
+
# NatsWave.logger.info "✅ Successfully subscribed to #{subject_pattern} (total: #{@nats_subscriptions.size})"
|
135
|
+
#
|
136
|
+
# rescue => e
|
137
|
+
# NatsWave.logger.error "Failed to subscribe to #{subject_pattern}: #{e.message}"
|
138
|
+
# raise
|
139
|
+
# end
|
140
|
+
# end
|
141
|
+
#
|
142
|
+
# def process_message(raw_message, custom_handler, model_mappings)
|
143
|
+
# return unless should_process_message?(raw_message)
|
144
|
+
#
|
145
|
+
# NatsWave.logger.debug "🔄 Processing message: #{raw_message[0..200]}..."
|
146
|
+
#
|
147
|
+
# message = parse_message(raw_message)
|
148
|
+
#
|
149
|
+
# if custom_handler
|
150
|
+
# custom_handler.call(message)
|
151
|
+
# else
|
152
|
+
# handle_model_sync(message, model_mappings)
|
153
|
+
# end
|
154
|
+
#
|
155
|
+
# NatsWave.logger.debug('✅ Successfully processed message')
|
156
|
+
# rescue StandardError => e
|
157
|
+
# handle_error(e, raw_message, message)
|
158
|
+
# # Don't re-raise - this would kill the subscription
|
159
|
+
# end
|
160
|
+
#
|
161
|
+
# def should_process_message?(raw_message_data)
|
162
|
+
# message = JSON.parse(raw_message_data)
|
163
|
+
# source = message['source'] || {}
|
164
|
+
#
|
165
|
+
# # Skip messages from same service instance
|
166
|
+
# if source['service'] == @config.service_name && source['instance_id'] == @config.instance_id
|
167
|
+
# NatsWave.logger.debug "🔄 Skipping message from same service instance"
|
168
|
+
# return false
|
169
|
+
# end
|
170
|
+
#
|
171
|
+
# true
|
172
|
+
# rescue JSON::ParserError => e
|
173
|
+
# NatsWave.logger.error "Failed to parse message for filtering: #{e.message}"
|
174
|
+
# false
|
175
|
+
# end
|
176
|
+
#
|
177
|
+
# def keep_alive
|
178
|
+
# NatsWave.logger.info "🔄 Starting keep-alive thread for persistent JetStream connection"
|
179
|
+
#
|
180
|
+
# @keep_alive_thread = Thread.new do
|
181
|
+
# while @running && !@shutdown
|
182
|
+
# begin
|
183
|
+
# sleep 30 # Check every 30 seconds
|
184
|
+
#
|
185
|
+
# connection_healthy = check_connection_health
|
186
|
+
#
|
187
|
+
# if connection_healthy
|
188
|
+
# NatsWave.logger.debug "💓 Subscriber connection healthy - #{@nats_subscriptions.size} active subscriptions"
|
189
|
+
# @connection_stats[:consecutive_failures] = 0
|
190
|
+
# @connection_stats[:last_successful_connection] = Time.current
|
191
|
+
# else
|
192
|
+
# @connection_stats[:consecutive_failures] += 1
|
193
|
+
# time_since_last_good = Time.current - @connection_stats[:last_successful_connection]
|
194
|
+
#
|
195
|
+
# NatsWave.logger.error "❌ Subscriber connection lost! (failure ##{@connection_stats[:consecutive_failures]}, last good: #{time_since_last_good.to_i}s ago)"
|
196
|
+
#
|
197
|
+
# log_connection_diagnostics
|
198
|
+
# attempt_reconnection
|
199
|
+
# end
|
200
|
+
#
|
201
|
+
# # If too many consecutive failures, escalate
|
202
|
+
# if @connection_stats[:consecutive_failures] > 5
|
203
|
+
# NatsWave.logger.error "🚨 Too many consecutive failures. Performing full reset."
|
204
|
+
# perform_full_reset
|
205
|
+
# end
|
206
|
+
#
|
207
|
+
# rescue => e
|
208
|
+
# NatsWave.logger.error "Error in keep_alive thread: #{e.message}\n#{e.backtrace.first(3).join("\n")}"
|
209
|
+
# sleep 10
|
210
|
+
# end
|
211
|
+
# end
|
212
|
+
#
|
213
|
+
# NatsWave.logger.info "🛑 Keep-alive thread shutting down"
|
214
|
+
# end
|
215
|
+
# end
|
216
|
+
#
|
217
|
+
# def check_connection_health
|
218
|
+
# return false unless @client
|
219
|
+
#
|
220
|
+
# begin
|
221
|
+
# # First check basic connectivity
|
222
|
+
# return false unless @client.connected?
|
223
|
+
#
|
224
|
+
# # Try to perform a simple operation with timeout
|
225
|
+
# Timeout.timeout(5) do
|
226
|
+
# @client.flush
|
227
|
+
# return true
|
228
|
+
# end
|
229
|
+
# rescue Timeout::Error
|
230
|
+
# NatsWave.logger.warn "⏰ Connection health check timed out"
|
231
|
+
# return false
|
232
|
+
# rescue => e
|
233
|
+
# NatsWave.logger.warn "🔍 Health check failed: #{e.message}"
|
234
|
+
# return false
|
235
|
+
# end
|
236
|
+
# end
|
237
|
+
#
|
238
|
+
# def log_connection_diagnostics
|
239
|
+
# begin
|
240
|
+
# NatsWave.logger.info "🔍 Connection Diagnostics:"
|
241
|
+
# NatsWave.logger.info " - Client connected?: #{@client&.connected? rescue 'unknown'}"
|
242
|
+
# NatsWave.logger.info " - Active subscriptions: #{@nats_subscriptions.size}"
|
243
|
+
# NatsWave.logger.info " - Reconnect count: #{@connection_stats[:reconnect_count]}"
|
244
|
+
# NatsWave.logger.info " - Thread count: #{Thread.list.count}"
|
245
|
+
# NatsWave.logger.info " - Last successful: #{@connection_stats[:last_successful_connection]}"
|
246
|
+
#
|
247
|
+
# # Check if subscriptions are actually valid
|
248
|
+
# valid_subs = @nats_subscriptions.count { |sub| sub.respond_to?(:unsubscribe) }
|
249
|
+
# NatsWave.logger.info " - Valid subscriptions: #{valid_subs}/#{@nats_subscriptions.size}"
|
250
|
+
#
|
251
|
+
# rescue => e
|
252
|
+
# NatsWave.logger.error "Failed to gather diagnostics: #{e.message}"
|
253
|
+
# end
|
254
|
+
# end
|
255
|
+
#
|
256
|
+
# def attempt_reconnection
|
257
|
+
# return if @shutdown || @reconnecting
|
258
|
+
#
|
259
|
+
# @reconnecting = true
|
260
|
+
# @connection_stats[:reconnect_count] += 1
|
261
|
+
#
|
262
|
+
# begin
|
263
|
+
# NatsWave.logger.info "🔄 Attempting to reconnect to NATS... (attempt ##{@connection_stats[:reconnect_count]})"
|
264
|
+
#
|
265
|
+
# # First, clean up existing connection and subscriptions
|
266
|
+
# cleanup_subscriptions
|
267
|
+
#
|
268
|
+
# # Close the existing client connection
|
269
|
+
# begin
|
270
|
+
# @client&.close if @client&.connected?
|
271
|
+
# rescue => e
|
272
|
+
# NatsWave.logger.debug "Error closing existing connection: #{e.message}"
|
273
|
+
# end
|
274
|
+
#
|
275
|
+
# # Wait before attempting to reconnect
|
276
|
+
# sleep_time = [2 ** [@connection_stats[:consecutive_failures] - 1, 4].min, 30].min
|
277
|
+
# NatsWave.logger.debug "⏱️ Waiting #{sleep_time}s before reconnection attempt"
|
278
|
+
# sleep sleep_time
|
279
|
+
#
|
280
|
+
# # Attempt to recreate the client connection
|
281
|
+
# if @original_client_factory
|
282
|
+
# @client = @original_client_factory.call
|
283
|
+
# NatsWave.logger.info "🔄 Created new NATS client using factory"
|
284
|
+
# else
|
285
|
+
# NatsWave.logger.warn "⚠️ No client factory available, hoping existing client recovers"
|
286
|
+
# end
|
287
|
+
#
|
288
|
+
# # Check if we have a working connection
|
289
|
+
# if @client&.connected?
|
290
|
+
# NatsWave.logger.info "✅ NATS client reconnected successfully"
|
291
|
+
#
|
292
|
+
# # Re-establish all subscriptions
|
293
|
+
# setup_subscriptions
|
294
|
+
#
|
295
|
+
# NatsWave.logger.info "✅ Subscriptions restored (#{@nats_subscriptions.size} active)"
|
296
|
+
# @connection_stats[:consecutive_failures] = 0
|
297
|
+
# @connection_stats[:last_successful_connection] = Time.current
|
298
|
+
#
|
299
|
+
# return true
|
300
|
+
# else
|
301
|
+
# raise "Connection failed - client not connected"
|
302
|
+
# end
|
303
|
+
#
|
304
|
+
# rescue => e
|
305
|
+
# NatsWave.logger.error "Reconnection failed: #{e.message}"
|
306
|
+
# return false
|
307
|
+
# ensure
|
308
|
+
# @reconnecting = false
|
309
|
+
# end
|
310
|
+
# end
|
311
|
+
#
|
312
|
+
# def perform_full_reset
|
313
|
+
# NatsWave.logger.info "🔄 Performing full connection reset"
|
314
|
+
#
|
315
|
+
# begin
|
316
|
+
# # Force close everything
|
317
|
+
# cleanup_subscriptions
|
318
|
+
# @client&.close rescue nil
|
319
|
+
#
|
320
|
+
# # Clear connection stats
|
321
|
+
# @connection_stats[:consecutive_failures] = 0
|
322
|
+
#
|
323
|
+
# # Force garbage collection
|
324
|
+
# GC.start
|
325
|
+
#
|
326
|
+
# # Wait longer before attempting reset
|
327
|
+
# sleep 10
|
328
|
+
#
|
329
|
+
# # Try to recreate everything
|
330
|
+
# if @original_client_factory
|
331
|
+
# @client = @original_client_factory.call
|
332
|
+
# setup_subscriptions if @client&.connected?
|
333
|
+
# end
|
334
|
+
#
|
335
|
+
# rescue => e
|
336
|
+
# NatsWave.logger.error "Full reset failed: #{e.message}"
|
337
|
+
# end
|
338
|
+
# end
|
339
|
+
#
|
340
|
+
# # ... rest of your existing methods (parse_message, handle_model_sync, handle_error)
|
341
|
+
# def parse_message(raw_message)
|
342
|
+
# @message_transformer.parse_message(raw_message)
|
343
|
+
# end
|
344
|
+
#
|
345
|
+
# def handle_model_sync(message, model_mappings)
|
346
|
+
# source_model = message['model']
|
347
|
+
# mapping = model_mappings[source_model] || @config.model_mappings[source_model]
|
348
|
+
#
|
349
|
+
# return unless mapping
|
350
|
+
#
|
351
|
+
# target_model = mapping[:target_model]
|
352
|
+
# field_mappings = mapping[:field_mappings] || {}
|
353
|
+
# transformations = mapping[:transformations] || {}
|
354
|
+
#
|
355
|
+
# # Transform the data
|
356
|
+
# transformed_data = @model_mapper.transform_data(
|
357
|
+
# message['data'],
|
358
|
+
# field_mappings,
|
359
|
+
# transformations
|
360
|
+
# )
|
361
|
+
#
|
362
|
+
# # Apply to database
|
363
|
+
# @database_connector.apply_change(
|
364
|
+
# model: target_model,
|
365
|
+
# action: message['action'],
|
366
|
+
# data: transformed_data,
|
367
|
+
# metadata: message['metadata']
|
368
|
+
# )
|
369
|
+
#
|
370
|
+
# NatsWave.logger.info "Synced #{source_model} -> #{target_model}: #{message['action']}"
|
371
|
+
# end
|
372
|
+
#
|
373
|
+
# def handle_error(error, raw_message, parsed_message = nil)
|
374
|
+
# event_id = parsed_message&.dig('event_id') || 'unknown'
|
375
|
+
# subject = parsed_message&.dig('subject') || 'unknown'
|
376
|
+
#
|
377
|
+
# NatsWave.logger.error("Error processing message #{event_id} from #{subject}: #{error.message}")
|
378
|
+
#
|
379
|
+
# # Send to dead letter queue
|
380
|
+
# @dead_letter_queue.store_failed_message(
|
381
|
+
# parsed_message || raw_message,
|
382
|
+
# error,
|
383
|
+
# 0 # retry count
|
384
|
+
# )
|
385
|
+
#
|
386
|
+
# # Continue processing - don't raise to avoid breaking subscription
|
387
|
+
# end
|
388
|
+
# end
|
389
|
+
# end
|
390
|
+
|
391
|
+
|
1
392
|
# frozen_string_literal: true
|
2
393
|
|
3
394
|
module NatsWave
|
@@ -7,6 +398,7 @@ module NatsWave
|
|
7
398
|
def initialize(config, client, middleware_stack = [])
|
8
399
|
@config = config
|
9
400
|
@client = client
|
401
|
+
@original_client_factory = nil # Store how to recreate the client
|
10
402
|
@database_connector = DatabaseConnector.new(config)
|
11
403
|
@model_mapper = ModelMapper.new(config)
|
12
404
|
@message_transformer = MessageTransformer.new(config)
|
@@ -16,6 +408,17 @@ module NatsWave
|
|
16
408
|
@nats_subscriptions = [] # NATS::Subscription objects
|
17
409
|
@running = false
|
18
410
|
@shutdown = false
|
411
|
+
@reconnecting = false
|
412
|
+
@connection_stats = {
|
413
|
+
reconnect_count: 0,
|
414
|
+
last_successful_connection: Time.current,
|
415
|
+
consecutive_failures: 0
|
416
|
+
}
|
417
|
+
end
|
418
|
+
|
419
|
+
# Add method to store client factory for reconnection
|
420
|
+
def set_client_factory(factory_proc)
|
421
|
+
@original_client_factory = factory_proc
|
19
422
|
end
|
20
423
|
|
21
424
|
def begin
|
@@ -27,12 +430,7 @@ module NatsWave
|
|
27
430
|
|
28
431
|
NatsWave.logger.info "Starting NATS subscriber for #{@config.service_name}"
|
29
432
|
|
30
|
-
|
31
|
-
@registry_subscriptions.each do |subscription|
|
32
|
-
subscription[:subjects].each do |subject|
|
33
|
-
subscribe_to_subject(subject, subscription[:handler])
|
34
|
-
end
|
35
|
-
end
|
433
|
+
setup_subscriptions
|
36
434
|
|
37
435
|
NatsWave.logger.info "Started #{@registry_subscriptions.size} subscriptions from Model Registry"
|
38
436
|
|
@@ -56,18 +454,12 @@ module NatsWave
|
|
56
454
|
# Stop keep alive thread
|
57
455
|
if @keep_alive_thread&.alive?
|
58
456
|
@keep_alive_thread.kill
|
457
|
+
@keep_alive_thread.join(5) # Wait up to 5 seconds for graceful shutdown
|
59
458
|
@keep_alive_thread = nil
|
60
459
|
end
|
61
460
|
|
62
461
|
# Unsubscribe from all subscriptions
|
63
|
-
|
64
|
-
begin
|
65
|
-
subscription.unsubscribe if subscription.respond_to?(:unsubscribe)
|
66
|
-
rescue => e
|
67
|
-
NatsWave.logger.error "Error unsubscribing: #{e.message}"
|
68
|
-
end
|
69
|
-
end
|
70
|
-
@nats_subscriptions.clear
|
462
|
+
cleanup_subscriptions
|
71
463
|
|
72
464
|
NatsWave.logger.info "🛑 Subscriber shutdown complete"
|
73
465
|
end
|
@@ -82,27 +474,60 @@ module NatsWave
|
|
82
474
|
|
83
475
|
private
|
84
476
|
|
85
|
-
def
|
86
|
-
|
477
|
+
def setup_subscriptions
|
478
|
+
return unless @client&.connected?
|
479
|
+
|
480
|
+
# Clear existing subscriptions first
|
481
|
+
cleanup_subscriptions
|
87
482
|
|
88
|
-
#
|
89
|
-
|
90
|
-
|
91
|
-
|
92
|
-
|
483
|
+
# Use ModelRegistry subscriptions
|
484
|
+
@registry_subscriptions.each do |subscription|
|
485
|
+
subscription[:subjects].each do |subject|
|
486
|
+
subscribe_to_subject(subject, subscription[:handler])
|
487
|
+
end
|
488
|
+
end
|
489
|
+
end
|
490
|
+
|
491
|
+
def cleanup_subscriptions
|
492
|
+
@nats_subscriptions.each do |subscription|
|
93
493
|
begin
|
94
|
-
|
95
|
-
process_message(msg.data, custom_handler, model_mappings)
|
494
|
+
subscription.unsubscribe if subscription.respond_to?(:unsubscribe)
|
96
495
|
rescue => e
|
97
|
-
NatsWave.logger.error "Error
|
98
|
-
NatsWave.logger.error e.backtrace.join("\n")
|
99
|
-
# Don't re-raise - this would kill the subscription
|
496
|
+
NatsWave.logger.error "Error unsubscribing: #{e.message}"
|
100
497
|
end
|
101
498
|
end
|
499
|
+
@nats_subscriptions.clear
|
500
|
+
end
|
501
|
+
|
502
|
+
def subscribe_to_subject(subject_pattern, custom_handler = nil, model_mappings = {})
|
503
|
+
return unless @client&.connected?
|
504
|
+
|
505
|
+
NatsWave.logger.info "🔍 Attempting to subscribe to: #{subject_pattern}"
|
506
|
+
|
507
|
+
begin
|
508
|
+
# Create the NATS subscription
|
509
|
+
nats_subscription = @client.subscribe(
|
510
|
+
subject_pattern,
|
511
|
+
queue: @config.queue_group
|
512
|
+
) do |msg|
|
513
|
+
begin
|
514
|
+
NatsWave.logger.debug "📨 Received message on #{msg.subject}"
|
515
|
+
process_message(msg.data, custom_handler, model_mappings)
|
516
|
+
rescue => e
|
517
|
+
NatsWave.logger.error "Error in subscription handler: #{e.message}"
|
518
|
+
NatsWave.logger.error e.backtrace.join("\n")
|
519
|
+
# Don't re-raise - this would kill the subscription
|
520
|
+
end
|
521
|
+
end
|
522
|
+
|
523
|
+
# Add to NATS subscriptions array
|
524
|
+
@nats_subscriptions << nats_subscription
|
525
|
+
NatsWave.logger.info "✅ Successfully subscribed to #{subject_pattern} (total: #{@nats_subscriptions.size})"
|
102
526
|
|
103
|
-
|
104
|
-
|
105
|
-
|
527
|
+
rescue => e
|
528
|
+
NatsWave.logger.error "Failed to subscribe to #{subject_pattern}: #{e.message}"
|
529
|
+
raise
|
530
|
+
end
|
106
531
|
end
|
107
532
|
|
108
533
|
def process_message(raw_message, custom_handler, model_mappings)
|
@@ -141,55 +566,168 @@ module NatsWave
|
|
141
566
|
end
|
142
567
|
|
143
568
|
def keep_alive
|
144
|
-
|
569
|
+
NatsWave.logger.info "🔄 Starting keep-alive thread for persistent JetStream connection"
|
145
570
|
|
146
571
|
@keep_alive_thread = Thread.new do
|
147
572
|
while @running && !@shutdown
|
148
573
|
begin
|
149
574
|
sleep 30 # Check every 30 seconds
|
150
575
|
|
151
|
-
|
152
|
-
|
576
|
+
connection_healthy = check_connection_health
|
577
|
+
|
578
|
+
if connection_healthy
|
579
|
+
NatsWave.logger.debug "💓 Subscriber connection healthy - #{@nats_subscriptions.size} active subscriptions"
|
580
|
+
@connection_stats[:consecutive_failures] = 0
|
581
|
+
@connection_stats[:last_successful_connection] = Time.current
|
153
582
|
else
|
154
|
-
|
583
|
+
@connection_stats[:consecutive_failures] += 1
|
584
|
+
time_since_last_good = Time.current - @connection_stats[:last_successful_connection]
|
585
|
+
|
586
|
+
NatsWave.logger.error "❌ Subscriber connection lost! (failure ##{@connection_stats[:consecutive_failures]}, last good: #{time_since_last_good.to_i}s ago)"
|
587
|
+
|
588
|
+
log_connection_diagnostics
|
155
589
|
attempt_reconnection
|
156
590
|
end
|
591
|
+
|
592
|
+
# If too many consecutive failures, escalate
|
593
|
+
if @connection_stats[:consecutive_failures] > 5
|
594
|
+
NatsWave.logger.error "🚨 Too many consecutive failures. Performing full reset."
|
595
|
+
perform_full_reset
|
596
|
+
end
|
157
597
|
|
158
598
|
rescue => e
|
159
|
-
|
599
|
+
NatsWave.logger.error "Error in keep_alive thread: #{e.message}\n#{e.backtrace.first(3).join("\n")}"
|
160
600
|
sleep 10
|
161
601
|
end
|
162
602
|
end
|
163
603
|
|
164
|
-
|
604
|
+
NatsWave.logger.info "🛑 Keep-alive thread shutting down"
|
605
|
+
end
|
606
|
+
end
|
607
|
+
|
608
|
+
def check_connection_health
|
609
|
+
return false unless @client
|
610
|
+
|
611
|
+
begin
|
612
|
+
# First check basic connectivity
|
613
|
+
return false unless @client.connected?
|
614
|
+
|
615
|
+
# Try to perform a simple operation with timeout
|
616
|
+
Timeout.timeout(5) do
|
617
|
+
@client.flush
|
618
|
+
return true
|
619
|
+
end
|
620
|
+
rescue Timeout::Error
|
621
|
+
NatsWave.logger.warn "⏰ Connection health check timed out"
|
622
|
+
return false
|
623
|
+
rescue => e
|
624
|
+
NatsWave.logger.warn "🔍 Health check failed: #{e.message}"
|
625
|
+
return false
|
626
|
+
end
|
627
|
+
end
|
628
|
+
|
629
|
+
def log_connection_diagnostics
|
630
|
+
begin
|
631
|
+
NatsWave.logger.info "🔍 Connection Diagnostics:"
|
632
|
+
NatsWave.logger.info " - Client connected?: #{@client&.connected? rescue 'unknown'}"
|
633
|
+
NatsWave.logger.info " - Active subscriptions: #{@nats_subscriptions.size}"
|
634
|
+
NatsWave.logger.info " - Reconnect count: #{@connection_stats[:reconnect_count]}"
|
635
|
+
NatsWave.logger.info " - Thread count: #{Thread.list.count}"
|
636
|
+
NatsWave.logger.info " - Last successful: #{@connection_stats[:last_successful_connection]}"
|
637
|
+
|
638
|
+
# Check if subscriptions are actually valid
|
639
|
+
valid_subs = @nats_subscriptions.count { |sub| sub.respond_to?(:unsubscribe) }
|
640
|
+
NatsWave.logger.info " - Valid subscriptions: #{valid_subs}/#{@nats_subscriptions.size}"
|
641
|
+
|
642
|
+
rescue => e
|
643
|
+
NatsWave.logger.error "Failed to gather diagnostics: #{e.message}"
|
165
644
|
end
|
166
645
|
end
|
167
646
|
|
168
647
|
def attempt_reconnection
|
169
|
-
return if @shutdown
|
648
|
+
return if @shutdown || @reconnecting
|
170
649
|
|
171
|
-
|
650
|
+
@reconnecting = true
|
651
|
+
@connection_stats[:reconnect_count] += 1
|
172
652
|
|
173
|
-
|
174
|
-
|
653
|
+
begin
|
654
|
+
NatsWave.logger.info "🔄 Attempting to reconnect to NATS... (attempt ##{@connection_stats[:reconnect_count]})"
|
655
|
+
|
656
|
+
# First, clean up existing connection and subscriptions
|
657
|
+
cleanup_subscriptions
|
658
|
+
|
659
|
+
# Close the existing client connection
|
660
|
+
begin
|
661
|
+
@client&.close if @client&.connected?
|
662
|
+
rescue => e
|
663
|
+
NatsWave.logger.debug "Error closing existing connection: #{e.message}"
|
664
|
+
end
|
665
|
+
|
666
|
+
# Wait before attempting to reconnect
|
667
|
+
sleep_time = [2 ** [@connection_stats[:consecutive_failures] - 1, 4].min, 30].min
|
668
|
+
NatsWave.logger.debug "⏱️ Waiting #{sleep_time}s before reconnection attempt"
|
669
|
+
sleep sleep_time
|
670
|
+
|
671
|
+
# Attempt to recreate the client connection
|
672
|
+
if @original_client_factory
|
673
|
+
@client = @original_client_factory.call
|
674
|
+
NatsWave.logger.info "🔄 Created new NATS client using factory"
|
675
|
+
else
|
676
|
+
NatsWave.logger.warn "⚠️ No client factory available, hoping existing client recovers"
|
677
|
+
end
|
678
|
+
|
679
|
+
# Check if we have a working connection
|
680
|
+
if @client&.connected?
|
681
|
+
NatsWave.logger.info "✅ NATS client reconnected successfully"
|
682
|
+
|
683
|
+
# Re-establish all subscriptions
|
684
|
+
setup_subscriptions
|
685
|
+
|
686
|
+
NatsWave.logger.info "✅ Subscriptions restored (#{@nats_subscriptions.size} active)"
|
687
|
+
@connection_stats[:consecutive_failures] = 0
|
688
|
+
@connection_stats[:last_successful_connection] = Time.current
|
689
|
+
|
690
|
+
return true
|
691
|
+
else
|
692
|
+
raise "Connection failed - client not connected"
|
693
|
+
end
|
175
694
|
|
176
|
-
|
177
|
-
|
695
|
+
rescue => e
|
696
|
+
NatsWave.logger.error "Reconnection failed: #{e.message}"
|
697
|
+
return false
|
698
|
+
ensure
|
699
|
+
@reconnecting = false
|
700
|
+
end
|
701
|
+
end
|
702
|
+
|
703
|
+
def perform_full_reset
|
704
|
+
NatsWave.logger.info "🔄 Performing full connection reset"
|
178
705
|
|
179
|
-
|
180
|
-
|
706
|
+
begin
|
707
|
+
# Force close everything
|
708
|
+
cleanup_subscriptions
|
709
|
+
@client&.close rescue nil
|
181
710
|
|
182
|
-
|
183
|
-
|
184
|
-
|
185
|
-
|
711
|
+
# Clear connection stats
|
712
|
+
@connection_stats[:consecutive_failures] = 0
|
713
|
+
|
714
|
+
# Force garbage collection
|
715
|
+
GC.start
|
716
|
+
|
717
|
+
# Wait longer before attempting reset
|
718
|
+
sleep 10
|
719
|
+
|
720
|
+
# Try to recreate everything
|
721
|
+
if @original_client_factory
|
722
|
+
@client = @original_client_factory.call
|
723
|
+
setup_subscriptions if @client&.connected?
|
186
724
|
end
|
725
|
+
|
726
|
+
rescue => e
|
727
|
+
NatsWave.logger.error "Full reset failed: #{e.message}"
|
187
728
|
end
|
188
|
-
rescue => e
|
189
|
-
NatsWave.logger.error "Failed to reconnect: #{e.message}"
|
190
729
|
end
|
191
730
|
|
192
|
-
# ... rest of your existing methods (parse_message, handle_model_sync, handle_error)
|
193
731
|
def parse_message(raw_message)
|
194
732
|
@message_transformer.parse_message(raw_message)
|
195
733
|
end
|