nats_wave 1.1.7 → 1.1.8

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 6c6127f961d72ff48417d654e302d3bf4f7899071660920554e4184b0ac39b53
-  data.tar.gz: 7e74bea372edd1f978f15e9d31ec3ce22f6b2c2a3d6af220ac17731d9e4d97d4
+  metadata.gz: 26e26c369d6d7d10d4f5c7bcd2f3cc9ba5560b759ff1c4478105c9e6e66853c7
+  data.tar.gz: 9786a3a34f636185b056ca4aed597efc69cd1e07897d57a10636fc51b754b3f3
 SHA512:
-  metadata.gz: 05a61fe3dbcc2ca4d64a33de79296c609f32bfc752c6436ae79950b4c0479d121687b86410359e2d82fa39fc3330cae3a0b8b2f77c8be304f9212dcc3e060224
-  data.tar.gz: 4e4e430505cd8dbf36586da200e5c6cdbf5b3fc086c484a88b90ad88dcd800fe846e0cc349ae68be0e281a4002984a3f5d736107d99253946a7649b97eae8902
+  metadata.gz: 10861846ae79ef25b5986752d5caa09a126c1b8d803f1a60df7cf95e2b2b4a288db150216cb827b2018ec7a2ea902188617dbd4beb1da7942ef0c6f052537e1c
+  data.tar.gz: 4e4bdf076e56692299090ad6f9f5dd22387a821f7795f4c5ebc4dc320dc864b5d9159b3a92747ca4b9bf6f2fbab981707d48056af9741a3c1c36ad9df7373032

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    nats_wave (1.1.7)
+    nats_wave (1.1.8)
       activerecord (>= 6.1, < 8.0)
       activesupport (>= 6.1, < 8.0)
       concurrent-ruby (~> 1.1)

data/lib/nats_wave/adapters/datadog_metrics.rb

@@ -77,7 +77,7 @@ module NatsWave
                 {
                         service: @service_name,
                         environment: ENV['RAILS_ENV'] || ENV['RACK_ENV'] || 'unknown',
-                        version: ENV['APP_VERSION'] || '1.1.7'
+                        version: ENV['APP_VERSION'] || '1.1.8'
                 }
             end
             

data/lib/nats_wave/client.rb

@@ -609,27 +609,90 @@ module NatsWave
             end
         end
         
+        def establish_nats_connection
+            NatsWave.logger.info "Attempting to connect to NATS at #{@config.nats_url}"
+
+            @client = NATS.connect(
+                    @config.nats_url,
+                    reconnect_time_wait: @config.retry_delay,
+                    max_reconnect_attempts: @config.reconnect_attempts,
+                    ping_interval: 20,
+                    max_outstanding_pings: 2,
+                    disconnected_cb: proc do |reason|
+                        NatsWave.logger.warn "🔌 NATS disconnected: #{reason}"
+                    end,
+                    reconnected_cb: proc do
+                        NatsWave.logger.info "🔌 NATS auto-reconnected"
+                    end,
+                    error_cb: proc do |error|
+                        NatsWave.logger.error "🔌 NATS error: #{error}"
+                    end
+            )
+
+            # Verify the connection actually worked
+            if @client&.connected?
+                NatsWave.logger.info "✅ Successfully connected to NATS at #{@config.nats_url}"
+            else
+                NatsWave.logger.error "❌ NATS client created but not connected"
+                @client = nil
+            end
+
+        rescue => e
+            NatsWave.logger.error "❌ Failed to connect to NATS: #{e.message}"
+            NatsWave.logger.error "NATS URL: #{@config.nats_url}"
+            NatsWave.logger.error "Error class: #{e.class}"
+
+            @client = nil
+
+            # Re-raise in development to catch issues early
+            if defined?(Rails) && Rails.env.development?
+                raise ConnectionError, "Failed to connect to NATS: #{e.message}"
+            end
+        end
+        
         # def establish_nats_connection
         #     NatsWave.logger.info "Attempting to connect to NATS at #{@config.nats_url}"
         #
-        #     @client = NATS.connect(
-        #             @config.nats_url,
+        #     # Add more detailed connection options for debugging
+        #     connection_options = {
         #             reconnect_time_wait: @config.retry_delay,
-        #             max_reconnect_attempts: @config.reconnect_attempts
-        #     )
+        #             max_reconnect_attempts: @config.reconnect_attempts,
+        #             dont_randomize_servers: true,
+        #             verbose: true,
+        #             pedantic: false
+        #     }
+        #
+        #     NatsWave.logger.info "Connection options: #{connection_options}"
+        #
+        #     @client = NATS.connect(@config.nats_url, connection_options)
         #
-        #     # Verify the connection actually worked
-        #     if @client&.connected?
-        #         NatsWave.logger.info "✅ Successfully connected to NATS at #{@config.nats_url}"
+        #     # Wait a moment for connection to establish
+        #     sleep 0.5
+        #
+        #     # Check connection status with more detail
+        #     if @client
+        #         NatsWave.logger.info "NATS client created: #{@client.class}"
+        #         NatsWave.logger.info "NATS client connected?: #{@client.connected?}"
+        #         NatsWave.logger.info "NATS client status: #{@client.status rescue 'unknown'}"
+        #
+        #         if @client.connected?
+        #             NatsWave.logger.info "✅ Successfully connected to NATS at #{@config.nats_url}"
+        #         else
+        #             NatsWave.logger.error "❌ NATS client created but not connected"
+        #             NatsWave.logger.error "Client last error: #{@client.last_error rescue 'none'}"
+        #             @client = nil
+        #         end
         #     else
-        #         NatsWave.logger.error "❌ NATS client created but not connected"
-        #         @client = nil
+        #         NatsWave.logger.error "❌ NATS client is nil after connection attempt"
         #     end
         #
         # rescue => e
         #     NatsWave.logger.error "❌ Failed to connect to NATS: #{e.message}"
-        #     NatsWave.logger.error "NATS URL: #{@config.nats_url}"
         #     NatsWave.logger.error "Error class: #{e.class}"
+        #     NatsWave.logger.error "Error backtrace: #{e.backtrace.first(5).join('\n')}"
+        #     NatsWave.logger.error "NATS URL: #{@config.nats_url}"
+        #     NatsWave.logger.error "Retry delay: #{@config.retry_delay}"
+        #     NatsWave.logger.error "Max reconnect attempts: #{@config.reconnect_attempts}"
         #
         #     @client = nil
         #
@@ -639,58 +702,6 @@ module NatsWave
         #     end
         # end
         
-        def establish_nats_connection
-            NatsWave.logger.info "Attempting to connect to NATS at #{@config.nats_url}"
-            
-            # Add more detailed connection options for debugging
-            connection_options = {
-                    reconnect_time_wait: @config.retry_delay,
-                    max_reconnect_attempts: @config.reconnect_attempts,
-                    dont_randomize_servers: true,
-                    verbose: true,
-                    pedantic: false
-            }
-            
-            NatsWave.logger.info "Connection options: #{connection_options}"
-            
-            @client = NATS.connect(@config.nats_url, connection_options)
-            
-            # Wait a moment for connection to establish
-            sleep 0.5
-            
-            # Check connection status with more detail
-            if @client
-                NatsWave.logger.info "NATS client created: #{@client.class}"
-                NatsWave.logger.info "NATS client connected?: #{@client.connected?}"
-                NatsWave.logger.info "NATS client status: #{@client.status rescue 'unknown'}"
-                
-                if @client.connected?
-                    NatsWave.logger.info "✅ Successfully connected to NATS at #{@config.nats_url}"
-                else
-                    NatsWave.logger.error "❌ NATS client created but not connected"
-                    NatsWave.logger.error "Client last error: #{@client.last_error rescue 'none'}"
-                    @client = nil
-                end
-            else
-                NatsWave.logger.error "❌ NATS client is nil after connection attempt"
-            end
-        
-        rescue => e
-            NatsWave.logger.error "❌ Failed to connect to NATS: #{e.message}"
-            NatsWave.logger.error "Error class: #{e.class}"
-            NatsWave.logger.error "Error backtrace: #{e.backtrace.first(5).join('\n')}"
-            NatsWave.logger.error "NATS URL: #{@config.nats_url}"
-            NatsWave.logger.error "Retry delay: #{@config.retry_delay}"
-            NatsWave.logger.error "Max reconnect attempts: #{@config.reconnect_attempts}"
-            
-            @client = nil
-            
-            # Re-raise in development to catch issues early
-            if defined?(Rails) && Rails.env.development?
-                raise ConnectionError, "Failed to connect to NATS: #{e.message}"
-            end
-        end
-        
         def ensure_connected!
             unless connected?
                 error_msg = if @client.nil?

data/lib/nats_wave/configuration.rb

@@ -18,7 +18,7 @@ module NatsWave
         def initialize(options = {})
             @nats_url = ENV['NATS_URL'] || "nats://localhost:4222"
             @service_name = ENV['NATS_SERVICE_NAME'] || "purplewave"
-            @version = ENV['NATS_SERVICE_VERSION'] || "1.1.7"
+            @version = ENV['NATS_SERVICE_VERSION'] || "1.1.8"
             @instance_id = ENV['NATS_INSTANCE_ID'] || Socket.gethostname
             @database_url = ENV['NATS_DATABASE_URL'] || nil
             @connection_pool_size = (ENV['NATS_CONNECTION_POOL_SIZE'] || 10).to_i

data/lib/nats_wave/subscriber.rb

@@ -7,6 +7,7 @@ module NatsWave
         def initialize(config, client, middleware_stack = [])
             @config = config
             @client = client
+            @original_client_factory = nil # Store how to recreate the client
             @database_connector = DatabaseConnector.new(config)
             @model_mapper = ModelMapper.new(config)
             @message_transformer = MessageTransformer.new(config)
@@ -16,6 +17,17 @@ module NatsWave
             @nats_subscriptions = [] # NATS::Subscription objects
             @running = false
             @shutdown = false
+            @reconnecting = false
+            @connection_stats = {
+                    reconnect_count: 0,
+                    last_successful_connection: Time.current,
+                    consecutive_failures: 0
+            }
+        end
+        
+        # Add method to store client factory for reconnection
+        def set_client_factory(factory_proc)
+            @original_client_factory = factory_proc
         end
         
         def begin
@@ -27,12 +39,7 @@ module NatsWave
             
             NatsWave.logger.info "Starting NATS subscriber for #{@config.service_name}"
             
-            # Use ModelRegistry subscriptions
-            @registry_subscriptions.each do |subscription|
-                subscription[:subjects].each do |subject|
-                    subscribe_to_subject(subject, subscription[:handler])
-                end
-            end
+            setup_subscriptions
             
             NatsWave.logger.info "Started #{@registry_subscriptions.size} subscriptions from Model Registry"
             
@@ -56,18 +63,12 @@ module NatsWave
             # Stop keep alive thread
             if @keep_alive_thread&.alive?
                 @keep_alive_thread.kill
+                @keep_alive_thread.join(5) # Wait up to 5 seconds for graceful shutdown
                 @keep_alive_thread = nil
             end
             
             # Unsubscribe from all subscriptions
-            @nats_subscriptions.each do |subscription|
-                begin
-                    subscription.unsubscribe if subscription.respond_to?(:unsubscribe)
-                rescue => e
-                    NatsWave.logger.error "Error unsubscribing: #{e.message}"
-                end
-            end
-            @nats_subscriptions.clear
+            cleanup_subscriptions
             
             NatsWave.logger.info "🛑 Subscriber shutdown complete"
         end
@@ -82,27 +83,60 @@ module NatsWave
         
         private
         
-        def subscribe_to_subject(subject_pattern, custom_handler = nil, model_mappings = {})
-            NatsWave.logger.info "🔍 Attempting to subscribe to: #{subject_pattern}"
+        def setup_subscriptions
+            return unless @client&.connected?
+            
+            # Clear existing subscriptions first
+            cleanup_subscriptions
             
-            # Create the NATS subscription
-            nats_subscription = @client.subscribe(
-                    subject_pattern,
-                    queue: @config.queue_group
-            ) do |msg|
+            # Use ModelRegistry subscriptions
+            @registry_subscriptions.each do |subscription|
+                subscription[:subjects].each do |subject|
+                    subscribe_to_subject(subject, subscription[:handler])
+                end
+            end
+        end
+        
+        def cleanup_subscriptions
+            @nats_subscriptions.each do |subscription|
                 begin
-                    NatsWave.logger.debug "📨 Received message on #{msg.subject}"
-                    process_message(msg.data, custom_handler, model_mappings)
+                    subscription.unsubscribe if subscription.respond_to?(:unsubscribe)
                 rescue => e
-                    NatsWave.logger.error "Error in subscription handler: #{e.message}"
-                    NatsWave.logger.error e.backtrace.join("\n")
-                    # Don't re-raise - this would kill the subscription
+                    NatsWave.logger.error "Error unsubscribing: #{e.message}"
                 end
             end
+            @nats_subscriptions.clear
+        end
+        
+        def subscribe_to_subject(subject_pattern, custom_handler = nil, model_mappings = {})
+            return unless @client&.connected?
+            
+            NatsWave.logger.info "🔍 Attempting to subscribe to: #{subject_pattern}"
+            
+            begin
+                # Create the NATS subscription
+                nats_subscription = @client.subscribe(
+                        subject_pattern,
+                        queue: @config.queue_group
+                ) do |msg|
+                    begin
+                        NatsWave.logger.debug "📨 Received message on #{msg.subject}"
+                        process_message(msg.data, custom_handler, model_mappings)
+                    rescue => e
+                        NatsWave.logger.error "Error in subscription handler: #{e.message}"
+                        NatsWave.logger.error e.backtrace.join("\n")
+                        # Don't re-raise - this would kill the subscription
+                    end
+                end
+                
+                # Add to NATS subscriptions array
+                @nats_subscriptions << nats_subscription
+                NatsWave.logger.info "✅ Successfully subscribed to #{subject_pattern} (total: #{@nats_subscriptions.size})"
             
-            # Add to NATS subscriptions array
-            @nats_subscriptions << nats_subscription
-            NatsWave.logger.info "✅ Successfully subscribed to #{subject_pattern} (total: #{@nats_subscriptions.size})"
+            rescue => e
+                NatsWave.logger.error "Failed to subscribe to #{subject_pattern}: #{e.message}"
+                raise
+            end
         end
         
         def process_message(raw_message, custom_handler, model_mappings)
@@ -141,52 +175,166 @@ module NatsWave
         end
         
         def keep_alive
-            Rails.logger.info "🔄 Starting keep-alive thread for persistent JetStream connection"
+            NatsWave.logger.info "🔄 Starting keep-alive thread for persistent JetStream connection"
             
             @keep_alive_thread = Thread.new do
                 while @running && !@shutdown
                     begin
                         sleep 30 # Check every 30 seconds
                         
-                        if @client&.connected?
-                            Rails.logger.debug "💓 Subscriber connection healthy - #{@nats_subscriptions.size} active subscriptions"
+                        connection_healthy = check_connection_health
+                        
+                        if connection_healthy
+                            NatsWave.logger.debug "💓 Subscriber connection healthy - #{@nats_subscriptions.size} active subscriptions"
+                            @connection_stats[:consecutive_failures] = 0
+                            @connection_stats[:last_successful_connection] = Time.current
                         else
-                            Rails.logger.error "❌ Subscriber connection lost! Attempting reconnection..."
+                            @connection_stats[:consecutive_failures] += 1
+                            time_since_last_good = Time.current - @connection_stats[:last_successful_connection]
+                            
+                            NatsWave.logger.error "❌ Subscriber connection lost! (failure ##{@connection_stats[:consecutive_failures]}, last good: #{time_since_last_good.to_i}s ago)"
+                            
+                            log_connection_diagnostics
                             attempt_reconnection
                         end
+                        
+                        # If too many consecutive failures, escalate
+                        if @connection_stats[:consecutive_failures] > 5
+                            NatsWave.logger.error "🚨 Too many consecutive failures. Performing full reset."
+                            perform_full_reset
+                        end
                     
                     rescue => e
-                        Rails.logger.error "Error in keep_alive thread: #{e.message}"
+                        NatsWave.logger.error "Error in keep_alive thread: #{e.message}\n#{e.backtrace.first(3).join("\n")}"
                         sleep 10
                     end
                 end
                 
-                Rails.logger.info "🛑 Keep-alive thread shutting down"
+                NatsWave.logger.info "🛑 Keep-alive thread shutting down"
+            end
+        end
+        
+        def check_connection_health
+            return false unless @client
+            
+            begin
+                # First check basic connectivity
+                return false unless @client.connected?
+                
+                # Try to perform a simple operation with timeout
+                Timeout.timeout(5) do
+                    @client.flush
+                    return true
+                end
+            rescue Timeout::Error
+                NatsWave.logger.warn "⏰ Connection health check timed out"
+                return false
+            rescue => e
+                NatsWave.logger.warn "🔍 Health check failed: #{e.message}"
+                return false
+            end
+        end
+        
+        def log_connection_diagnostics
+            begin
+                NatsWave.logger.info "🔍 Connection Diagnostics:"
+                NatsWave.logger.info "  - Client connected?: #{@client&.connected? rescue 'unknown'}"
+                NatsWave.logger.info "  - Active subscriptions: #{@nats_subscriptions.size}"
+                NatsWave.logger.info "  - Reconnect count: #{@connection_stats[:reconnect_count]}"
+                NatsWave.logger.info "  - Thread count: #{Thread.list.count}"
+                NatsWave.logger.info "  - Last successful: #{@connection_stats[:last_successful_connection]}"
+                
+                # Check if subscriptions are actually valid
+                valid_subs = @nats_subscriptions.count { |sub| sub.respond_to?(:unsubscribe) }
+                NatsWave.logger.info "  - Valid subscriptions: #{valid_subs}/#{@nats_subscriptions.size}"
+            
+            rescue => e
+                NatsWave.logger.error "Failed to gather diagnostics: #{e.message}"
             end
         end
         
         def attempt_reconnection
-            return if @shutdown
+            return if @shutdown || @reconnecting
             
-            NatsWave.logger.info "🔄 Attempting to reconnect to NATS..."
+            @reconnecting = true
+            @connection_stats[:reconnect_count] += 1
             
-            # Reset subscriptions
-            @nats_subscriptions.clear
+            begin
+                NatsWave.logger.info "🔄 Attempting to reconnect to NATS... (attempt ##{@connection_stats[:reconnect_count]})"
+                
+                # First, clean up existing connection and subscriptions
+                cleanup_subscriptions
+                
+                # Close the existing client connection
+                begin
+                    @client&.close if @client&.connected?
+                rescue => e
+                    NatsWave.logger.debug "Error closing existing connection: #{e.message}"
+                end
+                
+                # Wait before attempting to reconnect
+                sleep_time = [2 ** [@connection_stats[:consecutive_failures] - 1, 4].min, 30].min
+                NatsWave.logger.debug "⏱️  Waiting #{sleep_time}s before reconnection attempt"
+                sleep sleep_time
+                
+                # Attempt to recreate the client connection
+                if @original_client_factory
+                    @client = @original_client_factory.call
+                    NatsWave.logger.info "🔄 Created new NATS client using factory"
+                else
+                    NatsWave.logger.warn "⚠️  No client factory available, hoping existing client recovers"
+                end
+                
+                # Check if we have a working connection
+                if @client&.connected?
+                    NatsWave.logger.info "✅ NATS client reconnected successfully"
+                    
+                    # Re-establish all subscriptions
+                    setup_subscriptions
+                    
+                    NatsWave.logger.info "✅ Subscriptions restored (#{@nats_subscriptions.size} active)"
+                    @connection_stats[:consecutive_failures] = 0
+                    @connection_stats[:last_successful_connection] = Time.current
+                    
+                    return true
+                else
+                    raise "Connection failed - client not connected"
+                end
             
-            # Try to reestablish subscriptions
-            sleep 5 # Wait before reconnecting
+            rescue => e
+                NatsWave.logger.error "Reconnection failed: #{e.message}"
+                return false
+            ensure
+                @reconnecting = false
+            end
+        end
+        
+        def perform_full_reset
+            NatsWave.logger.info "🔄 Performing full connection reset"
             
-            if @client&.connected?
-                NatsWave.logger.info "✅ NATS reconnected, reestablishing subscriptions"
+            begin
+                # Force close everything
+                cleanup_subscriptions
+                @client&.close rescue nil
                 
-                @registry_subscriptions.each do |subscription|
-                    subscription[:subjects].each do |subject|
-                        subscribe_to_subject(subject, subscription[:handler])
-                    end
+                # Clear connection stats
+                @connection_stats[:consecutive_failures] = 0
+                
+                # Force garbage collection
+                GC.start
+                
+                # Wait longer before attempting reset
+                sleep 10
+                
+                # Try to recreate everything
+                if @original_client_factory
+                    @client = @original_client_factory.call
+                    setup_subscriptions if @client&.connected?
                 end
+            
+            rescue => e
+                NatsWave.logger.error "Full reset failed: #{e.message}"
             end
-        rescue => e
-            NatsWave.logger.error "Failed to reconnect: #{e.message}"
         end
         
         # ... rest of your existing methods (parse_message, handle_model_sync, handle_error)

data/lib/nats_wave/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module NatsWave
-    VERSION = '1.1.7'
+    VERSION = '1.1.8'
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: nats_wave
 version: !ruby/object:Gem::Version
-  version: 1.1.7
+  version: 1.1.8
 platform: ruby
 authors:
 - Jeffrey Dabo