smart_message 0.0.6 → 0.0.8

data/lib/smart_message/dead_letter_queue.rb

@@ -0,0 +1,344 @@
+# lib/smart_message/dead_letter_queue.rb
+# encoding: utf-8
+# frozen_string_literal: true
+
+require 'json'
+require 'fileutils'
+
+module SmartMessage
+  # File-based Dead Letter Queue implementation using JSON Lines format
+  # Provides FIFO queue operations with replay capabilities for failed messages
+  class DeadLetterQueue
+    attr_reader :file_path
+
+    # Default singleton instance
+    @@default_instance = nil
+
+    def self.default
+      @@default_instance ||= new
+    end
+
+    def self.configure_default(file_path)
+      @@default_instance = new(file_path)
+    end
+
+    def initialize(file_path = 'dead_letters.jsonl')
+      @file_path = File.expand_path(file_path)
+      @mutex = Mutex.new
+      ensure_directory_exists
+    end
+
+    # Core FIFO queue operations
+
+    # Add a failed message to the dead letter queue
+    # @param message_header [SmartMessage::Header] The message header
+    # @param message_payload [String] The serialized message payload
+    # @param error_info [Hash] Error details including :error, :retry_count, :transport, etc.
+    def enqueue(message_header, message_payload, error_info = {})
+      entry = {
+        timestamp: Time.now.iso8601,
+        header: message_header.to_hash,
+        payload: message_payload,
+        payload_format: error_info[:serializer] || 'json',  # Track serialization format
+        error: error_info[:error] || 'Unknown error',
+        retry_count: error_info[:retry_count] || 0,
+        transport: error_info[:transport],
+        stack_trace: error_info[:stack_trace]
+      }
+
+      @mutex.synchronize do
+        File.open(@file_path, 'a') do |file|
+          file.puts entry.to_json
+          file.fsync  # Ensure immediate write to disk
+        end
+      end
+
+      entry
+    end
+
+    # Remove and return the oldest message from the queue
+    # @return [Hash, nil] The oldest DLQ entry or nil if queue is empty
+    def dequeue
+      @mutex.synchronize do
+        return nil unless File.exist?(@file_path)
+
+        lines = File.readlines(@file_path)
+        return nil if lines.empty?
+
+        # Get first line (oldest)
+        oldest_line = lines.shift
+        oldest_entry = JSON.parse(oldest_line.strip, symbolize_names: true)
+
+        # Rewrite file without the first line
+        File.open(@file_path, 'w') do |file|
+          lines.each { |line| file.write(line) }
+        end
+
+        oldest_entry
+      end
+    rescue JSON::ParserError => e
+      puts "Warning: Corrupted DLQ entry skipped: #{e.message}" if $DEBUG
+      nil
+    end
+
+    # Look at the oldest message without removing it
+    # @return [Hash, nil] The oldest DLQ entry or nil if queue is empty
+    def peek
+      return nil unless File.exist?(@file_path)
+
+      File.open(@file_path, 'r') do |file|
+        first_line = file.readline
+        return nil if first_line.nil? || first_line.strip.empty?
+        JSON.parse(first_line.strip, symbolize_names: true)
+      end
+    rescue EOFError, JSON::ParserError
+      nil
+    end
+
+    # Get the number of messages in the queue
+    # @return [Integer] Number of messages in the DLQ
+    def size
+      return 0 unless File.exist?(@file_path)
+      File.readlines(@file_path).size
+    end
+
+    # Clear all messages from the queue
+    def clear
+      @mutex.synchronize do
+        File.delete(@file_path) if File.exist?(@file_path)
+      end
+    end
+
+    # Replay capabilities
+
+    # Replay all messages in the queue
+    # @param transport [SmartMessage::Transport::Base] Optional transport override
+    # @return [Hash] Results summary
+    def replay_all(transport = nil)
+      results = { success: 0, failed: 0, errors: [] }
+
+      while (entry = dequeue)
+        result = replay_entry(entry, transport)
+        if result[:success]
+          results[:success] += 1
+        else
+          results[:failed] += 1
+          results[:errors] << result[:error]
+          # Re-enqueue failed replay attempts
+          header = SmartMessage::Header.new(entry[:header])
+          enqueue(header, entry[:payload],
+            error: "Replay failed: #{result[:error]}",
+            retry_count: (entry[:retry_count] || 0) + 1)
+        end
+      end
+
+      results
+    end
+
+    # Replay the oldest message
+    # @param transport [SmartMessage::Transport::Base] Optional transport override
+    # @return [Hash] Result of replay attempt
+    def replay_one(transport = nil)
+      entry = dequeue
+      return { success: false, error: 'Queue is empty' } unless entry
+
+      replay_entry(entry, transport)
+    end
+
+    # Replay a batch of messages
+    # @param count [Integer] Number of messages to replay
+    # @param transport [SmartMessage::Transport::Base] Optional transport override
+    # @return [Hash] Results summary
+    def replay_batch(count = 10, transport = nil)
+      results = { success: 0, failed: 0, errors: [] }
+
+      count.times do
+        break if size == 0
+
+        result = replay_one(transport)
+        if result[:success]
+          results[:success] += 1
+        else
+          results[:failed] += 1
+          results[:errors] << result[:error]
+        end
+      end
+
+      results
+    end
+
+    # Administrative utilities
+
+    # Inspect messages in the queue without removing them
+    # @param limit [Integer] Maximum number of messages to show
+    # @return [Array<Hash>] Array of DLQ entries
+    def inspect_messages(limit: 10)
+      count = 0
+      read_entries_with_filter do |entry|
+        return [] if count >= limit
+        count += 1
+        entry
+      end
+    end
+
+    # Filter messages by message class
+    # @param message_class [String] The message class name to filter by
+    # @return [Array<Hash>] Filtered DLQ entries
+    def filter_by_class(message_class)
+      read_entries_with_filter do |entry|
+        entry if entry.dig(:header, :message_class) == message_class
+      end
+    end
+
+    # Filter messages by error pattern
+    # @param pattern [Regexp, String] Pattern to match against error messages
+    # @return [Array<Hash>] Filtered DLQ entries
+    def filter_by_error_pattern(pattern)
+      pattern = Regexp.new(pattern) if pattern.is_a?(String)
+      
+      read_entries_with_filter do |entry|
+        entry if pattern.match?(entry[:error].to_s)
+      end
+    end
+
+    # Get statistics about the dead letter queue
+    # @return [Hash] Statistics summary
+    def statistics
+      stats = { total: 0, by_class: Hash.new(0), by_error: Hash.new(0) }
+      
+      read_entries_with_filter do |entry|
+        stats[:total] += 1
+        
+        message_class = entry.dig(:header, :message_class) || 'Unknown'
+        stats[:by_class][message_class] += 1
+        
+        error = entry[:error] || 'Unknown error'
+        stats[:by_error][error] += 1
+        
+        nil  # Don't collect entries, just process for side effects
+      end
+      
+      stats
+    end
+
+    # Export messages within a time range
+    # @param start_time [Time] Start of time range
+    # @param end_time [Time] End of time range
+    # @return [Array<Hash>] DLQ entries within the time range
+    def export_range(start_time, end_time)
+      read_entries_with_filter do |entry|
+        begin
+          timestamp = Time.parse(entry[:timestamp])
+          entry if timestamp >= start_time && timestamp <= end_time
+        rescue ArgumentError
+          # Skip entries with invalid timestamps
+          nil
+        end
+      end
+    end
+
+    private
+
+    # Replay a single DLQ entry by recreating the message instance
+    # @param entry [Hash] The DLQ entry to replay
+    # @param transport_override [SmartMessage::Transport::Base] Optional transport override
+    # @return [Hash] Result of replay attempt
+    def replay_entry(entry, transport_override = nil)
+      message_class_name = entry.dig(:header, :message_class)
+      return { success: false, error: 'Missing message class' } unless message_class_name
+
+      # Get the message class
+      message_class = message_class_name.constantize
+
+      # Deserialize the payload using the appropriate format
+      payload_data = deserialize_payload(entry[:payload], entry[:payload_format] || 'json')
+      return { success: false, error: 'Failed to deserialize payload' } unless payload_data
+
+      # Remove the header from payload data (it's stored separately in DLQ)
+      payload_data.delete(:_sm_header)
+
+      # Create new message instance with original data
+      message = message_class.new(**payload_data)
+
+      # Restore complete header information
+      restore_header_fields(message, entry[:header])
+
+      # Override transport if provided - this must be done before publishing
+      if transport_override
+        message.transport(transport_override)
+      end
+
+      # Attempt to publish the message
+      message.publish
+
+      { success: true, message: message }
+    rescue => e
+      { success: false, error: "#{e.class.name}: #{e.message}" }
+    end
+
+    # Deserialize payload based on format
+    # @param payload [String] The serialized payload
+    # @param format [String] The serialization format (json, etc.)
+    # @return [Hash, nil] Deserialized data or nil if failed
+    def deserialize_payload(payload, format)
+      case format.to_s.downcase
+      when 'json'
+        JSON.parse(payload, symbolize_names: true)
+      else
+        # For unknown formats, assume JSON as fallback but log warning
+        puts "Warning: Unknown payload format '#{format}', attempting JSON" if $DEBUG
+        JSON.parse(payload, symbolize_names: true)
+      end
+    rescue JSON::ParserError => e
+      puts "Error: Failed to deserialize payload as #{format}: #{e.message}" if $DEBUG
+      nil
+    end
+
+    # Restore all header fields from DLQ entry
+    # @param message [SmartMessage::Base] The message instance
+    # @param header_data [Hash] The stored header data
+    def restore_header_fields(message, header_data)
+      return unless header_data
+      
+      # Restore all available header fields
+      message._sm_header.uuid = header_data[:uuid] if header_data[:uuid]
+      message._sm_header.message_class = header_data[:message_class] if header_data[:message_class]
+      message._sm_header.published_at = Time.parse(header_data[:published_at]) if header_data[:published_at]
+      message._sm_header.publisher_pid = header_data[:publisher_pid] if header_data[:publisher_pid]
+      message._sm_header.version = header_data[:version] if header_data[:version]
+      message._sm_header.from = header_data[:from] if header_data[:from]
+      message._sm_header.to = header_data[:to] if header_data[:to]
+      message._sm_header.reply_to = header_data[:reply_to] if header_data[:reply_to]
+    rescue => e
+      puts "Warning: Failed to restore some header fields: #{e.message}" if $DEBUG
+    end
+
+    # Generic file reading iterator with error handling
+    # @param block [Proc] Block to execute for each valid entry
+    # @return [Array] Results from the block
+    def read_entries_with_filter(&block)
+      return [] unless File.exist?(@file_path)
+      
+      results = []
+      File.open(@file_path, 'r') do |file|
+        file.each_line do |line|
+          begin
+            entry = JSON.parse(line.strip, symbolize_names: true)
+            result = block.call(entry)
+            results << result if result
+          rescue JSON::ParserError
+            # Skip corrupted lines
+          end
+        end
+      end
+      
+      results
+    end
+
+    # Ensure the directory for the DLQ file exists
+    def ensure_directory_exists
+      directory = File.dirname(@file_path)
+      FileUtils.mkdir_p(directory) unless Dir.exist?(directory)
+    end
+  end
+end

data/lib/smart_message/dispatcher.rb

@@ -3,26 +3,25 @@
 # frozen_string_literal: true
 
 require 'concurrent'
+require_relative 'circuit_breaker'
 
 module SmartMessage
 
   # The disoatcher routes incoming messages to all of the methods that
   # have been subscribed to the message.
   class Dispatcher
+    include BreakerMachines::DSL
 
     # TODO: setup forwardable for some @router_pool methods
 
-    def initialize
+    def initialize(circuit_breaker_options = {})
       @subscribers = Hash.new(Array.new)
       @router_pool = Concurrent::CachedThreadPool.new
+      
+      # Configure circuit breakers
+      configure_circuit_breakers(circuit_breaker_options)
       at_exit do
-        print "Shuttingdown down the dispatcher's @router_pool ..."
-        @router_pool.shutdown
-        while @router_pool.shuttingdown?
-          print '.'
-          sleep 1
-        end
-        puts " done."
+        shutdown_pool
       end
     end
 
@@ -139,7 +138,8 @@ module SmartMessage
         
         SS.add(message_klass, message_processor, 'routed' )
         @router_pool.post do
-          begin
+          # Use circuit breaker to protect message processing
+          circuit_result = circuit(:message_processor).wrap do
             # Check if this is a proc handler or a regular method call
             if proc_handler?(message_processor)
               # Call the proc handler via SmartMessage::Base
@@ -153,17 +153,63 @@ module SmartMessage
                           .method(class_method)
                           .call(message_header, message_payload)
             end
-          rescue Exception => e
-            # TODO: Add proper exception logging
-            # Exception details: #{e.message}
-            # Processor: #{message_processor}
-            puts "Error processing message: #{e.message}" if $DEBUG
+          end
+          
+          # Handle circuit breaker fallback responses
+          if circuit_result.is_a?(Hash) && circuit_result[:circuit_breaker]
+            handle_circuit_breaker_fallback(circuit_result, message_header, message_payload, message_processor)
+          end
+        end
+      end
+    end
+
+    # Get circuit breaker statistics
+    # @return [Hash] Circuit breaker statistics
+    def circuit_breaker_stats
+      stats = {}
+      
+      begin
+        if respond_to?(:circuit)
+          breaker = circuit(:message_processor)
+          if breaker
+            stats[:message_processor] = {
+              status: breaker.status,
+              closed: breaker.closed?,
+              open: breaker.open?,
+              half_open: breaker.half_open?,
+              last_error: breaker.last_error,
+              opened_at: breaker.opened_at,
+              stats: breaker.stats
+            }
           end
         end
+      rescue => e
+        stats[:error] = "Failed to get circuit breaker stats: #{e.message}"
       end
+      
+      stats
     end
 
-    private
+    # Reset circuit breakers
+    # @param circuit_name [Symbol] Optional specific circuit to reset
+    def reset_circuit_breakers!(circuit_name = nil)
+      if circuit_name
+        circuit(circuit_name)&.reset!
+      else
+        # Reset all known circuits
+        circuit(:message_processor)&.reset!
+      end
+    end
+
+    # Shutdown the router pool with timeout and fallback
+    def shutdown_pool
+      @router_pool.shutdown
+      
+      # Wait for graceful shutdown, force kill if timeout
+      unless @router_pool.wait_for_termination(3)
+        @router_pool.kill
+      end
+    end
 
     # Check if a message matches the subscription filters
     # @param message_header [SmartMessage::Header] The message header
@@ -199,6 +245,81 @@ module SmartMessage
       SmartMessage::Base.proc_handler?(message_processor)
     end
 
+    # Configure circuit breakers for the dispatcher
+    # @param options [Hash] Circuit breaker configuration options
+    def configure_circuit_breakers(options = {})
+      # Ensure CircuitBreaker module is available
+      return unless defined?(SmartMessage::CircuitBreaker::DEFAULT_CONFIGS)
+      
+      # Configure message processor circuit breaker
+      default_config = SmartMessage::CircuitBreaker::DEFAULT_CONFIGS[:message_processor]
+      return unless default_config
+      
+      processor_config = default_config.merge(options[:message_processor] || {})
+      
+      # Define the circuit using the class-level DSL
+      self.class.circuit :message_processor do
+        threshold failures: processor_config[:threshold][:failures], 
+                 within: processor_config[:threshold][:within].seconds
+        reset_after processor_config[:reset_after].seconds
+        
+        # Configure storage backend
+        case processor_config[:storage]
+        when :redis
+          # Use Redis storage if configured and available
+          if defined?(SmartMessage::Transport::RedisTransport)
+            begin
+              redis_transport = SmartMessage::Transport::RedisTransport.new
+              storage BreakerMachines::Storage::Redis.new(redis: redis_transport.redis_pub)
+            rescue
+              # Fall back to memory storage if Redis not available
+              storage BreakerMachines::Storage::Memory.new
+            end
+          else
+            storage BreakerMachines::Storage::Memory.new
+          end
+        else
+          storage BreakerMachines::Storage::Memory.new
+        end
+        
+        # Default fallback for message processing failures
+        fallback do |exception|
+          {
+            circuit_breaker: {
+              circuit: :message_processor,
+              state: 'open',
+              error: exception.message,
+              error_class: exception.class.name,
+              timestamp: Time.now.iso8601,
+              fallback_triggered: true
+            }
+          }
+        end
+      end
+      
+    end
+
+    # Handle circuit breaker fallback responses
+    # @param circuit_result [Hash] The circuit breaker fallback result
+    # @param message_header [SmartMessage::Header] The message header
+    # @param message_payload [String] The message payload
+    # @param message_processor [String] The processor that failed
+    def handle_circuit_breaker_fallback(circuit_result, message_header, message_payload, message_processor)
+      # Log circuit breaker activation
+      if $DEBUG
+        puts "Circuit breaker activated for processor: #{message_processor}"
+        puts "Error: #{circuit_result[:circuit_breaker][:error]}"
+        puts "Message: #{message_header.message_class} from #{message_header.from}"
+      end
+      
+      # TODO: Integrate with structured logging when implemented
+      # TODO: Send to dead letter queue when implemented
+      # TODO: Emit metrics/events for monitoring
+      
+      # For now, record the failure in simple stats
+      SS.add(message_header.message_class, message_processor, 'circuit_breaker_fallback')
+    end
+
 
     #######################################################
     ## Class methods

data/lib/smart_message/serializer/base.rb

@@ -2,22 +2,95 @@
 # encoding: utf-8
 # frozen_string_literal: true
 
+require_relative '../circuit_breaker'
+
 module SmartMessage::Serializer
   # the standard super class
   class Base
+    include BreakerMachines::DSL
+    
     # provide basic configuration
     def initialize
-      # TODO: write this
+      configure_serializer_circuit_breakers
     end
 
     def encode(message_instance)
-      # TODO: Add proper logging here
-      raise ::SmartMessage::Errors::NotImplemented
+      circuit(:serializer).wrap do
+        do_encode(message_instance)
+      end
+    rescue => e
+      # Handle circuit breaker fallback
+      if e.is_a?(Hash) && e[:circuit_breaker]
+        handle_serializer_fallback(e, :encode, message_instance)
+      else
+        raise
+      end
     end
 
     def decode(payload)
-      # TODO: Add proper logging here
+      circuit(:serializer).wrap do
+        do_decode(payload)
+      end
+    rescue => e
+      # Handle circuit breaker fallback
+      if e.is_a?(Hash) && e[:circuit_breaker]
+        handle_serializer_fallback(e, :decode, payload)
+      else
+        raise
+      end
+    end
+
+    # Template methods for actual serialization (implement in subclasses)
+    def do_encode(message_instance)
+      raise ::SmartMessage::Errors::NotImplemented
+    end
+
+    def do_decode(payload)
       raise ::SmartMessage::Errors::NotImplemented
     end
+
+    private
+
+    # Configure circuit breaker for serializer operations
+    def configure_serializer_circuit_breakers
+      serializer_config = SmartMessage::CircuitBreaker::DEFAULT_CONFIGS[:serializer]
+      
+      self.class.circuit :serializer do
+        threshold failures: serializer_config[:threshold][:failures], 
+                 within: serializer_config[:threshold][:within].seconds
+        reset_after serializer_config[:reset_after].seconds
+        
+        storage BreakerMachines::Storage::Memory.new
+        
+        # Fallback for serializer failures
+        fallback do |exception|
+          {
+            circuit_breaker: {
+              circuit: :serializer,
+              serializer_type: self.class.name,
+              state: 'open',
+              error: exception.message,
+              error_class: exception.class.name,
+              timestamp: Time.now.iso8601,
+              fallback_triggered: true
+            }
+          }
+        end
+      end
+    end
+
+    # Handle serializer circuit breaker fallback
+    def handle_serializer_fallback(fallback_result, operation, data)
+      if $DEBUG
+        puts "Serializer circuit breaker activated: #{self.class.name}"
+        puts "Operation: #{operation}"
+        puts "Error: #{fallback_result[:circuit_breaker][:error]}"
+      end
+      
+      # TODO: Integrate with structured logging when implemented
+      
+      # Return the fallback result
+      fallback_result
+    end
   end # class Base
 end # module SmartMessage::Serializer

data/lib/smart_message/serializer/json.rb

@@ -6,13 +6,13 @@ require 'json'  # STDLIB
 
 module SmartMessage::Serializer
   class JSON < Base
-    def encode(message_instance)
+    def do_encode(message_instance)
       # TODO: is this the right place to insert an automated-invisible
       #       message header?
       message_instance.to_json
     end
 
-    def decode(payload)
+    def do_decode(payload)
       # TODO: so how do I know to which message class this payload
       #       belongs?  The class needs to be in some kind of message
       #       header.