smart_message 0.0.7 → 0.0.9

data/lib/smart_message/circuit_breaker.rb

@@ -159,23 +159,40 @@ module SmartMessage
 
     # Configure fallback handlers for different scenarios
     module Fallbacks
-      # Dead letter queue fallback
-      def self.dead_letter_queue(dlq_transport = nil)
+      # Dead letter queue fallback - stores failed messages to file-based DLQ
+      def self.dead_letter_queue(dlq_instance = nil)
         proc do |exception, *args|
           # Extract message details from args if available
           message_header = args[0] if args[0].is_a?(SmartMessage::Header)
           message_payload = args[1] if args.length > 1
           
-          # Log to dead letter queue if transport provided
-          if dlq_transport && message_header && message_payload
-            dlq_transport.publish(message_header, message_payload)
+          # Use provided DLQ instance or default
+          dlq = dlq_instance || SmartMessage::DeadLetterQueue.default
+          
+          # Store failed message in dead letter queue
+          sent_to_dlq = false
+          if message_header && message_payload
+            begin
+              dlq.enqueue(message_header, message_payload, 
+                error: exception.message,
+                retry_count: 0,
+                serializer: 'json',  # Default to JSON, could be enhanced to detect actual serializer
+                stack_trace: exception.backtrace&.join("\n")
+              )
+              sent_to_dlq = true
+            rescue => dlq_error
+              # DLQ storage failed - log but don't raise
+              # Note: Logger might not be available in circuit breaker context
+              warn "Warning: Failed to store message in DLQ: #{dlq_error.message}"
+            end
           end
           
           {
             circuit_breaker: {
+              circuit: :transport_publish,  # Default circuit name, overridden by specific configurations
               state: 'open',
               error: exception.message,
-              sent_to_dlq: !!(dlq_transport && message_header && message_payload),
+              sent_to_dlq: sent_to_dlq,
               timestamp: Time.now.iso8601
             }
           }

data/lib/smart_message/configuration.rb

@@ -0,0 +1,199 @@
+# lib/smart_message/configuration.rb
+# encoding: utf-8
+# frozen_string_literal: true
+
+module SmartMessage
+  # Global configuration class for SmartMessage framework
+  #
+  # This class provides a centralized way for applications to configure
+  # default behavior for all SmartMessage classes. Applications can set
+  # global defaults for logger, transport, and serializer, which will be
+  # used by all message classes unless explicitly overridden.
+  #
+  # IMPORTANT: No configuration = NO LOGGING
+  # Applications must explicitly configure logging if they want it.
+  #
+  # Usage:
+  #   # No configuration block = NO LOGGING (default behavior)
+  #   
+  #   # Use framework default logger (Lumberjack) with custom log file:
+  #   SmartMessage.configure do |config|
+  #     config.logger = "log/my_app.log"              # String path = Lumberjack logger
+  #   end
+  #   
+  #   # Use framework default logger with STDOUT/STDERR:
+  #   SmartMessage.configure do |config|
+  #     config.logger = STDOUT                        # Log to STDOUT
+  #     config.logger = STDERR                        # Log to STDERR
+  #   end
+  #   
+  #   # Use framework default logger with default file (log/smart_message.log):
+  #   SmartMessage.configure do |config|
+  #     config.logger = :default                      # Framework default
+  #   end
+  #
+  #   # Configure Lumberjack logger options:
+  #   SmartMessage.configure do |config|
+  #     config.logger = :default                      # Use framework default
+  #     config.log_level = :debug                     # :debug, :info, :warn, :error, :fatal
+  #     config.log_format = :json                     # :text or :json
+  #     config.log_include_source = true              # Include file:line source info
+  #     config.log_structured_data = true             # Include structured message data
+  #     config.log_colorize = true                    # Enable colorized output (console only)
+  #     config.log_options = {                        # Additional Lumberjack options
+  #       roll_by_date: true,                         # Enable date-based log rolling
+  #       date_pattern: '%Y-%m-%d',                   # Date pattern for rolling
+  #       roll_by_size: true,                         # Enable size-based log rolling
+  #       max_file_size: 50 * 1024 * 1024,            # Max file size before rolling (50 MB)
+  #       keep_files: 10                              # Number of rolled files to keep
+  #     }
+  #   end
+  #
+  #   # Use custom logger:
+  #   SmartMessage.configure do |config|
+  #     config.logger = MyApp::Logger.new             # Custom logger object
+  #     config.transport = MyApp::Transport.new
+  #     config.serializer = MyApp::Serializer.new
+  #   end
+  #
+  #   # Explicitly disable logging:
+  #   SmartMessage.configure do |config|
+  #     config.logger = nil                           # Explicit no logging
+  #   end
+  #
+  #   # Individual message classes use these defaults automatically
+  #   class OrderMessage < SmartMessage::Base
+  #     property :order_id
+  #     # No config block needed - uses global defaults
+  #   end
+  #
+  #   # Override global defaults when needed
+  #   class SpecialMessage < SmartMessage::Base
+  #     config do
+  #       logger MyApp::SpecialLogger.new  # Override just the logger
+  #       # transport and serializer still use global defaults
+  #     end
+  #   end
+  class Configuration
+    attr_accessor :transport, :serializer, :log_level, :log_format, :log_include_source, :log_structured_data, :log_colorize, :log_options
+    attr_reader :logger
+    
+    def initialize
+      @logger = nil
+      @transport = nil
+      @serializer = nil
+      @logger_explicitly_set_to_nil = false
+      @log_level = nil
+      @log_format = nil
+      @log_include_source = nil
+      @log_structured_data = nil
+      @log_colorize = nil
+      @log_options = {}
+    end
+    
+    # Custom logger setter to track explicit nil assignment
+    def logger=(value)
+      @logger = value
+      @logger_explicitly_set_to_nil = value.nil?
+    end
+    
+    # Reset configuration to defaults
+    def reset!
+      @logger = nil
+      @transport = nil
+      @serializer = nil
+      @logger_explicitly_set_to_nil = false
+      @log_level = nil
+      @log_format = nil
+      @log_include_source = nil
+      @log_structured_data = nil
+      @log_colorize = nil
+      @log_options = {}
+    end
+    
+    # Check if logger is configured (including explicit nil for no logging)
+    def logger_configured?
+      !@logger.nil? || @logger_explicitly_set_to_nil || @logger == :default || @logger.is_a?(String) || @logger == STDOUT || @logger == STDERR
+    end
+    
+    # Check if transport is configured
+    def transport_configured?
+      !@transport.nil?
+    end
+    
+    # Check if serializer is configured
+    def serializer_configured?
+      !@serializer.nil?
+    end
+    
+    # Get the configured logger or no logging
+    def default_logger
+      case @logger
+      when nil
+        # If explicitly set to nil, use null logger (no logging)
+        if @logger_explicitly_set_to_nil
+          SmartMessage::Logger::Null.new
+        else
+          # Not configured, NO LOGGING
+          SmartMessage::Logger::Null.new
+        end
+      when :default
+        # Explicitly requested framework default
+        framework_default_logger
+      when String
+        # String path means use Lumberjack logger with that file path
+        SmartMessage::Logger::Lumberjack.new(**logger_options.merge(log_file: @logger))
+      when STDOUT, STDERR
+        # STDOUT/STDERR constants mean use Lumberjack logger with that output
+        SmartMessage::Logger::Lumberjack.new(**logger_options.merge(log_file: @logger))
+      else
+        @logger
+      end
+    end
+    
+    # Get the configured transport or framework default
+    def default_transport
+      @transport || framework_default_transport
+    end
+    
+    # Get the configured serializer or framework default
+    def default_serializer
+      @serializer || framework_default_serializer
+    end
+    
+    private
+    
+    # Framework's built-in default logger (Lumberjack)
+    def framework_default_logger
+      SmartMessage::Logger::Lumberjack.new(**logger_options)
+    end
+    
+    # Build logger options from configuration
+    def logger_options
+      options = {}
+      options[:level] = @log_level if @log_level
+      options[:format] = @log_format if @log_format
+      options[:include_source] = @log_include_source unless @log_include_source.nil?
+      options[:structured_data] = @log_structured_data unless @log_structured_data.nil?
+      options[:colorize] = @log_colorize unless @log_colorize.nil?
+      
+      # Merge in log_options (for roll_by_date, roll_by_size, max_file_size, etc.)
+      options.merge!(@log_options) if @log_options && @log_options.is_a?(Hash)
+      
+      options
+    end
+    
+    # Framework's built-in default transport (Redis)
+    def framework_default_transport
+      SmartMessage::Transport::RedisTransport.new
+    end
+    
+    # Framework's built-in default serializer (JSON)
+    def framework_default_serializer
+      SmartMessage::Serializer::Json.new
+    rescue => e
+      # Fallback if JSON serializer is not available
+      nil
+    end
+  end
+end

data/lib/smart_message/dead_letter_queue.rb

@@ -0,0 +1,361 @@
+# lib/smart_message/dead_letter_queue.rb
+# encoding: utf-8
+# frozen_string_literal: true
+
+require 'json'
+require 'fileutils'
+
+module SmartMessage
+  # File-based Dead Letter Queue implementation using JSON Lines format
+  # Provides FIFO queue operations with replay capabilities for failed messages
+  class DeadLetterQueue
+    attr_reader :file_path
+
+    # Default singleton instance
+    @@default_instance = nil
+
+    def self.default
+      @@default_instance ||= new
+    end
+
+    def self.configure_default(file_path)
+      @@default_instance = new(file_path)
+    end
+
+    def initialize(file_path = 'dead_letters.jsonl')
+      @file_path = File.expand_path(file_path)
+      @mutex = Mutex.new
+      ensure_directory_exists
+      
+      logger.debug { "[SmartMessage::DeadLetterQueue] Initialized with file path: #{@file_path}" }
+    rescue => e
+      logger&.error { "[SmartMessage] Error in dead letter queue initialization: #{e.class.name} - #{e.message}" }
+      raise
+    end
+    
+    private
+    
+    def logger
+      @logger ||= SmartMessage::Logger.default
+    end
+    
+    public
+
+    # Core FIFO queue operations
+
+    # Add a failed message to the dead letter queue
+    # @param message [SmartMessage::Base] The message instance
+    # @param error_info [Hash] Error details including :error, :retry_count, :transport, etc.
+    def enqueue(message, error_info = {})
+      message_header = message._sm_header
+      message_payload = message.encode
+
+      entry = {
+        timestamp: Time.now.iso8601,
+        header: message_header.to_hash,
+        payload: message_payload,
+        payload_format: error_info[:serializer] || 'json',
+        error: error_info[:error] || 'Unknown error',
+        retry_count: error_info[:retry_count] || 0,
+        transport: error_info[:transport],
+        stack_trace: error_info[:stack_trace]
+      }
+
+      @mutex.synchronize do
+        File.open(@file_path, 'a') do |file|
+          file.puts entry.to_json
+          file.fsync  # Ensure immediate write to disk
+        end
+      end
+
+      entry
+    end
+
+    # Remove and return the oldest message from the queue
+    # @return [Hash, nil] The oldest DLQ entry or nil if queue is empty
+    def dequeue
+      @mutex.synchronize do
+        return nil unless File.exist?(@file_path)
+
+        lines = File.readlines(@file_path)
+        return nil if lines.empty?
+
+        # Get first line (oldest)
+        oldest_line = lines.shift
+        oldest_entry = JSON.parse(oldest_line.strip, symbolize_names: true)
+
+        # Rewrite file without the first line
+        File.open(@file_path, 'w') do |file|
+          lines.each { |line| file.write(line) }
+        end
+
+        oldest_entry
+      end
+    rescue JSON::ParserError => e
+      logger.warn { "[SmartMessage] Warning: Corrupted DLQ entry skipped: #{e.message}" }
+      nil
+    end
+
+    # Look at the oldest message without removing it
+    # @return [Hash, nil] The oldest DLQ entry or nil if queue is empty
+    def peek
+      return nil unless File.exist?(@file_path)
+
+      File.open(@file_path, 'r') do |file|
+        first_line = file.readline
+        return nil if first_line.nil? || first_line.strip.empty?
+        JSON.parse(first_line.strip, symbolize_names: true)
+      end
+    rescue EOFError, JSON::ParserError
+      nil
+    end
+
+    # Get the number of messages in the queue
+    # @return [Integer] Number of messages in the DLQ
+    def size
+      return 0 unless File.exist?(@file_path)
+      File.readlines(@file_path).size
+    end
+
+    # Clear all messages from the queue
+    def clear
+      @mutex.synchronize do
+        File.delete(@file_path) if File.exist?(@file_path)
+      end
+    end
+
+    # Replay capabilities
+
+    # Replay all messages in the queue
+    # @param transport [SmartMessage::Transport::Base] Optional transport override
+    # @return [Hash] Results summary
+    def replay_all(transport = nil)
+      results = { success: 0, failed: 0, errors: [] }
+
+      while (entry = dequeue)
+        result = replay_entry(entry, transport)
+        if result[:success]
+          results[:success] += 1
+        else
+          results[:failed] += 1
+          results[:errors] << result[:error]
+          # Re-enqueue failed replay attempts
+          header = SmartMessage::Header.new(entry[:header])
+          enqueue(header, entry[:payload],
+            error: "Replay failed: #{result[:error]}",
+            retry_count: (entry[:retry_count] || 0) + 1)
+        end
+      end
+
+      results
+    end
+
+    # Replay the oldest message
+    # @param transport [SmartMessage::Transport::Base] Optional transport override
+    # @return [Hash] Result of replay attempt
+    def replay_one(transport = nil)
+      entry = dequeue
+      return { success: false, error: 'Queue is empty' } unless entry
+
+      replay_entry(entry, transport)
+    end
+
+    # Replay a batch of messages
+    # @param count [Integer] Number of messages to replay
+    # @param transport [SmartMessage::Transport::Base] Optional transport override
+    # @return [Hash] Results summary
+    def replay_batch(count = 10, transport = nil)
+      results = { success: 0, failed: 0, errors: [] }
+
+      count.times do
+        break if size == 0
+
+        result = replay_one(transport)
+        if result[:success]
+          results[:success] += 1
+        else
+          results[:failed] += 1
+          results[:errors] << result[:error]
+        end
+      end
+
+      results
+    end
+
+    # Administrative utilities
+
+    # Inspect messages in the queue without removing them
+    # @param limit [Integer] Maximum number of messages to show
+    # @return [Array<Hash>] Array of DLQ entries
+    def inspect_messages(limit: 10)
+      count = 0
+      read_entries_with_filter do |entry|
+        return [] if count >= limit
+        count += 1
+        entry
+      end
+    end
+
+    # Filter messages by message class
+    # @param message_class [String] The message class name to filter by
+    # @return [Array<Hash>] Filtered DLQ entries
+    def filter_by_class(message_class)
+      read_entries_with_filter do |entry|
+        entry if entry.dig(:header, :message_class) == message_class
+      end
+    end
+
+    # Filter messages by error pattern
+    # @param pattern [Regexp, String] Pattern to match against error messages
+    # @return [Array<Hash>] Filtered DLQ entries
+    def filter_by_error_pattern(pattern)
+      pattern = Regexp.new(pattern) if pattern.is_a?(String)
+      
+      read_entries_with_filter do |entry|
+        entry if pattern.match?(entry[:error].to_s)
+      end
+    end
+
+    # Get statistics about the dead letter queue
+    # @return [Hash] Statistics summary
+    def statistics
+      stats = { total: 0, by_class: Hash.new(0), by_error: Hash.new(0) }
+      
+      read_entries_with_filter do |entry|
+        stats[:total] += 1
+        
+        full_class_name = entry.dig(:header, :message_class) || 'Unknown'
+        # Extract short class name (everything after the last ::)
+        short_class_name = full_class_name.split('::').last || full_class_name
+        stats[:by_class][short_class_name] += 1
+        
+        error = entry[:error] || 'Unknown error'
+        stats[:by_error][error] += 1
+        
+        nil  # Don't collect entries, just process for side effects
+      end
+      
+      stats
+    end
+
+    # Export messages within a time range
+    # @param start_time [Time] Start of time range
+    # @param end_time [Time] End of time range
+    # @return [Array<Hash>] DLQ entries within the time range
+    def export_range(start_time, end_time)
+      read_entries_with_filter do |entry|
+        begin
+          timestamp = Time.parse(entry[:timestamp])
+          entry if timestamp >= start_time && timestamp <= end_time
+        rescue ArgumentError
+          # Skip entries with invalid timestamps
+          nil
+        end
+      end
+    end
+
+    private
+
+    # Replay a single DLQ entry by recreating the message instance
+    # @param entry [Hash] The DLQ entry to replay
+    # @param transport_override [SmartMessage::Transport::Base] Optional transport override
+    # @return [Hash] Result of replay attempt
+    def replay_entry(entry, transport_override = nil)
+      message_class_name = entry.dig(:header, :message_class)
+      return { success: false, error: 'Missing message class' } unless message_class_name
+
+      # Get the message class
+      message_class = message_class_name.constantize
+
+      # Deserialize the payload using the appropriate format
+      payload_data = deserialize_payload(entry[:payload], entry[:payload_format] || 'json')
+      return { success: false, error: 'Failed to deserialize payload' } unless payload_data
+
+      # Remove the header from payload data (it's stored separately in DLQ)
+      payload_data.delete(:_sm_header)
+
+      # Create new message instance with original data
+      message = message_class.new(**payload_data)
+
+      # Restore complete header information
+      restore_header_fields(message, entry[:header])
+
+      # Override transport if provided - this must be done before publishing
+      if transport_override
+        message.transport(transport_override)
+      end
+
+      # Attempt to publish the message
+      message.publish
+
+      { success: true, message: message }
+    rescue => e
+      { success: false, error: "#{e.class.name}: #{e.message}" }
+    end
+
+    # Deserialize payload based on format
+    # @param payload [String] The serialized payload
+    # @param format [String] The serialization format (json, etc.)
+    # @return [Hash, nil] Deserialized data or nil if failed
+    def deserialize_payload(payload, format)
+      case format.to_s.downcase
+      when 'json'
+        JSON.parse(payload, symbolize_names: true)
+      else
+        # For unknown formats, assume JSON as fallback but log warning
+        logger.warn { "[SmartMessage] Warning: Unknown payload format '#{format}', attempting JSON" }
+        JSON.parse(payload, symbolize_names: true)
+      end
+    rescue JSON::ParserError => e
+      logger.error { "[SmartMessage] Error in payload deserialization: #{e.class.name} - #{e.message}" }
+      nil
+    end
+
+    # Restore all header fields from DLQ entry
+    # @param message [SmartMessage::Base] The message instance
+    # @param header_data [Hash] The stored header data
+    def restore_header_fields(message, header_data)
+      return unless header_data
+      
+      # Restore all available header fields
+      message._sm_header.uuid = header_data[:uuid] if header_data[:uuid]
+      message._sm_header.message_class = header_data[:message_class] if header_data[:message_class]
+      message._sm_header.published_at = Time.parse(header_data[:published_at]) if header_data[:published_at]
+      message._sm_header.publisher_pid = header_data[:publisher_pid] if header_data[:publisher_pid]
+      message._sm_header.version = header_data[:version] if header_data[:version]
+      message._sm_header.from = header_data[:from] if header_data[:from]
+      message._sm_header.to = header_data[:to] if header_data[:to]
+      message._sm_header.reply_to = header_data[:reply_to] if header_data[:reply_to]
+    rescue => e
+      logger.warn { "[SmartMessage] Warning: Failed to restore some header fields: #{e.message}" }
+    end
+
+    # Generic file reading iterator with error handling
+    # @param block [Proc] Block to execute for each valid entry
+    # @return [Array] Results from the block
+    def read_entries_with_filter(&block)
+      return [] unless File.exist?(@file_path)
+      
+      results = []
+      File.open(@file_path, 'r') do |file|
+        file.each_line do |line|
+          begin
+            entry = JSON.parse(line.strip, symbolize_names: true)
+            result = block.call(entry)
+            results << result if result
+          rescue JSON::ParserError
+            # Skip corrupted lines
+          end
+        end
+      end
+      
+      results
+    end
+
+    # Ensure the directory for the DLQ file exists
+    def ensure_directory_exists
+      directory = File.dirname(@file_path)
+      FileUtils.mkdir_p(directory) unless Dir.exist?(directory)
+    end
+  end
+end