smart_message 0.0.12 → 0.0.16

data/lib/smart_message/transport/async_publish_queue.rb

@@ -0,0 +1,284 @@
+# lib/smart_message/transport/async_publish_queue.rb
+# encoding: utf-8
+# frozen_string_literal: true
+
+require 'timeout'
+
+module SmartMessage
+  module Transport
+    # Module for asynchronous publishing queue.
+    module AsyncPublishQueue
+      
+      private
+      
+      def logger
+        # Ensure we have a proper logger, not just an IO object
+        if @logger && @logger.respond_to?(:error) && @logger.respond_to?(:info) && @logger.respond_to?(:warn)
+          return @logger
+        end
+        @logger = SmartMessage::Logger.default
+      end
+      
+      public
+      def configure_async_publishing
+        return unless @options[:async]
+        
+        # Ensure file configuration is set up first
+        configure_file_output if respond_to?(:configure_file_output)
+        
+        @publish_queue = @options[:max_queue] ? SizedQueue.new(@options[:max_queue]) : Queue.new
+        @queue_stats = { queued: 0, processed: 0, failed: 0, dropped: 0, blocked_count: 0, max_queue_size_reached: 0 }
+        @last_warning_time = nil
+        
+        start_publish_worker
+        start_queue_monitoring if @options[:enable_queue_monitoring]
+      end
+
+      def async_publish(message_class, serialized_message)
+        check_queue_warning
+        
+        if queue_full?
+          handle_queue_overflow(message_class, serialized_message)
+          return false
+        end
+        
+        @publish_queue << { 
+          message_class: message_class, 
+          serialized_message: serialized_message, 
+          timestamp: Time.now, 
+          retry_count: 0 
+        }
+        @queue_stats[:queued] += 1
+        true
+      rescue => e
+        begin
+          logger.error { "[FileTransport] Error queuing message: #{e.message}" }
+        rescue
+          # Fallback if logger is not available
+        end
+        false
+      end
+
+      def queue_full?
+        @publish_queue.is_a?(SizedQueue) && @publish_queue.size >= @options[:max_queue]
+      end
+
+      def queue_usage_percentage
+        return 0 unless @publish_queue.is_a?(SizedQueue)
+        (@publish_queue.size.to_f / @options[:max_queue] * 100).round(1)
+      end
+
+      def publish_stats
+        return {} unless @options[:async]
+        @queue_stats.merge(
+          current_size: @publish_queue.size,
+          worker_alive: @publish_worker_thread&.alive? || false
+        )
+      end
+
+      private
+
+      def start_publish_worker
+        @publish_worker_thread = Thread.new do
+          Thread.current.name = "FileTransport-Publisher"
+          
+          loop do
+            begin
+              message_data = Timeout.timeout(@options[:worker_timeout] || 5) { @publish_queue.pop }
+              process_queued_message(message_data)
+            rescue Timeout::Error
+              next
+            rescue => e
+              begin
+                logger.error { "[FileTransport] Publish worker error: #{e.message}" }
+              rescue
+                # Fallback if logger is not available
+              end
+              sleep 1
+            end
+          end
+        end
+      end
+
+      def process_queued_message(message_data)
+        success = do_sync_publish(message_data[:message_class], message_data[:serialized_message])
+        if success
+          @queue_stats[:processed] += 1
+        else
+          handle_publish_failure(message_data)
+        end
+      end
+
+      def do_sync_publish(message_class, serialized_message)
+        begin
+          if @options[:file_type] == :fifo
+            write_to_fifo(serialized_message)
+          else
+            write_to_file(serialized_message)
+          end
+          true
+        rescue => e
+          begin
+            logger.error { "[FileTransport] Sync write error: #{e.message}" }
+          rescue
+            # Fallback if logger is not available
+          end
+          false
+        end
+      end
+
+      def handle_publish_failure(message_data)
+        retry_count = message_data[:retry_count] + 1
+        max_retries = @options[:max_retries] || 3
+        
+        if retry_count <= max_retries
+          sleep_time = [2 ** retry_count, @options[:max_retry_delay] || 30].min
+          Thread.new do
+            sleep sleep_time
+            message_data[:retry_count] = retry_count
+            @publish_queue << message_data
+          end
+          logger.warn { "[FileTransport] Retrying message publish (attempt #{retry_count}/#{max_retries})" }
+        else
+          @queue_stats[:failed] += 1
+          send_to_dead_letter_queue(message_data) if defined?(SmartMessage::DeadLetterQueue)
+        end
+      end
+
+      def handle_queue_overflow(message_class, serialized_message)
+        case @options[:queue_overflow_strategy]
+        when :drop_newest
+          @queue_stats[:dropped] += 1
+          logger.warn { "[FileTransport] Dropping newest message - queue full (#{@publish_queue.size}/#{@options[:max_queue]})" }
+        when :drop_oldest
+          begin
+            dropped_message = @publish_queue.pop(true)
+            @publish_queue << {
+              message_class: message_class,
+              serialized_message: serialized_message,
+              timestamp: Time.now,
+              retry_count: 0
+            }
+            @queue_stats[:dropped] += 1
+            logger.warn { "[FileTransport] Dropped oldest message to make room for new message" }
+            send_to_dead_letter_queue(dropped_message) if @options[:send_dropped_to_dlq]
+          rescue ThreadError
+            @queue_stats[:dropped] += 1
+            logger.error { "[FileTransport] Queue overflow but queue appears empty - dropping message" }
+          end
+        when :block
+          @queue_stats[:blocked_count] += 1
+          start_time = Time.now
+          logger.warn { "[FileTransport] Queue full (#{@publish_queue.size}/#{@options[:max_queue]}) - blocking until space available" }
+          
+          @publish_queue << {
+            message_class: message_class,
+            serialized_message: serialized_message,
+            timestamp: Time.now,
+            retry_count: 0
+          }
+          logger.info { "[FileTransport] Queue space available after #{(Time.now - start_time).round(2)}s - message queued" }
+          @queue_stats[:queued] += 1
+          return true
+        else
+          @queue_stats[:dropped] += 1
+          logger.error { "[FileTransport] Unknown overflow strategy '#{@options[:queue_overflow_strategy]}', dropping message" }
+        end
+        false
+      end
+
+      def send_to_dead_letter_queue(message_data)
+        SmartMessage::DeadLetterQueue.default.enqueue(
+          message_data[:message_class],
+          message_data[:serialized_message],
+          error: "Failed async publish after #{@options[:max_retries]} retries",
+          transport: self.class.name,
+          retry_count: message_data[:retry_count]
+        )
+      rescue => e
+        begin
+          logger.error { "[FileTransport] Failed to send to DLQ: #{e.message}" }
+        rescue
+          # Fallback if logger is not available
+        end
+      end
+
+      def check_queue_warning
+        return unless @options[:queue_warning_threshold]
+        return if @options[:max_queue] == 0
+        
+        usage = queue_usage_percentage
+        threshold = @options[:queue_warning_threshold] * 100
+        
+        if usage >= threshold
+          now = Time.now
+          if @last_warning_time.nil? || (now - @last_warning_time) > 60
+            logger.warn { "[FileTransport] Publish queue is #{usage}% full (#{@publish_queue.size}/#{@options[:max_queue]})" }
+            @last_warning_time = now
+          end
+        end
+      end
+
+      def start_queue_monitoring
+        @queue_monitoring_thread = Thread.new do
+          Thread.current.name = "FileTransport-QueueMonitor"
+          
+          loop do
+            sleep 30
+            
+            begin
+              stats = publish_stats
+              usage = queue_usage_percentage
+              
+              logger.info do
+                "[FileTransport] Queue stats: #{stats[:current_size]} messages " \
+                "(#{usage}% full), processed: #{stats[:processed]}, " \
+                "failed: #{stats[:failed]}, worker: #{stats[:worker_alive] ? 'alive' : 'dead'}"
+              end
+            rescue => e
+              logger.error { "[FileTransport] Queue monitoring error: #{e.message}" }
+            end
+          end
+        end
+      end
+
+      def stop_async_publishing
+        return unless @publish_worker_thread
+        
+        @publish_worker_thread.kill
+        @publish_worker_thread.join(@options[:shutdown_timeout] || 10)
+        
+        if @options[:drain_queue_on_shutdown]
+          drain_publish_queue
+        end
+        
+        @publish_worker_thread = nil
+        @queue_monitoring_thread&.kill
+        @queue_monitoring_thread&.join(5)
+      end
+
+      def drain_publish_queue
+        begin
+          logger.info { "[FileTransport] Draining #{@publish_queue.size} queued messages" }
+        rescue
+          # Fallback if logger is not available
+        end
+        
+        until @publish_queue.empty?
+          begin
+            message_data = @publish_queue.pop(true)
+            process_queued_message(message_data)
+          rescue ThreadError
+            break
+          end
+        end
+        
+        begin
+          logger.info { "[FileTransport] Queue drained" }
+        rescue
+          # Fallback if logger is not available
+        end
+      end
+    end
+  end
+end

data/lib/smart_message/transport/base.rb

@@ -11,15 +11,17 @@ module SmartMessage
     class Base
       include BreakerMachines::DSL
 
-      attr_reader :options, :dispatcher
+      attr_reader :options, :dispatcher, :serializer
 
       def initialize(**options)
         @options = default_options.merge(options)
         @dispatcher = options[:dispatcher] || SmartMessage::Dispatcher.new
+        @serializer = options[:serializer] || default_serializer
         configure
         configure_transport_circuit_breakers
         
         logger.debug { "[SmartMessage::Transport::#{self.class.name.split('::').last}] Initialized with options: #{@options}" }
+        logger.debug { "[SmartMessage::Transport::#{self.class.name.split('::').last}] Using serializer: #{@serializer.class.name}" }
       rescue => e
         logger&.error { "[SmartMessage] Error in transport initialization: #{e.class.name} - #{e.message}" }
         raise
@@ -43,10 +45,20 @@ module SmartMessage
         {}
       end
 
+      # Default serializer for this transport (override in subclasses)
+      def default_serializer
+        SmartMessage::Serializer::Json.new
+      end
+
       # Publish a message with circuit breaker protection
-      # @param message_class [String] The message class name (used for channel routing)
-      # @param serialized_message [String] Complete serialized message content
-      def publish(message_class, serialized_message)
+      # @param message [SmartMessage::Base] The message instance to publish
+      def publish(message)
+        # Extract routing info from message header
+        message_class = message._sm_header.message_class
+        
+        # Serialize the entire message (flat structure with _sm_header)
+        serialized_message = encode_message(message)
+        
         circuit(:transport_publish).wrap do
           do_publish(message_class, serialized_message)
         end
@@ -58,7 +70,7 @@ module SmartMessage
         raise unless e.is_a?(Hash) && e[:circuit_breaker]
 
         # Handle circuit breaker fallback
-        handle_publish_fallback(e, message_class, serialized_message)
+        handle_publish_fallback(e, message._sm_header.message_class, serialized_message)
       end
 
       # Template method for actual publishing (implement in subclasses)
@@ -150,13 +162,32 @@ module SmartMessage
         end
       end
 
+      # Encode a message using the transport's serializer
+      # @param message [SmartMessage::Base] The message to encode
+      # @return [String] The serialized message
+      def encode_message(message)
+        # Update header with serializer info
+        message._sm_header.serializer = @serializer.class.to_s
+        
+        # Serialize the entire message as a flat structure
+        @serializer.encode(message.to_hash)
+      end
+
+      # Decode a message using the transport's serializer
+      # @param serialized_message [String] The serialized message
+      # @return [Hash] The decoded message data
+      def decode_message(serialized_message)
+        @serializer.decode(serialized_message)
+      end
+
       # Receive and route a message (called by transport implementations)
       # @param message_class [String] The message class name
       # @param serialized_message [String] The serialized message content
       protected
 
       def receive(message_class, serialized_message)
-        # Decode the message using the class's configured serializer
+        # Decode the message using transport's serializer
+        decoded_data = decode_message(serialized_message)
         
         # Add defensive check for message_class type
         unless message_class.respond_to?(:constantize)
@@ -165,10 +196,13 @@ module SmartMessage
           raise ArgumentError, "message_class must be a String, got #{message_class.class.name}"
         end
         
+        # Reconstruct the message instance from flat structure
         message_class_obj = message_class.constantize
-        decoded_message = message_class_obj.decode(serialized_message)
+        # Convert string keys to symbols for keyword arguments
+        symbol_data = decoded_data.transform_keys(&:to_sym)
+        message = message_class_obj.new(**symbol_data)
         
-        @dispatcher.route(decoded_message)
+        @dispatcher.route(message)
       rescue => e
         logger.error { "[SmartMessage] Error in transport receive: #{e.class.name} - #{e.message}" }
         logger.error { "[SmartMessage] message_class: #{message_class.inspect} (#{message_class.class.name})" }

data/lib/smart_message/transport/fifo_operations.rb

@@ -0,0 +1,264 @@
+# lib/smart_message/transport/fifo_operations.rb
+# encoding: utf-8
+# frozen_string_literal: true
+
+require 'rbconfig'
+
+module SmartMessage
+  module Transport
+    # Module for FIFO operations, with cross-platform considerations.
+    module FifoOperations
+      
+      private
+      
+      def logger
+        # Ensure we have a proper logger, not just an IO object
+        if @logger && @logger.respond_to?(:error) && @logger.respond_to?(:info) && @logger.respond_to?(:warn)
+          return @logger
+        end
+        @logger = SmartMessage::Logger.default
+      end
+      
+      public
+      def configure_fifo
+        unless platform_supports_fifo?
+          begin
+            logger.warn { "[FileTransport] FIFO not supported, falling back to regular file" }
+          rescue
+            # Fallback if logger is not available
+          end
+          @options[:file_type] = :regular
+          return configure_file_output
+        end
+        
+        create_named_pipe if @options[:create_fifo]
+      end
+
+      def create_named_pipe
+        case RbConfig::CONFIG['host_os']
+        when /mswin|mingw|cygwin/
+          create_windows_named_pipe
+        else
+          File.mkfifo(@options[:file_path], @options[:fifo_permissions] || 0644)
+          begin
+            logger.info { "[FileTransport] Created FIFO: #{@options[:file_path]}" }
+          rescue
+            # Fallback if logger is not available
+          end
+        end
+      rescue NotImplementedError
+        begin
+          logger.error { "[FileTransport] Named pipes not supported on this platform" }
+        rescue
+          # Fallback if logger is not available
+        end
+        raise
+      rescue => e
+        begin
+          logger.error { "[FileTransport] Failed to create FIFO: #{e.message}" }
+        rescue
+          # Fallback if logger is not available
+        end
+        raise
+      end
+
+      def create_windows_named_pipe
+        require 'win32/pipe'
+        pipe_name = "\\\\.\\pipe\\#{File.basename(@options[:file_path])}"
+        @windows_pipe_server = Win32::Pipe::Server.new(pipe_name)
+        begin
+          logger.info { "[FileTransport] Created Windows named pipe: #{pipe_name}" }
+        rescue
+          # Fallback if logger is not available
+        end
+      rescue LoadError
+        raise "Windows named pipes require win32-pipe gem: gem install win32-pipe"
+      rescue => e
+        begin
+          logger.error { "[FileTransport] Failed to create Windows named pipe: #{e.message}" }
+        rescue
+          # Fallback if logger is not available
+        end
+        raise
+      end
+
+      def platform_supports_fifo?
+        case RbConfig::CONFIG['host_os']
+        when /mswin|mingw|cygwin/
+          defined?(Win32::Pipe)
+        else
+          true
+        end
+      end
+
+      def write_to_fifo(serialized_message)
+        handle = open_fifo_for_writing
+        unless handle
+          handle_fifo_write_failure(serialized_message)
+          return false
+        end
+        
+        content = prepare_file_content(serialized_message)
+        handle.write(content)
+        handle.flush
+        true
+      rescue Errno::EPIPE
+        begin
+          logger.warn { "[FileTransport] FIFO reader disconnected" }
+        rescue
+          # Fallback if logger is not available
+        end
+        handle_fifo_write_failure(serialized_message)
+        false
+      rescue => e
+        begin
+          logger.error { "[FileTransport] FIFO write error: #{e.message}" }
+        rescue
+          # Fallback if logger is not available
+        end
+        handle_fifo_write_failure(serialized_message)
+        false
+      ensure
+        handle&.close
+      end
+
+      def open_fifo_for_writing
+        mode = @options[:fifo_mode] == :non_blocking ? File::WRONLY | File::NONBLOCK : 'w'
+        File.open(@options[:file_path], mode)
+      rescue Errno::ENXIO, Errno::ENOENT
+        nil
+      end
+
+      def handle_fifo_write_failure(serialized_message)
+        if @options[:fallback_transport]
+          begin
+            @options[:fallback_transport].do_publish(@current_message_class, serialized_message)
+            begin
+              logger.info { "[FileTransport] Message sent to fallback transport" }
+            rescue
+              # Fallback if logger is not available
+            end
+          rescue => e
+            begin
+              logger.error { "[FileTransport] Fallback transport failed: #{e.message}" }
+            rescue
+              # Fallback if logger is not available
+            end
+          end
+        end
+      end
+
+      def start_fifo_reader(message_class, process_method, filter_options)
+        case @options[:subscription_mode]
+        when :fifo_blocking
+          start_blocking_fifo_reader(message_class, process_method)
+        when :fifo_select
+          start_select_fifo_reader(message_class, process_method)
+        when :fifo_polling
+          start_polling_fifo_reader(message_class, process_method)
+        else
+          begin
+            logger.warn { "[FileTransport] Invalid FIFO subscription mode: #{@options[:subscription_mode]}" }
+          rescue
+            # Fallback if logger is not available
+          end
+        end
+      end
+
+      def start_blocking_fifo_reader(message_class, process_method)
+        @fifo_reader_thread = Thread.new do
+          Thread.current.name = "FileTransport-FifoReader"
+          loop do
+            begin
+              File.open(@options[:file_path], 'r') do |fifo|
+                while line = fifo.gets
+                  next if line.strip.empty?
+                  receive(message_class, line.strip)
+                end
+              end
+            rescue => e
+              begin
+                logger.error { "[FileTransport] FIFO reader error: #{e.message}" }
+              rescue
+                # Fallback if logger is not available
+              end
+              sleep 1
+            end
+          end
+        end
+      end
+
+      def start_select_fifo_reader(message_class, process_method)
+        @fifo_select_thread = Thread.new do
+          Thread.current.name = "FileTransport-FifoSelect"
+          fifo = File.open(@options[:file_path], File::RDONLY | File::NONBLOCK)
+          
+          loop do
+            ready = IO.select([fifo], nil, nil, 1.0)
+            if ready
+              begin
+                while line = fifo.gets
+                  next if line.strip.empty?
+                  receive(message_class, line.strip)
+                end
+              rescue IO::WaitReadable
+                next
+              rescue => e
+                begin
+                  logger.error { "[FileTransport] FIFO select error: #{e.message}" }
+                rescue
+                  # Fallback if logger is not available
+                end
+              end
+            end
+          end
+        rescue => e
+          begin
+            logger.error { "[FileTransport] FIFO select thread error: #{e.message}" }
+          rescue
+            # Fallback if logger is not available
+          end
+        ensure
+          fifo&.close
+        end
+      end
+
+      def start_polling_fifo_reader(message_class, process_method)
+        @fifo_reader_thread = Thread.new do
+          Thread.current.name = "FileTransport-FifoPoller"
+          loop do
+            begin
+              File.open(@options[:file_path], File::RDONLY | File::NONBLOCK) do |fifo|
+                while line = fifo.gets
+                  next if line.strip.empty?
+                  receive(message_class, line.strip)
+                end
+              end
+            rescue Errno::EAGAIN
+              sleep(@options[:poll_interval] || 1.0)
+            rescue => e
+              begin
+                logger.error { "[FileTransport] FIFO polling error: #{e.message}" }
+              rescue
+                # Fallback if logger is not available
+              end
+              sleep 1
+            end
+          end
+        end
+      end
+
+      def stop_fifo_operations
+        @fifo_reader_thread&.kill
+        @fifo_reader_thread&.join(2)
+        @fifo_select_thread&.kill
+        @fifo_select_thread&.join(2)
+        @fifo_handle&.close
+      end
+
+      def fifo_active?
+        @fifo_reader_thread&.alive? || @fifo_select_thread&.alive?
+      end
+    end
+  end
+end