smart_message 0.0.12 → 0.0.16

data/lib/smart_message/transport/file_operations.rb

@@ -0,0 +1,200 @@
+# lib/smart_message/transport/file_operations.rb
+# encoding: utf-8
+# frozen_string_literal: true
+
+module SmartMessage
+  module Transport
+    # Module for shared file operations, including buffering, rotation, and basic I/O.
+    module FileOperations
+      def configure_file_output
+        ensure_directory_exists if @options[:create_directories]
+        @file_handle = open_file_handle
+        @write_buffer = []
+        @last_flush = Time.now
+        setup_rotation_timer if rotation_enabled?
+        @file_mutex = Mutex.new  # For thread-safety
+      end
+
+      def write_to_file(serialized_message)
+        content = prepare_file_content(serialized_message)
+        
+        @file_mutex.synchronize do
+          if buffered_mode?
+            buffer_write(content)
+          else
+            direct_write(content)
+          end
+          
+          rotate_file_if_needed
+        end
+      end
+
+      def flush_buffer
+        return if @write_buffer.empty?
+        
+        # Only synchronize if we're not already holding the lock
+        if @file_mutex.owned?
+          @file_handle.write(@write_buffer.join)
+          @file_handle.flush
+          @write_buffer.clear
+          @last_flush = Time.now
+        else
+          @file_mutex.synchronize do
+            @file_handle.write(@write_buffer.join)
+            @file_handle.flush
+            @write_buffer.clear
+            @last_flush = Time.now
+          end
+        end
+      end
+
+      def close_file_handle
+        flush_buffer if buffered_mode?
+        if @file_mutex
+          @file_mutex.synchronize do
+            @file_handle&.close
+            @file_handle = nil
+          end
+        else
+          @file_handle&.close
+          @file_handle = nil
+        end
+      end
+
+      private
+
+      def prepare_file_content(serialized_message)
+        case @options[:file_format]
+        when :lines
+          "#{serialized_message}\n"
+        when :raw
+          serialized_message
+        else
+          "#{serialized_message}\n"  # default to lines
+        end
+      end
+
+      def open_file_handle
+        File.open(current_file_path, file_mode, encoding: @options[:encoding])
+      end
+
+      def ensure_directory_exists
+        return unless @options[:create_directories]
+        
+        dir = File.dirname(current_file_path)
+        FileUtils.mkdir_p(dir) unless Dir.exist?(dir)
+      end
+
+      def current_file_path
+        if rotation_enabled? && time_based_rotation?
+          timestamped_file_path
+        else
+          @options[:file_path]
+        end
+      end
+
+      def file_mode
+        @options[:file_mode] || 'a'  # append by default
+      end
+
+      def buffered_mode?
+        @options[:buffer_size] && @options[:buffer_size] > 0
+      end
+
+      def buffer_write(content)
+        @write_buffer << content
+        
+        if buffer_full? || flush_interval_exceeded?
+          flush_buffer
+        end
+      end
+
+      def direct_write(content)
+        @file_handle.write(content)
+        @file_handle.flush if @options[:auto_flush]
+      end
+
+      def buffer_full?
+        @write_buffer.join.bytesize >= @options[:buffer_size]
+      end
+
+      def flush_interval_exceeded?
+        @options[:flush_interval] && 
+          (Time.now - @last_flush) >= @options[:flush_interval]
+      end
+
+      def rotation_enabled?
+        @options[:rotate_size] || @options[:rotate_time]
+      end
+
+      def time_based_rotation?
+        @options[:rotate_time]
+      end
+
+      def should_rotate?
+        size_rotation_needed? || time_rotation_needed?
+      end
+
+      def size_rotation_needed?
+        @options[:rotate_size] && 
+          File.exist?(current_file_path) && File.size(current_file_path) >= @options[:rotate_size]
+      end
+
+      def time_rotation_needed?
+        return false unless @options[:rotate_time]
+        
+        case @options[:rotate_time]
+        when :hourly
+          Time.now.min == 0 && Time.now.sec == 0
+        when :daily
+          Time.now.hour == 0 && Time.now.min == 0 && Time.now.sec == 0
+        else
+          false
+        end
+      end
+
+      def rotate_file_if_needed
+        return unless should_rotate?
+        
+        close_current_file
+        archive_current_file
+        @file_handle = open_file_handle
+      end
+
+      def close_current_file
+        flush_buffer if buffered_mode?
+        @file_handle&.close
+        @file_handle = nil
+      end
+
+      def archive_current_file
+        return unless File.exist?(current_file_path)
+        
+        timestamp = Time.now.strftime(@options[:timestamp_format] || '%Y%m%d_%H%M%S')
+        base = File.basename(@options[:file_path], '.*')
+        ext = File.extname(@options[:file_path])
+        dir = File.dirname(@options[:file_path])
+        archive_path = File.join(dir, "#{base}_#{timestamp}#{ext}")
+        
+        FileUtils.mv(current_file_path, archive_path)
+        
+        # Maintain rotation count
+        if @options[:rotate_count]
+          files = Dir.glob(File.join(dir, "#{base}_*#{ext}")).sort
+          while files.size > @options[:rotate_count]
+            File.delete(files.shift)
+          end
+        end
+      end
+
+      def timestamped_file_path
+        base = File.basename(@options[:file_path], '.*')
+        ext = File.extname(@options[:file_path])
+        dir = File.dirname(@options[:file_path])
+        timestamp = Time.now.strftime(@options[:timestamp_format] || '%Y%m%d_%H%M%S')
+        
+        File.join(dir, "#{base}_#{timestamp}#{ext}")
+      end
+    end
+  end
+end

data/lib/smart_message/transport/file_transport.rb

@@ -0,0 +1,149 @@
+# lib/smart_message/transport/file_transport.rb
+# encoding: utf-8
+# frozen_string_literal: true
+
+require_relative 'file_operations'
+require_relative 'file_watching'
+require_relative 'partitioned_files'
+require_relative 'async_publish_queue'
+require_relative 'fifo_operations'
+
+module SmartMessage
+  module Transport
+    class FileTransport < Base
+      include FileOperations
+      include FileWatching
+      include PartitionedFiles
+      include AsyncPublishQueue
+      include FifoOperations
+
+      # @param path [String, IO, Pathname] file path or IO-like object
+      # @param mode [String] file open mode ("a" for append, etc.)
+      # @param encoding [String, nil] file encoding
+      def initialize(options = {})
+        @current_message_class = nil
+        super(**options)
+      end
+
+      def default_options
+        {
+          file_path: 'messages.log',
+          file_mode: 'a',
+          encoding: nil,
+          file_format: :lines,
+          buffer_size: 0,
+          flush_interval: nil,
+          auto_flush: true,
+          rotate_size: nil,
+          rotate_time: nil,
+          rotate_count: 5,
+          timestamp_format: '%Y%m%d_%H%M%S',
+          create_directories: true,
+          async: false,
+          max_queue: nil,
+          drop_when_full: false,
+          queue_overflow_strategy: :block,
+          max_retries: 3,
+          max_retry_delay: 30,
+          worker_timeout: 5,
+          shutdown_timeout: 10,
+          queue_warning_threshold: 0.8,
+          enable_queue_monitoring: true,
+          drain_queue_on_shutdown: true,
+          send_dropped_to_dlq: false,
+          read_from_end: true,
+          poll_interval: 1.0,
+          file_type: :regular,
+          create_fifo: false,
+          fifo_mode: :blocking,
+          fifo_permissions: 0644,
+          fallback_transport: nil,
+          enable_subscriptions: false,
+          subscription_mode: :polling,
+          filename_selector: nil,
+          directory: nil,
+          subscription_file_path: nil
+        }
+      end
+
+      def configure
+        # Call parent configuration first
+        super if defined?(super)
+        
+        # Then configure our file-specific features
+        if @options[:async]
+          configure_async_publishing
+        elsif @options[:file_type] == :fifo
+          configure_fifo
+        else
+          configure_file_output
+        end
+      end
+
+      def publish(payload)
+        do_publish(nil, payload)
+      end
+
+      def do_publish(message_class, serialized_message)
+        @current_message_class = message_class
+        if @options[:async]
+          async_publish(message_class, serialized_message)
+        else
+          if @options[:filename_selector] || @options[:directory]
+            header = { message_class_name: message_class.to_s }
+            path = determine_file_path(serialized_message, header)
+            @file_handle = get_or_open_partition_handle(path)
+            write_to_file(serialized_message)
+          elsif @options[:file_type] == :fifo
+            write_to_fifo(serialized_message)
+          else
+            write_to_file(serialized_message)
+          end
+        end
+      end
+
+      def subscribe(message_class, process_method, filter_options = {})
+        unless @options[:enable_subscriptions]
+          logger.warn { "[FileTransport] Subscriptions disabled - set enable_subscriptions: true" }
+          return
+        end
+
+        if @options[:file_type] == :fifo
+          start_fifo_reader(message_class, process_method, filter_options)
+        else
+          start_file_polling(message_class, process_method, filter_options)
+        end
+
+        super(message_class, process_method, filter_options)
+      end
+
+      def connected?
+        case @options[:file_type]
+        when :fifo
+          subscription_active? || fifo_active?
+        else
+          (@file_handle && !@file_handle.closed?) || subscription_active?
+        end
+      end
+
+      def disconnect
+        stop_file_subscriptions
+        stop_fifo_operations if @options[:file_type] == :fifo
+        stop_async_publishing if @options[:async]
+        close_partition_handles if @options[:filename_selector] || @options[:directory]
+        close_file_handle
+      end
+
+      private
+
+      def subscription_active?
+        @polling_thread&.alive? || fifo_active?
+      end
+
+      def stop_file_subscriptions
+        @polling_thread&.kill
+        @polling_thread&.join(5)
+      end
+    end
+  end
+end

data/lib/smart_message/transport/file_watching.rb

@@ -0,0 +1,72 @@
+# lib/smart_message/transport/file_watching.rb
+# encoding: utf-8
+# frozen_string_literal: true
+
+module SmartMessage
+  module Transport
+    # Module for file watching and subscription support (reading/tailing).
+    module FileWatching
+      
+      private
+      
+      def logger
+        # Ensure we have a proper logger, not just an IO object
+        if @logger && @logger.respond_to?(:error) && @logger.respond_to?(:info) && @logger.respond_to?(:warn)
+          return @logger
+        end
+        @logger = SmartMessage::Logger.default
+      end
+      
+      public
+      def start_file_polling(message_class, process_method, filter_options)
+        poll_interval = filter_options[:poll_interval] || @options[:poll_interval] || 1.0
+        file_path = subscription_file_path(message_class, filter_options)
+        
+        @polling_thread = Thread.new do
+          Thread.current.name = "FileTransport-Poller"
+          last_position = @options[:read_from_end] ? (File.exist?(file_path) ? File.size(file_path) : 0) : 0
+          
+          loop do
+            sleep poll_interval
+            next unless File.exist?(file_path)
+            
+            current_size = File.size(file_path)
+            if current_size > last_position
+              process_new_file_content(file_path, last_position, current_size, message_class)
+              last_position = current_size
+            end
+          end
+        rescue => e
+          logger.error { "[FileTransport] Polling thread error: #{e.message}" }
+        end
+      end
+
+      def process_new_file_content(file_path, start_pos, end_pos, message_class)
+        File.open(file_path, 'r', encoding: @options[:encoding]) do |file|
+          file.seek(start_pos)
+          content = file.read(end_pos - start_pos)
+          
+          content.each_line do |line|
+            next if line.strip.empty?
+            
+            begin
+              receive(message_class, line.strip)
+            rescue => e
+              begin
+                logger.error { "[FileTransport] Error processing message: #{e.message}" }
+              rescue
+                # Fallback if logger is not available
+              end
+            end
+          end
+        end
+      end
+
+      def subscription_file_path(message_class, filter_options)
+        filter_options[:file_path] || 
+          @options[:subscription_file_path] ||
+          File.join(File.dirname(@options[:file_path]), "#{message_class.to_s.downcase}.jsonl")
+      end
+    end
+  end
+end

data/lib/smart_message/transport/memory_transport.rb

@@ -16,30 +16,49 @@ module SmartMessage
         }
       end
 
+      # Memory transport doesn't need serialization
+      def default_serializer
+        nil
+      end
+
       def configure
         @messages = []
         @message_mutex = Mutex.new
       end
 
-      # Publish message to memory queue
+      # Implement do_publish for memory transport (no serialization needed)
       def do_publish(message_class, serialized_message)
+        # For memory transport, serialized_message is actually the message object
+        message = serialized_message
+        
         @message_mutex.synchronize do
           # Prevent memory overflow
           @messages.shift if @messages.size >= @options[:max_messages]
           
+          # Store the actual message object, no serialization needed
           @messages << {
             message_class: message_class,
-            serialized_message: serialized_message,
+            message: message.dup,  # Store a copy to prevent mutation
             published_at: Time.now
           }
         end
 
         # Auto-process if enabled
         if @options[:auto_process]
-          receive(message_class, serialized_message)
+          # Route directly without serialization
+          @dispatcher.route(message)
         end
       end
 
+      # Override encode_message to return the message object directly
+      def encode_message(message)
+        # Update header with serializer info (even though we don't serialize)
+        message._sm_header.serializer = 'none'
+        
+        # Return the message object itself (no encoding needed)
+        message
+      end
+
       # Get all stored messages
       def all_messages
         @message_mutex.synchronize { @messages.dup }
@@ -59,7 +78,7 @@ module SmartMessage
       def process_all
         messages_to_process = @message_mutex.synchronize { @messages.dup }
         messages_to_process.each do |msg|
-          receive(msg[:message_class], msg[:serialized_message])
+          @dispatcher.route(msg[:message])
         end
       end
 

data/lib/smart_message/transport/partitioned_files.rb

@@ -0,0 +1,46 @@
+# lib/smart_message/transport/partitioned_files.rb
+# encoding: utf-8
+# frozen_string_literal: true
+
+module SmartMessage
+  module Transport
+    # Module for fan-out/partitioned file support.
+    module PartitionedFiles
+      def determine_file_path(payload, header)
+        if @options[:filename_selector]&.respond_to?(:call)
+          @options[:filename_selector].call(payload, header)
+        elsif @options[:directory]
+          File.join(@options[:directory], "#{header[:message_class_name].to_s.downcase}.log")
+        else
+          @options[:file_path]
+        end
+      end
+
+      def get_or_open_partition_handle(full_path)
+        @partition_handles ||= {}
+        @partition_mutexes ||= {}
+        
+        unless @partition_handles[full_path]
+          ensure_directory_exists_for(full_path)
+          @partition_handles[full_path] = File.open(full_path, file_mode, encoding: @options[:encoding])
+          @partition_mutexes[full_path] = Mutex.new
+        end
+        
+        @partition_handles[full_path]
+      end
+
+      def ensure_directory_exists_for(path)
+        dir = File.dirname(path)
+        FileUtils.mkdir_p(dir) if @options[:create_directories] && !Dir.exist?(dir)
+      end
+
+      def close_partition_handles
+        @partition_handles&.each do |_, handle|
+          handle.close
+        end
+        @partition_handles = {}
+        @partition_mutexes = {}
+      end
+    end
+  end
+end

data/lib/smart_message/transport/redis_transport.rb

@@ -24,6 +24,17 @@ module SmartMessage
         }
       end
 
+      # Default to MessagePack for Redis (efficient binary format)
+      def default_serializer
+        # Try MessagePack first, fall back to JSON
+        begin
+          require 'smart_message/serializer/message_pack'
+          SmartMessage::Serializer::MessagePack.new
+        rescue LoadError
+          SmartMessage::Serializer::Json.new
+        end
+      end
+
       def configure
         @redis_pub = Redis.new(url: @options[:url], db: @options[:db])
         @redis_sub = Redis.new(url: @options[:url], db: @options[:db])

data/lib/smart_message/transport/registry.rb

@@ -54,7 +54,6 @@ module SmartMessage
         register(:stdout, SmartMessage::Transport::StdoutTransport)
         register(:memory, SmartMessage::Transport::MemoryTransport)
         register(:redis, SmartMessage::Transport::RedisTransport)
-        register(:redis_queue, SmartMessage::Transport::RedisQueueTransport)
       end
     end
   end