smart_message 0.0.13 → 0.0.16

data/lib/smart_message/transport/fifo_operations.rb

@@ -0,0 +1,264 @@
+# lib/smart_message/transport/fifo_operations.rb
+# encoding: utf-8
+# frozen_string_literal: true
+
+require 'rbconfig'
+
+module SmartMessage
+  module Transport
+    # Module for FIFO operations, with cross-platform considerations.
+    module FifoOperations
+      
+      private
+      
+      def logger
+        # Ensure we have a proper logger, not just an IO object
+        if @logger && @logger.respond_to?(:error) && @logger.respond_to?(:info) && @logger.respond_to?(:warn)
+          return @logger
+        end
+        @logger = SmartMessage::Logger.default
+      end
+      
+      public
+      def configure_fifo
+        unless platform_supports_fifo?
+          begin
+            logger.warn { "[FileTransport] FIFO not supported, falling back to regular file" }
+          rescue
+            # Fallback if logger is not available
+          end
+          @options[:file_type] = :regular
+          return configure_file_output
+        end
+        
+        create_named_pipe if @options[:create_fifo]
+      end
+
+      def create_named_pipe
+        case RbConfig::CONFIG['host_os']
+        when /mswin|mingw|cygwin/
+          create_windows_named_pipe
+        else
+          File.mkfifo(@options[:file_path], @options[:fifo_permissions] || 0644)
+          begin
+            logger.info { "[FileTransport] Created FIFO: #{@options[:file_path]}" }
+          rescue
+            # Fallback if logger is not available
+          end
+        end
+      rescue NotImplementedError
+        begin
+          logger.error { "[FileTransport] Named pipes not supported on this platform" }
+        rescue
+          # Fallback if logger is not available
+        end
+        raise
+      rescue => e
+        begin
+          logger.error { "[FileTransport] Failed to create FIFO: #{e.message}" }
+        rescue
+          # Fallback if logger is not available
+        end
+        raise
+      end
+
+      def create_windows_named_pipe
+        require 'win32/pipe'
+        pipe_name = "\\\\.\\pipe\\#{File.basename(@options[:file_path])}"
+        @windows_pipe_server = Win32::Pipe::Server.new(pipe_name)
+        begin
+          logger.info { "[FileTransport] Created Windows named pipe: #{pipe_name}" }
+        rescue
+          # Fallback if logger is not available
+        end
+      rescue LoadError
+        raise "Windows named pipes require win32-pipe gem: gem install win32-pipe"
+      rescue => e
+        begin
+          logger.error { "[FileTransport] Failed to create Windows named pipe: #{e.message}" }
+        rescue
+          # Fallback if logger is not available
+        end
+        raise
+      end
+
+      def platform_supports_fifo?
+        case RbConfig::CONFIG['host_os']
+        when /mswin|mingw|cygwin/
+          defined?(Win32::Pipe)
+        else
+          true
+        end
+      end
+
+      def write_to_fifo(serialized_message)
+        handle = open_fifo_for_writing
+        unless handle
+          handle_fifo_write_failure(serialized_message)
+          return false
+        end
+        
+        content = prepare_file_content(serialized_message)
+        handle.write(content)
+        handle.flush
+        true
+      rescue Errno::EPIPE
+        begin
+          logger.warn { "[FileTransport] FIFO reader disconnected" }
+        rescue
+          # Fallback if logger is not available
+        end
+        handle_fifo_write_failure(serialized_message)
+        false
+      rescue => e
+        begin
+          logger.error { "[FileTransport] FIFO write error: #{e.message}" }
+        rescue
+          # Fallback if logger is not available
+        end
+        handle_fifo_write_failure(serialized_message)
+        false
+      ensure
+        handle&.close
+      end
+
+      def open_fifo_for_writing
+        mode = @options[:fifo_mode] == :non_blocking ? File::WRONLY | File::NONBLOCK : 'w'
+        File.open(@options[:file_path], mode)
+      rescue Errno::ENXIO, Errno::ENOENT
+        nil
+      end
+
+      def handle_fifo_write_failure(serialized_message)
+        if @options[:fallback_transport]
+          begin
+            @options[:fallback_transport].do_publish(@current_message_class, serialized_message)
+            begin
+              logger.info { "[FileTransport] Message sent to fallback transport" }
+            rescue
+              # Fallback if logger is not available
+            end
+          rescue => e
+            begin
+              logger.error { "[FileTransport] Fallback transport failed: #{e.message}" }
+            rescue
+              # Fallback if logger is not available
+            end
+          end
+        end
+      end
+
+      def start_fifo_reader(message_class, process_method, filter_options)
+        case @options[:subscription_mode]
+        when :fifo_blocking
+          start_blocking_fifo_reader(message_class, process_method)
+        when :fifo_select
+          start_select_fifo_reader(message_class, process_method)
+        when :fifo_polling
+          start_polling_fifo_reader(message_class, process_method)
+        else
+          begin
+            logger.warn { "[FileTransport] Invalid FIFO subscription mode: #{@options[:subscription_mode]}" }
+          rescue
+            # Fallback if logger is not available
+          end
+        end
+      end
+
+      def start_blocking_fifo_reader(message_class, process_method)
+        @fifo_reader_thread = Thread.new do
+          Thread.current.name = "FileTransport-FifoReader"
+          loop do
+            begin
+              File.open(@options[:file_path], 'r') do |fifo|
+                while line = fifo.gets
+                  next if line.strip.empty?
+                  receive(message_class, line.strip)
+                end
+              end
+            rescue => e
+              begin
+                logger.error { "[FileTransport] FIFO reader error: #{e.message}" }
+              rescue
+                # Fallback if logger is not available
+              end
+              sleep 1
+            end
+          end
+        end
+      end
+
+      def start_select_fifo_reader(message_class, process_method)
+        @fifo_select_thread = Thread.new do
+          Thread.current.name = "FileTransport-FifoSelect"
+          fifo = File.open(@options[:file_path], File::RDONLY | File::NONBLOCK)
+          
+          loop do
+            ready = IO.select([fifo], nil, nil, 1.0)
+            if ready
+              begin
+                while line = fifo.gets
+                  next if line.strip.empty?
+                  receive(message_class, line.strip)
+                end
+              rescue IO::WaitReadable
+                next
+              rescue => e
+                begin
+                  logger.error { "[FileTransport] FIFO select error: #{e.message}" }
+                rescue
+                  # Fallback if logger is not available
+                end
+              end
+            end
+          end
+        rescue => e
+          begin
+            logger.error { "[FileTransport] FIFO select thread error: #{e.message}" }
+          rescue
+            # Fallback if logger is not available
+          end
+        ensure
+          fifo&.close
+        end
+      end
+
+      def start_polling_fifo_reader(message_class, process_method)
+        @fifo_reader_thread = Thread.new do
+          Thread.current.name = "FileTransport-FifoPoller"
+          loop do
+            begin
+              File.open(@options[:file_path], File::RDONLY | File::NONBLOCK) do |fifo|
+                while line = fifo.gets
+                  next if line.strip.empty?
+                  receive(message_class, line.strip)
+                end
+              end
+            rescue Errno::EAGAIN
+              sleep(@options[:poll_interval] || 1.0)
+            rescue => e
+              begin
+                logger.error { "[FileTransport] FIFO polling error: #{e.message}" }
+              rescue
+                # Fallback if logger is not available
+              end
+              sleep 1
+            end
+          end
+        end
+      end
+
+      def stop_fifo_operations
+        @fifo_reader_thread&.kill
+        @fifo_reader_thread&.join(2)
+        @fifo_select_thread&.kill
+        @fifo_select_thread&.join(2)
+        @fifo_handle&.close
+      end
+
+      def fifo_active?
+        @fifo_reader_thread&.alive? || @fifo_select_thread&.alive?
+      end
+    end
+  end
+end

data/lib/smart_message/transport/file_operations.rb

@@ -0,0 +1,200 @@
+# lib/smart_message/transport/file_operations.rb
+# encoding: utf-8
+# frozen_string_literal: true
+
+module SmartMessage
+  module Transport
+    # Module for shared file operations, including buffering, rotation, and basic I/O.
+    module FileOperations
+      def configure_file_output
+        ensure_directory_exists if @options[:create_directories]
+        @file_handle = open_file_handle
+        @write_buffer = []
+        @last_flush = Time.now
+        setup_rotation_timer if rotation_enabled?
+        @file_mutex = Mutex.new  # For thread-safety
+      end
+
+      def write_to_file(serialized_message)
+        content = prepare_file_content(serialized_message)
+        
+        @file_mutex.synchronize do
+          if buffered_mode?
+            buffer_write(content)
+          else
+            direct_write(content)
+          end
+          
+          rotate_file_if_needed
+        end
+      end
+
+      def flush_buffer
+        return if @write_buffer.empty?
+        
+        # Only synchronize if we're not already holding the lock
+        if @file_mutex.owned?
+          @file_handle.write(@write_buffer.join)
+          @file_handle.flush
+          @write_buffer.clear
+          @last_flush = Time.now
+        else
+          @file_mutex.synchronize do
+            @file_handle.write(@write_buffer.join)
+            @file_handle.flush
+            @write_buffer.clear
+            @last_flush = Time.now
+          end
+        end
+      end
+
+      def close_file_handle
+        flush_buffer if buffered_mode?
+        if @file_mutex
+          @file_mutex.synchronize do
+            @file_handle&.close
+            @file_handle = nil
+          end
+        else
+          @file_handle&.close
+          @file_handle = nil
+        end
+      end
+
+      private
+
+      def prepare_file_content(serialized_message)
+        case @options[:file_format]
+        when :lines
+          "#{serialized_message}\n"
+        when :raw
+          serialized_message
+        else
+          "#{serialized_message}\n"  # default to lines
+        end
+      end
+
+      def open_file_handle
+        File.open(current_file_path, file_mode, encoding: @options[:encoding])
+      end
+
+      def ensure_directory_exists
+        return unless @options[:create_directories]
+        
+        dir = File.dirname(current_file_path)
+        FileUtils.mkdir_p(dir) unless Dir.exist?(dir)
+      end
+
+      def current_file_path
+        if rotation_enabled? && time_based_rotation?
+          timestamped_file_path
+        else
+          @options[:file_path]
+        end
+      end
+
+      def file_mode
+        @options[:file_mode] || 'a'  # append by default
+      end
+
+      def buffered_mode?
+        @options[:buffer_size] && @options[:buffer_size] > 0
+      end
+
+      def buffer_write(content)
+        @write_buffer << content
+        
+        if buffer_full? || flush_interval_exceeded?
+          flush_buffer
+        end
+      end
+
+      def direct_write(content)
+        @file_handle.write(content)
+        @file_handle.flush if @options[:auto_flush]
+      end
+
+      def buffer_full?
+        @write_buffer.join.bytesize >= @options[:buffer_size]
+      end
+
+      def flush_interval_exceeded?
+        @options[:flush_interval] && 
+          (Time.now - @last_flush) >= @options[:flush_interval]
+      end
+
+      def rotation_enabled?
+        @options[:rotate_size] || @options[:rotate_time]
+      end
+
+      def time_based_rotation?
+        @options[:rotate_time]
+      end
+
+      def should_rotate?
+        size_rotation_needed? || time_rotation_needed?
+      end
+
+      def size_rotation_needed?
+        @options[:rotate_size] && 
+          File.exist?(current_file_path) && File.size(current_file_path) >= @options[:rotate_size]
+      end
+
+      def time_rotation_needed?
+        return false unless @options[:rotate_time]
+        
+        case @options[:rotate_time]
+        when :hourly
+          Time.now.min == 0 && Time.now.sec == 0
+        when :daily
+          Time.now.hour == 0 && Time.now.min == 0 && Time.now.sec == 0
+        else
+          false
+        end
+      end
+
+      def rotate_file_if_needed
+        return unless should_rotate?
+        
+        close_current_file
+        archive_current_file
+        @file_handle = open_file_handle
+      end
+
+      def close_current_file
+        flush_buffer if buffered_mode?
+        @file_handle&.close
+        @file_handle = nil
+      end
+
+      def archive_current_file
+        return unless File.exist?(current_file_path)
+        
+        timestamp = Time.now.strftime(@options[:timestamp_format] || '%Y%m%d_%H%M%S')
+        base = File.basename(@options[:file_path], '.*')
+        ext = File.extname(@options[:file_path])
+        dir = File.dirname(@options[:file_path])
+        archive_path = File.join(dir, "#{base}_#{timestamp}#{ext}")
+        
+        FileUtils.mv(current_file_path, archive_path)
+        
+        # Maintain rotation count
+        if @options[:rotate_count]
+          files = Dir.glob(File.join(dir, "#{base}_*#{ext}")).sort
+          while files.size > @options[:rotate_count]
+            File.delete(files.shift)
+          end
+        end
+      end
+
+      def timestamped_file_path
+        base = File.basename(@options[:file_path], '.*')
+        ext = File.extname(@options[:file_path])
+        dir = File.dirname(@options[:file_path])
+        timestamp = Time.now.strftime(@options[:timestamp_format] || '%Y%m%d_%H%M%S')
+        
+        File.join(dir, "#{base}_#{timestamp}#{ext}")
+      end
+    end
+  end
+end

data/lib/smart_message/transport/file_transport.rb

@@ -0,0 +1,149 @@
+# lib/smart_message/transport/file_transport.rb
+# encoding: utf-8
+# frozen_string_literal: true
+
+require_relative 'file_operations'
+require_relative 'file_watching'
+require_relative 'partitioned_files'
+require_relative 'async_publish_queue'
+require_relative 'fifo_operations'
+
+module SmartMessage
+  module Transport
+    class FileTransport < Base
+      include FileOperations
+      include FileWatching
+      include PartitionedFiles
+      include AsyncPublishQueue
+      include FifoOperations
+
+      # @param path [String, IO, Pathname] file path or IO-like object
+      # @param mode [String] file open mode ("a" for append, etc.)
+      # @param encoding [String, nil] file encoding
+      def initialize(options = {})
+        @current_message_class = nil
+        super(**options)
+      end
+
+      def default_options
+        {
+          file_path: 'messages.log',
+          file_mode: 'a',
+          encoding: nil,
+          file_format: :lines,
+          buffer_size: 0,
+          flush_interval: nil,
+          auto_flush: true,
+          rotate_size: nil,
+          rotate_time: nil,
+          rotate_count: 5,
+          timestamp_format: '%Y%m%d_%H%M%S',
+          create_directories: true,
+          async: false,
+          max_queue: nil,
+          drop_when_full: false,
+          queue_overflow_strategy: :block,
+          max_retries: 3,
+          max_retry_delay: 30,
+          worker_timeout: 5,
+          shutdown_timeout: 10,
+          queue_warning_threshold: 0.8,
+          enable_queue_monitoring: true,
+          drain_queue_on_shutdown: true,
+          send_dropped_to_dlq: false,
+          read_from_end: true,
+          poll_interval: 1.0,
+          file_type: :regular,
+          create_fifo: false,
+          fifo_mode: :blocking,
+          fifo_permissions: 0644,
+          fallback_transport: nil,
+          enable_subscriptions: false,
+          subscription_mode: :polling,
+          filename_selector: nil,
+          directory: nil,
+          subscription_file_path: nil
+        }
+      end
+
+      def configure
+        # Call parent configuration first
+        super if defined?(super)
+        
+        # Then configure our file-specific features
+        if @options[:async]
+          configure_async_publishing
+        elsif @options[:file_type] == :fifo
+          configure_fifo
+        else
+          configure_file_output
+        end
+      end
+
+      def publish(payload)
+        do_publish(nil, payload)
+      end
+
+      def do_publish(message_class, serialized_message)
+        @current_message_class = message_class
+        if @options[:async]
+          async_publish(message_class, serialized_message)
+        else
+          if @options[:filename_selector] || @options[:directory]
+            header = { message_class_name: message_class.to_s }
+            path = determine_file_path(serialized_message, header)
+            @file_handle = get_or_open_partition_handle(path)
+            write_to_file(serialized_message)
+          elsif @options[:file_type] == :fifo
+            write_to_fifo(serialized_message)
+          else
+            write_to_file(serialized_message)
+          end
+        end
+      end
+
+      def subscribe(message_class, process_method, filter_options = {})
+        unless @options[:enable_subscriptions]
+          logger.warn { "[FileTransport] Subscriptions disabled - set enable_subscriptions: true" }
+          return
+        end
+
+        if @options[:file_type] == :fifo
+          start_fifo_reader(message_class, process_method, filter_options)
+        else
+          start_file_polling(message_class, process_method, filter_options)
+        end
+
+        super(message_class, process_method, filter_options)
+      end
+
+      def connected?
+        case @options[:file_type]
+        when :fifo
+          subscription_active? || fifo_active?
+        else
+          (@file_handle && !@file_handle.closed?) || subscription_active?
+        end
+      end
+
+      def disconnect
+        stop_file_subscriptions
+        stop_fifo_operations if @options[:file_type] == :fifo
+        stop_async_publishing if @options[:async]
+        close_partition_handles if @options[:filename_selector] || @options[:directory]
+        close_file_handle
+      end
+
+      private
+
+      def subscription_active?
+        @polling_thread&.alive? || fifo_active?
+      end
+
+      def stop_file_subscriptions
+        @polling_thread&.kill
+        @polling_thread&.join(5)
+      end
+    end
+  end
+end

data/lib/smart_message/transport/file_watching.rb

@@ -0,0 +1,72 @@
+# lib/smart_message/transport/file_watching.rb
+# encoding: utf-8
+# frozen_string_literal: true
+
+module SmartMessage
+  module Transport
+    # Module for file watching and subscription support (reading/tailing).
+    module FileWatching
+      
+      private
+      
+      def logger
+        # Ensure we have a proper logger, not just an IO object
+        if @logger && @logger.respond_to?(:error) && @logger.respond_to?(:info) && @logger.respond_to?(:warn)
+          return @logger
+        end
+        @logger = SmartMessage::Logger.default
+      end
+      
+      public
+      def start_file_polling(message_class, process_method, filter_options)
+        poll_interval = filter_options[:poll_interval] || @options[:poll_interval] || 1.0
+        file_path = subscription_file_path(message_class, filter_options)
+        
+        @polling_thread = Thread.new do
+          Thread.current.name = "FileTransport-Poller"
+          last_position = @options[:read_from_end] ? (File.exist?(file_path) ? File.size(file_path) : 0) : 0
+          
+          loop do
+            sleep poll_interval
+            next unless File.exist?(file_path)
+            
+            current_size = File.size(file_path)
+            if current_size > last_position
+              process_new_file_content(file_path, last_position, current_size, message_class)
+              last_position = current_size
+            end
+          end
+        rescue => e
+          logger.error { "[FileTransport] Polling thread error: #{e.message}" }
+        end
+      end
+
+      def process_new_file_content(file_path, start_pos, end_pos, message_class)
+        File.open(file_path, 'r', encoding: @options[:encoding]) do |file|
+          file.seek(start_pos)
+          content = file.read(end_pos - start_pos)
+          
+          content.each_line do |line|
+            next if line.strip.empty?
+            
+            begin
+              receive(message_class, line.strip)
+            rescue => e
+              begin
+                logger.error { "[FileTransport] Error processing message: #{e.message}" }
+              rescue
+                # Fallback if logger is not available
+              end
+            end
+          end
+        end
+      end
+
+      def subscription_file_path(message_class, filter_options)
+        filter_options[:file_path] || 
+          @options[:subscription_file_path] ||
+          File.join(File.dirname(@options[:file_path]), "#{message_class.to_s.downcase}.jsonl")
+      end
+    end
+  end
+end