smart_message 0.0.13 → 0.0.17

data/lib/smart_message/transport/file_operations.rb

@@ -0,0 +1,232 @@
+# lib/smart_message/transport/file_operations.rb
+# encoding: utf-8
+# frozen_string_literal: true
+
+module SmartMessage
+  module Transport
+    # Module for shared file operations, including buffering, rotation, and basic I/O.
+    module FileOperations
+      def configure_file_output
+        ensure_directory_exists if @options[:create_directories]
+        @file_handle = open_file_handle
+        @write_buffer = []
+        @last_flush = Time.now
+        setup_rotation_timer if rotation_enabled?
+        @file_mutex = Mutex.new  # For thread-safety
+      end
+
+      def write_to_file(serialized_message)
+        content = prepare_file_content(serialized_message)
+        
+        @file_mutex.synchronize do
+          if buffered_mode?
+            buffer_write(content)
+          else
+            direct_write(content)
+          end
+          
+          rotate_file_if_needed
+        end
+      end
+
+      def flush_buffer
+        return if @write_buffer.empty?
+        
+        # Only synchronize if we're not already holding the lock
+        if @file_mutex.owned?
+          @file_handle.write(@write_buffer.join)
+          @file_handle.flush
+          @write_buffer.clear
+          @last_flush = Time.now
+        else
+          @file_mutex.synchronize do
+            @file_handle.write(@write_buffer.join)
+            @file_handle.flush
+            @write_buffer.clear
+            @last_flush = Time.now
+          end
+        end
+      end
+
+      def close_file_handle
+        flush_buffer if buffered_mode?
+        if @file_mutex
+          @file_mutex.synchronize do
+            @file_handle&.flush unless @file_handle&.closed?
+            @file_handle&.close unless @file_handle&.closed?
+            @file_handle = nil
+          end
+        else
+          @file_handle&.flush unless @file_handle&.closed?
+          @file_handle&.close unless @file_handle&.closed?
+          @file_handle = nil
+        end
+      end
+
+      private
+
+      def prepare_file_content(serialized_message)
+        case @options[:format] || @options[:file_format] || :jsonl
+        when :json, :raw
+          serialized_message
+        when :jsonl, :lines
+          "#{serialized_message}\n"
+        when :pretty
+          begin
+            require 'amazing_print'
+            # Use the serializer to decode back to the original data structure
+            if @serializer.respond_to?(:decode)
+              data = @serializer.decode(serialized_message)
+              data.ai + "\n"
+            else
+              # Fallback: try to parse as JSON
+              begin
+                require 'json'
+                data = JSON.parse(serialized_message)
+                data.ai + "\n"
+              rescue JSON::ParserError
+                # If not JSON, pretty print the raw string
+                serialized_message.ai + "\n"
+              end
+            end
+          rescue LoadError
+            # Fallback if amazing_print not available
+            "#{serialized_message}\n"
+          rescue => e
+            # Handle any other errors (like circuit breaker issues)
+            "#{serialized_message}\n"
+          end
+        else
+          "#{serialized_message}\n"  # default to jsonl
+        end
+      end
+
+      def open_file_handle
+        # Return IO objects directly, don't try to open them
+        return @options[:file_path] if @options[:file_path].respond_to?(:write)
+        # Open file handle for string paths
+        File.open(current_file_path, file_mode, encoding: @options[:encoding])
+      end
+
+      def ensure_directory_exists
+        return unless @options[:create_directories]
+        return if @options[:file_path].respond_to?(:write)  # Skip for IO objects
+        
+        dir = File.dirname(current_file_path)
+        FileUtils.mkdir_p(dir) unless Dir.exist?(dir)
+      end
+
+      def current_file_path
+        if rotation_enabled? && time_based_rotation?
+          timestamped_file_path
+        else
+          @options[:file_path]
+        end
+      end
+
+      def file_mode
+        @options[:file_mode] || 'a'  # append by default
+      end
+
+      def buffered_mode?
+        @options[:buffer_size] && @options[:buffer_size] > 0
+      end
+
+      def buffer_write(content)
+        @write_buffer << content
+        
+        if buffer_full? || flush_interval_exceeded?
+          flush_buffer
+        end
+      end
+
+      def direct_write(content)
+        return unless @file_handle
+        @file_handle.write(content)
+        @file_handle.flush if @options[:auto_flush]
+      end
+
+      def buffer_full?
+        @write_buffer.join.bytesize >= @options[:buffer_size]
+      end
+
+      def flush_interval_exceeded?
+        @options[:flush_interval] && 
+          (Time.now - @last_flush) >= @options[:flush_interval]
+      end
+
+      def rotation_enabled?
+        @options[:rotate_size] || @options[:rotate_time]
+      end
+
+      def time_based_rotation?
+        @options[:rotate_time]
+      end
+
+      def should_rotate?
+        size_rotation_needed? || time_rotation_needed?
+      end
+
+      def size_rotation_needed?
+        @options[:rotate_size] && 
+          File.exist?(current_file_path) && File.size(current_file_path) >= @options[:rotate_size]
+      end
+
+      def time_rotation_needed?
+        return false unless @options[:rotate_time]
+        
+        case @options[:rotate_time]
+        when :hourly
+          Time.now.min == 0 && Time.now.sec == 0
+        when :daily
+          Time.now.hour == 0 && Time.now.min == 0 && Time.now.sec == 0
+        else
+          false
+        end
+      end
+
+      def rotate_file_if_needed
+        return unless should_rotate?
+        
+        close_current_file
+        archive_current_file
+        @file_handle = open_file_handle
+      end
+
+      def close_current_file
+        flush_buffer if buffered_mode?
+        @file_handle&.close
+        @file_handle = nil
+      end
+
+      def archive_current_file
+        return unless File.exist?(current_file_path)
+        
+        timestamp = Time.now.strftime(@options[:timestamp_format] || '%Y%m%d_%H%M%S')
+        base = File.basename(@options[:file_path], '.*')
+        ext = File.extname(@options[:file_path])
+        dir = File.dirname(@options[:file_path])
+        archive_path = File.join(dir, "#{base}_#{timestamp}#{ext}")
+        
+        FileUtils.mv(current_file_path, archive_path)
+        
+        # Maintain rotation count
+        if @options[:rotate_count]
+          files = Dir.glob(File.join(dir, "#{base}_*#{ext}")).sort
+          while files.size > @options[:rotate_count]
+            File.delete(files.shift)
+          end
+        end
+      end
+
+      def timestamped_file_path
+        base = File.basename(@options[:file_path], '.*')
+        ext = File.extname(@options[:file_path])
+        dir = File.dirname(@options[:file_path])
+        timestamp = Time.now.strftime(@options[:timestamp_format] || '%Y%m%d_%H%M%S')
+        
+        File.join(dir, "#{base}_#{timestamp}#{ext}")
+      end
+    end
+  end
+end

data/lib/smart_message/transport/file_transport.rb

@@ -0,0 +1,152 @@
+# lib/smart_message/transport/file_transport.rb
+# encoding: utf-8
+# frozen_string_literal: true
+
+require_relative 'file_operations'
+require_relative 'file_watching'
+require_relative 'partitioned_files'
+require_relative 'async_publish_queue'
+require_relative 'fifo_operations'
+
+module SmartMessage
+  module Transport
+    class FileTransport < Base
+      include FileOperations
+      include FileWatching
+      include PartitionedFiles
+      include AsyncPublishQueue
+      include FifoOperations
+
+      # @param path [String, IO, Pathname] file path or IO-like object
+      # @param mode [String] file open mode ("a" for append, etc.)
+      # @param encoding [String, nil] file encoding
+      def initialize(options = {})
+        @current_message_class = nil
+        super(**options)
+      end
+
+      def default_options
+        {
+          file_path: 'messages.log',
+          file_mode: 'a',
+          encoding: nil,
+          file_format: :lines,
+          buffer_size: 0,
+          flush_interval: nil,
+          auto_flush: true,
+          rotate_size: nil,
+          rotate_time: nil,
+          rotate_count: 5,
+          timestamp_format: '%Y%m%d_%H%M%S',
+          create_directories: true,
+          async: false,
+          max_queue: nil,
+          drop_when_full: false,
+          queue_overflow_strategy: :block,
+          max_retries: 3,
+          max_retry_delay: 30,
+          worker_timeout: 5,
+          shutdown_timeout: 10,
+          queue_warning_threshold: 0.8,
+          enable_queue_monitoring: true,
+          drain_queue_on_shutdown: true,
+          send_dropped_to_dlq: false,
+          read_from_end: true,
+          poll_interval: 1.0,
+          file_type: :regular,
+          create_fifo: false,
+          fifo_mode: :blocking,
+          fifo_permissions: 0644,
+          fallback_transport: nil,
+          enable_subscriptions: false,
+          subscription_mode: :polling,
+          filename_selector: nil,
+          directory: nil,
+          subscription_file_path: nil
+        }
+      end
+
+      def configure
+        # Call parent configuration first
+        super if defined?(super)
+        
+        # Then configure our file-specific features
+        if @options[:async]
+          configure_async_publishing
+        elsif @options[:file_type] == :fifo
+          configure_fifo
+        else
+          configure_file_output
+        end
+      end
+
+      def publish(message)
+        # Extract message class and serialize the message
+        message_class = message._sm_header.message_class
+        serialized_message = encode_message(message)
+        do_publish(message_class, serialized_message)
+      end
+
+      def do_publish(message_class, serialized_message)
+        @current_message_class = message_class
+        if @options[:async]
+          async_publish(message_class, serialized_message)
+        else
+          if @options[:filename_selector] || @options[:directory]
+            header = { message_class_name: message_class.to_s }
+            path = determine_file_path(serialized_message, header)
+            @file_handle = get_or_open_partition_handle(path)
+            write_to_file(serialized_message)
+          elsif @options[:file_type] == :fifo
+            write_to_fifo(serialized_message)
+          else
+            write_to_file(serialized_message)
+          end
+        end
+      end
+
+      def subscribe(message_class, process_method, filter_options = {})
+        unless @options[:enable_subscriptions]
+          logger.warn { "[FileTransport] Subscriptions disabled - set enable_subscriptions: true" }
+          return
+        end
+
+        if @options[:file_type] == :fifo
+          start_fifo_reader(message_class, process_method, filter_options)
+        else
+          start_file_polling(message_class, process_method, filter_options)
+        end
+
+        super(message_class, process_method, filter_options)
+      end
+
+      def connected?
+        case @options[:file_type]
+        when :fifo
+          subscription_active? || fifo_active?
+        else
+          (@file_handle && !@file_handle.closed?) || subscription_active?
+        end
+      end
+
+      def disconnect
+        stop_file_subscriptions
+        stop_fifo_operations if @options[:file_type] == :fifo
+        stop_async_publishing if @options[:async]
+        close_partition_handles if @options[:filename_selector] || @options[:directory]
+        close_file_handle
+      end
+
+      private
+
+      def subscription_active?
+        @polling_thread&.alive? || fifo_active?
+      end
+
+      def stop_file_subscriptions
+        @polling_thread&.kill
+        @polling_thread&.join(5)
+      end
+    end
+  end
+end

data/lib/smart_message/transport/file_watching.rb

@@ -0,0 +1,72 @@
+# lib/smart_message/transport/file_watching.rb
+# encoding: utf-8
+# frozen_string_literal: true
+
+module SmartMessage
+  module Transport
+    # Module for file watching and subscription support (reading/tailing).
+    module FileWatching
+      
+      private
+      
+      def logger
+        # Ensure we have a proper logger, not just an IO object
+        if @logger && @logger.respond_to?(:error) && @logger.respond_to?(:info) && @logger.respond_to?(:warn)
+          return @logger
+        end
+        @logger = SmartMessage::Logger.default
+      end
+      
+      public
+      def start_file_polling(message_class, process_method, filter_options)
+        poll_interval = filter_options[:poll_interval] || @options[:poll_interval] || 1.0
+        file_path = subscription_file_path(message_class, filter_options)
+        
+        @polling_thread = Thread.new do
+          Thread.current.name = "FileTransport-Poller"
+          last_position = @options[:read_from_end] ? (File.exist?(file_path) ? File.size(file_path) : 0) : 0
+          
+          loop do
+            sleep poll_interval
+            next unless File.exist?(file_path)
+            
+            current_size = File.size(file_path)
+            if current_size > last_position
+              process_new_file_content(file_path, last_position, current_size, message_class)
+              last_position = current_size
+            end
+          end
+        rescue => e
+          logger.error { "[FileTransport] Polling thread error: #{e.message}" }
+        end
+      end
+
+      def process_new_file_content(file_path, start_pos, end_pos, message_class)
+        File.open(file_path, 'r', encoding: @options[:encoding]) do |file|
+          file.seek(start_pos)
+          content = file.read(end_pos - start_pos)
+          
+          content.each_line do |line|
+            next if line.strip.empty?
+            
+            begin
+              receive(message_class, line.strip)
+            rescue => e
+              begin
+                logger.error { "[FileTransport] Error processing message: #{e.message}" }
+              rescue
+                # Fallback if logger is not available
+              end
+            end
+          end
+        end
+      end
+
+      def subscription_file_path(message_class, filter_options)
+        filter_options[:file_path] || 
+          @options[:subscription_file_path] ||
+          File.join(File.dirname(@options[:file_path]), "#{message_class.to_s.downcase}.jsonl")
+      end
+    end
+  end
+end

data/lib/smart_message/transport/partitioned_files.rb

@@ -0,0 +1,46 @@
+# lib/smart_message/transport/partitioned_files.rb
+# encoding: utf-8
+# frozen_string_literal: true
+
+module SmartMessage
+  module Transport
+    # Module for fan-out/partitioned file support.
+    module PartitionedFiles
+      def determine_file_path(payload, header)
+        if @options[:filename_selector]&.respond_to?(:call)
+          @options[:filename_selector].call(payload, header)
+        elsif @options[:directory]
+          File.join(@options[:directory], "#{header[:message_class_name].to_s.downcase}.log")
+        else
+          @options[:file_path]
+        end
+      end
+
+      def get_or_open_partition_handle(full_path)
+        @partition_handles ||= {}
+        @partition_mutexes ||= {}
+        
+        unless @partition_handles[full_path]
+          ensure_directory_exists_for(full_path)
+          @partition_handles[full_path] = File.open(full_path, file_mode, encoding: @options[:encoding])
+          @partition_mutexes[full_path] = Mutex.new
+        end
+        
+        @partition_handles[full_path]
+      end
+
+      def ensure_directory_exists_for(path)
+        dir = File.dirname(path)
+        FileUtils.mkdir_p(dir) if @options[:create_directories] && !Dir.exist?(dir)
+      end
+
+      def close_partition_handles
+        @partition_handles&.each do |_, handle|
+          handle.close
+        end
+        @partition_handles = {}
+        @partition_mutexes = {}
+      end
+    end
+  end
+end

data/lib/smart_message/transport/stdout_transport.rb

@@ -2,89 +2,15 @@
 # encoding: utf-8
 # frozen_string_literal: true
 
-module SmartMessage
-  module Transport
-    # STDOUT transport for testing and development
-    # This transport outputs messages to STDOUT and optionally loops them back
-    class StdoutTransport < Base
-      def default_options
-        {
-          loopback: false,
-          output: $stdout,
-          format: :pretty  # :pretty or :json
-        }
-      end
-
-      # Default to JSON for readability in STDOUT
-      def default_serializer
-        SmartMessage::Serializer::Json.new
-      end
-
-      def configure
-        @output = @options[:output].is_a?(String) ? File.open(@options[:output], 'w') : @options[:output]
-      end
-
-      # Enable/disable loopback mode
-      def loopback=(enabled)
-        @options[:loopback] = enabled
-      end
-
-      def loopback?
-        @options[:loopback]
-      end
-
-      # Publish message to STDOUT
-      def do_publish(message_class, serialized_message)
-        logger.debug { "[SmartMessage::StdoutTransport] do_publish called" }
-        logger.debug { "[SmartMessage::StdoutTransport] message_class: #{message_class}" }
-        
-        @output.puts format_message(message_class, serialized_message)
-        @output.flush
+require_relative 'file_transport'
 
-        # If loopback is enabled, route the message back through the dispatcher
-        if loopback?
-          logger.debug { "[SmartMessage::StdoutTransport] Loopback enabled, calling receive" }
-          receive(message_class, serialized_message)
-        end
-      rescue => e
-        logger.error { "[SmartMessage] Error in stdout transport do_publish: #{e.class.name} - #{e.message}" }
-        raise
-      end
-
-      def connected?
-        !@output.closed?
-      end
-
-      def disconnect
-        @output.close if @output.respond_to?(:close) && @output != $stdout && @output != $stderr
-      end
-
-      private
-
-      def format_message(message_class, serialized_message)
-        if @options[:format] == :json
-          # Output as JSON for machine parsing
-          {
-            transport: 'stdout',
-            message_class: message_class,
-            serialized_message: serialized_message,
-            timestamp: Time.now.iso8601
-          }.to_json
-        else
-          # Pretty format for human reading
-          <<~MESSAGE
-
-            ===================================================
-            == SmartMessage Published via STDOUT Transport
-            == Message Class: #{message_class}
-            == Serializer: #{@serializer.class.name}
-            == Serialized Message:
-            #{serialized_message}
-            ===================================================
-
-          MESSAGE
+module SmartMessage
+    module Transport
+      class StdoutTransport < FileTransport
+        def initialize(options = {})
+          defaults = { file_path: $stdout, file_mode: 'w', file_type: :regular }
+          super(defaults.merge(options))
         end
       end
     end
   end
-end

data/lib/smart_message/transport/stdout_transport.rb.backup

@@ -0,0 +1,88 @@
+# lib/smart_message/transport/stdout_transport.rb
+# encoding: utf-8
+# frozen_string_literal: true
+
+module SmartMessage
+  module Transport
+    # STDOUT transport for testing and development
+    # This is a publish-only transport that outputs messages to STDOUT
+    class StdoutTransport < Base
+      def default_options
+        {
+          output: $stdout,
+          format: :pretty  # :pretty or :json
+        }
+      end
+
+      # Default to JSON for readability in STDOUT
+      def default_serializer
+        SmartMessage::Serializer::Json.new
+      end
+
+      def configure
+        @output = @options[:output].is_a?(String) ? File.open(@options[:output], 'w') : @options[:output]
+      end
+
+
+      # Publish message to STDOUT
+      def do_publish(message_class, serialized_message)
+        logger.debug { "[SmartMessage::StdoutTransport] do_publish called" }
+        logger.debug { "[SmartMessage::StdoutTransport] message_class: #{message_class}" }
+        
+        @output.puts format_message(message_class, serialized_message)
+        @output.flush
+      rescue => e
+        logger.error { "[SmartMessage] Error in stdout transport do_publish: #{e.class.name} - #{e.message}" }
+        raise
+      end
+
+      def connected?
+        !@output.closed?
+      end
+
+      def disconnect
+        @output.close if @output.respond_to?(:close) && @output != $stdout && @output != $stderr
+      end
+
+      # Override subscribe methods to log warnings since this is a publish-only transport
+      def subscribe(message_class, process_method, filter_options = {})
+        logger.warn { "[SmartMessage::StdoutTransport] Subscription attempt ignored - STDOUT transport is publish-only (message_class: #{message_class}, process_method: #{process_method})" }
+      end
+
+      def unsubscribe(message_class, process_method)
+        logger.warn { "[SmartMessage::StdoutTransport] Unsubscribe attempt ignored - STDOUT transport is publish-only (message_class: #{message_class}, process_method: #{process_method})" }
+      end
+
+      def unsubscribe!(message_class)
+        logger.warn { "[SmartMessage::StdoutTransport] Unsubscribe all attempt ignored - STDOUT transport is publish-only (message_class: #{message_class})" }
+      end
+
+      private
+
+      def format_message(message_class, serialized_message)
+        if @options[:format] == :json
+          # Output as JSON for machine parsing
+          {
+            transport: 'stdout',
+            message_class: message_class,
+            serialized_message: serialized_message,
+            timestamp: Time.now.iso8601
+          }.to_json
+        else
+          # Pretty format for human reading
+          <<~MESSAGE
+
+            ===================================================
+            == SmartMessage Published via STDOUT Transport
+            == Message Class: #{message_class}
+            == Serializer: #{@serializer.class.name}
+            == Serialized Message:
+            #{serialized_message}
+            ===================================================
+
+          MESSAGE
+        end
+      end
+    end
+  end
+end

data/lib/smart_message/version.rb

@@ -3,5 +3,5 @@
 # frozen_string_literal: true
 
 module SmartMessage
-  VERSION = '0.0.13'
+  VERSION = '0.0.17'
 end