smart_message 0.0.12 → 0.0.13

data/lib/smart_message/transport/redis_queue_transport.rb

@@ -1,555 +0,0 @@
-# lib/smart_message/transport/redis_queue_transport_async.rb
-# encoding: utf-8
-# frozen_string_literal: true
-
-require 'async'
-require 'async/redis'
-require 'json'
-
-module SmartMessage
-  module Transport
-    # Redis Queue Transport - Async-powered routing with RabbitMQ-style patterns
-    # This transport provides intelligent routing using Redis Lists as queues with pattern matching
-    # Built on Ruby's Async framework for modern concurrency and testing
-    # 
-    # Key Features:
-    # - Async/Fiber-based concurrency (thousands of subscriptions)
-    # - RabbitMQ-style topic exchange pattern matching (#.*.my_uuid)
-    # - Load balancing via consumer groups  
-    # - Queue persistence using Redis Lists
-    # - FIFO message ordering
-    # - 10x faster than RabbitMQ with same routing intelligence
-    # - Test-friendly with proper async lifecycle management
-    #
-    # Usage:
-    #   Async do
-    #     transport = SmartMessage::Transport::RedisQueueTransport.new
-    #     transport.subscribe_pattern("#.*.my_service") do |msg_class, data|
-    #       puts "Processing: #{msg_class}"
-    #     end
-    #   end
-    class RedisQueueTransport < Base
-      
-      DEFAULT_CONFIG = {
-        url: 'redis://localhost:6379',
-        db: 0,
-        exchange_name: 'smart_message',
-        queue_prefix: 'smart_message.queue',
-        routing_prefix: 'smart_message.routing',
-        consumer_timeout: 1,        # 1 second blocking pop timeout
-        max_queue_size: 10000,      # Max messages per queue (circular buffer)
-        cleanup_on_disconnect: true, # Remove queues on shutdown
-        reconnect_attempts: 5,
-        reconnect_delay: 1,
-        async_timeout: 30           # Global async task timeout
-      }.freeze
-
-      attr_reader :redis, :exchange_name, :active_queues, :consumer_tasks
-
-      def initialize(**options)
-        @active_queues = {}      # queue_name => consumer_info
-        @consumer_tasks = {}     # queue_name => async task
-        @routing_table = {}      # pattern => [queue_names]
-        @shutdown = false
-        # Don't initialize @redis to nil here since super() calls configure which sets it
-        super(**options)
-      end
-
-      def configure
-        @exchange_name = @options[:exchange_name]
-        
-        # Setup Redis connection synchronously for immediate availability
-        begin
-          redis_client = nil
-          Async do
-            endpoint = Async::Redis::Endpoint.parse(@options[:url], db: @options[:db])
-            redis_client = Async::Redis::Client.new(endpoint)
-            
-            # Test connection
-            redis_client.call('PING')
-            
-            # Initialize exchange metadata with local redis client
-            @redis = redis_client  # Set instance variable
-            setup_exchange
-            
-            logger.info { "[RedisQueue] Async transport configured with exchange: #{@exchange_name}" }
-          end.wait
-          
-          # Ensure @redis is set outside the async block too
-          @redis = redis_client if redis_client
-        rescue => e
-          logger.error { "[RedisQueue] Failed to configure transport: #{e.message}" }
-          @redis = nil
-          raise
-        end
-      end
-
-      def default_options
-        DEFAULT_CONFIG
-      end
-
-      # Publish message with intelligent routing to matching queues (Async)
-      # @param message_class [String] The message class name
-      # @param serialized_message [String] The serialized message content
-      def do_publish(message_class, serialized_message)
-        async_task do
-          routing_info = extract_routing_info(serialized_message)
-          routing_key = build_enhanced_routing_key(message_class, routing_info)
-          
-          # Find all queues that match this routing key (like RabbitMQ topic exchange)
-          matching_queues = find_matching_queues(routing_key)
-          
-          if matching_queues.empty?
-            logger.debug { "[RedisQueue] No queues match routing key: #{routing_key}" }
-            next
-          end
-
-          # Create message envelope with metadata
-          message_envelope = {
-            routing_key: routing_key,
-            message_class: message_class.to_s,
-            data: serialized_message,
-            timestamp: Time.now.to_f,
-            headers: routing_info
-          }.to_json
-
-          # Publish to all matching queues atomically using async redis
-          published_count = 0
-          
-          # Use pipelined operations for better performance
-          commands = []
-          matching_queues.each do |queue_name|
-            commands << [:lpush, queue_name, message_envelope]
-            commands << [:ltrim, queue_name, 0, @options[:max_queue_size] - 1]
-            published_count += 1
-          end
-          
-          # Execute all commands in pipeline
-          @redis.pipelined(commands) if commands.any?
-
-          logger.debug { "[RedisQueue] Published to #{published_count} queues with key: #{routing_key}" }
-        rescue => e
-          logger.error { "[RedisQueue] Publish error: #{e.message}" }
-          raise
-        end
-      end
-
-      # Subscribe to messages using RabbitMQ-style pattern matching (Async)
-      # @param pattern [String] Routing pattern (e.g., "#.*.my_service", "order.#.*.*")
-      # @param process_method [String] Method identifier for processing
-      # @param filter_options [Hash] Additional filtering options
-      # @param block [Proc] Optional block for message processing
-      def subscribe_pattern(pattern, process_method = :process, filter_options = {}, &block)
-        queue_name = derive_queue_name(pattern, filter_options)
-        
-        # Add pattern to routing table (no mutex needed with Fibers)
-        @routing_table[pattern] ||= []
-        @routing_table[pattern] << queue_name unless @routing_table[pattern].include?(queue_name)
-        
-        # Store queue metadata
-        @active_queues[queue_name] = {
-          pattern: pattern,
-          process_method: process_method,
-          filter_options: filter_options,
-          created_at: Time.now,
-          block_handler: block
-        }
-        
-        # Start async consumer task for this queue (unless in test mode)
-        start_queue_consumer(queue_name) unless @options[:test_mode]
-        
-        logger.info { "[RedisQueue] Subscribed to pattern '#{pattern}' via queue '#{queue_name}'" }
-      end
-
-      # Subscribe to all messages sent to a specific recipient
-      # @param recipient_id [String] The recipient identifier
-      def subscribe_to_recipient(recipient_id, process_method = :process, &block)
-        pattern = "#.*.#{sanitize_for_routing_key(recipient_id)}"
-        subscribe_pattern(pattern, process_method, {}, &block)
-      end
-
-      # Subscribe to all messages from a specific sender
-      # @param sender_id [String] The sender identifier  
-      def subscribe_from_sender(sender_id, process_method = :process, &block)
-        pattern = "#.#{sanitize_for_routing_key(sender_id)}.*"
-        subscribe_pattern(pattern, process_method, {}, &block)
-      end
-
-      # Subscribe to all messages of a specific type regardless of routing
-      # @param message_type [String] The message class name
-      def subscribe_to_type(message_type, process_method = :process, &block)
-        pattern = "*.#{message_type.to_s.gsub('::', '.').downcase}.*.*"
-        subscribe_pattern(pattern, process_method, {}, &block)
-      end
-
-      # Subscribe to all alert/emergency messages
-      def subscribe_to_alerts(process_method = :process, &block)
-        patterns = [
-          "emergency.#.*.*",
-          "#.alert.*.*", 
-          "#.alarm.*.*",
-          "#.critical.*.*"
-        ]
-        
-        patterns.each { |pattern| subscribe_pattern(pattern, process_method, {}, &block) }
-      end
-
-      # Subscribe to all broadcast messages
-      def subscribe_to_broadcasts(process_method = :process, &block)
-        pattern = "#.*.broadcast"
-        subscribe_pattern(pattern, process_method, {}, &block)
-      end
-
-      # Test-friendly disconnect method
-      def disconnect
-        @shutdown = true
-        
-        # Stop all consumer tasks
-        @consumer_tasks.each do |queue_name, task|
-          task.stop if task&.running?
-        rescue => e
-          logger.debug { "[RedisQueue] Error stopping consumer task for #{queue_name}: #{e.message}" }
-        end
-        
-        @consumer_tasks.clear
-        
-        # Close Redis connection
-        async_task do
-          @redis&.close
-          @redis = nil
-        rescue => e
-          logger.debug { "[RedisQueue] Error closing Redis connection: #{e.message}" }
-        end
-      end
-
-      def connected?
-        return false unless @redis
-        
-        # Test connection with async ping
-        begin
-          if Async::Task.current?
-            # Already in async context
-            return @redis.call('PING') == 'PONG'
-          else
-            # Need to create async context
-            result = Async do
-              @redis.call('PING') == 'PONG'
-            end.wait
-            return result
-          end
-        rescue => e
-          logger.debug { "[RedisQueue] Connection test failed: #{e.message}" }
-          return false
-        end
-      end
-
-      # Get statistics about all active queues
-      def queue_stats
-        return {} unless @redis
-        
-        stats = {}
-        
-        begin
-          Async do
-            @active_queues.each do |queue_name, queue_info|
-              length = @redis.llen(queue_name).to_i
-              stats[queue_name] = {
-                length: length,
-                pattern: queue_info[:pattern],
-                created_at: queue_info[:created_at],
-                consumers: @consumer_tasks.key?(queue_name) ? 1 : 0
-              }
-            end
-          end.wait
-        rescue => e
-          logger.error { "[RedisQueue] Error getting queue stats: #{e.message}" }
-        end
-        
-        stats
-      end
-
-      # Get the current routing table
-      def routing_table
-        @routing_table.dup
-      end
-
-      # Fluent API builder for complex subscriptions
-      def where
-        RedisQueueSubscriptionBuilder.new(self)
-      end
-
-      private
-
-      # Ensure async operations run in proper context
-      def async_task(&block)
-        if Async::Task.current?
-          # Already in async context
-          yield
-        else
-          # Need to create async context
-          Async { yield }
-        end
-      end
-
-      # Start async consumer for a queue
-      def start_queue_consumer(queue_name)
-        return if @consumer_tasks.key?(queue_name)
-
-        @consumer_tasks[queue_name] = async_task do |task|
-          begin
-            consume_from_queue(queue_name)
-          rescue => e
-            logger.error { "[RedisQueue] Consumer task error for #{queue_name}: #{e.message}" }
-          ensure
-            @consumer_tasks.delete(queue_name)
-          end
-        end
-      end
-
-      # Async consumer loop for a specific queue
-      def consume_from_queue(queue_name)
-        queue_info = @active_queues[queue_name]
-        return unless queue_info
-        
-        while !@shutdown
-          begin
-            # Use BRPOP for blocking read with timeout (cooperative blocking)
-            result = @redis.brpop(queue_name, timeout: @options[:consumer_timeout])
-            
-            if result && result.length >= 2
-              _, message_envelope = result
-              process_queue_message(message_envelope, queue_info)
-            end
-            
-          rescue => e
-            logger.error { "[RedisQueue] Redis connection error in consumer: #{e.message}" }
-            # Async will handle reconnection automatically
-            sleep(1) unless @shutdown
-          rescue => e
-            logger.error { "[RedisQueue] Consumer error for #{queue_name}: #{e.message}" }
-            sleep(1) unless @shutdown
-          end
-        end
-      end
-
-      # Process a message from the queue
-      def process_queue_message(message_envelope, queue_info)
-        begin
-          message_data = JSON.parse(message_envelope)
-          
-          message_class = message_data['message_class']
-          payload = message_data['data']
-          headers = message_data['headers'] || {}
-          
-          # Apply additional filtering if specified
-          if should_process_message?(headers, queue_info[:filter_options])
-            # Use block handler if provided, otherwise route through dispatcher
-            if queue_info[:block_handler]
-              queue_info[:block_handler].call(message_class, payload)
-            else
-              # Route through dispatcher (inherited from Base)
-              receive(message_class, payload)
-            end
-          else
-            logger.debug { "[RedisQueue] Message filtered out by queue rules" }
-          end
-          
-        rescue JSON::ParserError => e
-          logger.error { "[RedisQueue] Invalid message envelope: #{e.message}" }
-        rescue => e
-          logger.error { "[RedisQueue] Error processing message: #{e.message}" }
-        end
-      end
-
-      # Check if message should be processed based on filters
-      def should_process_message?(headers, filter_options)
-        return true if filter_options.empty?
-        
-        # Apply from/to filters if specified
-        if filter_options[:from] && headers['from'] != filter_options[:from]
-          return false
-        end
-        
-        if filter_options[:to] && headers['to'] != filter_options[:to]
-          return false
-        end
-        
-        true
-      end
-
-      # Setup exchange metadata in Redis
-      def setup_exchange
-        exchange_key = "#{@options[:routing_prefix]}:#{@exchange_name}:metadata"
-        @redis.hset(exchange_key, 'type', 'topic')
-        @redis.hset(exchange_key, 'created_at', Time.now.to_f)
-        @redis.expire(exchange_key, 86400)  # Expire in 24 hours
-      end
-
-      # Generate queue name from pattern and options
-      def derive_queue_name(pattern, filter_options = {})
-        base_name = pattern.gsub(/[#*]/, 'wildcard')
-        filter_suffix = filter_options.empty? ? '' : "_#{filter_options.hash.abs}"
-        "#{@options[:queue_prefix]}.#{base_name}#{filter_suffix}"
-      end
-
-      # Extract routing information from message
-      def extract_routing_info(serialized_message)
-        data = JSON.parse(serialized_message)
-        header = data['_sm_header'] || {}
-        {
-          from: header['from'],
-          to: header['to']
-        }
-      rescue JSON::ParserError
-        { from: nil, to: nil }
-      end
-
-      # Build enhanced routing key similar to RabbitMQ topic exchange
-      def build_enhanced_routing_key(message_class, routing_info)
-        # Create hierarchical routing key: namespace.message_type.from.to
-        parts = []
-        
-        # Add exchange name as namespace
-        parts << @exchange_name
-        
-        # Add message class (normalized)
-        normalized_class = message_class.to_s.gsub('::', '.').downcase
-        parts << normalized_class
-        
-        # Add from/to if present (default to 'any')
-        parts << (routing_info[:from] || 'any')
-        parts << (routing_info[:to] || 'any')
-        
-        parts.join('.')
-      end
-
-      # Find queues that match the routing key pattern
-      def find_matching_queues(routing_key)
-        matching = []
-        
-        @routing_table.each do |pattern, queue_names|
-          if routing_key_matches_pattern?(routing_key, pattern)
-            matching.concat(queue_names)
-          end
-        end
-        
-        matching.uniq
-      end
-
-      # Check if routing key matches the pattern (RabbitMQ-style)
-      def routing_key_matches_pattern?(routing_key, pattern)
-        return false if routing_key.nil? || routing_key.empty?
-        return false if pattern.nil? || pattern.empty?
-        
-        # Split into segments for proper RabbitMQ-style matching
-        routing_segments = routing_key.split('.')
-        pattern_segments = pattern.split('.')
-        
-        match_segments(routing_segments, pattern_segments)
-      end
-      
-      # Recursively match routing key segments against pattern segments
-      def match_segments(routing_segments, pattern_segments)
-        return routing_segments.empty? && pattern_segments.empty? if pattern_segments.empty?
-        return pattern_segments.all? { |seg| seg == '#' } if routing_segments.empty?
-        
-        pattern_seg = pattern_segments[0]
-        
-        case pattern_seg
-        when '#'
-          # # matches zero or more segments
-          # Try matching with zero segments (skip the # pattern)
-          if match_segments(routing_segments, pattern_segments[1..-1])
-            return true
-          end
-          # Try matching with one or more segments (consume one routing segment)
-          return match_segments(routing_segments[1..-1], pattern_segments)
-          
-        when '*'
-          # * matches exactly one segment
-          return false if routing_segments.empty?
-          return match_segments(routing_segments[1..-1], pattern_segments[1..-1])
-          
-        else
-          # Literal match required
-          return false if routing_segments.empty? || routing_segments[0] != pattern_seg
-          return match_segments(routing_segments[1..-1], pattern_segments[1..-1])
-        end
-      end
-
-      # Sanitize string for use in routing key
-      def sanitize_for_routing_key(str)
-        str.to_s.gsub(/[^a-zA-Z0-9_-]/, '_').downcase
-      end
-    end
-
-    # Fluent API builder for complex subscription patterns
-    class RedisQueueSubscriptionBuilder
-      def initialize(transport)
-        @transport = transport
-        @conditions = {}
-      end
-
-      def from(sender_uuid)
-        @conditions[:from] = sender_uuid
-        self
-      end
-
-      def to(recipient_uuid)
-        @conditions[:to] = recipient_uuid
-        self
-      end
-
-      def type(message_type)
-        @conditions[:type] = message_type
-        self
-      end
-
-      def broadcast
-        @conditions[:to] = 'broadcast'
-        self
-      end
-
-      def alerts
-        @conditions[:alerts] = true
-        self
-      end
-
-      def consumer_group(group_name)
-        @conditions[:consumer_group] = group_name
-        self
-      end
-
-      def build
-        parts = []
-        
-        # Build pattern based on conditions
-        if @conditions[:alerts]
-          return "alert.#.*.*"
-        end
-        
-        if @conditions[:type]
-          parts << "*"
-          parts << @conditions[:type].to_s.gsub('::', '.').downcase
-        else
-          parts << "#"
-        end
-        
-        parts << (@conditions[:from] || "*")
-        parts << (@conditions[:to] || "*")
-        
-        parts.join(".")
-      end
-
-      def subscribe(process_method = :process, &block)
-        pattern = build
-        
-        filter_options = {}
-        filter_options[:from] = @conditions[:from] if @conditions[:from]
-        filter_options[:to] = @conditions[:to] if @conditions[:to]
-        
-        @transport.subscribe_pattern(pattern, process_method, filter_options, &block)
-      end
-    end
-  end
-end