smart_message 0.0.8 → 0.0.10

data/lib/smart_message/deduplication.rb

@@ -0,0 +1,174 @@
+# lib/smart_message/deduplication.rb
+# encoding: utf-8
+# frozen_string_literal: true
+
+require_relative 'ddq'
+
+module SmartMessage
+  # Deduplication functionality for message classes
+  # 
+  # Provides class-level configuration and instance-level deduplication
+  # checking using a Deduplication Queue (DDQ).
+  module Deduplication
+    def self.included(base)
+      base.extend(ClassMethods)
+      base.instance_variable_set(:@ddq_size, DDQ::DEFAULT_SIZE)
+      base.instance_variable_set(:@ddq_storage, DDQ::DEFAULT_STORAGE)
+      base.instance_variable_set(:@ddq_options, {})
+      base.instance_variable_set(:@ddq_enabled, false)
+      base.instance_variable_set(:@ddq_instance, nil)
+    end
+    
+    module ClassMethods
+      # Configure DDQ size for this message class
+      # @param size [Integer] Maximum number of UUIDs to track
+      def ddq_size(size)
+        unless size.is_a?(Integer) && size > 0
+          raise ArgumentError, "DDQ size must be a positive integer, got: #{size.inspect}"
+        end
+        @ddq_size = size
+        reset_ddq_instance! if ddq_enabled?
+      end
+      
+      # Configure DDQ storage type for this message class
+      # @param storage [Symbol] Storage type (:memory or :redis)
+      # @param options [Hash] Additional options for the storage backend
+      def ddq_storage(storage, **options)
+        unless [:memory, :redis].include?(storage.to_sym)
+          raise ArgumentError, "DDQ storage must be :memory or :redis, got: #{storage.inspect}"
+        end
+        @ddq_storage = storage.to_sym
+        @ddq_options = options
+        reset_ddq_instance! if ddq_enabled?
+      end
+      
+      # Enable deduplication for this message class
+      def enable_deduplication!
+        @ddq_enabled = true
+        get_ddq_instance # Initialize the DDQ
+      end
+      
+      # Disable deduplication for this message class
+      def disable_deduplication!
+        @ddq_enabled = false
+        @ddq_instance = nil
+      end
+      
+      # Check if deduplication is enabled
+      # @return [Boolean] true if DDQ is enabled
+      def ddq_enabled?
+        !!@ddq_enabled
+      end
+      
+      # Get the current DDQ configuration
+      # @return [Hash] Current DDQ configuration
+      def ddq_config
+        {
+          enabled: ddq_enabled?,
+          size: @ddq_size,
+          storage: @ddq_storage,
+          options: @ddq_options
+        }
+      end
+      
+      # Get DDQ statistics
+      # @return [Hash] DDQ statistics
+      def ddq_stats
+        return { enabled: false } unless ddq_enabled?
+        
+        ddq = get_ddq_instance
+        if ddq
+          ddq.stats.merge(enabled: true)
+        else
+          { enabled: true, error: "DDQ instance not available" }
+        end
+      end
+      
+      # Clear the DDQ
+      def clear_ddq!
+        return unless ddq_enabled?
+        
+        ddq = get_ddq_instance
+        ddq&.clear
+      end
+      
+      # Check if a UUID is a duplicate (for external use)
+      # @param uuid [String] The UUID to check
+      # @return [Boolean] true if UUID is a duplicate
+      def duplicate_uuid?(uuid)
+        return false unless ddq_enabled?
+        
+        ddq = get_ddq_instance
+        ddq ? ddq.contains?(uuid) : false
+      end
+      
+      # Get the DDQ instance (exposed for testing)
+      def get_ddq_instance
+        return nil unless ddq_enabled?
+        
+        # Return cached instance if available and configuration hasn't changed
+        if @ddq_instance
+          return @ddq_instance
+        end
+        
+        # Create new DDQ instance
+        size = @ddq_size
+        storage = @ddq_storage
+        options = @ddq_options
+        
+        ddq = DDQ.create(storage, size, options)
+        @ddq_instance = ddq
+        
+        SmartMessage::Logger.default.debug do
+          "[SmartMessage::Deduplication] Created DDQ for #{self.name}: " \
+          "storage=#{storage}, size=#{size}, options=#{options}"
+        end
+        
+        ddq
+      rescue => e
+        SmartMessage::Logger.default.error do
+          "[SmartMessage::Deduplication] Failed to create DDQ for #{self.name}: #{e.message}"
+        end
+        nil
+      end
+      
+      private
+      
+      def reset_ddq_instance!
+        @ddq_instance = nil
+      end
+    end
+    
+    # Instance methods for deduplication checking
+    
+    # Check if this message is a duplicate based on its UUID
+    # @return [Boolean] true if this message UUID has been seen before
+    def duplicate?
+      return false unless self.class.ddq_enabled?
+      return false unless uuid
+      
+      self.class.duplicate_uuid?(uuid)
+    end
+    
+    # Mark this message as processed (add UUID to DDQ)
+    # @return [void]
+    def mark_as_processed!
+      return unless self.class.ddq_enabled?
+      return unless uuid
+      
+      ddq = self.class.send(:get_ddq_instance)
+      if ddq
+        ddq.add(uuid)
+        SmartMessage::Logger.default.debug do
+          "[SmartMessage::Deduplication] Marked UUID as processed: #{uuid}"
+        end
+      end
+    end
+    
+    # Get the message UUID
+    # @return [String, nil] The message UUID
+    def uuid
+      _sm_header&.uuid
+    end
+  end
+end

data/lib/smart_message/dispatcher.rb

@@ -15,15 +15,29 @@ module SmartMessage
     # TODO: setup forwardable for some @router_pool methods
 
     def initialize(circuit_breaker_options = {})
-      @subscribers = Hash.new(Array.new)
+      @subscribers = Hash.new { |h, k| h[k] = [] }
+      @subscriber_ddqs = {}  # Hash to store DDQs per subscriber: "MessageClass:subscriber_id" => DDQ
       @router_pool = Concurrent::CachedThreadPool.new
-      
+
       # Configure circuit breakers
       configure_circuit_breakers(circuit_breaker_options)
       at_exit do
         shutdown_pool
       end
+      
+      logger.debug { "[SmartMessage::Dispatcher] Initialized with circuit breaker options: #{circuit_breaker_options}" }
+    rescue => e
+      logger.error { "[SmartMessage] Error in dispatcher initialization: #{e.class.name} - #{e.message}" }
+      raise
+    end
+    
+    private
+    
+    def logger
+      @logger ||= SmartMessage::Logger.default
     end
+    
+    public
 
 
     def what_can_i_do?
@@ -85,57 +99,128 @@ module SmartMessage
     def add(message_class, process_method_as_string, filter_options = {})
       klass = String(message_class)
       
-      # Create subscription entry with filter options
+      # Subscriber ID is derived from the process method (handler-only scoping)
+      subscriber_id = process_method_as_string
+      
+      # Initialize DDQ for this handler if message class supports deduplication
+      begin
+        message_class_obj = klass.constantize
+        if message_class_obj.respond_to?(:ddq_enabled?) && message_class_obj.ddq_enabled?
+          initialize_ddq_for_subscriber(klass, subscriber_id, message_class_obj)
+        end
+      rescue NameError
+        # Message class doesn't exist as a constant (e.g., in tests with fake class names)
+        # Skip DDQ initialization - this is fine since the class wouldn't support deduplication anyway
+      end
+
+      # Create subscription entry with filter options and derived subscriber ID
       subscription = {
+        subscriber_id: subscriber_id,
         process_method: process_method_as_string,
         filters: filter_options
       }
-      
+
       # Check if this exact subscription already exists
       existing_subscription = @subscribers[klass].find do |sub|
-        sub[:process_method] == process_method_as_string && sub[:filters] == filter_options
+        sub[:subscriber_id] == subscriber_id &&
+        sub[:process_method] == process_method_as_string && 
+        sub[:filters] == filter_options
       end
-      
+
       unless existing_subscription
-        @subscribers[klass] += [subscription]
+        @subscribers[klass] << subscription
+        logger.debug { "[SmartMessage::Dispatcher] Added subscription: #{klass} for #{subscriber_id}" }
       end
     end
 
 
     # drop a processer from a subscribed message
-    def drop(message_class, process_method_as_string)
+    def drop(message_class, process_method_as_string, filter_options = {})
       klass = String(message_class)
-      @subscribers[klass].reject! { |sub| sub[:process_method] == process_method_as_string }
+      
+      # Subscriber ID is derived from the process method
+      subscriber_id = process_method_as_string
+      
+      # Drop the specific method (now uniquely identified by handler)
+      @subscribers[klass].reject! do |sub| 
+        sub[:subscriber_id] == subscriber_id && sub[:process_method] == process_method_as_string
+      end
+      
+      # Clean up DDQ if no more subscriptions for this handler
+      cleanup_ddq_if_empty(klass, subscriber_id)
     end
 
+    # drop all processors for a specific handler (subscriber)
+    def drop_subscriber(message_class, handler_method)
+      klass = String(message_class)
+      unless handler_method.is_a?(String) && !handler_method.empty?
+        raise ArgumentError, "handler_method must be a non-empty String, got: #{handler_method.inspect}"
+      end
+      
+      # Handler method is the subscriber ID
+      subscriber_id = handler_method
+      @subscribers[klass].reject! { |sub| sub[:subscriber_id] == subscriber_id }
+      cleanup_ddq_if_empty(klass, subscriber_id)
+    end
 
-    # drop all processer from a subscribed message
+    # drop all processors from a subscribed message
     def drop_all(message_class)
-      @subscribers.delete String(message_class)
+      klass = String(message_class)
+      
+      # Clean up all DDQs for this message class
+      if @subscribers[klass]
+        @subscribers[klass].each do |sub|
+          cleanup_ddq_if_empty(klass, sub[:subscriber_id], force: true)
+        end
+      end
+      
+      @subscribers.delete klass
     end
 
-
     # complete reset all subscriptions
     def drop_all!
-      @subscribers = Hash.new(Array.new)
+      # Clean up all DDQs
+      @subscriber_ddqs&.clear
+      @subscribers = Hash.new { |h, k| h[k] = [] }
     end
 
 
-    # message_header is of class SmartMessage::Header
-    # message_payload is a string buffer that is a serialized
-    # SmartMessage
-    def route(message_header, message_payload)
+    # Route a decoded message to appropriate message processors
+    # @param decoded_message [SmartMessage::Base] The decoded message instance
+    def route(decoded_message)
+      message_header = decoded_message._sm_header
       message_klass = message_header.message_class
-      return nil if @subscribers[message_klass].empty?
       
+      # Try to get the message class object for deduplication checks
+      message_class_obj = nil
+      begin
+        message_class_obj = message_klass.constantize
+      rescue NameError
+        # Message class doesn't exist as a constant - skip deduplication checks
+      end
+      
+      logger.debug { "[SmartMessage::Dispatcher] Routing message #{message_klass} to #{@subscribers[message_klass]&.size || 0} subscribers" }
+      logger.debug { "[SmartMessage::Dispatcher] Available subscribers: #{@subscribers.keys}" }
+      return nil if @subscribers[message_klass].nil? || @subscribers[message_klass].empty?
+
       @subscribers[message_klass].each do |subscription|
         # Extract subscription details
+        subscriber_id = subscription[:subscriber_id]
         message_processor = subscription[:process_method]
         filters = subscription[:filters]
-        
+
         # Check if message matches filters
         next unless message_matches_filters?(message_header, filters)
-        
+
+        # Subscriber-specific deduplication check (only if class exists and supports it)
+        if message_class_obj&.respond_to?(:ddq_enabled?) && message_class_obj.ddq_enabled?
+          ddq = get_ddq_for_subscriber(message_klass, subscriber_id)
+          if ddq&.contains?(decoded_message.uuid)
+            logger.warn { "[SmartMessage::Dispatcher] Skipping duplicate for #{subscriber_id}: #{decoded_message.uuid} | Header: #{decoded_message._sm_header.to_h}" }
+            next  # Skip this subscriber, continue to next
+          end
+        end
+
         SS.add(message_klass, message_processor, 'routed' )
         @router_pool.post do
           # Use circuit breaker to protect message processing
@@ -143,21 +228,30 @@ module SmartMessage
             # Check if this is a proc handler or a regular method call
             if proc_handler?(message_processor)
               # Call the proc handler via SmartMessage::Base
-              SmartMessage::Base.call_proc_handler(message_processor, message_header, message_payload)
+              SmartMessage::Base.call_proc_handler(message_processor, decoded_message)
             else
-              # Original method call logic
+              # Method call logic with decoded message
               parts         = message_processor.split('.')
               target_klass  = parts[0]
               class_method  = parts[1]
               target_klass.constantize
                           .method(class_method)
-                          .call(message_header, message_payload)
+                          .call(decoded_message)
             end
           end
-          
+
           # Handle circuit breaker fallback responses
           if circuit_result.is_a?(Hash) && circuit_result[:circuit_breaker]
-            handle_circuit_breaker_fallback(circuit_result, message_header, message_payload, message_processor)
+            handle_circuit_breaker_fallback(circuit_result, decoded_message, message_processor)
+          else
+            # Mark message as processed in subscriber's DDQ after successful processing
+            if message_class_obj&.respond_to?(:ddq_enabled?) && message_class_obj.ddq_enabled?
+              ddq = get_ddq_for_subscriber(message_klass, subscriber_id)
+              if ddq && decoded_message.uuid
+                ddq.add(decoded_message.uuid)
+                logger.debug { "[SmartMessage::Dispatcher] Marked UUID as processed for #{subscriber_id}: #{decoded_message.uuid}" }
+              end
+            end
           end
         end
       end
@@ -167,7 +261,7 @@ module SmartMessage
     # @return [Hash] Circuit breaker statistics
     def circuit_breaker_stats
       stats = {}
-      
+
       begin
         if respond_to?(:circuit)
           breaker = circuit(:message_processor)
@@ -186,7 +280,7 @@ module SmartMessage
       rescue => e
         stats[:error] = "Failed to get circuit breaker stats: #{e.message}"
       end
-      
+
       stats
     end
 
@@ -204,7 +298,7 @@ module SmartMessage
     # Shutdown the router pool with timeout and fallback
     def shutdown_pool
       @router_pool.shutdown
-      
+
       # Wait for graceful shutdown, force kill if timeout
       unless @router_pool.wait_for_termination(3)
         @router_pool.kill
@@ -216,26 +310,45 @@ module SmartMessage
     # @param filters [Hash] The filter criteria
     # @return [Boolean] True if the message matches all filters
     def message_matches_filters?(message_header, filters)
-      # If no filters specified, accept all messages (backward compatibility)
-      return true if filters.nil? || filters.empty? || filters.values.all?(&:nil?)
-      
-      # Check from filter
+      # Check from filter first (if specified)
       if filters[:from]
-        from_match = filters[:from].include?(message_header.from)
+        from_match = filter_value_matches?(message_header.from, filters[:from])
         return false unless from_match
       end
-      
+
       # Check to/broadcast filters (OR logic between them)
-      if filters[:broadcast] || filters[:to]
+      # Only consider explicit filtering if the values are not nil
+      if filters[:to] || filters[:broadcast]
+        # Explicit filtering specified
         broadcast_match = filters[:broadcast] && message_header.to.nil?
-        to_match = filters[:to] && filters[:to].include?(message_header.to)
+        to_match = filters[:to] && filter_value_matches?(message_header.to, filters[:to])
         
-        # If either broadcast or to filter is specified, at least one must match
-        combined_match = (broadcast_match || to_match)
-        return false unless combined_match
+        # At least one must match
+        return (broadcast_match || to_match)
+      else
+        # No to/broadcast filtering - accept all messages (backward compatibility)
+        return true
+      end
+    end
+
+    # Check if a value matches any of the filter criteria
+    # Supports both exact string matching and regex pattern matching
+    # @param value [String, nil] The value to match against
+    # @param filter_array [Array] Array of strings and/or regexps to match against
+    # @return [Boolean] True if the value matches any filter in the array
+    def filter_value_matches?(value, filter_array)
+      return false if value.nil? || filter_array.nil?
+
+      filter_array.any? do |filter|
+        case filter
+        when String
+          filter == value
+        when Regexp
+          filter.match?(value)
+        else
+          false
+        end
       end
-      
-      true
     end
 
     # Check if a message processor is a proc handler
@@ -250,19 +363,19 @@ module SmartMessage
     def configure_circuit_breakers(options = {})
       # Ensure CircuitBreaker module is available
       return unless defined?(SmartMessage::CircuitBreaker::DEFAULT_CONFIGS)
-      
+
       # Configure message processor circuit breaker
       default_config = SmartMessage::CircuitBreaker::DEFAULT_CONFIGS[:message_processor]
       return unless default_config
-      
+
       processor_config = default_config.merge(options[:message_processor] || {})
-      
+
       # Define the circuit using the class-level DSL
       self.class.circuit :message_processor do
-        threshold failures: processor_config[:threshold][:failures], 
+        threshold failures: processor_config[:threshold][:failures],
                  within: processor_config[:threshold][:within].seconds
         reset_after processor_config[:reset_after].seconds
-        
+
         # Configure storage backend
         case processor_config[:storage]
         when :redis
@@ -281,7 +394,7 @@ module SmartMessage
         else
           storage BreakerMachines::Storage::Memory.new
         end
-        
+
         # Default fallback for message processing failures
         fallback do |exception|
           {
@@ -296,30 +409,115 @@ module SmartMessage
           }
         end
       end
-      
+
     end
 
     # Handle circuit breaker fallback responses
     # @param circuit_result [Hash] The circuit breaker fallback result
-    # @param message_header [SmartMessage::Header] The message header
-    # @param message_payload [String] The message payload
+    # @param decoded_message [SmartMessage::Base] The decoded message instance
     # @param message_processor [String] The processor that failed
-    def handle_circuit_breaker_fallback(circuit_result, message_header, message_payload, message_processor)
-      # Log circuit breaker activation
-      if $DEBUG
-        puts "Circuit breaker activated for processor: #{message_processor}"
-        puts "Error: #{circuit_result[:circuit_breaker][:error]}"
-        puts "Message: #{message_header.message_class} from #{message_header.from}"
-      end
-      
+    def handle_circuit_breaker_fallback(circuit_result, decoded_message, message_processor)
+      message_header = decoded_message._sm_header
+
+      # Always log circuit breaker activation for debugging
+      error_msg = circuit_result[:circuit_breaker][:error]
+      logger.error { "[SmartMessage::Dispatcher] Circuit breaker activated for processor: #{message_processor}" }
+      logger.error { "[SmartMessage::Dispatcher] Error: #{error_msg}" }
+      logger.error { "[SmartMessage::Dispatcher] Message: #{message_header.message_class} from #{message_header.from}" }
+
+      # Send to dead letter queue
+      SmartMessage::DeadLetterQueue.default.enqueue(decoded_message,
+        error: circuit_result[:circuit_breaker][:error],
+        retry_count: 0,
+        transport: 'circuit_breaker'
+      )
+
       # TODO: Integrate with structured logging when implemented
-      # TODO: Send to dead letter queue when implemented
       # TODO: Emit metrics/events for monitoring
-      
-      # For now, record the failure in simple stats
+
+      # Record the failure in simple stats
       SS.add(message_header.message_class, message_processor, 'circuit_breaker_fallback')
     end
 
+    # Initialize DDQ for a subscriber if message class supports deduplication
+    # @param message_class [String] The message class name
+    # @param subscriber_id [String] The subscriber identifier
+    # @param message_class_obj [Class] The message class object
+    def initialize_ddq_for_subscriber(message_class, subscriber_id, message_class_obj)
+      ddq_key = "#{message_class}:#{subscriber_id}"
+      return if @subscriber_ddqs[ddq_key]  # Already initialized
+      
+      # Get DDQ configuration from message class
+      ddq_config = message_class_obj.ddq_config
+      
+      begin
+        # Create DDQ instance using message class configuration
+        ddq = SmartMessage::DDQ.create(
+          ddq_config[:storage], 
+          ddq_config[:size], 
+          ddq_config[:options].merge(key_prefix: ddq_key)
+        )
+        
+        @subscriber_ddqs[ddq_key] = ddq
+        
+        logger.debug do
+          "[SmartMessage::Dispatcher] Initialized DDQ for #{subscriber_id}: " \
+          "storage=#{ddq_config[:storage]}, size=#{ddq_config[:size]}"
+        end
+      rescue => e
+        logger.error do
+          "[SmartMessage::Dispatcher] Failed to initialize DDQ for #{subscriber_id}: #{e.message}"
+        end
+      end
+    end
+
+    # Get DDQ instance for a specific subscriber
+    # @param message_class [String] The message class name
+    # @param subscriber_id [String] The subscriber identifier
+    # @return [SmartMessage::DDQ::Base, nil] The DDQ instance or nil
+    def get_ddq_for_subscriber(message_class, subscriber_id)
+      ddq_key = "#{message_class}:#{subscriber_id}"
+      @subscriber_ddqs[ddq_key]
+    end
+
+    # Clean up DDQ if no more subscriptions exist for a subscriber
+    # @param message_class [String] The message class name
+    # @param subscriber_id [String] The subscriber identifier
+    # @param force [Boolean] Force cleanup even if subscriptions exist
+    def cleanup_ddq_if_empty(message_class, subscriber_id, force: false)
+      ddq_key = "#{message_class}:#{subscriber_id}"
+      
+      # Check if any subscriptions still exist for this subscriber
+      has_subscriptions = @subscribers[message_class]&.any? do |sub|
+        sub[:subscriber_id] == subscriber_id
+      end
+      
+      if force || !has_subscriptions
+        ddq = @subscriber_ddqs.delete(ddq_key)
+        if ddq
+          logger.debug do
+            "[SmartMessage::Dispatcher] Cleaned up DDQ for #{subscriber_id}"
+          end
+        end
+      end
+    end
+
+    # Get DDQ statistics for all subscribers
+    # @return [Hash] DDQ statistics keyed by subscriber
+    def ddq_stats
+      stats = {}
+      
+      @subscriber_ddqs.each do |ddq_key, ddq|
+        begin
+          stats[ddq_key] = ddq.stats
+        rescue => e
+          stats[ddq_key] = { error: e.message }
+        end
+      end
+      
+      stats
+    end
+
 
     #######################################################
     ## Class methods

data/lib/smart_message/header.rb

@@ -57,5 +57,10 @@ module SmartMessage
     property :reply_to,
       required: false,
       description: "Optional unique identifier of the entity that should receive replies to this message. Defaults to 'from' entity if not specified"
+    
+    # Serialization tracking for wrapper architecture
+    property :serializer,
+      required: false,
+      description: "Class name of the serializer used to encode the payload (e.g., 'SmartMessage::Serializer::Json'). Used by DLQ and cross-serializer gateway patterns"
   end
 end

data/lib/smart_message/logger/base.rb

@@ -4,5 +4,25 @@
 
 module SmartMessage::Logger
   class Base
+    # Standard logging methods that subclasses should implement
+    def debug(message = nil, &block)
+      raise NotImplementedError, "Subclass must implement #debug"
+    end
+
+    def info(message = nil, &block)
+      raise NotImplementedError, "Subclass must implement #info"
+    end
+
+    def warn(message = nil, &block)
+      raise NotImplementedError, "Subclass must implement #warn"
+    end
+
+    def error(message = nil, &block)
+      raise NotImplementedError, "Subclass must implement #error"
+    end
+
+    def fatal(message = nil, &block)
+      raise NotImplementedError, "Subclass must implement #fatal"
+    end
   end
-end # module SmartMessage::Logger
+end # module SmartMessage::Logger