smart_message 0.0.9 → 0.0.10

data/lib/smart_message/ddq/base.rb

@@ -0,0 +1,71 @@
+# lib/smart_message/ddq/base.rb
+# encoding: utf-8
+# frozen_string_literal: true
+
+module SmartMessage
+  module DDQ
+    # Base class for Deduplication Queue implementations
+    # 
+    # Defines the interface that all DDQ storage backends must implement.
+    # Provides circular queue semantics with O(1) lookup performance.
+    class Base
+      attr_reader :size, :logger
+      
+      def initialize(size)
+        @size = size
+        @logger = SmartMessage::Logger.default
+        validate_size!
+      end
+      
+      # Check if a UUID exists in the queue
+      # @param uuid [String] The UUID to check
+      # @return [Boolean] true if UUID exists, false otherwise
+      def contains?(uuid)
+        raise NotImplementedError, "Subclasses must implement #contains?"
+      end
+      
+      # Add a UUID to the queue (removes oldest if full)
+      # @param uuid [String] The UUID to add
+      # @return [void]
+      def add(uuid)
+        raise NotImplementedError, "Subclasses must implement #add"
+      end
+      
+      # Get current queue statistics
+      # @return [Hash] Statistics about the queue
+      def stats
+        {
+          size: @size,
+          storage_type: storage_type,
+          implementation: self.class.name
+        }
+      end
+      
+      # Clear all entries from the queue
+      # @return [void]
+      def clear
+        raise NotImplementedError, "Subclasses must implement #clear"
+      end
+      
+      # Get the storage type identifier
+      # @return [Symbol] Storage type (:memory, :redis, etc.)
+      def storage_type
+        raise NotImplementedError, "Subclasses must implement #storage_type"
+      end
+      
+      private
+      
+      def validate_size!
+        unless @size.is_a?(Integer) && @size > 0
+          raise ArgumentError, "DDQ size must be a positive integer, got: #{@size.inspect}"
+        end
+      end
+      
+      def validate_uuid!(uuid)
+        unless uuid.is_a?(String) && !uuid.empty?
+          raise ArgumentError, "UUID must be a non-empty string, got: #{uuid.inspect}"
+        end
+      end
+    end
+  end
+end

data/lib/smart_message/ddq/memory.rb

@@ -0,0 +1,109 @@
+# lib/smart_message/ddq/memory.rb
+# encoding: utf-8
+# frozen_string_literal: true
+
+require 'set'
+require_relative 'base'
+
+module SmartMessage
+  module DDQ
+    # Memory-based Deduplication Queue implementation
+    # 
+    # Uses a hybrid approach with Array for circular queue behavior 
+    # and Set for O(1) lookup performance. Thread-safe with Mutex protection.
+    class Memory < Base
+      def initialize(size)
+        super(size)
+        @circular_array = Array.new(@size)
+        @lookup_set = Set.new
+        @index = 0
+        @mutex = Mutex.new
+        @count = 0
+        
+        logger.debug { "[SmartMessage::DDQ::Memory] Initialized with size: #{@size}" }
+      end
+      
+      # Check if a UUID exists in the queue (O(1) operation)
+      # @param uuid [String] The UUID to check
+      # @return [Boolean] true if UUID exists, false otherwise
+      def contains?(uuid)
+        validate_uuid!(uuid)
+        
+        @mutex.synchronize do
+          @lookup_set.include?(uuid)
+        end
+      end
+      
+      # Add a UUID to the queue, removing oldest if full (O(1) operation)
+      # @param uuid [String] The UUID to add
+      # @return [void]
+      def add(uuid)
+        validate_uuid!(uuid)
+        
+        @mutex.synchronize do
+          # Don't add if already exists
+          return if @lookup_set.include?(uuid)
+          
+          # Remove old entry if slot is occupied
+          old_uuid = @circular_array[@index]
+          if old_uuid
+            @lookup_set.delete(old_uuid)
+            logger.debug { "[SmartMessage::DDQ::Memory] Evicted UUID: #{old_uuid}" }
+          end
+          
+          # Add new entry
+          @circular_array[@index] = uuid
+          @lookup_set.add(uuid)
+          @index = (@index + 1) % @size
+          @count = [@count + 1, @size].min
+          
+          logger.debug { "[SmartMessage::DDQ::Memory] Added UUID: #{uuid}, count: #{@count}" }
+        end
+      end
+      
+      # Get current queue statistics
+      # @return [Hash] Statistics about the queue
+      def stats
+        @mutex.synchronize do
+          super.merge(
+            current_count: @count,
+            utilization: (@count.to_f / @size * 100).round(2),
+            next_index: @index
+          )
+        end
+      end
+      
+      # Clear all entries from the queue
+      # @return [void]
+      def clear
+        @mutex.synchronize do
+          @circular_array = Array.new(@size)
+          @lookup_set.clear
+          @index = 0
+          @count = 0
+          
+          logger.debug { "[SmartMessage::DDQ::Memory] Cleared all entries" }
+        end
+      end
+      
+      # Get the storage type identifier
+      # @return [Symbol] Storage type
+      def storage_type
+        :memory
+      end
+      
+      # Get current entries (for debugging/testing)
+      # @return [Array<String>] Current UUIDs in insertion order
+      def entries
+        @mutex.synchronize do
+          result = []
+          @count.times do |i|
+            idx = (@index - @count + i) % @size
+            result << @circular_array[idx] if @circular_array[idx]
+          end
+          result
+        end
+      end
+    end
+  end
+end

data/lib/smart_message/ddq/redis.rb

@@ -0,0 +1,168 @@
+# lib/smart_message/ddq/redis.rb
+# encoding: utf-8
+# frozen_string_literal: true
+
+require_relative 'base'
+
+module SmartMessage
+  module DDQ
+    # Redis-based Deduplication Queue implementation
+    # 
+    # Uses Redis SET for O(1) lookup and LIST for circular queue behavior.
+    # Supports distributed deduplication across multiple processes/servers.
+    class Redis < Base
+      attr_reader :redis, :key_prefix, :ttl
+      
+      def initialize(size, options = {})
+        super(size)
+        @redis = options[:redis] || default_redis_connection
+        @key_prefix = options[:key_prefix] || 'smart_message:ddq'
+        @ttl = options[:ttl] || 3600  # 1 hour default TTL
+        
+        validate_redis_connection!
+        
+        logger.debug { "[SmartMessage::DDQ::Redis] Initialized with size: #{@size}, TTL: #{@ttl}s" }
+      end
+      
+      # Check if a UUID exists in the queue (O(1) operation)
+      # @param uuid [String] The UUID to check
+      # @return [Boolean] true if UUID exists, false otherwise
+      def contains?(uuid)
+        validate_uuid!(uuid)
+        
+        result = @redis.sismember(set_key, uuid)
+        logger.debug { "[SmartMessage::DDQ::Redis] UUID #{uuid} exists: #{result}" }
+        result
+      rescue ::Redis::BaseError => e
+        logger.error { "[SmartMessage::DDQ::Redis] Error checking UUID #{uuid}: #{e.message}" }
+        # Fail open - allow processing if Redis is down
+        false
+      end
+      
+      # Add a UUID to the queue, removing oldest if full (O(1) amortized)
+      # @param uuid [String] The UUID to add
+      # @return [void]
+      def add(uuid)
+        validate_uuid!(uuid)
+        
+        # Check if UUID already exists first (avoid unnecessary work)
+        return if @redis.sismember(set_key, uuid)
+        
+        # Use Redis transaction for atomicity
+        @redis.multi do |pipeline|
+          # Add to set for O(1) lookup
+          pipeline.sadd(set_key, uuid)
+          
+          # Add to list for ordering/eviction
+          pipeline.lpush(list_key, uuid)
+          
+          # Trim list to maintain size (removes oldest)
+          pipeline.ltrim(list_key, 0, @size - 1)
+          
+          # Set TTL on both keys
+          pipeline.expire(set_key, @ttl)
+          pipeline.expire(list_key, @ttl)
+        end
+        
+        # Get and remove evicted items from set (outside transaction)
+        list_length = @redis.llen(list_key)
+        if list_length > @size
+          evicted_uuids = @redis.lrange(list_key, @size, -1)
+          evicted_uuids.each do |evicted_uuid|
+            @redis.srem(set_key, evicted_uuid)
+          end
+        end
+        
+        logger.debug { "[SmartMessage::DDQ::Redis] Added UUID: #{uuid}" }
+      rescue ::Redis::BaseError => e
+        logger.error { "[SmartMessage::DDQ::Redis] Error adding UUID #{uuid}: #{e.message}" }
+        # Don't raise - deduplication failure shouldn't break message processing
+      end
+      
+      # Get current queue statistics
+      # @return [Hash] Statistics about the queue
+      def stats
+        set_size = @redis.scard(set_key)
+        list_size = @redis.llen(list_key)
+        
+        super.merge(
+          current_count: set_size,
+          list_count: list_size,
+          utilization: (set_size.to_f / @size * 100).round(2),
+          ttl_remaining: @redis.ttl(set_key),
+          redis_info: {
+            host: redis_host,
+            port: redis_port,
+            db: redis_db
+          }
+        )
+      rescue ::Redis::BaseError => e
+        logger.error { "[SmartMessage::DDQ::Redis] Error getting stats: #{e.message}" }
+        super.merge(error: e.message)
+      end
+      
+      # Clear all entries from the queue
+      # @return [void]
+      def clear
+        @redis.multi do |pipeline|
+          pipeline.del(set_key)
+          pipeline.del(list_key)
+        end
+        
+        logger.debug { "[SmartMessage::DDQ::Redis] Cleared all entries" }
+      rescue ::Redis::BaseError => e
+        logger.error { "[SmartMessage::DDQ::Redis] Error clearing queue: #{e.message}" }
+      end
+      
+      # Get the storage type identifier
+      # @return [Symbol] Storage type
+      def storage_type
+        :redis
+      end
+      
+      # Get current entries (for debugging/testing)
+      # @return [Array<String>] Current UUIDs in insertion order (newest first)
+      def entries
+        @redis.lrange(list_key, 0, -1)
+      rescue ::Redis::BaseError => e
+        logger.error { "[SmartMessage::DDQ::Redis] Error getting entries: #{e.message}" }
+        []
+      end
+      
+      private
+      
+      def set_key
+        "#{@key_prefix}:set"
+      end
+      
+      def list_key
+        "#{@key_prefix}:list"
+      end
+      
+      def default_redis_connection
+        require 'redis'
+        ::Redis.new
+      rescue LoadError
+        raise LoadError, "Redis gem not available. Install with: gem install redis"
+      end
+      
+      def validate_redis_connection!
+        @redis.ping
+      rescue ::Redis::BaseError => e
+        raise ConnectionError, "Failed to connect to Redis: #{e.message}"
+      end
+      
+      def redis_host
+        @redis.connection[:host] rescue 'unknown'
+      end
+      
+      def redis_port
+        @redis.connection[:port] rescue 'unknown'
+      end
+      
+      def redis_db
+        @redis.connection[:db] rescue 'unknown'
+      end
+    end
+  end
+end

data/lib/smart_message/ddq.rb

@@ -0,0 +1,31 @@
+# lib/smart_message/ddq.rb
+# encoding: utf-8
+# frozen_string_literal: true
+
+require_relative 'ddq/base'
+require_relative 'ddq/memory'
+require_relative 'ddq/redis'
+
+module SmartMessage
+  # Deduplication Queue (DDQ) for preventing duplicate message processing
+  # 
+  # Provides a circular queue with O(1) lookup performance for detecting
+  # duplicate messages based on UUID. Supports both memory and Redis storage.
+  module DDQ
+    # Default configuration
+    DEFAULT_SIZE = 100
+    DEFAULT_STORAGE = :memory
+    
+    # Create a DDQ instance based on storage type
+    def self.create(storage_type, size = DEFAULT_SIZE, options = {})
+      case storage_type.to_sym
+      when :memory
+        Memory.new(size)
+      when :redis
+        Redis.new(size, options)
+      else
+        raise ArgumentError, "Unknown DDQ storage type: #{storage_type}. Supported types: :memory, :redis"
+      end
+    end
+  end
+end

data/lib/smart_message/deduplication.rb

@@ -0,0 +1,174 @@
+# lib/smart_message/deduplication.rb
+# encoding: utf-8
+# frozen_string_literal: true
+
+require_relative 'ddq'
+
+module SmartMessage
+  # Deduplication functionality for message classes
+  # 
+  # Provides class-level configuration and instance-level deduplication
+  # checking using a Deduplication Queue (DDQ).
+  module Deduplication
+    def self.included(base)
+      base.extend(ClassMethods)
+      base.instance_variable_set(:@ddq_size, DDQ::DEFAULT_SIZE)
+      base.instance_variable_set(:@ddq_storage, DDQ::DEFAULT_STORAGE)
+      base.instance_variable_set(:@ddq_options, {})
+      base.instance_variable_set(:@ddq_enabled, false)
+      base.instance_variable_set(:@ddq_instance, nil)
+    end
+    
+    module ClassMethods
+      # Configure DDQ size for this message class
+      # @param size [Integer] Maximum number of UUIDs to track
+      def ddq_size(size)
+        unless size.is_a?(Integer) && size > 0
+          raise ArgumentError, "DDQ size must be a positive integer, got: #{size.inspect}"
+        end
+        @ddq_size = size
+        reset_ddq_instance! if ddq_enabled?
+      end
+      
+      # Configure DDQ storage type for this message class
+      # @param storage [Symbol] Storage type (:memory or :redis)
+      # @param options [Hash] Additional options for the storage backend
+      def ddq_storage(storage, **options)
+        unless [:memory, :redis].include?(storage.to_sym)
+          raise ArgumentError, "DDQ storage must be :memory or :redis, got: #{storage.inspect}"
+        end
+        @ddq_storage = storage.to_sym
+        @ddq_options = options
+        reset_ddq_instance! if ddq_enabled?
+      end
+      
+      # Enable deduplication for this message class
+      def enable_deduplication!
+        @ddq_enabled = true
+        get_ddq_instance # Initialize the DDQ
+      end
+      
+      # Disable deduplication for this message class
+      def disable_deduplication!
+        @ddq_enabled = false
+        @ddq_instance = nil
+      end
+      
+      # Check if deduplication is enabled
+      # @return [Boolean] true if DDQ is enabled
+      def ddq_enabled?
+        !!@ddq_enabled
+      end
+      
+      # Get the current DDQ configuration
+      # @return [Hash] Current DDQ configuration
+      def ddq_config
+        {
+          enabled: ddq_enabled?,
+          size: @ddq_size,
+          storage: @ddq_storage,
+          options: @ddq_options
+        }
+      end
+      
+      # Get DDQ statistics
+      # @return [Hash] DDQ statistics
+      def ddq_stats
+        return { enabled: false } unless ddq_enabled?
+        
+        ddq = get_ddq_instance
+        if ddq
+          ddq.stats.merge(enabled: true)
+        else
+          { enabled: true, error: "DDQ instance not available" }
+        end
+      end
+      
+      # Clear the DDQ
+      def clear_ddq!
+        return unless ddq_enabled?
+        
+        ddq = get_ddq_instance
+        ddq&.clear
+      end
+      
+      # Check if a UUID is a duplicate (for external use)
+      # @param uuid [String] The UUID to check
+      # @return [Boolean] true if UUID is a duplicate
+      def duplicate_uuid?(uuid)
+        return false unless ddq_enabled?
+        
+        ddq = get_ddq_instance
+        ddq ? ddq.contains?(uuid) : false
+      end
+      
+      # Get the DDQ instance (exposed for testing)
+      def get_ddq_instance
+        return nil unless ddq_enabled?
+        
+        # Return cached instance if available and configuration hasn't changed
+        if @ddq_instance
+          return @ddq_instance
+        end
+        
+        # Create new DDQ instance
+        size = @ddq_size
+        storage = @ddq_storage
+        options = @ddq_options
+        
+        ddq = DDQ.create(storage, size, options)
+        @ddq_instance = ddq
+        
+        SmartMessage::Logger.default.debug do
+          "[SmartMessage::Deduplication] Created DDQ for #{self.name}: " \
+          "storage=#{storage}, size=#{size}, options=#{options}"
+        end
+        
+        ddq
+      rescue => e
+        SmartMessage::Logger.default.error do
+          "[SmartMessage::Deduplication] Failed to create DDQ for #{self.name}: #{e.message}"
+        end
+        nil
+      end
+      
+      private
+      
+      def reset_ddq_instance!
+        @ddq_instance = nil
+      end
+    end
+    
+    # Instance methods for deduplication checking
+    
+    # Check if this message is a duplicate based on its UUID
+    # @return [Boolean] true if this message UUID has been seen before
+    def duplicate?
+      return false unless self.class.ddq_enabled?
+      return false unless uuid
+      
+      self.class.duplicate_uuid?(uuid)
+    end
+    
+    # Mark this message as processed (add UUID to DDQ)
+    # @return [void]
+    def mark_as_processed!
+      return unless self.class.ddq_enabled?
+      return unless uuid
+      
+      ddq = self.class.send(:get_ddq_instance)
+      if ddq
+        ddq.add(uuid)
+        SmartMessage::Logger.default.debug do
+          "[SmartMessage::Deduplication] Marked UUID as processed: #{uuid}"
+        end
+      end
+    end
+    
+    # Get the message UUID
+    # @return [String, nil] The message UUID
+    def uuid
+      _sm_header&.uuid
+    end
+  end
+end