legion-cache 1.4.2 → 1.4.3

data/lib/legion/cache/memory.rb

@@ -5,16 +5,181 @@ require 'legion/logging/helper'
 
 module Legion
   module Cache
+    # Shared bounded storage for Memory and Cacheable's process-local fallback.
+    # Cleanup exists only while expiring entries exist; clear/close joins it.
+    class ExpiringStore
+      DEFAULT_MAX_ENTRIES = 10_000
+
+      def initialize(max_entries: DEFAULT_MAX_ENTRIES)
+        @entries = {}
+        @expiry = {}
+        @mutex = Mutex.new
+        @condition = ConditionVariable.new
+        self.max_entries = max_entries
+      end
+
+      def max_entries=(limit)
+        raise ArgumentError, 'max_entries must be a positive integer' unless limit.is_a?(Integer) && limit.positive?
+
+        @mutex.synchronize do
+          @max_entries = limit
+          evict_oldest while @entries.size > limit
+        end
+      end
+
+      def get(key)
+        @mutex.synchronize do
+          expire_key(key)
+          @entries[key]
+        end
+      end
+
+      def set(key, value, ttl: nil, unless_exist: false)
+        deadline = expiry_deadline(ttl)
+        @mutex.synchronize do
+          expire_key(key)
+          return false if unless_exist && @entries.key?(key)
+
+          write_entry(key, value, deadline)
+        end
+        true
+      end
+
+      def cas(key, ttl: nil)
+        @mutex.synchronize do
+          expire_key(key)
+          value = yield @entries[key]
+          write_entry(key, value, expiry_deadline(ttl))
+          value
+        end
+      end
+
+      def mget(keys)
+        @mutex.synchronize do
+          keys.to_h do |key|
+            expire_key(key)
+            [key, @entries[key]]
+          end
+        end
+      end
+
+      def mset(values, ttl: nil)
+        deadline = expiry_deadline(ttl)
+        @mutex.synchronize { values.each { |key, value| write_entry(key, value, deadline) } }
+        true
+      end
+
+      def delete(key)
+        @mutex.synchronize do
+          expire_key(key)
+          present = @entries.key?(key)
+          @entries.delete(key)
+          @expiry.delete(key)
+          @condition.signal
+          present
+        end
+      end
+
+      def clear
+        worker = @mutex.synchronize do
+          @entries.clear
+          @expiry.clear
+          @next_expiry = nil
+          thread = @cleanup_thread
+          @cleanup_thread = nil
+          @condition.broadcast
+          thread
+        end
+        worker&.join
+        true
+      end
+
+      def close
+        clear
+      end
+
+      # Does not opportunistically expire entries: cleanup reclaims inactive keys.
+      def size
+        @mutex.synchronize { @entries.size }
+      end
+
+      private
+
+      def expiry_deadline(ttl)
+        return if ttl.nil? || ttl.zero?
+        raise ArgumentError, 'ttl must be a finite nonnegative number' unless ttl.is_a?(Numeric) && ttl.real? && ttl.finite? && ttl.positive?
+
+        monotonic_now + ttl
+      end
+
+      def monotonic_now
+        ::Process.clock_gettime(::Process::CLOCK_MONOTONIC)
+      end
+
+      def expire_key(key)
+        return unless @expiry.key?(key) && @expiry[key] <= monotonic_now
+
+        @entries.delete(key)
+        @expiry.delete(key)
+        @condition.signal
+      end
+
+      def write_entry(key, value, deadline)
+        if !@entries.key?(key) && @entries.size >= @max_entries
+          purge_expired if @next_expiry && @next_expiry <= monotonic_now
+          evict_oldest if @entries.size >= @max_entries
+        end
+        @entries[key] = value
+        if deadline
+          @expiry[key] = deadline
+          @next_expiry = deadline if @next_expiry.nil? || deadline < @next_expiry
+          @cleanup_thread ||= Thread.new { cleanup_loop }
+        else
+          @expiry.delete(key)
+        end
+        @condition.signal
+      end
+
+      def evict_oldest
+        key, = @entries.shift
+        @expiry.delete(key)
+        @condition.signal
+      end
+
+      def purge_expired
+        now = monotonic_now
+        @expiry.delete_if do |key, deadline|
+          next false if deadline > now
+
+          @entries.delete(key)
+          true
+        end
+        @next_expiry = @expiry.each_value.min
+      end
+
+      def cleanup_loop
+        @mutex.synchronize do
+          while @cleanup_thread.equal?(Thread.current)
+            purge_expired if @next_expiry && @next_expiry <= monotonic_now
+            break if @expiry.empty?
+
+            delay = @next_expiry - monotonic_now
+            @condition.wait(@mutex, delay) if delay.positive?
+          end
+          @cleanup_thread = nil if @cleanup_thread.equal?(Thread.current)
+        end
+      end
+    end
+
     module Memory
       extend self
       extend Legion::Logging::Helper
 
-      @store = {}
-      @expiry = {}
-      @mutex = Mutex.new
+      @store = ExpiringStore.new
       @connected = Concurrent::AtomicBoolean.new(false)
 
-      def setup(**)
+      def setup(max_entries: nil, **)
+        @store.max_entries = max_entries unless max_entries.nil?
         @connected.make_true
         log.info 'Legion::Cache::Memory connected'
         true
@@ -32,7 +197,7 @@ module Legion
 
       def restart(**)
         shutdown
-        setup
+        setup(**)
       rescue StandardError => e
         @connected.make_false
         handle_exception(e, level: :warn, handled: true, operation: :memory_restart)
@@ -40,12 +205,9 @@ module Legion
       end
 
       def get(key)
-        @mutex.synchronize do
-          expire_if_needed(key)
-          result = @store[key]
-          log.debug { "[cache:memory] GET #{key} hit=#{!result.nil?}" }
-          result
-        end
+        result = @store.get(key)
+        log.debug { "[cache:memory] GET #{key} hit=#{!result.nil?}" }
+        result
       rescue StandardError => e
         handle_exception(e, level: :warn, handled: true, operation: :memory_get)
         nil
@@ -55,50 +217,40 @@ module Legion
         set_sync(key, value, ttl: ttl, phi: phi)
       end
 
-      def set_nx(key, value, ttl: nil)
-        @mutex.synchronize do
-          expire_if_needed(key)
-          return false if @store.key?(key)
-
-          @store[key] = value
-          if ttl&.positive?
-            @expiry[key] = Time.now + ttl
-          else
-            @expiry.delete(key)
-          end
-          log.debug { "[cache:memory] SET_NX #{key} ttl=#{ttl.inspect} result=true" }
-          true
-        end
-      rescue StandardError => e
-        handle_exception(e, level: :warn, handled: true, operation: :memory_set_nx)
-        false
+      def set_nx(key, value, ttl: nil, phi: false)
+        ttl = enforce_phi_ttl(ttl) if phi
+        result = @store.set(key, value, ttl: ttl, unless_exist: true)
+        log.debug { "[cache:memory] SET_NX #{key} ttl=#{ttl.inspect} result=#{result}" }
+        result
       end
 
       def set_sync(key, value, ttl: nil, phi: false)
-        ttl = enforce_phi_ttl(ttl, phi: phi) if phi
-        @mutex.synchronize do
-          @store[key] = value
-          if ttl&.positive?
-            @expiry[key] = Time.now + ttl
-          else
-            @expiry.delete(key)
-          end
-          log.debug { "[cache:memory] SET #{key} ttl=#{ttl.inspect}" }
-          true
-        end
+        ttl = enforce_phi_ttl(ttl) if phi
+        @store.set(key, value, ttl: ttl)
+        log.debug { "[cache:memory] SET #{key} ttl=#{ttl.inspect}" }
+        true
+      end
+
+      def cas(key, ttl: nil, phi: false, &)
+        ttl = enforce_phi_ttl(ttl) if phi
+        value = @store.cas(key, ttl: ttl, &)
+        log.debug { "[cache:memory] CAS #{key} ttl=#{ttl.inspect}" }
+        value
       end
 
-      def fetch(key, ttl: nil, &block)
+      def fetch(key, ttl: nil, phi: false, &block)
+        ttl = enforce_phi_ttl(ttl) if phi
         val = get(key)
         return val unless val.nil?
 
         log.debug { "[cache:memory] FETCH #{key} miss=true" }
         val = block&.call
-        set(key, val, ttl: ttl)
+        begin
+          set(key, val, ttl: ttl)
+        rescue StandardError => e
+          handle_exception(e, level: :warn, handled: true, operation: :memory_fetch_write)
+        end
         val
-      rescue StandardError => e
-        handle_exception(e, level: :warn, handled: true, operation: :memory_fetch)
-        nil
       end
 
       def delete(key, async: true) # rubocop:disable Lint/UnusedMethodArgument
@@ -106,59 +258,29 @@ module Legion
       end
 
       def delete_sync(key)
-        @mutex.synchronize do
-          removed = @store.delete(key)
-          @expiry.delete(key)
-          log.debug { "[cache:memory] DELETE #{key} success=#{!removed.nil?}" }
-          !removed.nil?
-        end
+        removed = @store.delete(key)
+        log.debug { "[cache:memory] DELETE #{key} success=#{removed}" }
+        removed
       end
 
       def mget(*keys)
-        keys = keys.flatten
-        return {} if keys.empty?
-
-        @mutex.synchronize do
-          keys.each { |k| expire_if_needed(k) }
-          keys.to_h { |k| [k, @store[k]] }
-        end
+        @store.mget(keys.flatten)
       rescue StandardError => e
         handle_exception(e, level: :warn, handled: true, operation: :memory_mget)
         {}
       end
 
-      def mset(hash, ttl: nil, async: true)
-        return true if hash.empty?
-
-        hash.each { |k, v| set(k, v, ttl: ttl, async: async) }
-        true
-      rescue StandardError => e
-        handle_exception(e, level: :warn, handled: true, operation: :memory_mset)
-        true
+      def mset(hash, ttl: nil, async: true, phi: false) # rubocop:disable Lint/UnusedMethodArgument
+        mset_sync(hash, ttl: ttl, phi: phi)
       end
 
       def mset_sync(hash, ttl: nil, phi: false)
-        return true if hash.empty?
-
-        @mutex.synchronize do
-          hash.each do |key, value|
-            effective_ttl = phi ? enforce_phi_ttl(ttl, phi: true) : ttl
-            @store[key] = value
-            if effective_ttl&.positive?
-              @expiry[key] = Time.now + effective_ttl
-            else
-              @expiry.delete(key)
-            end
-          end
-          true
-        end
+        ttl = enforce_phi_ttl(ttl) if phi
+        @store.mset(hash, ttl: ttl)
       end
 
       def flush
-        @mutex.synchronize do
-          @store.clear
-          @expiry.clear
-        end
+        @store.clear
         log.info 'Legion::Cache::Memory flushed'
         true
       rescue StandardError => e
@@ -166,10 +288,12 @@ module Legion
         false
       end
 
-      def close = nil
+      def close
+        shutdown
+      end
 
       def shutdown
-        flush
+        @store.close
         @connected.make_false
         log.info 'Legion::Cache::Memory shut down'
         false
@@ -180,11 +304,9 @@ module Legion
       end
 
       def reset!
-        @mutex.synchronize do
-          @connected.make_false
-          @store.clear
-          @expiry.clear
-        end
+        @store.clear
+        @store.max_entries = ExpiringStore::DEFAULT_MAX_ENTRIES
+        @connected.make_false
         log.info 'Legion::Cache::Memory state reset'
         false
       end
@@ -194,24 +316,18 @@ module Legion
 
       private
 
-      def expire_if_needed(key)
-        return unless @expiry.key?(key) && Time.now > @expiry[key]
-
-        @store.delete(key)
-        @expiry.delete(key)
-        log.debug { "[cache:memory] EXPIRE #{key}" }
-      end
-
-      def enforce_phi_ttl(ttl, phi: false)
-        return ttl unless phi
-
+      def enforce_phi_ttl(ttl)
         max = if defined?(Legion::Settings)
-                Legion::Settings.dig(:cache, :compliance, :phi_max_ttl) || 3600
+                Legion::Settings.dig(:cache, :compliance, :phi_max_ttl)
               else
                 3600
               end
-        result = ttl.nil? ? max : [ttl, max].min
-        [result, 1].max
+        max = 3600 if max.nil?
+        raise ArgumentError, 'phi_max_ttl must be a finite positive number' unless max.is_a?(Numeric) && max.real? && max.finite? && max.positive?
+        return max if ttl.nil?
+        raise ArgumentError, 'ttl must be a finite number' unless ttl.is_a?(Numeric) && ttl.real? && ttl.finite?
+
+        ttl.positive? ? [ttl, max].min : max
       end
     end
   end

data/lib/legion/cache/pool.rb

@@ -1,6 +1,9 @@
 # frozen_string_literal: true
 
 require 'connection_pool'
+require 'concurrent'
+require 'timeout'
+require 'socket'
 require 'legion/logging/helper'
 
 module Legion
@@ -10,10 +13,7 @@ module Legion
       extend Legion::Logging::Helper
 
       def connected?
-        return false unless defined?(@connected)
-        return @connected.true? if @connected.respond_to?(:true?)
-
-        @connected == true
+        @connected&.value == true
       end
 
       def size
@@ -33,27 +33,44 @@ module Legion
       end
 
       def close
-        return unless @client
-
-        @client.shutdown(&:close)
+        pool = @client
         @client = nil
-        @connected = false
+        cache_mark_connected(false)
+        pool&.shutdown(&:close)
         log.info "#{pool_log_name} pool closed"
       end
 
-      def restart(**opts)
+      def restart(**)
         close
-        @client = nil
-        client_hash = opts
-        client_hash[:pool_size] = opts[:pool_size] if opts.key? :pool_size
-        client_hash[:timeout] = opts[:timeout] if opts.key? :timeout
-        client(**client_hash)
-        @connected = true
+        client(**)
         log.info "#{pool_log_name} pool restarted"
       end
 
       private
 
+      def cache_mark_connected(value)
+        @connected ||= Concurrent::AtomicBoolean.new(false)
+        @connected.value = value
+      end
+
+      def cache_backend_failed(error)
+        return unless cache_connection_error?(error)
+
+        cache_mark_connected(false)
+        @cache_failure_callback&.call(error)
+      end
+
+      def cache_connection_error?(error)
+        return true if error.is_a?(IOError) || error.is_a?(SystemCallError) || error.is_a?(Timeout::Error) || error.is_a?(SocketError)
+        return true if error.is_a?(ConnectionPool::TimeoutError)
+        return error.errors.values.any? { |nested| cache_connection_error?(nested) } if error.instance_of?(::Redis::Cluster::CommandErrorCollection)
+
+        error.class.ancestors.any? do |type|
+          %w[Dalli::NetworkError Dalli::RingError Redis::BaseConnectionError RedisClient::ConnectionError
+             Redis::Cluster::InitialSetupError Redis::Cluster::NodeMightBeDown].include?(type.name)
+        end
+      end
+
       def pool_log_name
         if respond_to?(:name)
           label = name.to_s

data/lib/legion/cache/reconnector.rb

@@ -10,6 +10,7 @@ module Legion
 
       DEFAULT_INITIAL_DELAY = 1
       DEFAULT_MAX_DELAY = 60
+      STOP_TIMEOUT = 1
 
       def initialize(tier:, connect_block:, enabled_block:, settings_key: :cache)
         @tier = tier
@@ -19,13 +20,17 @@ module Legion
         @attempts = Concurrent::AtomicFixnum.new(0)
         @thread = nil
         @mutex = Mutex.new
+        @wake = ConditionVariable.new
+        @stopping = false
         @stop_signal = Concurrent::AtomicBoolean.new(false)
         @next_retry_at = nil
       end
 
       def start
+        return unless configured_enabled?
+
         @mutex.synchronize do
-          return if running?
+          return if @stopping || running?
 
           @stop_signal.make_false
           @thread = Thread.new { reconnect_loop }
@@ -34,13 +39,25 @@ module Legion
       end
 
       def stop
-        thread_to_join = nil
-        @mutex.synchronize do
+        thread_to_join = @mutex.synchronize do
+          @stopping = true unless @thread == Thread.current
           @stop_signal.make_true
-          thread_to_join = @thread
-          @thread = nil
+          @wake.broadcast
+          @thread
+        end
+        Thread.exit if thread_to_join == Thread.current
+
+        if thread_to_join && !thread_to_join.join(STOP_TIMEOUT)
+          thread_to_join.kill
+          raise ThreadError, "Legion::Cache::Reconnector[#{@tier}] could not stop its connection callback" unless thread_to_join.join(STOP_TIMEOUT)
+        end
+
+        @mutex.synchronize do
+          if @thread == thread_to_join
+            @thread = nil
+            @stopping = false
+          end
         end
-        thread_to_join&.join(5)
         log.info "Legion::Cache::Reconnector[#{@tier}] stopped"
       end
 
@@ -60,19 +77,19 @@ module Legion
         delay = configured_initial_delay
 
         until @stop_signal.true?
-          unless @enabled_block.call
-            sleep 1
+          unless configured_enabled? && @enabled_block.call
+            wait_for_retry(1)
             next
           end
 
           begin
             @next_retry_at = Time.now + delay
-            sleep delay
-            return if @stop_signal.true?
+            return unless wait_for_retry(delay)
+            next unless configured_enabled? && @enabled_block.call
 
             @connect_block.call
             count = @attempts.value
-            @attempts = Concurrent::AtomicFixnum.new(0)
+            @attempts.value = 0
             @next_retry_at = nil
             log.info "Legion::Cache::Reconnector[#{@tier}] reconnected after #{count} attempts"
             return
@@ -86,6 +103,28 @@ module Legion
         end
       rescue StandardError => e
         handle_exception(e, level: :error, handled: true, operation: :"reconnector_#{@tier}_loop")
+      ensure
+        @next_retry_at = nil
+      end
+
+      def wait_for_retry(delay)
+        deadline = ::Process.clock_gettime(::Process::CLOCK_MONOTONIC) + delay
+        @mutex.synchronize do
+          until @stop_signal.true?
+            remaining = deadline - ::Process.clock_gettime(::Process::CLOCK_MONOTONIC)
+            return true unless remaining.positive?
+
+            @wake.wait(@mutex, remaining)
+          end
+        end
+        false
+      end
+
+      def configured_enabled?
+        return true unless defined?(Legion::Settings)
+
+        Legion::Settings.dig(@settings_key, :enabled) != false &&
+          Legion::Settings.dig(@settings_key, :reconnect, :enabled) != false
       end
 
       def configured_initial_delay