redis_queued_locks 1.7.0 → 1.9.0

data/lib/redis_queued_locks/resource.rb

@@ -21,6 +21,12 @@ module RedisQueuedLocks::Resource
   # @since 1.0.0
   LOCK_QUEUE_PATTERN = 'rql:lock_queue:*'
 
+  # @return [String]
+  #
+  # @api private
+  # @since 1.9.0
+  SWARM_KEY = 'rql:swarm:hsts'
+
   # @return [Integer] Redis time error (in milliseconds).
   #
   # @api private
@@ -53,6 +59,22 @@ module RedisQueuedLocks::Resource
       "rql:acq:#{process_id}/#{thread_id}/#{fiber_id}/#{ractor_id}/#{identity}"
     end
 
+    # @param process_id [Integer,String]
+    # @param thread_id [Integer,String]
+    # @param ractor_id [Integer,String]
+    # @param identity [String]
+    # @return [String]
+    #
+    # @api private
+    # @since 1.9.0
+    def host_identifier(process_id, thread_id, ractor_id, identity)
+      # NOTE:
+      #   - fiber's object_id is not used cuz we can't analyze fiber objects via ObjectSpace
+      #     after the any new Ractor object is created in the current process
+      #     (ObjectSpace no longer sees objects of Thread and Fiber classes after that);
+      "rql:hst:#{process_id}/#{thread_id}/#{ractor_id}/#{identity}"
+    end
+
     # @param lock_name [String]
     # @return [String]
     #
@@ -88,8 +110,17 @@ module RedisQueuedLocks::Resource
       Time.now.to_f - queue_ttl
     end
 
+    # @param zombie_ttl [Float] In seconds with milliseconds.
+    # @return [Float]
+    #
+    # @api private
+    # @since 1.9.0
+    def calc_zombie_score(zombie_ttl)
+      Time.now.to_f - zombie_ttl
+    end
+
     # @param acquier_position [Float]
-    #   A time (epoch, seconds.microseconds) that represents
+    #   A time (epoch, seconds.milliseconds) that represents
     #   the acquier position in lock request queue.
     # @parma queue_ttl [Integer]
     #   In second.
@@ -143,5 +174,32 @@ module RedisQueuedLocks::Resource
     def get_process_id
       ::Process.pid
     end
+
+    # @return [Array<String>]
+    #
+    # @api private
+    # @since 1.9.0
+    def possible_host_identifiers(identity)
+      # NOTE №1: we can not use ObjectSpace.each_object for Thread and Fiber cuz after the any
+      #   ractor creation the ObjectSpace stops seeing ::Thread and ::Fiber objects: each_object
+      #   for each of them returns `0`;
+      # NOTE №2: we have no any approach to count Fiber objects in the current process without
+      #   object space API (or super memory-expensive) so host identification works without fibers;
+      # NOTE №3: we still can extract thread objects via Thread.list API;
+      current_process_id = get_process_id
+      current_threads = ::Thread.list
+      current_ractor_id = get_ractor_id
+
+      [].tap do |acquiers|
+        current_threads.each do |thread|
+          acquiers << host_identifier(
+            current_process_id,
+            thread.object_id,
+            current_ractor_id,
+            identity
+          )
+        end
+      end
+    end
   end
 end

data/lib/redis_queued_locks/swarm/acquirers.rb

@@ -0,0 +1,44 @@
+# frozen_string_literal: true
+
+# @api private
+# @since 1.9.0
+module RedisQueuedLocks::Swarm::Acquirers
+  class << self
+    # Returns the list of swarm acquirers stored as HASH.
+    # Format:
+    #   {
+    #     <acquirer id #1> => {
+    #       zombie: <Boolean>,
+    #       last_probe_time: <Time>,
+    #       last_probe_score: <Numeric>
+    #      },
+    #     <acquirer id #2> => {
+    #       zombie: <Boolean>,
+    #       last_probe_time: <Time>,
+    #       last_probe_score: <Numeric>
+    #      },
+    #     ...
+    #   }
+    # Liveness probe time is represented as a float value (Time.now.to_f initially).
+    #
+    # @param redis_client [RedisClient]
+    # @param zombie_ttl [Integer]
+    # @return [Hash<String,Hash<Symbol,Float|Time>>]
+    #
+    # @api private
+    # @since 1.9.0
+    def acquirers(redis_client, zombie_ttl)
+      redis_client.with do |rconn|
+        rconn.call('HGETALL', RedisQueuedLocks::Resource::SWARM_KEY).tap do |swarm_acqs|
+          swarm_acqs.transform_values! do |last_probe|
+            last_probe_score = last_probe.to_f
+            last_probe_time = Time.at(last_probe_score)
+            zombie_score = RedisQueuedLocks::Resource.calc_zombie_score(zombie_ttl / 1_000)
+            is_zombie = last_probe_score < zombie_score
+            { zombie: is_zombie, last_probe_time:, last_probe_score: }
+          end
+        end
+      end
+    end
+  end
+end

data/lib/redis_queued_locks/swarm/flush_zombies.rb

@@ -0,0 +1,133 @@
+# frozen_string_literal: true
+
+# @api private
+# @since 1.9.0
+class RedisQueuedLocks::Swarm::FlushZombies < RedisQueuedLocks::Swarm::SwarmElement::Isolated
+  class << self
+    # @param redis_client [RedisClient]
+    # @parma zombie_ttl [Integer]
+    # @param lock_scan_size [Integer]
+    # @param queue_scan_size [Integer]
+    # @return [
+    #   RedisQueuedLocks::Data[
+    #     ok: <Boolean>,
+    #     deleted_zombie_hosts: <Set<String>>,
+    #     deleted_zombie_acquiers: <Set<String>>,
+    #     deleted_zombie_locks: <Set<String>>
+    #   ]
+    # ]
+    #
+    # @api private
+    # @since 1.9.0
+    # rubocop:disable Metrics/MethodLength
+    def flush_zombies(
+      redis_client,
+      zombie_ttl,
+      lock_scan_size,
+      queue_scan_size
+    )
+      redis_client.with do |rconn|
+        # Step 1:
+        #   calculate zombie score (the time marker that shows acquirers that
+        #   have not announced live probes for a long time)
+        zombie_score = RedisQueuedLocks::Resource.calc_zombie_score(zombie_ttl / 1_000)
+
+        # Step 2: extract zombie acquirers from the swarm list
+        zombie_hosts = rconn.call('HGETALL', RedisQueuedLocks::Resource::SWARM_KEY)
+        zombie_hosts = zombie_hosts.each_with_object(Set.new) do |(hst_id, ts), zombies|
+          (zombies << hst_id) if (zombie_score > ts.to_f)
+        end
+
+        # Step X: exit if we have no any zombie acquirer
+        next RedisQueuedLocks::Data[
+          ok: true,
+          deleted_zombie_hosts: Set.new,
+          deleted_zombie_acquirers: Set.new,
+          deleted_zombie_locks: Set.new,
+        ] if zombie_hosts.empty?
+
+        # Step 3: find zombie locks held by zombies and delete them
+        # TODO: indexing (in order to prevent full database scan);
+        # NOTE: original redis does not support indexing so we need to use
+        #   internal data structers to simulate data indexing (such as sorted sets or lists);
+        zombie_locks = Set.new
+        zombie_acquiers = Set.new
+
+        rconn.scan(
+          'MATCH', RedisQueuedLocks::Resource::LOCK_PATTERN, count: lock_scan_size
+        ) do |lock_key|
+          acquier_id, host_id = rconn.call('HMGET', lock_key, 'acq_id', 'hst_id')
+          if zombie_hosts.include?(host_id)
+            zombie_locks << lock_key
+            zombie_acquiers << acquier_id
+          end
+        end
+        rconn.call('DEL', *zombie_locks) if zombie_locks.any?
+
+        # Step 4: find zombie requests => and drop them
+        # TODO: indexing (in order to prevent full database scan);
+        # NOTE: original redis does not support indexing so we need to use
+        #   internal data structers to simulate data indexing (such as sorted sets or lists);
+        rconn.scan(
+          'MATCH', RedisQueuedLocks::Resource::LOCK_QUEUE_PATTERN, count: queue_scan_size
+        ) do |lock_queue|
+          zombie_acquiers.each do |zombie_acquier|
+            rconn.call('ZREM', lock_queue, zombie_acquier)
+          end
+        end
+
+        # Step 5: drop zombies from the swarm
+        rconn.call('HDEL', RedisQueuedLocks::Resource::SWARM_KEY, *zombie_hosts)
+
+        # Step 6: inform about deleted zombies
+        RedisQueuedLocks::Data[
+          ok: true,
+          deleted_zombie_hosts: zombie_hosts,
+          deleted_zombie_acquiers: zombie_acquiers,
+          deleted_zombie_locks: zombie_locks
+        ]
+      end
+    end
+    # rubocop:enable Metrics/MethodLength
+  end
+
+  # @return [Boolean]
+  #
+  # @api private
+  # @since 1.9.0
+  def enabled?
+    rql_client.config[:swarm][:flush_zombies][:enabled_for_swarm]
+  end
+
+  # @return [void]
+  #
+  # @api private
+  # @since 1.9.0
+  def swarm!
+    @swarm_element = Ractor.new(
+      rql_client.config.slice_value('swarm.flush_zombies.redis_config'),
+      rql_client.config[:swarm][:flush_zombies][:zombie_ttl],
+      rql_client.config[:swarm][:flush_zombies][:zombie_lock_scan_size],
+      rql_client.config[:swarm][:flush_zombies][:zombie_queue_scan_size],
+      rql_client.config[:swarm][:flush_zombies][:zombie_flush_period]
+    ) do |rc, z_ttl, z_lss, z_qss, z_fl_prd|
+      RedisQueuedLocks::Swarm::FlushZombies.swarm_loop do
+        Thread.new do
+          redis_client = RedisQueuedLocks::Swarm::RedisClientBuilder.build(
+            pooled: rc['pooled'],
+            sentinel: rc['sentinel'],
+            config: rc['config'],
+            pool_config: rc['pool_config']
+          )
+
+          loop do
+            RedisQueuedLocks::Swarm::FlushZombies.flush_zombies(
+              redis_client, z_ttl, z_lss, z_qss
+            )
+            sleep(z_fl_prd)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/redis_queued_locks/swarm/probe_hosts.rb

@@ -0,0 +1,69 @@
+# frozen_string_literal: true
+
+# @api private
+# @since 1.9.0
+class RedisQueuedLocks::Swarm::ProbeHosts < RedisQueuedLocks::Swarm::SwarmElement::Threaded
+  class << self
+    # @param redis_client [RedisClient]
+    # @param uniq_identity [String]
+    # @return [
+    #   RedisQueuedLocks::Data[
+    #     ok: <Boolean>,
+    #     result: {
+    #       host_id1 <String> => score1 <String>,
+    #       host_id2 <String> => score2 <String>,
+    #       etc...
+    #     }
+    #   ]
+    # ]
+    #
+    # @api private
+    # @since 1.9.0
+    def probe_hosts(redis_client, uniq_identity)
+      possible_hosts = RedisQueuedLocks::Resource.possible_host_identifiers(uniq_identity)
+      probed_hosts = {}
+
+      redis_client.with do |rconn|
+        possible_hosts.each do |host_id|
+          rconn.call(
+            'HSET',
+            RedisQueuedLocks::Resource::SWARM_KEY,
+            host_id,
+            probe_score = Time.now.to_f
+          )
+          probed_hosts[host_id] = probe_score
+        end
+
+        RedisQueuedLocks::Data[ok: true, result: probed_hosts]
+      end
+    end
+  end
+
+  # @return [Boolean]
+  #
+  # @api private
+  # @since 1.9.0
+  def enabled?
+    rql_client.config[:swarm][:probe_hosts][:enabled_for_swarm]
+  end
+
+  # @return [Thread]
+  #
+  # @api private
+  # @since 1.9.0
+  def spawn_main_loop!
+    Thread.new do
+      redis_client = RedisQueuedLocks::Swarm::RedisClientBuilder.build(
+        pooled: rql_client.config[:swarm][:probe_hosts][:redis_config][:pooled],
+        sentinel: rql_client.config[:swarm][:probe_hosts][:redis_config][:sentinel],
+        config: rql_client.config[:swarm][:probe_hosts][:redis_config][:config],
+        pool_config: rql_client.config[:swarm][:probe_hosts][:redis_config][:pool_config]
+      )
+
+      loop do
+        RedisQueuedLocks::Swarm::ProbeHosts.probe_hosts(redis_client, rql_client.uniq_identity)
+        sleep(rql_client.config[:swarm][:probe_hosts][:probe_period])
+      end
+    end
+  end
+end

data/lib/redis_queued_locks/swarm/redis_client_builder.rb

@@ -0,0 +1,67 @@
+# frozen_string_literal: true
+
+# @api private
+# @since 1.9.0
+module RedisQueuedLocks::Swarm::RedisClientBuilder
+  class << self
+    # @option pooled [Boolean]
+    # @option sentinel [Boolean]
+    # @option config [Hash]
+    # @return [RedisClient]
+    #
+    # @api private
+    # @since 1.9.0
+    # rubocop:disable Style/RedundantAssignment
+    def build(pooled: false, sentinel: false, config: {}, pool_config: {})
+      config.transform_keys!(&:to_sym)
+      pool_config.transform_keys!(&:to_sym)
+
+      redis_config =
+        sentinel ? sentinel_config(config) : non_sentinel_config(config)
+      redis_client =
+        pooled ? pooled_client(redis_config, pool_config) : non_pooled_client(redis_config)
+
+      redis_client
+    end
+    # rubocop:enable Style/RedundantAssignment
+
+    private
+
+    # @param config [Hash]
+    # @return [RedisClient::Config]
+    #
+    # @api private
+    # @since 1.9.0
+    def sentinel_config(config)
+      RedisClient.sentinel(**config)
+    end
+
+    # @param config [Hash]
+    # @return [RedisClient::Config]
+    #
+    # @api private
+    # @since 1.9.0
+    def non_sentinel_config(config)
+      RedisClient.config(**config)
+    end
+
+    # @param redis_config [ReidsClient::Config]
+    # @param pool_config [Hash]
+    # @return [RedisClient]
+    #
+    # @api private
+    # @since 1.9.0
+    def pooled_client(redis_config, pool_config)
+      redis_config.new_pool(**pool_config)
+    end
+
+    # @param redis_config [ReidsClient::Config]
+    # @return [RedisClient]
+    #
+    # @api private
+    # @since 1.9.0
+    def non_pooled_client(redis_config)
+      redis_config.new_client
+    end
+  end
+end

data/lib/redis_queued_locks/swarm/supervisor.rb

@@ -0,0 +1,83 @@
+# frozen_string_literal: true
+
+# @api private
+# @since 1.9.0
+class RedisQueuedLocks::Swarm::Supervisor
+  # @since 1.9.0
+  include RedisQueuedLocks::Utilities
+
+  # @return [RedisQueuedLocks::Client]
+  #
+  # @api private
+  # @since 1.9.0
+  attr_reader :rql_client
+
+  # @return [Thread,NilClass]
+  #
+  # @api private
+  # @since 1.9.0
+  attr_reader :visor
+
+  # @return [Proc,NilClass]
+  #
+  # @api private
+  # @since 1.9.0
+  attr_reader :observable
+
+  # @return [void]
+  #
+  # @api private
+  # @since 1.9.0
+  def initialize(rql_client)
+    @rql_client = rql_client
+    @visor = nil
+    @observable = nil
+  end
+
+  # @param observable [Block]
+  # @return [void]
+  #
+  # @api private
+  # @since 1.9.0
+  def observe!(&observable)
+    @observable = observable
+    @visor = Thread.new do
+      loop do
+        yield rescue nil # TODO/CHECK: may be we need to process exceptions here
+        sleep(rql_client.config[:swarm][:supervisor][:liveness_probing_period])
+      end
+    end
+    # NOTE: need to give a timespot to initialize visor thread;
+    sleep(0.1)
+  end
+
+  # @return [Boolean]
+  #
+  # @api private
+  # @since 1.9.0
+  def running?
+    visor != nil && visor.alive?
+  end
+
+  # @return [void]
+  #
+  # @api private
+  # @since 1.9.0
+  def stop!
+    visor.kill if running?
+    @visor = nil
+    @observable = nil
+  end
+
+  # @return [Hash<Symbol|Hash<Symbol,String|Boolean>>]
+  #
+  # @api private
+  # @since 1.9.0
+  def status
+    {
+      running: running?,
+      state: (visor == nil) ? 'non_initialized' : thread_state(visor),
+      observable: (observable == nil) ? 'non_initialized' : 'initialized'
+    }
+  end
+end