redis_queued_locks 1.8.0 → 1.9.0

data/lib/redis_queued_locks/swarm/acquirers.rb

@@ -0,0 +1,44 @@
+# frozen_string_literal: true
+
+# @api private
+# @since 1.9.0
+module RedisQueuedLocks::Swarm::Acquirers
+  class << self
+    # Returns the list of swarm acquirers stored as HASH.
+    # Format:
+    #   {
+    #     <acquirer id #1> => {
+    #       zombie: <Boolean>,
+    #       last_probe_time: <Time>,
+    #       last_probe_score: <Numeric>
+    #      },
+    #     <acquirer id #2> => {
+    #       zombie: <Boolean>,
+    #       last_probe_time: <Time>,
+    #       last_probe_score: <Numeric>
+    #      },
+    #     ...
+    #   }
+    # Liveness probe time is represented as a float value (Time.now.to_f initially).
+    #
+    # @param redis_client [RedisClient]
+    # @param zombie_ttl [Integer]
+    # @return [Hash<String,Hash<Symbol,Float|Time>>]
+    #
+    # @api private
+    # @since 1.9.0
+    def acquirers(redis_client, zombie_ttl)
+      redis_client.with do |rconn|
+        rconn.call('HGETALL', RedisQueuedLocks::Resource::SWARM_KEY).tap do |swarm_acqs|
+          swarm_acqs.transform_values! do |last_probe|
+            last_probe_score = last_probe.to_f
+            last_probe_time = Time.at(last_probe_score)
+            zombie_score = RedisQueuedLocks::Resource.calc_zombie_score(zombie_ttl / 1_000)
+            is_zombie = last_probe_score < zombie_score
+            { zombie: is_zombie, last_probe_time:, last_probe_score: }
+          end
+        end
+      end
+    end
+  end
+end

data/lib/redis_queued_locks/swarm/flush_zombies.rb

@@ -0,0 +1,133 @@
+# frozen_string_literal: true
+
+# @api private
+# @since 1.9.0
+class RedisQueuedLocks::Swarm::FlushZombies < RedisQueuedLocks::Swarm::SwarmElement::Isolated
+  class << self
+    # @param redis_client [RedisClient]
+    # @parma zombie_ttl [Integer]
+    # @param lock_scan_size [Integer]
+    # @param queue_scan_size [Integer]
+    # @return [
+    #   RedisQueuedLocks::Data[
+    #     ok: <Boolean>,
+    #     deleted_zombie_hosts: <Set<String>>,
+    #     deleted_zombie_acquiers: <Set<String>>,
+    #     deleted_zombie_locks: <Set<String>>
+    #   ]
+    # ]
+    #
+    # @api private
+    # @since 1.9.0
+    # rubocop:disable Metrics/MethodLength
+    def flush_zombies(
+      redis_client,
+      zombie_ttl,
+      lock_scan_size,
+      queue_scan_size
+    )
+      redis_client.with do |rconn|
+        # Step 1:
+        #   calculate zombie score (the time marker that shows acquirers that
+        #   have not announced live probes for a long time)
+        zombie_score = RedisQueuedLocks::Resource.calc_zombie_score(zombie_ttl / 1_000)
+
+        # Step 2: extract zombie acquirers from the swarm list
+        zombie_hosts = rconn.call('HGETALL', RedisQueuedLocks::Resource::SWARM_KEY)
+        zombie_hosts = zombie_hosts.each_with_object(Set.new) do |(hst_id, ts), zombies|
+          (zombies << hst_id) if (zombie_score > ts.to_f)
+        end
+
+        # Step X: exit if we have no any zombie acquirer
+        next RedisQueuedLocks::Data[
+          ok: true,
+          deleted_zombie_hosts: Set.new,
+          deleted_zombie_acquirers: Set.new,
+          deleted_zombie_locks: Set.new,
+        ] if zombie_hosts.empty?
+
+        # Step 3: find zombie locks held by zombies and delete them
+        # TODO: indexing (in order to prevent full database scan);
+        # NOTE: original redis does not support indexing so we need to use
+        #   internal data structers to simulate data indexing (such as sorted sets or lists);
+        zombie_locks = Set.new
+        zombie_acquiers = Set.new
+
+        rconn.scan(
+          'MATCH', RedisQueuedLocks::Resource::LOCK_PATTERN, count: lock_scan_size
+        ) do |lock_key|
+          acquier_id, host_id = rconn.call('HMGET', lock_key, 'acq_id', 'hst_id')
+          if zombie_hosts.include?(host_id)
+            zombie_locks << lock_key
+            zombie_acquiers << acquier_id
+          end
+        end
+        rconn.call('DEL', *zombie_locks) if zombie_locks.any?
+
+        # Step 4: find zombie requests => and drop them
+        # TODO: indexing (in order to prevent full database scan);
+        # NOTE: original redis does not support indexing so we need to use
+        #   internal data structers to simulate data indexing (such as sorted sets or lists);
+        rconn.scan(
+          'MATCH', RedisQueuedLocks::Resource::LOCK_QUEUE_PATTERN, count: queue_scan_size
+        ) do |lock_queue|
+          zombie_acquiers.each do |zombie_acquier|
+            rconn.call('ZREM', lock_queue, zombie_acquier)
+          end
+        end
+
+        # Step 5: drop zombies from the swarm
+        rconn.call('HDEL', RedisQueuedLocks::Resource::SWARM_KEY, *zombie_hosts)
+
+        # Step 6: inform about deleted zombies
+        RedisQueuedLocks::Data[
+          ok: true,
+          deleted_zombie_hosts: zombie_hosts,
+          deleted_zombie_acquiers: zombie_acquiers,
+          deleted_zombie_locks: zombie_locks
+        ]
+      end
+    end
+    # rubocop:enable Metrics/MethodLength
+  end
+
+  # @return [Boolean]
+  #
+  # @api private
+  # @since 1.9.0
+  def enabled?
+    rql_client.config[:swarm][:flush_zombies][:enabled_for_swarm]
+  end
+
+  # @return [void]
+  #
+  # @api private
+  # @since 1.9.0
+  def swarm!
+    @swarm_element = Ractor.new(
+      rql_client.config.slice_value('swarm.flush_zombies.redis_config'),
+      rql_client.config[:swarm][:flush_zombies][:zombie_ttl],
+      rql_client.config[:swarm][:flush_zombies][:zombie_lock_scan_size],
+      rql_client.config[:swarm][:flush_zombies][:zombie_queue_scan_size],
+      rql_client.config[:swarm][:flush_zombies][:zombie_flush_period]
+    ) do |rc, z_ttl, z_lss, z_qss, z_fl_prd|
+      RedisQueuedLocks::Swarm::FlushZombies.swarm_loop do
+        Thread.new do
+          redis_client = RedisQueuedLocks::Swarm::RedisClientBuilder.build(
+            pooled: rc['pooled'],
+            sentinel: rc['sentinel'],
+            config: rc['config'],
+            pool_config: rc['pool_config']
+          )
+
+          loop do
+            RedisQueuedLocks::Swarm::FlushZombies.flush_zombies(
+              redis_client, z_ttl, z_lss, z_qss
+            )
+            sleep(z_fl_prd)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/redis_queued_locks/swarm/probe_hosts.rb

@@ -0,0 +1,69 @@
+# frozen_string_literal: true
+
+# @api private
+# @since 1.9.0
+class RedisQueuedLocks::Swarm::ProbeHosts < RedisQueuedLocks::Swarm::SwarmElement::Threaded
+  class << self
+    # @param redis_client [RedisClient]
+    # @param uniq_identity [String]
+    # @return [
+    #   RedisQueuedLocks::Data[
+    #     ok: <Boolean>,
+    #     result: {
+    #       host_id1 <String> => score1 <String>,
+    #       host_id2 <String> => score2 <String>,
+    #       etc...
+    #     }
+    #   ]
+    # ]
+    #
+    # @api private
+    # @since 1.9.0
+    def probe_hosts(redis_client, uniq_identity)
+      possible_hosts = RedisQueuedLocks::Resource.possible_host_identifiers(uniq_identity)
+      probed_hosts = {}
+
+      redis_client.with do |rconn|
+        possible_hosts.each do |host_id|
+          rconn.call(
+            'HSET',
+            RedisQueuedLocks::Resource::SWARM_KEY,
+            host_id,
+            probe_score = Time.now.to_f
+          )
+          probed_hosts[host_id] = probe_score
+        end
+
+        RedisQueuedLocks::Data[ok: true, result: probed_hosts]
+      end
+    end
+  end
+
+  # @return [Boolean]
+  #
+  # @api private
+  # @since 1.9.0
+  def enabled?
+    rql_client.config[:swarm][:probe_hosts][:enabled_for_swarm]
+  end
+
+  # @return [Thread]
+  #
+  # @api private
+  # @since 1.9.0
+  def spawn_main_loop!
+    Thread.new do
+      redis_client = RedisQueuedLocks::Swarm::RedisClientBuilder.build(
+        pooled: rql_client.config[:swarm][:probe_hosts][:redis_config][:pooled],
+        sentinel: rql_client.config[:swarm][:probe_hosts][:redis_config][:sentinel],
+        config: rql_client.config[:swarm][:probe_hosts][:redis_config][:config],
+        pool_config: rql_client.config[:swarm][:probe_hosts][:redis_config][:pool_config]
+      )
+
+      loop do
+        RedisQueuedLocks::Swarm::ProbeHosts.probe_hosts(redis_client, rql_client.uniq_identity)
+        sleep(rql_client.config[:swarm][:probe_hosts][:probe_period])
+      end
+    end
+  end
+end

data/lib/redis_queued_locks/swarm/redis_client_builder.rb

@@ -0,0 +1,67 @@
+# frozen_string_literal: true
+
+# @api private
+# @since 1.9.0
+module RedisQueuedLocks::Swarm::RedisClientBuilder
+  class << self
+    # @option pooled [Boolean]
+    # @option sentinel [Boolean]
+    # @option config [Hash]
+    # @return [RedisClient]
+    #
+    # @api private
+    # @since 1.9.0
+    # rubocop:disable Style/RedundantAssignment
+    def build(pooled: false, sentinel: false, config: {}, pool_config: {})
+      config.transform_keys!(&:to_sym)
+      pool_config.transform_keys!(&:to_sym)
+
+      redis_config =
+        sentinel ? sentinel_config(config) : non_sentinel_config(config)
+      redis_client =
+        pooled ? pooled_client(redis_config, pool_config) : non_pooled_client(redis_config)
+
+      redis_client
+    end
+    # rubocop:enable Style/RedundantAssignment
+
+    private
+
+    # @param config [Hash]
+    # @return [RedisClient::Config]
+    #
+    # @api private
+    # @since 1.9.0
+    def sentinel_config(config)
+      RedisClient.sentinel(**config)
+    end
+
+    # @param config [Hash]
+    # @return [RedisClient::Config]
+    #
+    # @api private
+    # @since 1.9.0
+    def non_sentinel_config(config)
+      RedisClient.config(**config)
+    end
+
+    # @param redis_config [ReidsClient::Config]
+    # @param pool_config [Hash]
+    # @return [RedisClient]
+    #
+    # @api private
+    # @since 1.9.0
+    def pooled_client(redis_config, pool_config)
+      redis_config.new_pool(**pool_config)
+    end
+
+    # @param redis_config [ReidsClient::Config]
+    # @return [RedisClient]
+    #
+    # @api private
+    # @since 1.9.0
+    def non_pooled_client(redis_config)
+      redis_config.new_client
+    end
+  end
+end

data/lib/redis_queued_locks/swarm/supervisor.rb

@@ -0,0 +1,83 @@
+# frozen_string_literal: true
+
+# @api private
+# @since 1.9.0
+class RedisQueuedLocks::Swarm::Supervisor
+  # @since 1.9.0
+  include RedisQueuedLocks::Utilities
+
+  # @return [RedisQueuedLocks::Client]
+  #
+  # @api private
+  # @since 1.9.0
+  attr_reader :rql_client
+
+  # @return [Thread,NilClass]
+  #
+  # @api private
+  # @since 1.9.0
+  attr_reader :visor
+
+  # @return [Proc,NilClass]
+  #
+  # @api private
+  # @since 1.9.0
+  attr_reader :observable
+
+  # @return [void]
+  #
+  # @api private
+  # @since 1.9.0
+  def initialize(rql_client)
+    @rql_client = rql_client
+    @visor = nil
+    @observable = nil
+  end
+
+  # @param observable [Block]
+  # @return [void]
+  #
+  # @api private
+  # @since 1.9.0
+  def observe!(&observable)
+    @observable = observable
+    @visor = Thread.new do
+      loop do
+        yield rescue nil # TODO/CHECK: may be we need to process exceptions here
+        sleep(rql_client.config[:swarm][:supervisor][:liveness_probing_period])
+      end
+    end
+    # NOTE: need to give a timespot to initialize visor thread;
+    sleep(0.1)
+  end
+
+  # @return [Boolean]
+  #
+  # @api private
+  # @since 1.9.0
+  def running?
+    visor != nil && visor.alive?
+  end
+
+  # @return [void]
+  #
+  # @api private
+  # @since 1.9.0
+  def stop!
+    visor.kill if running?
+    @visor = nil
+    @observable = nil
+  end
+
+  # @return [Hash<Symbol|Hash<Symbol,String|Boolean>>]
+  #
+  # @api private
+  # @since 1.9.0
+  def status
+    {
+      running: running?,
+      state: (visor == nil) ? 'non_initialized' : thread_state(visor),
+      observable: (observable == nil) ? 'non_initialized' : 'initialized'
+    }
+  end
+end

data/lib/redis_queued_locks/swarm/swarm_element/isolated.rb

@@ -0,0 +1,287 @@
+# frozen_string_literal: true
+
+# @api private
+# @since 1.9.0
+# rubocop:disable Metrics/ClassLength
+class RedisQueuedLocks::Swarm::SwarmElement::Isolated
+  # @since 1.9.0
+  include RedisQueuedLocks::Utilities
+
+  # @return [RedisQueuedLocks::Client]
+  #
+  # @api private
+  # @since 1.9.0
+  attr_reader :rql_client
+
+  # @return [Ractor,NilClass]
+  #
+  # @api private
+  # @since 1.9.0
+  attr_reader :swarm_element
+
+  # @return [RedisQueuedLocks::Utilities::Lock]
+  #
+  # @api private
+  # @since 1.9.0
+  attr_reader :sync
+
+  # @return [void]
+  #
+  # @api private
+  # @since 1.9.0
+  def initialize(rql_client)
+    @rql_client = rql_client
+    @swarm_element = nil
+    @sync = RedisQueuedLocks::Utilities::Lock.new
+  end
+
+  # @return [void]
+  #
+  # @api private
+  # @since 1.9.0
+  def try_swarm!
+    return unless enabled?
+
+    sync.synchronize do
+      swarm_loop__kill
+      swarm!
+      swarm_loop__start
+    end
+  end
+
+  # @return [void]
+  #
+  # @api private
+  # @since 1.9.0
+  def reswarm_if_dead!
+    return unless enabled?
+
+    sync.synchronize do
+      if swarmed__stopped?
+        swarm_loop__start
+      elsif swarmed__dead? || idle?
+        swarm!
+        swarm_loop__start
+      end
+    end
+  end
+
+  # @return [void]
+  #
+  # @api private
+  # @since 1.9.0
+  def try_kill!
+    sync.synchronize do
+      swarm_loop__kill
+    end
+  end
+
+  # @return [Boolean]
+  #
+  # @api private
+  # @since 1.9.0
+  def enabled?
+    # NOTE: provide an <is enabled> logic here by analyzing the redis queued locks config.
+  end
+
+  # @return [Hash<Symbol,Boolean|Hash<Symbol,String|Boolean>>]
+  #   Format: {
+  #     enabled: <Boolean>,
+  #     ractor: {
+  #       running: <Boolean>,
+  #       state: <String>,
+  #     },
+  #     main_loop: {
+  #       running: <Boolean>,
+  #       state: <String>
+  #     }
+  #   }
+  #
+  # @api private
+  # @since 1.9.0
+  def status
+    sync.synchronize do
+      ractor_running = swarmed__alive?
+      ractor_state = swarmed? ? ractor_status(swarm_element) : 'non_initialized'
+
+      main_loop_running = nil
+      main_loop_state = nil
+      begin
+        main_loop_running = swarmed__running?
+        main_loop_state =
+          main_loop_running ? swarm_loop__status[:main_loop][:state] : 'non_initialized'
+      rescue Ractor::ClosedError
+        # NOTE: it can happend when you run RedisQueuedLocks::Swarm#deswarm!;
+        main_loop_running = false
+        main_loop_state = 'non_initialized'
+      end
+
+      {
+        enabled: enabled?,
+        ractor: {
+          running: ractor_running,
+          state: ractor_state
+        },
+        main_loop: {
+          running: main_loop_running,
+          state: main_loop_state
+        }
+      }
+    end
+  end
+
+  private
+
+  # Swarm element lifecycle have the following scheme:
+  # => 1) init (swarm!): create a ractor, main loop is not started;
+  # => 2) start (swarm_loop__start!): run main lopp inside the ractor;
+  # => 3) stop (swarm_loop__stop!): stop the main loop inside a ractor;
+  # => 4) kill (swarm_loop__kill!): kill the main loop inside teh ractor and kill a ractor;
+  #
+  # @return [void]
+  #
+  # @api private
+  # @since 1.9.0
+  def swarm!
+    # IMPORTANT №1: initialize @swarm_element here with Ractor;
+    # IMPORTANT №2: your Ractor should invoke .swarm_loop inside (see below);
+    # IMPORTANT №3: you should pass the main loop logic as a block to .swarm_loop;
+  end
+
+  # @param main_loop_spawner [Block]
+  # @return [void]
+  #
+  # @api private
+  # @since 1.9.0
+  # rubocop:disable Layout/ClassStructure, Lint/IneffectiveAccessModifier
+  def self.swarm_loop(&main_loop_spawner)
+    # NOTE:
+    #   This self.-related part of code is placed in the middle of class in order
+    #   to provide better code readability (it is placed next to the method inside
+    #   wich it should be called (see #swarm!)). That's why some rubocop cops are disabled.
+
+    main_loop = nil
+
+    loop do
+      command = Ractor.receive
+
+      case command
+      when :status
+        main_loop_alive = main_loop != nil && main_loop.alive?
+        main_loop_state =
+          if main_loop == nil
+            'non_initialized'
+          else
+            # NOTE: (full name resolution): ractor has no syntax-based constatnt context;
+            RedisQueuedLocks::Utilities.thread_state(main_loop)
+          end
+        Ractor.yield({ main_loop: { alive: main_loop_alive, state: main_loop_state } })
+      when :is_active
+        Ractor.yield(main_loop != nil && main_loop.alive?)
+      when :start
+        main_loop.kill if main_loop != nil
+        main_loop = yield # REFERENCE: `main_loop_spawner.call`
+      when :stop
+        main_loop.kill if main_loop != nil
+      when :kill
+        main_loop.kill if main_loop != nil
+        exit
+      end
+    end
+  end
+  # rubocop:enable Layout/ClassStructure, Lint/IneffectiveAccessModifier
+
+  # @return [Boolean]
+  #
+  # @api private
+  # @since 1.9.0
+  def idle?
+    swarm_element == nil
+  end
+
+  # @return [Boolean]
+  #
+  # @api private
+  # @since 1.9.0
+  def swarmed?
+    swarm_element != nil
+  end
+
+  # @return [Boolean]
+  #
+  # @api private
+  # @since 1.9.0
+  def swarmed__alive?
+    swarm_element != nil && ractor_alive?(swarm_element)
+  end
+
+  # @return [Boolean]
+  #
+  # @api private
+  # @since 1.9.0
+  def swarmed__dead?
+    swarm_element != nil && !ractor_alive?(swarm_element)
+  end
+
+  # @return [Boolean]
+  #
+  # @api private
+  # @since 1.9.0
+  def swarmed__running?
+    swarm_element != nil && ractor_alive?(swarm_element) && swarm_loop__is_active
+  end
+
+  # @return [Boolean]
+  #
+  # @api private
+  # @since 1.9.0
+  def swarmed__stopped?
+    swarm_element != nil && ractor_alive?(swarm_element) && !swarm_loop__is_active
+  end
+
+  # @return [Boolean]
+  #
+  # @api private
+  # @since 1.9.0
+  def swarm_loop__is_active
+    return if idle? || swarmed__dead?
+    sync.synchronize { swarm_element.send(:is_active).take }
+  end
+
+  # @return [Hash]
+  #
+  # @api private
+  # @since 1.9.0
+  def swarm_loop__status
+    return if idle? || swarmed__dead?
+    sync.synchronize { swarm_element.send(:status).take }
+  end
+
+  # @return [void]
+  #
+  # @api private
+  # @since 1.9.0
+  def swarm_loop__start
+    return if idle? || swarmed__dead?
+    sync.synchronize { swarm_element.send(:start) }
+  end
+
+  # @return [void]
+  #
+  # @api private
+  # @since 1.9.0
+  def swarm_loop__pause
+    return if idle? || swarmed__dead?
+    sync.synchronize { swarm_element.send(:stop) }
+  end
+
+  # @return [void]
+  #
+  # @api private
+  # @since 1.9.0
+  def swarm_loop__kill
+    return if idle? || swarmed__dead?
+    sync.synchronize { swarm_element.send(:kill) }
+  end
+end
+# rubocop:enable Metrics/ClassLength