karafka 2.2.13 → 2.3.0.alpha1

data/lib/karafka/routing/features/dead_letter_queue/topic.rb

@@ -14,12 +14,19 @@ module Karafka
           # @param max_retries [Integer] after how many retries should we move data to dlq
           # @param topic [String, false] where the messages should be moved if failing or false
           #   if we do not want to move it anywhere and just skip
+          # @param independent [Boolean] needs to be true in order for each marking as consumed
+          #   in a retry flow to reset the errors counter
           # @return [Config] defined config
-          def dead_letter_queue(max_retries: DEFAULT_MAX_RETRIES, topic: nil)
+          def dead_letter_queue(
+            max_retries: DEFAULT_MAX_RETRIES,
+            topic: nil,
+            independent: false
+          )
             @dead_letter_queue ||= Config.new(
               active: !topic.nil?,
               max_retries: max_retries,
-              topic: topic
+              topic: topic,
+              independent: independent
             )
           end
 

data/lib/karafka/routing/proxy.rb

@@ -10,11 +10,12 @@ module Karafka
       # @param target [Object] target object to which we proxy any DSL call
       # @param defaults [Proc] defaults for target that should be applicable after the proper
       #   proxy context (if needed)
-      # @param block [Proc] block that we want to evaluate in the proxy context
+      # @param block [Proc, nil] block that we want to evaluate in the proxy context or nil if no
+      #   proxy block context for example because whole context is taken from defaults
       def initialize(target, defaults = ->(_) {}, &block)
         @target = target
-        instance_eval(&block)
-        instance_eval(&defaults)
+        instance_eval(&block) if block
+        instance_eval(&defaults) if defaults
       end
 
       # Ruby 2.7.0 to 2.7.2 do not have arg forwarding, so we fallback to the old way

data/lib/karafka/routing/subscription_group.rb

@@ -37,7 +37,8 @@ module Karafka
       # @param topics [Karafka::Routing::Topics] all the topics that share the same key settings
       # @return [SubscriptionGroup] built subscription group
       def initialize(position, topics)
-        @name = topics.first.subscription_group_name
+        @details = topics.first.subscription_group_details
+        @name = @details.fetch(:name)
         @consumer_group = topics.first.consumer_group
         # We include the consumer group id here because we want to have unique ids of subscription
         # groups across the system. Otherwise user could set the same name for multiple
@@ -47,7 +48,6 @@ module Karafka
         @position = position
         @topics = topics
         @kafka = build_kafka
-        freeze
       end
 
       # @return [String] consumer group id

data/lib/karafka/routing/subscription_groups_builder.rb

@@ -19,7 +19,7 @@ module Karafka
         max_messages
         max_wait_time
         initial_offset
-        subscription_group_name
+        subscription_group_details
       ].freeze
 
       private_constant :DISTRIBUTION_KEYS
@@ -37,7 +37,7 @@ module Karafka
           .group_by(&:first)
           .values
           .map { |value| value.map(&:last) }
-          .map { |topics_array| Routing::Topics.new(topics_array) }
+          .flat_map { |value| expand(value) }
           .map { |grouped_topics| SubscriptionGroup.new(@position += 1, grouped_topics) }
           .tap do |subscription_groups|
             subscription_groups.each do |subscription_group|
@@ -60,6 +60,15 @@ module Karafka
 
         accu.hash
       end
+
+      # Hook for optional expansion of groups based on subscription group features
+      #
+      # @param topics_array [Array<Routing::Topic>] group of topics that have the same settings
+      #   and can use the same connection
+      # @return [Array<Array<Routing::Topics>>] expanded groups
+      def expand(topics_array)
+        [Routing::Topics.new(topics_array)]
+      end
     end
   end
 end

data/lib/karafka/routing/topic.rb

@@ -9,7 +9,7 @@ module Karafka
       attr_reader :id, :name, :consumer_group
       attr_writer :consumer
 
-      attr_accessor :subscription_group_name
+      attr_accessor :subscription_group_details
 
       # Full subscription group reference can be built only when we have knowledge about the
       # whole routing tree, this is why it is going to be set later on
@@ -46,15 +46,13 @@ module Karafka
       INHERITABLE_ATTRIBUTES.each do |attribute|
         attr_writer attribute
 
-        define_method attribute do
-          current_value = instance_variable_get(:"@#{attribute}")
+        class_eval <<~RUBY, __FILE__, __LINE__ + 1
+          def #{attribute}
+            return @#{attribute} unless @#{attribute}.nil?
 
-          return current_value unless current_value.nil?
-
-          value = Karafka::App.config.send(attribute)
-
-          instance_variable_set(:"@#{attribute}", value)
-        end
+            @#{attribute} = Karafka::App.config.send(:#{attribute})
+          end
+        RUBY
       end
 
       # @return [String] name of subscription that will go to librdkafka
@@ -117,7 +115,7 @@ module Karafka
           active: active?,
           consumer: consumer,
           consumer_group_id: consumer_group.id,
-          subscription_group_name: subscription_group_name
+          subscription_group_details: subscription_group_details
         ).freeze
       end
     end

data/lib/karafka/routing/topics.rb

@@ -9,7 +9,7 @@ module Karafka
       include Enumerable
       extend Forwardable
 
-      def_delegators :@accumulator, :[], :size, :empty?, :last, :<<
+      def_delegators :@accumulator, :[], :size, :empty?, :last, :<<, :map!, :sort_by!, :reverse!
 
       # @param topics_array [Array<Karafka::Routing::Topic>] array with topics
       def initialize(topics_array)

data/lib/karafka/runner.rb

@@ -3,6 +3,11 @@
 module Karafka
   # Class used to run the Karafka listeners in separate threads
   class Runner
+    def initialize
+      @manager = App.config.internal.connection.manager
+      @conductor = App.config.internal.connection.conductor
+    end
+
     # Starts listening on all the listeners asynchronously and handles the jobs queue closing
     # after listeners are done with their work.
     def call
@@ -13,16 +18,21 @@ module Karafka
       workers = Processing::WorkersBatch.new(jobs_queue)
       listeners = Connection::ListenersBatch.new(jobs_queue)
 
+      # Register all the listeners so they can be started and managed
+      @manager.register(listeners)
+
       workers.each(&:async_call)
-      listeners.each(&:async_call)
 
       # We aggregate threads here for a supervised shutdown process
       Karafka::Server.workers = workers
       Karafka::Server.listeners = listeners
       Karafka::Server.jobs_queue = jobs_queue
 
-      # All the listener threads need to finish
-      listeners.each(&:join)
+      until @manager.done?
+        @conductor.wait
+
+        @manager.control
+      end
 
       # We close the jobs queue only when no listener threads are working.
       # This ensures, that everything was closed prior to us not accepting anymore jobs and that

data/lib/karafka/server.rb

@@ -88,7 +88,10 @@ module Karafka
         # their work and if so, we can just return and normal shutdown process will take place
         # We divide it by 1000 because we use time in ms.
         ((timeout / 1_000) * SUPERVISION_CHECK_FACTOR).to_i.times do
-          return if listeners.count(&:alive?).zero? && workers.count(&:alive?).zero?
+          all_listeners_stopped = listeners.all?(&:stopped?)
+          all_workers_stopped = workers.none?(&:alive?)
+
+          return if all_listeners_stopped && all_workers_stopped
 
           sleep SUPERVISION_SLEEP
         end
@@ -104,7 +107,7 @@ module Karafka
 
         # We're done waiting, lets kill them!
         workers.each(&:terminate)
-        listeners.each(&:terminate)
+        listeners.active.each(&:terminate)
         # We always need to shutdown clients to make sure we do not force the GC to close consumer.
         # This can cause memory leaks and crashes.
         listeners.each(&:shutdown)
@@ -137,13 +140,6 @@ module Karafka
         # We don't have to safe-guard it with check states as the state transitions work only
         # in one direction
         Karafka::App.quiet!
-
-        # We need one more thread to monitor the process and move to quieted once everything
-        # is quiet and no processing is happening anymore
-        Thread.new do
-          sleep(0.1) until listeners.coordinators.all?(&:finished?)
-          Karafka::App.quieted!
-        end
       end
 
       private

data/lib/karafka/setup/config.rb

@@ -184,8 +184,23 @@ module Karafka
 
         # Namespace for internal connection related settings
         setting :connection do
+          # Manages starting up and stopping Kafka connections
+          setting :manager, default: Connection::Manager.new
+          # Controls frequency of connections management checks
+          setting :conductor, default: Connection::Conductor.new
+
           # Settings that are altered by our client proxy layer
           setting :proxy do
+            # Committed offsets for given CG query
+            setting :committed do
+              # timeout for this request. For busy or remote clusters, this should be high enough
+              setting :timeout, default: 5_000
+              # How many times should we try to run this call before raising an error
+              setting :max_attempts, default: 3
+              # How long should we wait before next attempt in case of a failure
+              setting :wait_time, default: 1_000
+            end
+
             # Watermark offsets request settings
             setting :query_watermark_offsets do
               # timeout for this request. For busy or remote clusters, this should be high enough
@@ -211,7 +226,7 @@ module Karafka
         setting :processing do
           setting :jobs_queue_class, default: Processing::JobsQueue
           # option scheduler [Object] scheduler we will be using
-          setting :scheduler_class, default: Processing::Scheduler
+          setting :scheduler_class, default: Processing::Schedulers::Default
           # option jobs_builder [Object] jobs builder we want to use
           setting :jobs_builder, default: Processing::JobsBuilder.new
           # option coordinator [Class] work coordinator we want to user for processing coordination
@@ -222,6 +237,8 @@ module Karafka
           setting :strategy_selector, default: Processing::StrategySelector.new
           # option expansions_selector [Object] processing expansions selector to be used
           setting :expansions_selector, default: Processing::ExpansionsSelector.new
+          # option [Class] executor class
+          setting :executor_class, default: Processing::Executor
         end
 
         # Things related to operating on messages
@@ -278,6 +295,9 @@ module Karafka
           # are also configured
           Pro::Loader.post_setup_all(config) if Karafka.pro?
 
+          # Subscribe the assignments tracker so we can always query all current assignments
+          config.monitor.subscribe(Instrumentation::AssignmentsTracker.instance)
+
           Karafka::App.initialized!
         end
 

data/lib/karafka/status.rb

@@ -42,25 +42,34 @@ module Karafka
     end
 
     STATES.each do |state, transition|
-      define_method :"#{state}?" do
-        @status == state
-      end
+      class_eval <<~RUBY, __FILE__, __LINE__ + 1
+        def #{state}?
+          @status == :#{state}
+        end
+
+        def #{transition}
+          MUTEX.synchronize do
+            # Do not allow reverse state transitions (we always go one way) or transition to the same
+            # state as currently
+            return if @status && STATES.keys.index(:#{state}) <= STATES.keys.index(@status)
 
-      define_method transition do
-        MUTEX.synchronize do
-          # Do not allow reverse state transitions (we always go one way) or transition to the same
-          # state as currently
-          return if @status && STATES.keys.index(state) <= STATES.keys.index(@status)
+            @status = :#{state}
 
-          @status = state
+            # Skip on creation (initializing)
+            # We skip as during this state we do not have yet a monitor
+            return if initializing?
 
-          # Skip on creation (initializing)
-          # We skip as during this state we do not have yet a monitor
-          return if initializing?
+            # We do not set conductor in the initializer because this status object is created
+            # before the configuration kicks in
+            # We need to signal conductor on each state change as those may be relevant to
+            # listeners operations
+            @conductor ||= Karafka::App.config.internal.connection.conductor
+            @conductor.signal
 
-          Karafka.monitor.instrument("app.#{state}")
+            Karafka.monitor.instrument("app.#{state}")
+          end
         end
-      end
+      RUBY
     end
 
     # @return [Boolean] true if we are in any of the status that would indicate we should no longer

data/lib/karafka/templates/karafka.rb.erb

@@ -68,3 +68,10 @@ class KarafkaApp < Karafka::App
     end
   end
 end
+
+# Karafka now features a Web UI!
+# Visit the setup documentation to get started and enhance your experience.
+#
+# https://karafka.io/docs/Web-UI-Getting-Started
+#
+# Karafka::Web.enable!

data/lib/karafka/time_trackers/partition_usage.rb

@@ -0,0 +1,56 @@
+# frozen_string_literal: true
+
+module Karafka
+  module TimeTrackers
+    # Tracker used to keep time reference when we last time dispatched any job related to a given
+    # topic partition.
+    #
+    # We can use it to know when last time a job was scheduled
+    #
+    # @note We do not track revocation as on revocation we clear given topic partition reference
+    #   not to have a potential memory leak
+    #
+    # @note We do not track shutdown jobs as shutdown is finishing the process, so no time
+    #   sensitive operations remain that would use this
+    #
+    # @note We consider partition as active if we scheduled any job related to it within the tick
+    #   interval. This has nothing to do whether a partition is assigned.
+    class PartitionUsage < Base
+      # Creates new partition usage time tracker
+      def initialize
+        super
+
+        @last_usage = Hash.new do |topics_hash, topic_name|
+          topics_hash[topic_name] = Hash.new do |partitions_hash, partition_id|
+            partitions_hash[partition_id] = 0
+          end
+        end
+      end
+
+      # @param topic [String]
+      # @param partition [Integer]
+      # @param interval [Integer] minimum interval
+      # @return [Boolean] was this topic partition active
+      def active?(topic, partition, interval)
+        monotonic_now - @last_usage[topic][partition] < interval
+      end
+
+      # Marks usage of given partition
+      #
+      # @param topic [String]
+      # @param partition [Integer]
+      def track(topic, partition)
+        @last_usage[topic][partition] = monotonic_now
+      end
+
+      # Clears references about given partition. Useful on revocation so we do not store old
+      # unassigned partitions data
+      #
+      # @param topic [String]
+      # @param partition [Integer]
+      def revoke(topic, partition)
+        @last_usage[topic].delete(partition)
+      end
+    end
+  end
+end

data/lib/karafka/version.rb

@@ -3,5 +3,5 @@
 # Main module namespace
 module Karafka
   # Current Karafka version
-  VERSION = '2.2.13'
+  VERSION = '2.3.0.alpha1'
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: karafka
 version: !ruby/object:Gem::Version
-  version: 2.2.13
+  version: 2.3.0.alpha1
 platform: ruby
 authors:
 - Maciej Mensfeld
@@ -35,7 +35,7 @@ cert_chain:
   AnG1dJU+yL2BK7vaVytLTstJME5mepSZ46qqIJXMuWob/YPDmVaBF39TDSG9e34s
   msG3BiCqgOgHAnL23+CN3Rt8MsuRfEtoTKpJVcCfoEoNHOkc
   -----END CERTIFICATE-----
-date: 2023-11-17 00:00:00.000000000 Z
+date: 2024-01-15 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: karafka-core
@@ -43,27 +43,27 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 2.2.7
+        version: 2.3.0.alpha1
     - - "<"
       - !ruby/object:Gem::Version
-        version: 2.3.0
+        version: 2.4.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 2.2.7
+        version: 2.3.0.alpha1
     - - "<"
       - !ruby/object:Gem::Version
-        version: 2.3.0
+        version: 2.4.0
 - !ruby/object:Gem::Dependency
   name: waterdrop
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 2.6.11
+        version: 2.6.12
     - - "<"
       - !ruby/object:Gem::Version
         version: 3.0.0
@@ -73,7 +73,7 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 2.6.11
+        version: 2.6.12
     - - "<"
       - !ruby/object:Gem::Version
         version: 3.0.0
@@ -123,6 +123,7 @@ files:
 - LICENSE-COMM
 - LICENSE-LGPL
 - README.md
+- SECURITY.md
 - bin/benchmarks
 - bin/create_token
 - bin/integrations
@@ -151,6 +152,7 @@ files:
 - lib/karafka/active_job/job_extensions.rb
 - lib/karafka/active_job/job_options_contract.rb
 - lib/karafka/admin.rb
+- lib/karafka/admin/acl.rb
 - lib/karafka/app.rb
 - lib/karafka/base_consumer.rb
 - lib/karafka/cli.rb
@@ -162,14 +164,16 @@ files:
 - lib/karafka/cli/server.rb
 - lib/karafka/cli/topics.rb
 - lib/karafka/connection/client.rb
-- lib/karafka/connection/consumer_group_coordinator.rb
+- lib/karafka/connection/conductor.rb
 - lib/karafka/connection/listener.rb
 - lib/karafka/connection/listeners_batch.rb
+- lib/karafka/connection/manager.rb
 - lib/karafka/connection/messages_buffer.rb
 - lib/karafka/connection/pauses_manager.rb
 - lib/karafka/connection/proxy.rb
 - lib/karafka/connection/raw_messages_buffer.rb
 - lib/karafka/connection/rebalance_manager.rb
+- lib/karafka/connection/status.rb
 - lib/karafka/contracts.rb
 - lib/karafka/contracts/base.rb
 - lib/karafka/contracts/config.rb
@@ -183,6 +187,7 @@ files:
 - lib/karafka/helpers/colorize.rb
 - lib/karafka/helpers/interval_runner.rb
 - lib/karafka/helpers/multi_delegator.rb
+- lib/karafka/instrumentation/assignments_tracker.rb
 - lib/karafka/instrumentation/callbacks/error.rb
 - lib/karafka/instrumentation/callbacks/rebalance.rb
 - lib/karafka/instrumentation/callbacks/statistics.rb
@@ -216,10 +221,13 @@ files:
 - lib/karafka/pro/active_job/consumer.rb
 - lib/karafka/pro/active_job/dispatcher.rb
 - lib/karafka/pro/active_job/job_options_contract.rb
+- lib/karafka/pro/base_consumer.rb
 - lib/karafka/pro/cleaner.rb
 - lib/karafka/pro/cleaner/errors.rb
 - lib/karafka/pro/cleaner/messages/message.rb
 - lib/karafka/pro/cleaner/messages/messages.rb
+- lib/karafka/pro/connection/manager.rb
+- lib/karafka/pro/connection/multiplexing/listener.rb
 - lib/karafka/pro/contracts/base.rb
 - lib/karafka/pro/contracts/server_cli_options.rb
 - lib/karafka/pro/encryption.rb
@@ -229,13 +237,15 @@ files:
 - lib/karafka/pro/encryption/messages/middleware.rb
 - lib/karafka/pro/encryption/messages/parser.rb
 - lib/karafka/pro/encryption/setup/config.rb
+- lib/karafka/pro/instrumentation/performance_tracker.rb
 - lib/karafka/pro/iterator.rb
 - lib/karafka/pro/iterator/expander.rb
 - lib/karafka/pro/iterator/tpl_builder.rb
 - lib/karafka/pro/loader.rb
-- lib/karafka/pro/performance_tracker.rb
 - lib/karafka/pro/processing/collapser.rb
 - lib/karafka/pro/processing/coordinator.rb
+- lib/karafka/pro/processing/executor.rb
+- lib/karafka/pro/processing/expansions_selector.rb
 - lib/karafka/pro/processing/filters/base.rb
 - lib/karafka/pro/processing/filters/delayer.rb
 - lib/karafka/pro/processing/filters/expirer.rb
@@ -244,11 +254,17 @@ files:
 - lib/karafka/pro/processing/filters/virtual_limiter.rb
 - lib/karafka/pro/processing/filters_applier.rb
 - lib/karafka/pro/processing/jobs/consume_non_blocking.rb
+- lib/karafka/pro/processing/jobs/periodic.rb
+- lib/karafka/pro/processing/jobs/periodic_non_blocking.rb
 - lib/karafka/pro/processing/jobs/revoked_non_blocking.rb
 - lib/karafka/pro/processing/jobs_builder.rb
 - lib/karafka/pro/processing/jobs_queue.rb
+- lib/karafka/pro/processing/offset_metadata/consumer.rb
+- lib/karafka/pro/processing/offset_metadata/fetcher.rb
+- lib/karafka/pro/processing/offset_metadata/listener.rb
 - lib/karafka/pro/processing/partitioner.rb
-- lib/karafka/pro/processing/scheduler.rb
+- lib/karafka/pro/processing/schedulers/base.rb
+- lib/karafka/pro/processing/schedulers/default.rb
 - lib/karafka/pro/processing/strategies.rb
 - lib/karafka/pro/processing/strategies/aj/dlq_ftr_lrj_mom.rb
 - lib/karafka/pro/processing/strategies/aj/dlq_ftr_lrj_mom_vp.rb
@@ -326,6 +342,19 @@ files:
 - lib/karafka/pro/routing/features/long_running_job/config.rb
 - lib/karafka/pro/routing/features/long_running_job/contracts/topic.rb
 - lib/karafka/pro/routing/features/long_running_job/topic.rb
+- lib/karafka/pro/routing/features/multiplexing.rb
+- lib/karafka/pro/routing/features/multiplexing/config.rb
+- lib/karafka/pro/routing/features/multiplexing/contracts/topic.rb
+- lib/karafka/pro/routing/features/multiplexing/patches/contracts/consumer_group.rb
+- lib/karafka/pro/routing/features/multiplexing/proxy.rb
+- lib/karafka/pro/routing/features/multiplexing/subscription_group.rb
+- lib/karafka/pro/routing/features/multiplexing/subscription_groups_builder.rb
+- lib/karafka/pro/routing/features/non_blocking_job.rb
+- lib/karafka/pro/routing/features/non_blocking_job/topic.rb
+- lib/karafka/pro/routing/features/offset_metadata.rb
+- lib/karafka/pro/routing/features/offset_metadata/config.rb
+- lib/karafka/pro/routing/features/offset_metadata/contracts/topic.rb
+- lib/karafka/pro/routing/features/offset_metadata/topic.rb
 - lib/karafka/pro/routing/features/patterns.rb
 - lib/karafka/pro/routing/features/patterns/builder.rb
 - lib/karafka/pro/routing/features/patterns/config.rb
@@ -341,6 +370,10 @@ files:
 - lib/karafka/pro/routing/features/pausing.rb
 - lib/karafka/pro/routing/features/pausing/contracts/topic.rb
 - lib/karafka/pro/routing/features/pausing/topic.rb
+- lib/karafka/pro/routing/features/periodic_job.rb
+- lib/karafka/pro/routing/features/periodic_job/config.rb
+- lib/karafka/pro/routing/features/periodic_job/contracts/topic.rb
+- lib/karafka/pro/routing/features/periodic_job/topic.rb
 - lib/karafka/pro/routing/features/throttling.rb
 - lib/karafka/pro/routing/features/throttling/config.rb
 - lib/karafka/pro/routing/features/throttling/contracts/topic.rb
@@ -367,7 +400,7 @@ files:
 - lib/karafka/processing/jobs_queue.rb
 - lib/karafka/processing/partitioner.rb
 - lib/karafka/processing/result.rb
-- lib/karafka/processing/scheduler.rb
+- lib/karafka/processing/schedulers/default.rb
 - lib/karafka/processing/strategies/aj_dlq_mom.rb
 - lib/karafka/processing/strategies/aj_mom.rb
 - lib/karafka/processing/strategies/base.rb
@@ -425,6 +458,7 @@ files:
 - lib/karafka/templates/example_consumer.rb.erb
 - lib/karafka/templates/karafka.rb.erb
 - lib/karafka/time_trackers/base.rb
+- lib/karafka/time_trackers/partition_usage.rb
 - lib/karafka/time_trackers/pause.rb
 - lib/karafka/time_trackers/poll.rb
 - lib/karafka/version.rb
@@ -457,7 +491,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.4.19
+rubygems_version: 3.5.3
 signing_key: 
 specification_version: 4
 summary: Karafka is Ruby and Rails efficient Kafka processing framework.

data/lib/karafka/connection/consumer_group_coordinator.rb

@@ -1,48 +0,0 @@
-# frozen_string_literal: true
-
-module Karafka
-  module Connection
-    # This object represents a collective status of execution of group of listeners running inside
-    # of one consumer group but in separate subscription groups.
-    #
-    # There are cases when we do not want to close a given client when others from the same
-    # consumer group are running because it can cause instabilities due to early shutdown of some
-    # of the clients out of same consumer group.
-    #
-    # We also want to make sure, we close one consumer at a time while others can continue polling.
-    #
-    # This prevents a scenario, where a rebalance is not acknowledged and we loose assignment
-    # without having a chance to commit changes.
-    class ConsumerGroupCoordinator
-      # @param group_size [Integer] number of separate subscription groups in a consumer group
-      def initialize(group_size)
-        @shutdown_mutex = Mutex.new
-        @group_size = group_size
-        @finished = Set.new
-      end
-
-      # @return [Boolean] true if all the subscription groups from a given consumer group are
-      #   finished
-      def finished?
-        @finished.size == @group_size
-      end
-
-      # @return [Boolean] can we start shutdown on a given listener
-      # @note If true, will also obtain a lock so no-one else will be closing the same time we do
-      def shutdown?
-        finished? && @shutdown_mutex.try_lock
-      end
-
-      # Unlocks the shutdown lock
-      def unlock
-        @shutdown_mutex.unlock if @shutdown_mutex.owned?
-      end
-
-      # Marks given listener as finished
-      # @param listener_id [String]
-      def finish_work(listener_id)
-        @finished << listener_id
-      end
-    end
-  end
-end