karafka 2.0.18 → 2.0.20

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 17800cf4d60529269b31340d76245ed2035b463f03daceeb482241528eb5d681
-  data.tar.gz: 9caed11b4f7221d9c5e9c33607ba965a0fd9c75c7ce037d40e38df119a8255ea
+  metadata.gz: c3b857c4c930396d4cac4682e4350c4c5fdc2888128dfade3f06e21a02e90c29
+  data.tar.gz: 4f8fdb66df24164ec65886d5b4c4ba766dda9c7d4d23a7a301920a8adb21e16e
 SHA512:
-  metadata.gz: 5de87acf0d5dbef037c2e980d1354c398fe47bf9a14312f772b346f59bd6f638aa0d4444b9d32984f494cd66dc3cb752f843dc72534ad9ec7df367ecde100dfc
-  data.tar.gz: 8577247eeb4eae1d5abc9e74de9c2c95787535662ed8a2a664824871fe1907abb7aa6e14619926cde218cfb06db5496ec0c38dd6fc3fc3ad0efe0b5702fe067e
+  metadata.gz: d4ed9d036f7dae85b3ce2f9c45de64eed8a8da110f0a826baf100ec9c90dc7e7dbaf5c74f89b472eb89ee7d3bdc9e265fee259621d93e21c972776fa3f682cc0
+  data.tar.gz: aed5fcd7447be757cd2ca641d1d4f6029db11296ac6b4012b4a02047126f27be4aef8d42da700f736c4f70eba2b0aff252a4ccf332fb6c8b88e6a911e3432171

data/.github/FUNDING.yml

@@ -0,0 +1 @@
+custom: ['https://karafka.io/#become-pro']

data/CHANGELOG.md

@@ -1,5 +1,15 @@
 # Karafka framework changelog
 
+## 2.0.20 (2022-11-24)
+- [Improvement] Support `group.instance.id` assignment (static group membership) for a case where a single consumer group has multiple subscription groups (#1173).
+
+## 2.0.19 (2022-11-20)
+- **[Feature]** Provide ability to skip failing messages without dispatching them to an alternative topic (DLQ).
+- [Improvement] Improve the integration with Ruby on Rails by preventing double-require of components.
+- [Improvement] Improve stability of the shutdown process upon critical errors.
+- [Improvement] Improve stability of the integrations spec suite.
+- [Fix] Fix an issue where upon fast startup of multiple subscription groups from the same consumer group, a ghost queue would be created due to problems in `Concurrent::Hash`.
+
 ## 2.0.18 (2022-11-18)
 - **[Feature]** Support quiet mode via `TSTP` signal. When used, Karafka will finish processing current messages, run `shutdown` jobs, and switch to a quiet mode where no new work is being accepted. At the same time, it will keep the consumer group quiet, and thus no rebalance will be triggered. This can be particularly useful during deployments.
 - [Improvement] Trigger `#revoked` for jobs in case revocation would happen during shutdown when jobs are still running. This should ensure, we get a notion of revocation for Pro LRJ jobs even when revocation happening upon shutdown (#1150).

data/Gemfile.lock

@@ -1,8 +1,8 @@
 PATH
   remote: .
   specs:
-    karafka (2.0.18)
-      karafka-core (>= 2.0.2, < 3.0.0)
+    karafka (2.0.20)
+      karafka-core (>= 2.0.4, < 3.0.0)
       rdkafka (>= 0.12)
       thor (>= 0.20)
       waterdrop (>= 2.4.1, < 3.0.0)
@@ -30,7 +30,7 @@ GEM
       activesupport (>= 5.0)
     i18n (1.12.0)
       concurrent-ruby (~> 1.0)
-    karafka-core (2.0.3)
+    karafka-core (2.0.4)
       concurrent-ruby (>= 1.1)
     mini_portile2 (2.8.0)
     minitest (5.16.3)

data/karafka.gemspec

@@ -16,7 +16,7 @@ Gem::Specification.new do |spec|
   spec.description = 'Framework used to simplify Apache Kafka based Ruby applications development'
   spec.licenses    = ['LGPL-3.0', 'Commercial']
 
-  spec.add_dependency 'karafka-core', '>= 2.0.2', '< 3.0.0'
+  spec.add_dependency 'karafka-core', '>= 2.0.4', '< 3.0.0'
   spec.add_dependency 'rdkafka', '>= 0.12'
   spec.add_dependency 'thor', '>= 0.20'
   spec.add_dependency 'waterdrop', '>= 2.4.1', '< 3.0.0'
@@ -34,7 +34,12 @@ Gem::Specification.new do |spec|
   spec.require_paths = %w[lib]
 
   spec.metadata = {
+    'funding_uri' => 'https://karafka.io/#become-pro',
+    'homepage_uri' => 'https://karafka.io',
+    'changelog_uri' => 'https://github.com/karafka/karafka/blob/master/CHANGELOG.md',
+    'bug_tracker_uri' => 'https://github.com/karafka/karafka/issues',
     'source_code_uri' => 'https://github.com/karafka/karafka',
+    'documentation_uri' => 'https://karafka.io/docs',
     'rubygems_mfa_required' => 'true'
   }
 end

data/lib/active_job/karafka.rb

@@ -1,7 +1,9 @@
 # frozen_string_literal: true
 
 begin
-  require 'active_job'
+  # Do not load active job if already loaded
+  require 'active_job' unless Object.const_defined?('ActiveJob')
+
   require_relative 'queue_adapters/karafka_adapter'
 
   module ActiveJob

data/lib/karafka/connection/client.rb

@@ -241,6 +241,17 @@ module Karafka
         end
       end
 
+      # Runs a single poll ignoring all the potential errors
+      # This is used as a keep-alive in the shutdown stage and any errors that happen here are
+      # irrelevant from the shutdown process perspective
+      #
+      # This is used only to trigger rebalance callbacks
+      def ping
+        poll(100)
+      rescue Rdkafka::RdkafkaError
+        nil
+      end
+
       private
 
       # When we cannot store an offset, it means we no longer own the partition
@@ -302,8 +313,8 @@ module Karafka
 
             @kafka.close
             @buffer.clear
-            # @note We do not clear rebalance manager here as we may still have revocation info here
-            # that we want to consider valid prior to running another reconnection
+            # @note We do not clear rebalance manager here as we may still have revocation info
+            # here that we want to consider valid prior to running another reconnection
           end
         end
       end

data/lib/karafka/connection/consumer_group_coordinator.rb

@@ -0,0 +1,42 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Connection
+    # This object represents a collective status of execution of group of listeners running inside
+    # of one consumer group but in separate subscription groups.
+    #
+    # There are cases when we do not want to close a given client when others from the same
+    # consumer group are running because it can cause instabilities due to early shutdown of some
+    # of the clients out of same consumer group.
+    #
+    # We also want to make sure, we close one consumer at a time while others can continue polling.
+    #
+    # This prevents a scenario, where a rebalance is not acknowledged and we loose assignment
+    # without having a chance to commit changes.
+    class ConsumerGroupCoordinator
+      # @param group_size [Integer] number of separate subscription groups in a consumer group
+      def initialize(group_size)
+        @shutdown_lock = Mutex.new
+        @group_size = group_size
+        @finished = Set.new
+      end
+
+      # @return [Boolean] can we start shutdown on a given listener
+      # @note If true, will also obtain a lock so no-one else will be closing the same time we do
+      def shutdown?
+        @finished.size == @group_size && @shutdown_lock.try_lock
+      end
+
+      # Unlocks the shutdown lock
+      def unlock
+        @shutdown_lock.unlock if @shutdown_lock.owned?
+      end
+
+      # Marks given listener as finished
+      # @param listener_id [String]
+      def finish_work(listener_id)
+        @finished << listener_id
+      end
+    end
+  end
+end

data/lib/karafka/connection/listener.rb

@@ -14,20 +14,15 @@ module Karafka
       # @return [String] id of this listener
       attr_reader :id
 
-      # Mutex for things we do not want to do in parallel even in different consumer groups
-      MUTEX = Mutex.new
-
-      private_constant :MUTEX
-
-      # @param consumer_group_status [Karafka::Connection::ConsumerGroupStatus]
+      # @param consumer_group_coordinator [Karafka::Connection::ConsumerGroupCoordinator]
       # @param subscription_group [Karafka::Routing::SubscriptionGroup]
       # @param jobs_queue [Karafka::Processing::JobsQueue] queue where we should push work
       # @return [Karafka::Connection::Listener] listener instance
-      def initialize(consumer_group_status, subscription_group, jobs_queue)
+      def initialize(consumer_group_coordinator, subscription_group, jobs_queue)
         proc_config = ::Karafka::App.config.internal.processing
 
         @id = SecureRandom.uuid
-        @consumer_group_status = consumer_group_status
+        @consumer_group_coordinator = consumer_group_coordinator
         @subscription_group = subscription_group
         @jobs_queue = jobs_queue
         @coordinators = Processing::CoordinatorsBuffer.new
@@ -68,17 +63,13 @@ module Karafka
       #
       # @note We wrap it with a mutex exactly because of the above case of forceful shutdown
       def shutdown
-        # We want to make sure that we never close two librdkafka clients at the same time. I'm not
-        # particularly fond of it's shutdown API being fully thread-safe
-        MUTEX.synchronize do
-          return if @stopped
-
-          @mutex.synchronize do
-            @stopped = true
-            @executors.clear
-            @coordinators.reset
-            @client.stop
-          end
+        return if @stopped
+
+        @mutex.synchronize do
+          @stopped = true
+          @executors.clear
+          @coordinators.reset
+          @client.stop
         end
       end
 
@@ -147,9 +138,9 @@ module Karafka
         # What we do care however is the ability to still run revocation jobs in case anything
         # would change in the cluster. We still want to notify the long-running jobs about changes
         # that occurred in the cluster.
-        wait_polling(
+        wait_pinging(
           wait_until: -> { @jobs_queue.empty?(@subscription_group.id) },
-          after_poll: -> { build_and_schedule_revoke_lost_partitions_jobs }
+          after_ping: -> { build_and_schedule_revoke_lost_partitions_jobs }
         )
 
         # We do not want to schedule the shutdown jobs prior to finishing all the jobs
@@ -159,19 +150,23 @@ module Karafka
         build_and_schedule_shutdown_jobs
 
         # Wait until all the shutdown jobs are done
-        wait_polling(wait_until: -> { @jobs_queue.empty?(@subscription_group.id) })
+        wait_pinging(wait_until: -> { @jobs_queue.empty?(@subscription_group.id) })
 
         # Once all the work is done, we need to decrement counter of active subscription groups
         # within this consumer group
-        @consumer_group_status.finish
+        @consumer_group_coordinator.finish_work(id)
 
         # Wait if we're in the quiet mode
-        wait_polling(wait_until: -> { !Karafka::App.quieting? })
+        wait_pinging(wait_until: -> { !Karafka::App.quieting? })
 
         # We need to wait until all the work in the whole consumer group (local to the process)
         # is done. Otherwise we may end up with locks and `Timed out LeaveGroupRequest in flight`
         # warning notifications.
-        wait_polling(wait_until: -> { !@consumer_group_status.working? })
+        wait_pinging(wait_until: -> { @consumer_group_coordinator.shutdown? })
+
+        # This extra ping will make sure we've refreshed the rebalance state after other instances
+        # potentially shutdown. This will prevent us from closing with a dangling callback
+        @client.ping
 
         shutdown
 
@@ -189,6 +184,8 @@ module Karafka
         restart
 
         sleep(1) && retry
+      ensure
+        @consumer_group_coordinator.unlock
       end
 
       # Resumes processing of partitions that were paused due to an error.
@@ -293,15 +290,15 @@ module Karafka
       # can safely discard it. We can however use the rebalance information if needed.
       #
       # @param wait_until [Proc] until this evaluates to true, we will poll data
-      # @param after_poll [Proc] code that we want to run after each batch poll (if any)
+      # @param after_ping [Proc] code that we want to run after each ping (if any)
       #
       # @note Performance of this is not relevant (in regards to blocks) because it is used only
       #   on shutdown and quiet, hence not in the running mode
-      def wait_polling(wait_until:, after_poll: -> {})
+      def wait_pinging(wait_until:, after_ping: -> {})
         until wait_until.call
-          @client.batch_poll
-
-          after_poll.call
+          @client.ping
+          after_ping.call
+          sleep(0.2)
         end
       end
 

data/lib/karafka/connection/listeners_batch.rb

@@ -10,13 +10,13 @@ module Karafka
       # @return [ListenersBatch]
       def initialize(jobs_queue)
         @batch = App.subscription_groups.flat_map do |_consumer_group, subscription_groups|
-          consumer_group_status = Connection::ConsumerGroupStatus.new(
+          consumer_group_coordinator = Connection::ConsumerGroupCoordinator.new(
             subscription_groups.size
           )
 
           subscription_groups.map do |subscription_group|
             Connection::Listener.new(
-              consumer_group_status,
+              consumer_group_coordinator,
               subscription_group,
               jobs_queue
             )

data/lib/karafka/errors.rb

@@ -44,9 +44,6 @@ module Karafka
     # This should never happen. Please open an issue if it does.
     InvalidCoordinatorStateError = Class.new(BaseError)
 
-    # This should never happen. Please open an issue if it does.
-    InvalidConsumerGroupStatusError = Class.new(BaseError)
-
     # This should never happen. Please open an issue if it does.
     StrategyNotFoundError = Class.new(BaseError)
 

data/lib/karafka/pro/performance_tracker.rb

@@ -25,11 +25,13 @@ module Karafka
 
       # Builds up nested concurrent hash for data tracking
       def initialize
-        @processing_times = Concurrent::Hash.new do |topics_hash, topic|
-          topics_hash[topic] = Concurrent::Hash.new do |partitions_hash, partition|
-            # This array does not have to be concurrent because we always access single partition
-            # data via instrumentation that operates in a single thread via consumer
-            partitions_hash[partition] = []
+        @processing_times = Concurrent::Map.new do |topics_hash, topic|
+          topics_hash.compute_if_absent(topic) do
+            Concurrent::Map.new do |partitions_hash, partition|
+              # This array does not have to be concurrent because we always access single
+              # partition data via instrumentation that operates in a single thread via consumer
+              partitions_hash.compute_if_absent(partition) { [] }
+            end
           end
         end
       end

data/lib/karafka/pro/processing/strategies/aj_dlq_mom.rb

@@ -21,7 +21,7 @@ module Karafka
         #
         # AJ has manual offset management on by default and the offset management is delegated to
         # the AJ consumer. This means, we cannot mark as consumed always. We can only mark as
-        # consumed when we skip given job upon errors. In all the other scenarions marking as
+        # consumed when we skip given job upon errors. In all the other scenarios marking as
         # consumed needs to happen in the AJ consumer on a per job basis.
         module AjDlqMom
           include DlqMom
@@ -46,7 +46,7 @@ module Karafka
               else
                 coordinator.pause_tracker.reset
                 skippable_message = find_skippable_message
-                dispatch_to_dlq(skippable_message)
+                dispatch_to_dlq(skippable_message) if dispatch_to_dlq?
                 # We can commit the offset here because we know that we skip it "forever" and
                 # since AJ consumer commits the offset after each job, we also know that the
                 # previous job was successful

data/lib/karafka/pro/processing/strategies/dlq.rb

@@ -42,7 +42,7 @@ module Karafka
                 # We reset the pause to indicate we will now consider it as "ok".
                 coordinator.pause_tracker.reset
                 skippable_message = find_skippable_message
-                dispatch_to_dlq(skippable_message)
+                dispatch_to_dlq(skippable_message) if dispatch_to_dlq?
                 mark_as_consumed(skippable_message)
                 pause(coordinator.seek_offset)
               end
@@ -59,7 +59,6 @@ module Karafka
 
           # Moves the broken message into a separate queue defined via the settings
           #
-          # @private
           # @param skippable_message [Array<Karafka::Messages::Message>] message we want to
           #   dispatch to DLQ
           def dispatch_to_dlq(skippable_message)
@@ -81,6 +80,13 @@ module Karafka
               message: skippable_message
             )
           end
+
+          # @return [Boolean] should we dispatch the message to DLQ or not. When the dispatch topic
+          #   is set to false, we will skip the dispatch, effectively ignoring the broken message
+          #   without taking any action.
+          def dispatch_to_dlq?
+            topic.dead_letter_queue.topic
+          end
         end
       end
     end

data/lib/karafka/pro/processing/strategies/dlq_lrj.rb

@@ -43,10 +43,9 @@ module Karafka
               else
                 coordinator.pause_tracker.reset
 
-                skippable_message = find_skippable_message
-
                 unless revoked?
-                  dispatch_to_dlq(skippable_message)
+                  skippable_message = find_skippable_message
+                  dispatch_to_dlq(skippable_message) if dispatch_to_dlq?
                   mark_as_consumed(skippable_message)
                 end
 

data/lib/karafka/pro/processing/strategies/dlq_lrj_mom.rb

@@ -42,10 +42,12 @@ module Karafka
               else
                 coordinator.pause_tracker.reset
 
-                skippable_message = find_skippable_message
-
                 unless revoked?
-                  dispatch_to_dlq(skippable_message)
+                  if dispatch_to_dlq?
+                    skippable_message = find_skippable_message
+                    dispatch_to_dlq(skippable_message)
+                  end
+
                   seek(coordinator.seek_offset)
                 end
 

data/lib/karafka/pro/processing/strategies/dlq_mom.rb

@@ -45,8 +45,12 @@ module Karafka
               else
                 # We reset the pause to indicate we will now consider it as "ok".
                 coordinator.pause_tracker.reset
-                skippable_message = find_skippable_message
-                dispatch_to_dlq(skippable_message)
+
+                if dispatch_to_dlq?
+                  skippable_message = find_skippable_message
+                  dispatch_to_dlq(skippable_message)
+                end
+
                 pause(coordinator.seek_offset)
               end
             end

data/lib/karafka/processing/jobs_queue.rb

@@ -20,8 +20,12 @@ module Karafka
         # scheduled by Ruby hundreds of thousands of times per group.
         # We cannot use a single semaphore as it could potentially block in listeners that should
         # process with their data and also could unlock when a given group needs to remain locked
-        @semaphores = Concurrent::Map.new { |h, k| h[k] = Queue.new }
+        @semaphores = Concurrent::Map.new do |h, k|
+          h.compute_if_absent(k) { Queue.new }
+        end
+
         @in_processing = Hash.new { |h, k| h[k] = [] }
+
         @mutex = Mutex.new
       end
 

data/lib/karafka/railtie.rb

@@ -5,7 +5,8 @@
 rails = false
 
 begin
-  require 'rails'
+  # Do not load Rails again if already loaded
+  Object.const_defined?('Rails::Railtie') || require('rails')
 
   rails = true
 rescue LoadError

data/lib/karafka/routing/builder.rb

@@ -40,6 +40,9 @@ module Karafka
           consumer_group.topics.each do |topic|
             Contracts::Topic.new.validate!(topic.to_h)
           end
+
+          # Initialize subscription groups after all the routing is done
+          consumer_group.subscription_groups
         end
       end
 

data/lib/karafka/routing/consumer_group.rb

@@ -14,7 +14,7 @@ module Karafka
       # It allows us to store the "current" subscription group defined in the routing
       # This subscription group id is then injected into topics, so we can compute the subscription
       # groups
-      attr_accessor :current_subscription_group_name
+      attr_accessor :current_subscription_group_id
 
       # @param name [String, Symbol] raw name of this consumer group. Raw means, that it does not
       #   yet have an application client_id namespace, this will be added here by default.
@@ -24,6 +24,9 @@ module Karafka
         @name = name.to_s
         @id = Karafka::App.config.consumer_mapper.call(name)
         @topics = Topics.new([])
+        # Initialize the subscription group so there's always a value for it, since even if not
+        # defined directly, a subscription group will be created
+        @current_subscription_group_id = SecureRandom.uuid
       end
 
       # @return [Boolean] true if this consumer group should be active in our current process
@@ -41,7 +44,7 @@ module Karafka
         built_topic = @topics.last
         # We overwrite it conditionally in case it was not set by the user inline in the topic
         # block definition
-        built_topic.subscription_group ||= current_subscription_group_name
+        built_topic.subscription_group ||= current_subscription_group_id
         built_topic
       end
 
@@ -52,19 +55,24 @@ module Karafka
       def subscription_group=(name, &block)
         # We cast it here, so the routing supports symbol based but that's anyhow later on
         # validated as a string
-        self.current_subscription_group_name = name
+        @current_subscription_group_id = name
 
         Proxy.new(self, &block)
 
         # We need to reset the current subscription group after it is used, so it won't leak
         # outside to other topics that would be defined without a defined subscription group
-        self.current_subscription_group_name = nil
+        @current_subscription_group_id = SecureRandom.uuid
       end
 
       # @return [Array<Routing::SubscriptionGroup>] all the subscription groups build based on
       #   the consumer group topics
       def subscription_groups
-        App.config.internal.routing.subscription_groups_builder.call(topics)
+        @subscription_groups ||= App
+                                 .config
+                                 .internal
+                                 .routing
+                                 .subscription_groups_builder
+                                 .call(topics)
       end
 
       # Hashed version of consumer group that can be used for validation purposes

data/lib/karafka/routing/features/dead_letter_queue/contract.rb

@@ -29,6 +29,8 @@ module Karafka
 
             topic = dead_letter_queue[:topic]
 
+            # When topic is set to false, it means we just want to skip dispatch on DLQ
+            next if topic == false
             next if topic.is_a?(String) && Contracts::TOPIC_REGEXP.match?(topic)
 
             [[%i[dead_letter_queue topic], :format]]

data/lib/karafka/routing/features/dead_letter_queue/topic.rb

@@ -12,7 +12,8 @@ module Karafka
           private_constant :DEFAULT_MAX_RETRIES
 
           # @param max_retries [Integer] after how many retries should we move data to dlq
-          # @param topic [String] where the messages should be moved if failing
+          # @param topic [String, false] where the messages should be moved if failing or false
+          #   if we do not want to move it anywhere and just skip
           # @return [Config] defined config
           def dead_letter_queue(max_retries: DEFAULT_MAX_RETRIES, topic: nil)
             @dead_letter_queue ||= Config.new(

data/lib/karafka/routing/subscription_group.rb

@@ -8,13 +8,18 @@ module Karafka
     # @note One subscription group will always belong to one consumer group, but one consumer
     #   group can have multiple subscription groups.
     class SubscriptionGroup
-      attr_reader :id, :topics
+      attr_reader :id, :topics, :kafka
 
+      # @param position [Integer] position of this subscription group in all the subscriptions
+      #   groups array. We need to have this value for sake of static group memberships, where
+      #   we need a "in-between" restarts unique identifier
       # @param topics [Karafka::Routing::Topics] all the topics that share the same key settings
       # @return [SubscriptionGroup] built subscription group
-      def initialize(topics)
-        @id = SecureRandom.uuid
+      def initialize(position, topics)
+        @id = "#{topics.first.subscription_group}_#{position}"
+        @position = position
         @topics = topics
+        @kafka = build_kafka
         freeze
       end
 
@@ -33,12 +38,22 @@ module Karafka
         @topics.first.max_wait_time
       end
 
+      private
+
       # @return [Hash] kafka settings are a bit special. They are exactly the same for all of the
       #   topics but they lack the group.id (unless explicitly) provided. To make it compatible
       #   with our routing engine, we inject it before it will go to the consumer
-      def kafka
+      def build_kafka
         kafka = Setup::AttributesMap.consumer(@topics.first.kafka.dup)
 
+        # If we use static group memberships, there can be a case, where same instance id would
+        # be set on many subscription groups as the group instance id from Karafka perspective is
+        # set per config. Each instance even if they are subscribed to different topics needs to
+        # have if fully unique. To make sure of that, we just add extra postfix at the end that
+        # increments.
+        group_instance_id = kafka.fetch(:'group.instance.id', false)
+
+        kafka[:'group.instance.id'] = "#{group_instance_id}_#{@position}" if group_instance_id
         kafka[:'client.id'] ||= Karafka::App.config.client_id
         kafka[:'group.id'] ||= @topics.first.consumer_group.id
         kafka[:'auto.offset.reset'] ||= @topics.first.initial_offset

data/lib/karafka/routing/subscription_groups_builder.rb

@@ -24,6 +24,10 @@ module Karafka
 
       private_constant :DISTRIBUTION_KEYS
 
+      def initialize
+        @position = -1
+      end
+
       # @param topics [Karafka::Routing::Topics] all the topics based on which we want to build
       #   subscription groups
       # @return [Array<SubscriptionGroup>] all subscription groups we need in separate threads
@@ -34,7 +38,7 @@ module Karafka
           .values
           .map { |value| value.map(&:last) }
           .map { |topics_array| Routing::Topics.new(topics_array) }
-          .map { |grouped_topics| SubscriptionGroup.new(grouped_topics) }
+          .map { |grouped_topics| SubscriptionGroup.new(@position += 1, grouped_topics) }
       end
 
       private

data/lib/karafka/setup/attributes_map.rb

@@ -84,6 +84,7 @@ module Karafka
         reconnect.backoff.jitter.ms
         reconnect.backoff.max.ms
         reconnect.backoff.ms
+        resolve_cb
         sasl.kerberos.keytab
         sasl.kerberos.kinit.cmd
         sasl.kerberos.min.time.before.relogin
@@ -215,6 +216,7 @@ module Karafka
         reconnect.backoff.ms
         request.required.acks
         request.timeout.ms
+        resolve_cb
         retries
         retry.backoff.ms
         sasl.kerberos.keytab

data/lib/karafka/version.rb

@@ -3,5 +3,5 @@
 # Main module namespace
 module Karafka
   # Current Karafka version
-  VERSION = '2.0.18'
+  VERSION = '2.0.20'
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: karafka
 version: !ruby/object:Gem::Version
-  version: 2.0.18
+  version: 2.0.20
 platform: ruby
 authors:
 - Maciej Mensfeld
@@ -35,7 +35,7 @@ cert_chain:
   Qf04B9ceLUaC4fPVEz10FyobjaFoY4i32xRto3XnrzeAgfEe4swLq8bQsR3w/EF3
   MGU0FeSV2Yj7Xc2x/7BzLK8xQn5l7Yy75iPF+KP3vVmDHnNl
   -----END CERTIFICATE-----
-date: 2022-11-18 00:00:00.000000000 Z
+date: 2022-11-24 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: karafka-core
@@ -43,7 +43,7 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 2.0.2
+        version: 2.0.4
     - - "<"
       - !ruby/object:Gem::Version
         version: 3.0.0
@@ -53,7 +53,7 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 2.0.2
+        version: 2.0.4
     - - "<"
       - !ruby/object:Gem::Version
         version: 3.0.0
@@ -130,6 +130,7 @@ files:
 - ".coditsu/ci.yml"
 - ".console_irbrc"
 - ".diffend.yml"
+- ".github/FUNDING.yml"
 - ".github/ISSUE_TEMPLATE/bug_report.md"
 - ".github/ISSUE_TEMPLATE/feature_request.md"
 - ".github/workflows/ci.yml"
@@ -176,7 +177,7 @@ files:
 - lib/karafka/cli/install.rb
 - lib/karafka/cli/server.rb
 - lib/karafka/connection/client.rb
-- lib/karafka/connection/consumer_group_status.rb
+- lib/karafka/connection/consumer_group_coordinator.rb
 - lib/karafka/connection/listener.rb
 - lib/karafka/connection/listeners_batch.rb
 - lib/karafka/connection/messages_buffer.rb
@@ -327,7 +328,12 @@ licenses:
 - LGPL-3.0
 - Commercial
 metadata:
+  funding_uri: https://karafka.io/#become-pro
+  homepage_uri: https://karafka.io
+  changelog_uri: https://github.com/karafka/karafka/blob/master/CHANGELOG.md
+  bug_tracker_uri: https://github.com/karafka/karafka/issues
   source_code_uri: https://github.com/karafka/karafka
+  documentation_uri: https://karafka.io/docs
   rubygems_mfa_required: 'true'
 post_install_message: 
 rdoc_options: []

data/lib/karafka/connection/consumer_group_status.rb

@@ -1,38 +0,0 @@
-# frozen_string_literal: true
-
-module Karafka
-  module Connection
-    # This object represents a collective status of execution of group of listeners running inside
-    # of one consumer group but potentially in separate subscription groups.
-    #
-    # There are cases when we do not want to close a given client when others from the same
-    # consumer group are running because it can cause instabilities due to early shutdown of some
-    # of the clients out of same consumer group.
-    #
-    # Here we can track it and only shutdown listeners when all work in a group is done.
-    class ConsumerGroupStatus
-      # @param group_size [Integer] number of separate subscription groups in a consumer group
-      def initialize(group_size)
-        @mutex = Mutex.new
-        @active_size = group_size
-      end
-
-      # @return [Boolean] Are there any listeners that are still doing any type of work. If not,
-      #   it means a consumer group is safe to be shutdown fully.
-      def working?
-        @active_size.positive?
-      end
-
-      # Decrements number of working listeners in the group by one until there's none
-      def finish
-        @mutex.synchronize do
-          @active_size -= 1
-
-          return if @active_size >= 0
-
-          raise Errors::InvalidConsumerGroupStatusError, @active_size
-        end
-      end
-    end
-  end
-end