karafka 2.0.19 → 2.0.20

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: be91c3848b757c6af4c25f905df2b081629532bd29dbcea23ed2ef0af2e4e4a2
-  data.tar.gz: 6823d4335e4b395546642101d6754b97958c86810cbcd12819559acff74bd90d
+  metadata.gz: c3b857c4c930396d4cac4682e4350c4c5fdc2888128dfade3f06e21a02e90c29
+  data.tar.gz: 4f8fdb66df24164ec65886d5b4c4ba766dda9c7d4d23a7a301920a8adb21e16e
 SHA512:
-  metadata.gz: fce0259ee987e37c01ea037f81ea91b4eb770ea8eabcb9f93c66aa1a1960c903030648b5441945ef28f43a88660d18240e6db61f6885a169d70eb46174543616
-  data.tar.gz: f93985c98daba5965f8f0597da4744d1dae0603f24a6a44f4b337462f66c8b0f08d0c22dd734205a0aebe7c3dbdb73237ff568f7f1ff842b38c05a7f4b5ce463
+  metadata.gz: d4ed9d036f7dae85b3ce2f9c45de64eed8a8da110f0a826baf100ec9c90dc7e7dbaf5c74f89b472eb89ee7d3bdc9e265fee259621d93e21c972776fa3f682cc0
+  data.tar.gz: aed5fcd7447be757cd2ca641d1d4f6029db11296ac6b4012b4a02047126f27be4aef8d42da700f736c4f70eba2b0aff252a4ccf332fb6c8b88e6a911e3432171

data/.github/FUNDING.yml

@@ -0,0 +1 @@
+custom: ['https://karafka.io/#become-pro']

data/CHANGELOG.md

@@ -1,5 +1,8 @@
 # Karafka framework changelog
 
+## 2.0.20 (2022-11-24)
+- [Improvement] Support `group.instance.id` assignment (static group membership) for a case where a single consumer group has multiple subscription groups (#1173).
+
 ## 2.0.19 (2022-11-20)
 - **[Feature]** Provide ability to skip failing messages without dispatching them to an alternative topic (DLQ).
 - [Improvement] Improve the integration with Ruby on Rails by preventing double-require of components.

data/Gemfile.lock

@@ -1,8 +1,8 @@
 PATH
   remote: .
   specs:
-    karafka (2.0.19)
-      karafka-core (>= 2.0.2, < 3.0.0)
+    karafka (2.0.20)
+      karafka-core (>= 2.0.4, < 3.0.0)
       rdkafka (>= 0.12)
       thor (>= 0.20)
       waterdrop (>= 2.4.1, < 3.0.0)
@@ -30,7 +30,7 @@ GEM
       activesupport (>= 5.0)
     i18n (1.12.0)
       concurrent-ruby (~> 1.0)
-    karafka-core (2.0.3)
+    karafka-core (2.0.4)
       concurrent-ruby (>= 1.1)
     mini_portile2 (2.8.0)
     minitest (5.16.3)

data/karafka.gemspec

@@ -16,7 +16,7 @@ Gem::Specification.new do |spec|
   spec.description = 'Framework used to simplify Apache Kafka based Ruby applications development'
   spec.licenses    = ['LGPL-3.0', 'Commercial']
 
-  spec.add_dependency 'karafka-core', '>= 2.0.2', '< 3.0.0'
+  spec.add_dependency 'karafka-core', '>= 2.0.4', '< 3.0.0'
   spec.add_dependency 'rdkafka', '>= 0.12'
   spec.add_dependency 'thor', '>= 0.20'
   spec.add_dependency 'waterdrop', '>= 2.4.1', '< 3.0.0'

data/lib/karafka/connection/consumer_group_coordinator.rb

@@ -16,11 +16,6 @@ module Karafka
     class ConsumerGroupCoordinator
       # @param group_size [Integer] number of separate subscription groups in a consumer group
       def initialize(group_size)
-        # We need two locks here:
-        # - first one is to decrement the number of listeners doing work
-        # - second to ensure only one client is being closed the same time and that others can
-        #   wait actively (not locked)
-        @work_mutex = Mutex.new
         @shutdown_lock = Mutex.new
         @group_size = group_size
         @finished = Set.new

data/lib/karafka/routing/builder.rb

@@ -40,6 +40,9 @@ module Karafka
           consumer_group.topics.each do |topic|
             Contracts::Topic.new.validate!(topic.to_h)
           end
+
+          # Initialize subscription groups after all the routing is done
+          consumer_group.subscription_groups
         end
       end
 

data/lib/karafka/routing/consumer_group.rb

@@ -14,7 +14,7 @@ module Karafka
       # It allows us to store the "current" subscription group defined in the routing
       # This subscription group id is then injected into topics, so we can compute the subscription
       # groups
-      attr_accessor :current_subscription_group_name
+      attr_accessor :current_subscription_group_id
 
       # @param name [String, Symbol] raw name of this consumer group. Raw means, that it does not
       #   yet have an application client_id namespace, this will be added here by default.
@@ -24,6 +24,9 @@ module Karafka
         @name = name.to_s
         @id = Karafka::App.config.consumer_mapper.call(name)
         @topics = Topics.new([])
+        # Initialize the subscription group so there's always a value for it, since even if not
+        # defined directly, a subscription group will be created
+        @current_subscription_group_id = SecureRandom.uuid
       end
 
       # @return [Boolean] true if this consumer group should be active in our current process
@@ -41,7 +44,7 @@ module Karafka
         built_topic = @topics.last
         # We overwrite it conditionally in case it was not set by the user inline in the topic
         # block definition
-        built_topic.subscription_group ||= current_subscription_group_name
+        built_topic.subscription_group ||= current_subscription_group_id
         built_topic
       end
 
@@ -52,19 +55,24 @@ module Karafka
       def subscription_group=(name, &block)
         # We cast it here, so the routing supports symbol based but that's anyhow later on
         # validated as a string
-        self.current_subscription_group_name = name
+        @current_subscription_group_id = name
 
         Proxy.new(self, &block)
 
         # We need to reset the current subscription group after it is used, so it won't leak
         # outside to other topics that would be defined without a defined subscription group
-        self.current_subscription_group_name = nil
+        @current_subscription_group_id = SecureRandom.uuid
       end
 
       # @return [Array<Routing::SubscriptionGroup>] all the subscription groups build based on
       #   the consumer group topics
       def subscription_groups
-        App.config.internal.routing.subscription_groups_builder.call(topics)
+        @subscription_groups ||= App
+                                 .config
+                                 .internal
+                                 .routing
+                                 .subscription_groups_builder
+                                 .call(topics)
       end
 
       # Hashed version of consumer group that can be used for validation purposes

data/lib/karafka/routing/subscription_group.rb

@@ -8,13 +8,18 @@ module Karafka
     # @note One subscription group will always belong to one consumer group, but one consumer
     #   group can have multiple subscription groups.
     class SubscriptionGroup
-      attr_reader :id, :topics
+      attr_reader :id, :topics, :kafka
 
+      # @param position [Integer] position of this subscription group in all the subscriptions
+      #   groups array. We need to have this value for sake of static group memberships, where
+      #   we need a "in-between" restarts unique identifier
       # @param topics [Karafka::Routing::Topics] all the topics that share the same key settings
       # @return [SubscriptionGroup] built subscription group
-      def initialize(topics)
-        @id = SecureRandom.uuid
+      def initialize(position, topics)
+        @id = "#{topics.first.subscription_group}_#{position}"
+        @position = position
         @topics = topics
+        @kafka = build_kafka
         freeze
       end
 
@@ -33,12 +38,22 @@ module Karafka
         @topics.first.max_wait_time
       end
 
+      private
+
       # @return [Hash] kafka settings are a bit special. They are exactly the same for all of the
       #   topics but they lack the group.id (unless explicitly) provided. To make it compatible
       #   with our routing engine, we inject it before it will go to the consumer
-      def kafka
+      def build_kafka
         kafka = Setup::AttributesMap.consumer(@topics.first.kafka.dup)
 
+        # If we use static group memberships, there can be a case, where same instance id would
+        # be set on many subscription groups as the group instance id from Karafka perspective is
+        # set per config. Each instance even if they are subscribed to different topics needs to
+        # have if fully unique. To make sure of that, we just add extra postfix at the end that
+        # increments.
+        group_instance_id = kafka.fetch(:'group.instance.id', false)
+
+        kafka[:'group.instance.id'] = "#{group_instance_id}_#{@position}" if group_instance_id
         kafka[:'client.id'] ||= Karafka::App.config.client_id
         kafka[:'group.id'] ||= @topics.first.consumer_group.id
         kafka[:'auto.offset.reset'] ||= @topics.first.initial_offset

data/lib/karafka/routing/subscription_groups_builder.rb

@@ -24,6 +24,10 @@ module Karafka
 
       private_constant :DISTRIBUTION_KEYS
 
+      def initialize
+        @position = -1
+      end
+
       # @param topics [Karafka::Routing::Topics] all the topics based on which we want to build
       #   subscription groups
       # @return [Array<SubscriptionGroup>] all subscription groups we need in separate threads
@@ -34,7 +38,7 @@ module Karafka
           .values
           .map { |value| value.map(&:last) }
           .map { |topics_array| Routing::Topics.new(topics_array) }
-          .map { |grouped_topics| SubscriptionGroup.new(grouped_topics) }
+          .map { |grouped_topics| SubscriptionGroup.new(@position += 1, grouped_topics) }
       end
 
       private

data/lib/karafka/version.rb

@@ -3,5 +3,5 @@
 # Main module namespace
 module Karafka
   # Current Karafka version
-  VERSION = '2.0.19'
+  VERSION = '2.0.20'
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: karafka
 version: !ruby/object:Gem::Version
-  version: 2.0.19
+  version: 2.0.20
 platform: ruby
 authors:
 - Maciej Mensfeld
@@ -35,7 +35,7 @@ cert_chain:
   Qf04B9ceLUaC4fPVEz10FyobjaFoY4i32xRto3XnrzeAgfEe4swLq8bQsR3w/EF3
   MGU0FeSV2Yj7Xc2x/7BzLK8xQn5l7Yy75iPF+KP3vVmDHnNl
   -----END CERTIFICATE-----
-date: 2022-11-20 00:00:00.000000000 Z
+date: 2022-11-24 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: karafka-core
@@ -43,7 +43,7 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 2.0.2
+        version: 2.0.4
     - - "<"
       - !ruby/object:Gem::Version
         version: 3.0.0
@@ -53,7 +53,7 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 2.0.2
+        version: 2.0.4
     - - "<"
       - !ruby/object:Gem::Version
         version: 3.0.0
@@ -130,6 +130,7 @@ files:
 - ".coditsu/ci.yml"
 - ".console_irbrc"
 - ".diffend.yml"
+- ".github/FUNDING.yml"
 - ".github/ISSUE_TEMPLATE/bug_report.md"
 - ".github/ISSUE_TEMPLATE/feature_request.md"
 - ".github/workflows/ci.yml"