karafka 2.0.19 → 2.0.21

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: be91c3848b757c6af4c25f905df2b081629532bd29dbcea23ed2ef0af2e4e4a2
-  data.tar.gz: 6823d4335e4b395546642101d6754b97958c86810cbcd12819559acff74bd90d
+  metadata.gz: 9de6cb2d19b24258eb60548b383e664dc035f1f48ffc7077ca5f238b3a85aa3e
+  data.tar.gz: 706d4e81a74b1aad9d1f9b13a67d168a7ef3c45088b89ec710d29abb3a767f11
 SHA512:
-  metadata.gz: fce0259ee987e37c01ea037f81ea91b4eb770ea8eabcb9f93c66aa1a1960c903030648b5441945ef28f43a88660d18240e6db61f6885a169d70eb46174543616
-  data.tar.gz: f93985c98daba5965f8f0597da4744d1dae0603f24a6a44f4b337462f66c8b0f08d0c22dd734205a0aebe7c3dbdb73237ff568f7f1ff842b38c05a7f4b5ce463
+  metadata.gz: a22be6ff489f9f7cce39e0e4d6d374a70680124842afeeb962e33eaa8c6cd3db8cbcd6f3251543653c98777d025b6d9b0385f4580d92fd4b807e835a68296a43
+  data.tar.gz: 466d9723427771594701071e5a27a3a1a53b6b4efce9ffa6b241f5261c9a887136b2a8a4a90b35e216b2af01539336bde5377f4edc1ab99837087dd751b830a8

data/.github/FUNDING.yml

@@ -0,0 +1 @@
+custom: ['https://karafka.io/#become-pro']

data/CHANGELOG.md

@@ -1,5 +1,11 @@
 # Karafka framework changelog
 
+## 20.0.21 (2022-11-25)
+- [Improvement] Make revocation jobs for LRJ topics non-blocking to prevent blocking polling when someone uses non-revocation aware LRJ jobs and revocation happens.
+
+## 2.0.20 (2022-11-24)
+- [Improvement] Support `group.instance.id` assignment (static group membership) for a case where a single consumer group has multiple subscription groups (#1173).
+
 ## 2.0.19 (2022-11-20)
 - **[Feature]** Provide ability to skip failing messages without dispatching them to an alternative topic (DLQ).
 - [Improvement] Improve the integration with Ruby on Rails by preventing double-require of components.

data/Gemfile.lock

@@ -1,8 +1,8 @@
 PATH
   remote: .
   specs:
-    karafka (2.0.19)
-      karafka-core (>= 2.0.2, < 3.0.0)
+    karafka (2.0.21)
+      karafka-core (>= 2.0.4, < 3.0.0)
       rdkafka (>= 0.12)
       thor (>= 0.20)
       waterdrop (>= 2.4.1, < 3.0.0)
@@ -30,7 +30,7 @@ GEM
       activesupport (>= 5.0)
     i18n (1.12.0)
       concurrent-ruby (~> 1.0)
-    karafka-core (2.0.3)
+    karafka-core (2.0.4)
       concurrent-ruby (>= 1.1)
     mini_portile2 (2.8.0)
     minitest (5.16.3)
@@ -79,4 +79,4 @@ DEPENDENCIES
   simplecov
 
 BUNDLED WITH
-   2.3.24
+   2.3.26

data/karafka.gemspec

@@ -12,11 +12,16 @@ Gem::Specification.new do |spec|
   spec.authors     = ['Maciej Mensfeld']
   spec.email       = %w[contact@karafka.io]
   spec.homepage    = 'https://karafka.io'
-  spec.summary     = 'Efficient Kafka processing framework for Ruby and Rails'
-  spec.description = 'Framework used to simplify Apache Kafka based Ruby applications development'
   spec.licenses    = ['LGPL-3.0', 'Commercial']
+  spec.summary     = 'Karafka is Ruby and Rails efficient Kafka processing framework.'
+  spec.description = <<-DESC
+    Karafka is Ruby and Rails efficient Kafka processing framework.
 
-  spec.add_dependency 'karafka-core', '>= 2.0.2', '< 3.0.0'
+    Karafka allows you to capture everything that happens in your systems in large scale,
+    without having to focus on things that are not your business domain.
+  DESC
+
+  spec.add_dependency 'karafka-core', '>= 2.0.4', '< 3.0.0'
   spec.add_dependency 'rdkafka', '>= 0.12'
   spec.add_dependency 'thor', '>= 0.20'
   spec.add_dependency 'waterdrop', '>= 2.4.1', '< 3.0.0'

data/lib/karafka/connection/consumer_group_coordinator.rb

@@ -16,11 +16,6 @@ module Karafka
     class ConsumerGroupCoordinator
       # @param group_size [Integer] number of separate subscription groups in a consumer group
       def initialize(group_size)
-        # We need two locks here:
-        # - first one is to decrement the number of listeners doing work
-        # - second to ensure only one client is being closed the same time and that others can
-        #   wait actively (not locked)
-        @work_mutex = Mutex.new
         @shutdown_lock = Mutex.new
         @group_size = group_size
         @finished = Set.new

data/lib/karafka/pro/processing/jobs/consume_non_blocking.rb

@@ -18,8 +18,7 @@ module Karafka
       # Pro jobs
       module Jobs
         # The main job type in a non-blocking variant.
-        # This variant works "like" the regular consumption but pauses the partition for as long
-        # as it is needed until a job is done.
+        # This variant works "like" the regular consumption but does not block the queue.
         #
         # It can be useful when having long lasting jobs that would exceed `max.poll.interval`
         # if would block.

data/lib/karafka/pro/processing/jobs/revoked_non_blocking.rb

@@ -0,0 +1,37 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    # Pro components related to processing part of Karafka
+    module Processing
+      # Pro jobs
+      module Jobs
+        # The revoked job type in a non-blocking variant.
+        # This variant works "like" the regular revoked but does not block the queue.
+        #
+        # It can be useful when having long lasting jobs that would exceed `max.poll.interval`
+        # in scenarios where there are more jobs than threads, without this being async we
+        # would potentially stop polling
+        class RevokedNonBlocking < ::Karafka::Processing::Jobs::Revoked
+          # Makes this job non-blocking from the start
+          # @param args [Array] any arguments accepted by `::Karafka::Processing::Jobs::Revoked`
+          def initialize(*args)
+            super
+            @non_blocking = true
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/processing/jobs_builder.rb

@@ -28,6 +28,18 @@ module Karafka
             super
           end
         end
+
+        # @param executor [Karafka::Processing::Executor]
+        # @return [Karafka::Processing::Jobs::Revoked] revocation job for non LRJ
+        # @return [Karafka::Processing::Jobs::RevokedNonBlocking] revocation job that is
+        #   non-blocking, so when revocation job is scheduled for LRJ it also will not block
+        def revoked(executor)
+          if executor.topic.long_running_job?
+            Jobs::RevokedNonBlocking.new(executor)
+          else
+            super
+          end
+        end
       end
     end
   end

data/lib/karafka/routing/builder.rb

@@ -40,6 +40,9 @@ module Karafka
           consumer_group.topics.each do |topic|
             Contracts::Topic.new.validate!(topic.to_h)
           end
+
+          # Initialize subscription groups after all the routing is done
+          consumer_group.subscription_groups
         end
       end
 

data/lib/karafka/routing/consumer_group.rb

@@ -14,7 +14,7 @@ module Karafka
       # It allows us to store the "current" subscription group defined in the routing
       # This subscription group id is then injected into topics, so we can compute the subscription
       # groups
-      attr_accessor :current_subscription_group_name
+      attr_accessor :current_subscription_group_id
 
       # @param name [String, Symbol] raw name of this consumer group. Raw means, that it does not
       #   yet have an application client_id namespace, this will be added here by default.
@@ -24,6 +24,9 @@ module Karafka
         @name = name.to_s
         @id = Karafka::App.config.consumer_mapper.call(name)
         @topics = Topics.new([])
+        # Initialize the subscription group so there's always a value for it, since even if not
+        # defined directly, a subscription group will be created
+        @current_subscription_group_id = SecureRandom.uuid
       end
 
       # @return [Boolean] true if this consumer group should be active in our current process
@@ -41,7 +44,7 @@ module Karafka
         built_topic = @topics.last
         # We overwrite it conditionally in case it was not set by the user inline in the topic
         # block definition
-        built_topic.subscription_group ||= current_subscription_group_name
+        built_topic.subscription_group ||= current_subscription_group_id
         built_topic
       end
 
@@ -52,19 +55,24 @@ module Karafka
       def subscription_group=(name, &block)
         # We cast it here, so the routing supports symbol based but that's anyhow later on
         # validated as a string
-        self.current_subscription_group_name = name
+        @current_subscription_group_id = name
 
         Proxy.new(self, &block)
 
         # We need to reset the current subscription group after it is used, so it won't leak
         # outside to other topics that would be defined without a defined subscription group
-        self.current_subscription_group_name = nil
+        @current_subscription_group_id = SecureRandom.uuid
       end
 
       # @return [Array<Routing::SubscriptionGroup>] all the subscription groups build based on
       #   the consumer group topics
       def subscription_groups
-        App.config.internal.routing.subscription_groups_builder.call(topics)
+        @subscription_groups ||= App
+                                 .config
+                                 .internal
+                                 .routing
+                                 .subscription_groups_builder
+                                 .call(topics)
       end
 
       # Hashed version of consumer group that can be used for validation purposes

data/lib/karafka/routing/subscription_group.rb

@@ -8,13 +8,18 @@ module Karafka
     # @note One subscription group will always belong to one consumer group, but one consumer
     #   group can have multiple subscription groups.
     class SubscriptionGroup
-      attr_reader :id, :topics
+      attr_reader :id, :topics, :kafka
 
+      # @param position [Integer] position of this subscription group in all the subscriptions
+      #   groups array. We need to have this value for sake of static group memberships, where
+      #   we need a "in-between" restarts unique identifier
       # @param topics [Karafka::Routing::Topics] all the topics that share the same key settings
       # @return [SubscriptionGroup] built subscription group
-      def initialize(topics)
-        @id = SecureRandom.uuid
+      def initialize(position, topics)
+        @id = "#{topics.first.subscription_group}_#{position}"
+        @position = position
         @topics = topics
+        @kafka = build_kafka
         freeze
       end
 
@@ -33,12 +38,22 @@ module Karafka
         @topics.first.max_wait_time
       end
 
+      private
+
       # @return [Hash] kafka settings are a bit special. They are exactly the same for all of the
       #   topics but they lack the group.id (unless explicitly) provided. To make it compatible
       #   with our routing engine, we inject it before it will go to the consumer
-      def kafka
+      def build_kafka
         kafka = Setup::AttributesMap.consumer(@topics.first.kafka.dup)
 
+        # If we use static group memberships, there can be a case, where same instance id would
+        # be set on many subscription groups as the group instance id from Karafka perspective is
+        # set per config. Each instance even if they are subscribed to different topics needs to
+        # have if fully unique. To make sure of that, we just add extra postfix at the end that
+        # increments.
+        group_instance_id = kafka.fetch(:'group.instance.id', false)
+
+        kafka[:'group.instance.id'] = "#{group_instance_id}_#{@position}" if group_instance_id
         kafka[:'client.id'] ||= Karafka::App.config.client_id
         kafka[:'group.id'] ||= @topics.first.consumer_group.id
         kafka[:'auto.offset.reset'] ||= @topics.first.initial_offset

data/lib/karafka/routing/subscription_groups_builder.rb

@@ -24,6 +24,10 @@ module Karafka
 
       private_constant :DISTRIBUTION_KEYS
 
+      def initialize
+        @position = -1
+      end
+
       # @param topics [Karafka::Routing::Topics] all the topics based on which we want to build
       #   subscription groups
       # @return [Array<SubscriptionGroup>] all subscription groups we need in separate threads
@@ -34,7 +38,7 @@ module Karafka
           .values
           .map { |value| value.map(&:last) }
           .map { |topics_array| Routing::Topics.new(topics_array) }
-          .map { |grouped_topics| SubscriptionGroup.new(grouped_topics) }
+          .map { |grouped_topics| SubscriptionGroup.new(@position += 1, grouped_topics) }
       end
 
       private

data/lib/karafka/version.rb

@@ -3,5 +3,5 @@
 # Main module namespace
 module Karafka
   # Current Karafka version
-  VERSION = '2.0.19'
+  VERSION = '2.0.21'
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: karafka
 version: !ruby/object:Gem::Version
-  version: 2.0.19
+  version: 2.0.21
 platform: ruby
 authors:
 - Maciej Mensfeld
@@ -35,7 +35,7 @@ cert_chain:
   Qf04B9ceLUaC4fPVEz10FyobjaFoY4i32xRto3XnrzeAgfEe4swLq8bQsR3w/EF3
   MGU0FeSV2Yj7Xc2x/7BzLK8xQn5l7Yy75iPF+KP3vVmDHnNl
   -----END CERTIFICATE-----
-date: 2022-11-20 00:00:00.000000000 Z
+date: 2022-11-25 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: karafka-core
@@ -43,7 +43,7 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 2.0.2
+        version: 2.0.4
     - - "<"
       - !ruby/object:Gem::Version
         version: 3.0.0
@@ -53,7 +53,7 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 2.0.2
+        version: 2.0.4
     - - "<"
       - !ruby/object:Gem::Version
         version: 3.0.0
@@ -119,7 +119,11 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '2.3'
-description: Framework used to simplify Apache Kafka based Ruby applications development
+description: |2
+      Karafka is Ruby and Rails efficient Kafka processing framework.
+
+      Karafka allows you to capture everything that happens in your systems in large scale,
+      without having to focus on things that are not your business domain.
 email:
 - contact@karafka.io
 executables:
@@ -130,6 +134,7 @@ files:
 - ".coditsu/ci.yml"
 - ".console_irbrc"
 - ".diffend.yml"
+- ".github/FUNDING.yml"
 - ".github/ISSUE_TEMPLATE/bug_report.md"
 - ".github/ISSUE_TEMPLATE/feature_request.md"
 - ".github/workflows/ci.yml"
@@ -226,6 +231,7 @@ files:
 - lib/karafka/pro/performance_tracker.rb
 - lib/karafka/pro/processing/coordinator.rb
 - lib/karafka/pro/processing/jobs/consume_non_blocking.rb
+- lib/karafka/pro/processing/jobs/revoked_non_blocking.rb
 - lib/karafka/pro/processing/jobs_builder.rb
 - lib/karafka/pro/processing/partitioner.rb
 - lib/karafka/pro/processing/scheduler.rb
@@ -352,5 +358,5 @@ requirements: []
 rubygems_version: 3.3.7
 signing_key: 
 specification_version: 4
-summary: Efficient Kafka processing framework for Ruby and Rails
+summary: Karafka is Ruby and Rails efficient Kafka processing framework.
 test_files: []