karafka 2.4.5 → 2.4.7

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: fe76e139b2007e11b0cb9393bc94e2b5b9cad5dc40d69b5ad2bfa34a0555e2b8
-  data.tar.gz: e288c63648d17388f577a45b51267e30fceb4726def0a42c40a9133de27d93f8
+  metadata.gz: 6c94c3a4e646233f535e63bcf7a65c88d656f534bfccd21cad87be59b525adb1
+  data.tar.gz: 63b6ad0491880325d6ac0e5a7367f2a938e8a36245bd1cff48d3de73da6266c6
 SHA512:
-  metadata.gz: b5e46439a216e66c728ceb74070832407d39b1ebe06d21c858f3e3a0e8e599df15b749318131aa82a5bc994e6dd7ed3094f44639e706a57231f031f0e52186b6
-  data.tar.gz: dc44b6a2f0e5d21bdcbe156f0cc94c324d586c74e9fb757454157ac014897fc3cbd683acb19e6397d668f67a9c68e97d683521f1ceecdbf338e24f3d497c688d
+  metadata.gz: 6c04bcd1f7ed17855140e519a82a52c0cae93a3c41126120ab797bbcf172315f7f7720f12e233cc85eed4af2dd437a3364f7f5613f730c9b2e32f8b79225570f
+  data.tar.gz: 3441c4ac2230c5903dacd210b54c80c05f72c2804490229334d10656672c4ef9d9b3160c12cdd4d22916d55f1687a730980b0b3fc21099a003f36238f58025eb

data/CHANGELOG.md

@@ -1,4 +1,18 @@
-# Karafka framework changelog
+# Karafka Framework Changelog
+
+## 2.4.7 (2024-08-01)
+- [Enhancement] Introduce `Karafka::Server.mode` to check in what mode Karafka process operates (`standalone`, `swarm`, `supervisor`, `embedded`).
+- [Enhancement] Ensure `max.poll.interval.ms` is always present and populate it with librdkafka default.
+- [Enhancement] Introduce a shutdown time limit for unsubscription wait.
+- [Enhancement] Tag with `mode:swarm` each of the running swarm consumers.
+- [Change] Tag with `mode:embedded` instead of `embedded` the embedded consumers.
+- [Fix] License identifier `LGPL-3.0` is deprecated for SPDX (#2177).
+- [Fix] Fix an issue where custom clusters would not have default settings populated same as the primary cluster.
+- [Fix] Fix Rspec warnings of nil mocks.
+- [Maintenance] Cover `cooperative-sticky` librdkafka issues with integration spec.
+
+## 2.4.6 (2024-07-22)
+- [Fix] Mitigate `rd_kafka_cgrp_terminated` and other `librdkafka` shutdown issues by unsubscribing fully prior to shutdown.
 
 ## 2.4.5 (2024-07-18)
 - [Change] Inject `client.id` when building subscription group and not during the initial setup.

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    karafka (2.4.5)
+    karafka (2.4.7)
       base64 (~> 0.2)
       karafka-core (>= 2.4.3, < 2.5.0)
       waterdrop (>= 2.7.3, < 3.0.0)
@@ -39,13 +39,13 @@ GEM
       activesupport (>= 6.1)
     i18n (1.14.5)
       concurrent-ruby (~> 1.0)
-    karafka-core (2.4.3)
-      karafka-rdkafka (>= 0.15.0, < 0.17.0)
-    karafka-rdkafka (0.16.0)
+    karafka-core (2.4.4)
+      karafka-rdkafka (>= 0.15.0, < 0.18.0)
+    karafka-rdkafka (0.17.1)
       ffi (~> 1.15)
       mini_portile2 (~> 2.6)
       rake (> 12)
-    karafka-testing (2.4.4)
+    karafka-testing (2.4.6)
       karafka (>= 2.4.0, < 2.5.0)
       waterdrop (>= 2.7.0)
     karafka-web (0.9.1)
@@ -58,7 +58,7 @@ GEM
     minitest (5.24.0)
     mutex_m (0.2.0)
     ostruct (0.6.0)
-    rack (3.1.4)
+    rack (3.1.5)
     rake (13.2.1)
     roda (3.81.0)
       rack
@@ -84,11 +84,11 @@ GEM
     tilt (2.4.0)
     tzinfo (2.0.6)
       concurrent-ruby (~> 1.0)
-    waterdrop (2.7.3)
-      karafka-core (>= 2.4.0, < 3.0.0)
+    waterdrop (2.7.4)
+      karafka-core (>= 2.4.3, < 3.0.0)
       karafka-rdkafka (>= 0.15.1)
       zeitwerk (~> 2.3)
-    zeitwerk (2.6.16)
+    zeitwerk (2.6.17)
 
 PLATFORMS
   ruby
@@ -106,4 +106,4 @@ DEPENDENCIES
   simplecov
 
 BUNDLED WITH
-   2.5.14
+   2.5.11

data/LICENSE

@@ -1,7 +1,7 @@
 Copyright (c) Maciej Mensfeld
 
 Karafka is an Open Source project licensed under the terms of
-the LGPLv3 license. Please see <https://github.com/karafka/karafka/blob/master/LGPL>
+the LGPLv3 license. Please see <https://github.com/karafka/karafka/blob/master/LICENSE-LGPL>
 for license text.
 
 Karafka has also commercial-friendly license, commercial support and commercial components.

data/docker-compose.yml

@@ -3,7 +3,7 @@ version: '2'
 services:
   kafka:
     container_name: kafka
-    image: confluentinc/cp-kafka:7.6.2
+    image: confluentinc/cp-kafka:7.7.0
 
     ports:
       - 9092:9092

data/karafka.gemspec

@@ -12,7 +12,7 @@ Gem::Specification.new do |spec|
   spec.authors     = ['Maciej Mensfeld']
   spec.email       = %w[contact@karafka.io]
   spec.homepage    = 'https://karafka.io'
-  spec.licenses    = %w[LGPL-3.0 Commercial]
+  spec.licenses    = %w[LGPL-3.0-only Commercial]
   spec.summary     = 'Karafka is Ruby and Rails efficient Kafka processing framework.'
   spec.description = <<-DESC
     Karafka is Ruby and Rails efficient Kafka processing framework.

data/lib/karafka/cli/server.rb

@@ -89,6 +89,7 @@ module Karafka
         register_inclusions
         register_exclusions
 
+        Karafka::Server.execution_mode = :standalone
         Karafka::Server.run
       end
 

data/lib/karafka/cli/swarm.rb

@@ -23,6 +23,7 @@ module Karafka
         server.register_inclusions
         server.register_exclusions
 
+        Karafka::Server.execution_mode = :supervisor
         Karafka::Swarm::Supervisor.new.run
       end
     end

data/lib/karafka/connection/client.rb

@@ -8,6 +8,8 @@ module Karafka
     # It is threadsafe and provides some security measures so we won't end up operating on a
     # closed consumer instance as it causes Ruby VM process to crash.
     class Client
+      include ::Karafka::Core::Helpers::Time
+
       attr_reader :rebalance_manager
 
       # @return [Karafka::Routing::SubscriptionGroup] subscription group to which this client
@@ -24,7 +26,12 @@ module Karafka
       # How many times should we retry polling in case of a failure
       MAX_POLL_RETRIES = 20
 
-      private_constant :MAX_POLL_RETRIES
+      # How much time of the total shutdown time can we wait for our manual unsubscribe before
+      # attempting to close without unsubscribe. We try to wait for 50% of the shutdown time
+      # before we move to a regular unsubscribe.
+      COOP_UNSUBSCRIBE_FACTOR = 0.5
+
+      private_constant :MAX_POLL_RETRIES, :COOP_UNSUBSCRIBE_FACTOR
 
       # Creates a new consumer instance.
       #
@@ -257,14 +264,36 @@ module Karafka
 
       # Gracefully stops topic consumption.
       def stop
-        # In case of cooperative-sticky, there is a bug in librdkafka that may hang it.
-        # To mitigate it we first need to unsubscribe so we will not receive any assignments and
-        # only then we should be good to go.
+        # librdkafka has several constant issues when shutting down during rebalance. This is
+        # an issue that gets back every few versions of librdkafka in a limited scope, for example
+        # for cooperative-sticky or in a general scope. This is why we unsubscribe and wait until
+        # we no longer have any assignments. That way librdkafka consumer shutdown should never
+        # happen with rebalance associated with the given consumer instance. Since we do not want
+        # to wait forever, we also impose a limit on how long should we wait. This prioritizes
+        # shutdown stability over endless wait.
+        #
+        # The `@unsubscribing` ensures that when there would be a direct close attempt, it
+        # won't get into this loop again. This can happen when supervision decides it should close
+        # things faster
+        #
+        # @see https://github.com/confluentinc/librdkafka/issues/4792
         # @see https://github.com/confluentinc/librdkafka/issues/4527
-        if @subscription_group.kafka[:'partition.assignment.strategy'] == 'cooperative-sticky'
+        if unsubscribe?
+          @unsubscribing = true
+
+          # Give 50% of time for the final close before we reach the forceful
+          max_wait = ::Karafka::App.config.shutdown_timeout * COOP_UNSUBSCRIBE_FACTOR
+          used = 0
+          stopped_at = monotonic_now
+
           unsubscribe
 
           until assignment.empty?
+            used += monotonic_now - stopped_at
+            stopped_at = monotonic_now
+
+            break if used >= max_wait
+
             sleep(0.1)
 
             ping
@@ -658,8 +687,13 @@ module Karafka
         subscriptions = @subscription_group.subscriptions
         assignments = @subscription_group.assignments(consumer)
 
-        consumer.subscribe(*subscriptions) if subscriptions
-        consumer.assign(assignments) if assignments
+        if subscriptions
+          consumer.subscribe(*subscriptions)
+          @mode = :subscribe
+        elsif assignments
+          consumer.assign(assignments)
+          @mode = :assign
+        end
 
         consumer
       end
@@ -690,6 +724,23 @@ module Karafka
         @kafka.start
         @kafka
       end
+
+      # Decides whether or not we should unsubscribe prior to closing.
+      #
+      # We cannot do it when there is a static group membership assignment as it would be
+      # reassigned.
+      # We cannot do it also for assign mode because then there are no subscriptions
+      # We also do not do it if there are no assignments at all as it does not make sense
+      #
+      # @return [Boolean] should we unsubscribe prior to shutdown
+      def unsubscribe?
+        return false if @unsubscribing
+        return false if @subscription_group.kafka.key?(:'group.instance.id')
+        return false if @mode != :subscribe
+        return false if assignment.empty?
+
+        true
+      end
     end
   end
 end

data/lib/karafka/embedded.rb

@@ -27,7 +27,8 @@ module Karafka
         Thread.new do
           Thread.current.name = 'karafka.embedded'
 
-          Karafka::Process.tags.add(:execution_mode, 'embedded')
+          Karafka::Process.tags.add(:execution_mode, 'mode:embedded')
+          Karafka::Server.execution_mode = :embedded
           Karafka::Server.start
         end
       end

data/lib/karafka/routing/consumer_group.rb

@@ -8,6 +8,12 @@ module Karafka
     # @note A single consumer group represents Kafka consumer group, but it may not match 1:1 with
     #   subscription groups. There can be more subscription groups than consumer groups
     class ConsumerGroup
+      include Helpers::ConfigImporter.new(
+        activity_manager: %i[internal routing activity_manager],
+        builder: %i[internal routing builder],
+        subscription_groups_builder: %i[internal routing subscription_groups_builder]
+      )
+
       attr_reader :id, :topics, :name
 
       # This is a "virtual" attribute that is not building subscription groups.
@@ -32,7 +38,7 @@ module Karafka
 
       # @return [Boolean] true if this consumer group should be active in our current process
       def active?
-        config.internal.routing.activity_manager.active?(:consumer_groups, name)
+        activity_manager.active?(:consumer_groups, name)
       end
 
       # Builds a topic representation inside of a current consumer group route
@@ -43,7 +49,7 @@ module Karafka
         topic = Topic.new(name, self)
         @topics << Proxy.new(
           topic,
-          config.internal.routing.builder.defaults,
+          builder.defaults,
           &block
         ).target
         built_topic = @topics.last
@@ -72,11 +78,7 @@ module Karafka
       # @return [Array<Routing::SubscriptionGroup>] all the subscription groups build based on
       #   the consumer group topics
       def subscription_groups
-        @subscription_groups ||= config
-                                 .internal
-                                 .routing
-                                 .subscription_groups_builder
-                                 .call(topics)
+        @subscription_groups ||= subscription_groups_builder.call(topics)
       end
 
       # Hashed version of consumer group that can be used for validation purposes
@@ -88,13 +90,6 @@ module Karafka
           id: id
         }.freeze
       end
-
-      private
-
-      # @return [Karafka::Core::Configurable::Node] root node config
-      def config
-        ::Karafka::App.config
-      end
     end
   end
 end

data/lib/karafka/routing/subscription_group.rb

@@ -120,6 +120,7 @@ module Karafka
       def build_kafka
         kafka = Setup::AttributesMap.consumer(@topics.first.kafka.dup)
 
+        inject_defaults(kafka)
         inject_group_instance_id(kafka)
         inject_client_id(kafka)
 
@@ -132,6 +133,13 @@ module Karafka
         kafka
       end
 
+      # Injects (if needed) defaults
+      #
+      # @param kafka [Hash] kafka level config
+      def inject_defaults(kafka)
+        Setup::DefaultsInjector.consumer(kafka)
+      end
+
       # Sets (if needed) the client.id attribute
       #
       # @param kafka [Hash] kafka level config

data/lib/karafka/server.rb

@@ -19,6 +19,17 @@ module Karafka
       # Jobs queue
       attr_accessor :jobs_queue
 
+      # Mode in which the Karafka server is executed. It can be:
+      #
+      # - :standalone - regular karafka consumer process
+      # - :embedded - embedded in a different process and not supervised
+      # - :supervisor - swarm supervisor process
+      # - :swarm - one of swarm processes
+      #
+      # Sometimes it is important to know in what mode we operate, especially from UI perspective
+      # as not everything is possible when operating in non-standalone mode, etc.
+      attr_accessor :execution_mode
+
       # Method which runs app
       def run
         self.listeners = []
@@ -171,5 +182,10 @@ module Karafka
         config.internal.process
       end
     end
+
+    # Always start with standalone so there always is a value for the execution mode.
+    # This is overwritten quickly during boot, but just in case someone would reach it prior to
+    # booting, we want to have the default value.
+    self.execution_mode = :standalone
   end
 end

data/lib/karafka/setup/attributes_map.rb

@@ -38,6 +38,7 @@ module Karafka
         default_topic_conf
         enable.auto.commit
         enable.auto.offset.store
+        enable.metrics.push
         enable.partition.eof
         enable.random.seed
         enable.sasl.oauthbearer.unsecure.jwt
@@ -89,6 +90,8 @@ module Karafka
         reconnect.backoff.max.ms
         reconnect.backoff.ms
         resolve_cb
+        retry.backoff.max.ms
+        retry.backoff.ms
         sasl.kerberos.keytab
         sasl.kerberos.kinit.cmd
         sasl.kerberos.min.time.before.relogin
@@ -182,6 +185,7 @@ module Karafka
         dr_msg_cb
         enable.gapless.guarantee
         enable.idempotence
+        enable.metrics.push
         enable.random.seed
         enable.sasl.oauthbearer.unsecure.jwt
         enable.ssl.certificate.verification

data/lib/karafka/setup/config.rb

@@ -14,34 +14,6 @@ module Karafka
     class Config
       extend ::Karafka::Core::Configurable
 
-      # Defaults for kafka settings, that will be overwritten only if not present already
-      KAFKA_DEFAULTS = {
-        # We emit the statistics by default, so all the instrumentation and web-ui work out of
-        # the box, without requiring users to take any extra actions aside from enabling.
-        'statistics.interval.ms': 5_000,
-        'client.software.name': 'karafka',
-        'client.software.version': [
-          "v#{Karafka::VERSION}",
-          "rdkafka-ruby-v#{Rdkafka::VERSION}",
-          "librdkafka-v#{Rdkafka::LIBRDKAFKA_VERSION}"
-        ].join('-')
-      }.freeze
-
-      # Contains settings that should not be used in production but make life easier in dev
-      KAFKA_DEV_DEFAULTS = {
-        # Will create non-existing topics automatically.
-        # Note that the broker needs to be configured with `auto.create.topics.enable=true`
-        # While it is not recommended in prod, it simplifies work in dev
-        'allow.auto.create.topics': 'true',
-        # We refresh the cluster state often as newly created topics in dev may not be detected
-        # fast enough. Fast enough means within reasonable time to provide decent user experience
-        # While it's only a one time thing for new topics, it can still be irritating to have to
-        # restart the process.
-        'topic.metadata.refresh.interval.ms': 5_000
-      }.freeze
-
-      private_constant :KAFKA_DEFAULTS, :KAFKA_DEV_DEFAULTS
-
       # Available settings
 
       # Namespace for Pro version related license management. If you use LGPL, no need to worry
@@ -350,7 +322,6 @@ module Karafka
           Pro::Loader.pre_setup_all(config) if Karafka.pro?
 
           configure(&block)
-          merge_kafka_defaults!(config)
 
           Contracts::Config.new.validate!(config.to_h)
 
@@ -374,26 +345,6 @@ module Karafka
 
         private
 
-        # Propagates the kafka setting defaults unless they are already present
-        # This makes it easier to set some values that users usually don't change but still allows
-        # them to overwrite the whole hash if they want to
-        # @param config [Karafka::Core::Configurable::Node] config of this producer
-        def merge_kafka_defaults!(config)
-          KAFKA_DEFAULTS.each do |key, value|
-            next if config.kafka.key?(key)
-
-            config.kafka[key] = value
-          end
-
-          return if Karafka::App.env.production?
-
-          KAFKA_DEV_DEFAULTS.each do |key, value|
-            next if config.kafka.key?(key)
-
-            config.kafka[key] = value
-          end
-        end
-
         # Sets up all the components that are based on the user configuration
         # @note At the moment it is only WaterDrop
         def configure_components

data/lib/karafka/setup/defaults_injector.rb

@@ -0,0 +1,64 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Setup
+    # Injector that enriches each Kafka cluster with needed defaults. User may use more than one
+    # cluster and define them on a per-topic basis. We use this when we build the final config
+    # per subscription group.
+    module DefaultsInjector
+      # Defaults for kafka settings, that will be overwritten only if not present already
+      CONSUMER_KAFKA_DEFAULTS = {
+        # We emit the statistics by default, so all the instrumentation and web-ui work out of
+        # the box, without requiring users to take any extra actions aside from enabling.
+        'statistics.interval.ms': 5_000,
+        'client.software.name': 'karafka',
+        # Same as librdkafka default, we inject it nonetheless to have it always available as
+        # some features may use this value for computation and it is better to ensure, we do
+        # always have it
+        'max.poll.interval.ms': 300_000,
+        'client.software.version': [
+          "v#{Karafka::VERSION}",
+          "rdkafka-ruby-v#{Rdkafka::VERSION}",
+          "librdkafka-v#{Rdkafka::LIBRDKAFKA_VERSION}"
+        ].join('-')
+      }.freeze
+
+      # Contains settings that should not be used in production but make life easier in dev
+      CONSUMER_KAFKA_DEV_DEFAULTS = {
+        # Will create non-existing topics automatically.
+        # Note that the broker needs to be configured with `auto.create.topics.enable=true`
+        # While it is not recommended in prod, it simplifies work in dev
+        'allow.auto.create.topics': 'true',
+        # We refresh the cluster state often as newly created topics in dev may not be detected
+        # fast enough. Fast enough means within reasonable time to provide decent user experience
+        # While it's only a one time thing for new topics, it can still be irritating to have to
+        # restart the process.
+        'topic.metadata.refresh.interval.ms': 5_000
+      }.freeze
+
+      private_constant :CONSUMER_KAFKA_DEFAULTS, :CONSUMER_KAFKA_DEV_DEFAULTS
+
+      class << self
+        # Propagates the kafka setting defaults unless they are already present for consumer config
+        # This makes it easier to set some values that users usually don't change but still allows
+        # them to overwrite the whole hash if they want to
+        # @param kafka_config [Hash] kafka scoped config
+        def consumer(kafka_config)
+          CONSUMER_KAFKA_DEFAULTS.each do |key, value|
+            next if kafka_config.key?(key)
+
+            kafka_config[key] = value
+          end
+
+          return if Karafka::App.env.production?
+
+          CONSUMER_KAFKA_DEV_DEFAULTS.each do |key, value|
+            next if kafka_config.key?(key)
+
+            kafka_config[key] = value
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/swarm/node.rb

@@ -68,6 +68,8 @@ module Karafka
           monitor.subscribe(liveness_listener)
           monitor.instrument('swarm.node.after_fork', caller: self)
 
+          Karafka::Process.tags.add(:execution_mode, 'mode:swarm')
+          Server.execution_mode = :swarm
           Server.run
 
           @writer.close

data/lib/karafka/version.rb

@@ -3,5 +3,5 @@
 # Main module namespace
 module Karafka
   # Current Karafka version
-  VERSION = '2.4.5'
+  VERSION = '2.4.7'
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: karafka
 version: !ruby/object:Gem::Version
-  version: 2.4.5
+  version: 2.4.7
 platform: ruby
 authors:
 - Maciej Mensfeld
@@ -35,7 +35,7 @@ cert_chain:
   AnG1dJU+yL2BK7vaVytLTstJME5mepSZ46qqIJXMuWob/YPDmVaBF39TDSG9e34s
   msG3BiCqgOgHAnL23+CN3Rt8MsuRfEtoTKpJVcCfoEoNHOkc
   -----END CERTIFICATE-----
-date: 2024-07-18 00:00:00.000000000 Z
+date: 2024-08-01 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: base64
@@ -505,6 +505,7 @@ files:
 - lib/karafka/server.rb
 - lib/karafka/setup/attributes_map.rb
 - lib/karafka/setup/config.rb
+- lib/karafka/setup/defaults_injector.rb
 - lib/karafka/setup/dsl.rb
 - lib/karafka/status.rb
 - lib/karafka/swarm.rb
@@ -524,7 +525,7 @@ files:
 - renovate.json
 homepage: https://karafka.io
 licenses:
-- LGPL-3.0
+- LGPL-3.0-only
 - Commercial
 metadata:
   funding_uri: https://karafka.io/#become-pro