karafka 2.4.6 → 2.4.7

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: e550fa0b7395eab3b961181319d195809baeca82c02935d2735cf98353f6ef8b
-  data.tar.gz: f734562bfb86db59a64f4fbeb596bfe126eacd428a4ade00da90d238d959ab64
+  metadata.gz: 6c94c3a4e646233f535e63bcf7a65c88d656f534bfccd21cad87be59b525adb1
+  data.tar.gz: 63b6ad0491880325d6ac0e5a7367f2a938e8a36245bd1cff48d3de73da6266c6
 SHA512:
-  metadata.gz: f80b521d063653c93caee987358573c992af2488ffec9c6465dbb04f9e51e8334548cf669b2b5f8f5783540be01fa8b52e067042cf72afda35f14181148ec87d
-  data.tar.gz: 32fd0f08102a2687f4b646d15cd07599f9aaefac957f828396af1f2d91bc91d5c922119bfec04d94cba63af86724d43cb50852de140d78e289fdbb612892dc78
+  metadata.gz: 6c04bcd1f7ed17855140e519a82a52c0cae93a3c41126120ab797bbcf172315f7f7720f12e233cc85eed4af2dd437a3364f7f5613f730c9b2e32f8b79225570f
+  data.tar.gz: 3441c4ac2230c5903dacd210b54c80c05f72c2804490229334d10656672c4ef9d9b3160c12cdd4d22916d55f1687a730980b0b3fc21099a003f36238f58025eb

data/CHANGELOG.md

@@ -1,5 +1,16 @@
 # Karafka Framework Changelog
 
+## 2.4.7 (2024-08-01)
+- [Enhancement] Introduce `Karafka::Server.mode` to check in what mode Karafka process operates (`standalone`, `swarm`, `supervisor`, `embedded`).
+- [Enhancement] Ensure `max.poll.interval.ms` is always present and populate it with librdkafka default.
+- [Enhancement] Introduce a shutdown time limit for unsubscription wait.
+- [Enhancement] Tag with `mode:swarm` each of the running swarm consumers.
+- [Change] Tag with `mode:embedded` instead of `embedded` the embedded consumers.
+- [Fix] License identifier `LGPL-3.0` is deprecated for SPDX (#2177).
+- [Fix] Fix an issue where custom clusters would not have default settings populated same as the primary cluster.
+- [Fix] Fix Rspec warnings of nil mocks.
+- [Maintenance] Cover `cooperative-sticky` librdkafka issues with integration spec.
+
 ## 2.4.6 (2024-07-22)
 - [Fix] Mitigate `rd_kafka_cgrp_terminated` and other `librdkafka` shutdown issues by unsubscribing fully prior to shutdown.
 

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    karafka (2.4.6)
+    karafka (2.4.7)
       base64 (~> 0.2)
       karafka-core (>= 2.4.3, < 2.5.0)
       waterdrop (>= 2.7.3, < 3.0.0)
@@ -41,11 +41,11 @@ GEM
       concurrent-ruby (~> 1.0)
     karafka-core (2.4.4)
       karafka-rdkafka (>= 0.15.0, < 0.18.0)
-    karafka-rdkafka (0.17.0)
+    karafka-rdkafka (0.17.1)
       ffi (~> 1.15)
       mini_portile2 (~> 2.6)
       rake (> 12)
-    karafka-testing (2.4.5)
+    karafka-testing (2.4.6)
       karafka (>= 2.4.0, < 2.5.0)
       waterdrop (>= 2.7.0)
     karafka-web (0.9.1)
@@ -88,7 +88,7 @@ GEM
       karafka-core (>= 2.4.3, < 3.0.0)
       karafka-rdkafka (>= 0.15.1)
       zeitwerk (~> 2.3)
-    zeitwerk (2.6.16)
+    zeitwerk (2.6.17)
 
 PLATFORMS
   ruby

data/LICENSE

@@ -1,7 +1,7 @@
 Copyright (c) Maciej Mensfeld
 
 Karafka is an Open Source project licensed under the terms of
-the LGPLv3 license. Please see <https://github.com/karafka/karafka/blob/master/LGPL>
+the LGPLv3 license. Please see <https://github.com/karafka/karafka/blob/master/LICENSE-LGPL>
 for license text.
 
 Karafka has also commercial-friendly license, commercial support and commercial components.

data/docker-compose.yml

@@ -3,7 +3,7 @@ version: '2'
 services:
   kafka:
     container_name: kafka
-    image: confluentinc/cp-kafka:7.6.2
+    image: confluentinc/cp-kafka:7.7.0
 
     ports:
       - 9092:9092

data/karafka.gemspec

@@ -12,7 +12,7 @@ Gem::Specification.new do |spec|
   spec.authors     = ['Maciej Mensfeld']
   spec.email       = %w[contact@karafka.io]
   spec.homepage    = 'https://karafka.io'
-  spec.licenses    = %w[LGPL-3.0 Commercial]
+  spec.licenses    = %w[LGPL-3.0-only Commercial]
   spec.summary     = 'Karafka is Ruby and Rails efficient Kafka processing framework.'
   spec.description = <<-DESC
     Karafka is Ruby and Rails efficient Kafka processing framework.

data/lib/karafka/cli/server.rb

@@ -89,6 +89,7 @@ module Karafka
         register_inclusions
         register_exclusions
 
+        Karafka::Server.execution_mode = :standalone
         Karafka::Server.run
       end
 

data/lib/karafka/cli/swarm.rb

@@ -23,6 +23,7 @@ module Karafka
         server.register_inclusions
         server.register_exclusions
 
+        Karafka::Server.execution_mode = :supervisor
         Karafka::Swarm::Supervisor.new.run
       end
     end

data/lib/karafka/connection/client.rb

@@ -8,6 +8,8 @@ module Karafka
     # It is threadsafe and provides some security measures so we won't end up operating on a
     # closed consumer instance as it causes Ruby VM process to crash.
     class Client
+      include ::Karafka::Core::Helpers::Time
+
       attr_reader :rebalance_manager
 
       # @return [Karafka::Routing::SubscriptionGroup] subscription group to which this client
@@ -24,7 +26,12 @@ module Karafka
       # How many times should we retry polling in case of a failure
       MAX_POLL_RETRIES = 20
 
-      private_constant :MAX_POLL_RETRIES
+      # How much time of the total shutdown time can we wait for our manual unsubscribe before
+      # attempting to close without unsubscribe. We try to wait for 50% of the shutdown time
+      # before we move to a regular unsubscribe.
+      COOP_UNSUBSCRIBE_FACTOR = 0.5
+
+      private_constant :MAX_POLL_RETRIES, :COOP_UNSUBSCRIBE_FACTOR
 
       # Creates a new consumer instance.
       #
@@ -261,14 +268,32 @@ module Karafka
         # an issue that gets back every few versions of librdkafka in a limited scope, for example
         # for cooperative-sticky or in a general scope. This is why we unsubscribe and wait until
         # we no longer have any assignments. That way librdkafka consumer shutdown should never
-        # happen with rebalance associated with the given consumer instance
+        # happen with rebalance associated with the given consumer instance. Since we do not want
+        # to wait forever, we also impose a limit on how long should we wait. This prioritizes
+        # shutdown stability over endless wait.
+        #
+        # The `@unsubscribing` ensures that when there would be a direct close attempt, it
+        # won't get into this loop again. This can happen when supervision decides it should close
+        # things faster
         #
         # @see https://github.com/confluentinc/librdkafka/issues/4792
         # @see https://github.com/confluentinc/librdkafka/issues/4527
         if unsubscribe?
+          @unsubscribing = true
+
+          # Give 50% of time for the final close before we reach the forceful
+          max_wait = ::Karafka::App.config.shutdown_timeout * COOP_UNSUBSCRIBE_FACTOR
+          used = 0
+          stopped_at = monotonic_now
+
           unsubscribe
 
           until assignment.empty?
+            used += monotonic_now - stopped_at
+            stopped_at = monotonic_now
+
+            break if used >= max_wait
+
             sleep(0.1)
 
             ping
@@ -709,6 +734,7 @@ module Karafka
       #
       # @return [Boolean] should we unsubscribe prior to shutdown
       def unsubscribe?
+        return false if @unsubscribing
         return false if @subscription_group.kafka.key?(:'group.instance.id')
         return false if @mode != :subscribe
         return false if assignment.empty?

data/lib/karafka/embedded.rb

@@ -27,7 +27,8 @@ module Karafka
         Thread.new do
           Thread.current.name = 'karafka.embedded'
 
-          Karafka::Process.tags.add(:execution_mode, 'embedded')
+          Karafka::Process.tags.add(:execution_mode, 'mode:embedded')
+          Karafka::Server.execution_mode = :embedded
           Karafka::Server.start
         end
       end

data/lib/karafka/routing/consumer_group.rb

@@ -8,6 +8,12 @@ module Karafka
     # @note A single consumer group represents Kafka consumer group, but it may not match 1:1 with
     #   subscription groups. There can be more subscription groups than consumer groups
     class ConsumerGroup
+      include Helpers::ConfigImporter.new(
+        activity_manager: %i[internal routing activity_manager],
+        builder: %i[internal routing builder],
+        subscription_groups_builder: %i[internal routing subscription_groups_builder]
+      )
+
       attr_reader :id, :topics, :name
 
       # This is a "virtual" attribute that is not building subscription groups.
@@ -32,7 +38,7 @@ module Karafka
 
       # @return [Boolean] true if this consumer group should be active in our current process
       def active?
-        config.internal.routing.activity_manager.active?(:consumer_groups, name)
+        activity_manager.active?(:consumer_groups, name)
       end
 
       # Builds a topic representation inside of a current consumer group route
@@ -43,7 +49,7 @@ module Karafka
         topic = Topic.new(name, self)
         @topics << Proxy.new(
           topic,
-          config.internal.routing.builder.defaults,
+          builder.defaults,
           &block
         ).target
         built_topic = @topics.last
@@ -72,11 +78,7 @@ module Karafka
       # @return [Array<Routing::SubscriptionGroup>] all the subscription groups build based on
       #   the consumer group topics
       def subscription_groups
-        @subscription_groups ||= config
-                                 .internal
-                                 .routing
-                                 .subscription_groups_builder
-                                 .call(topics)
+        @subscription_groups ||= subscription_groups_builder.call(topics)
       end
 
       # Hashed version of consumer group that can be used for validation purposes
@@ -88,13 +90,6 @@ module Karafka
           id: id
         }.freeze
       end
-
-      private
-
-      # @return [Karafka::Core::Configurable::Node] root node config
-      def config
-        ::Karafka::App.config
-      end
     end
   end
 end

data/lib/karafka/routing/subscription_group.rb

@@ -120,6 +120,7 @@ module Karafka
       def build_kafka
         kafka = Setup::AttributesMap.consumer(@topics.first.kafka.dup)
 
+        inject_defaults(kafka)
         inject_group_instance_id(kafka)
         inject_client_id(kafka)
 
@@ -132,6 +133,13 @@ module Karafka
         kafka
       end
 
+      # Injects (if needed) defaults
+      #
+      # @param kafka [Hash] kafka level config
+      def inject_defaults(kafka)
+        Setup::DefaultsInjector.consumer(kafka)
+      end
+
       # Sets (if needed) the client.id attribute
       #
       # @param kafka [Hash] kafka level config

data/lib/karafka/server.rb

@@ -19,6 +19,17 @@ module Karafka
       # Jobs queue
       attr_accessor :jobs_queue
 
+      # Mode in which the Karafka server is executed. It can be:
+      #
+      # - :standalone - regular karafka consumer process
+      # - :embedded - embedded in a different process and not supervised
+      # - :supervisor - swarm supervisor process
+      # - :swarm - one of swarm processes
+      #
+      # Sometimes it is important to know in what mode we operate, especially from UI perspective
+      # as not everything is possible when operating in non-standalone mode, etc.
+      attr_accessor :execution_mode
+
       # Method which runs app
       def run
         self.listeners = []
@@ -171,5 +182,10 @@ module Karafka
         config.internal.process
       end
     end
+
+    # Always start with standalone so there always is a value for the execution mode.
+    # This is overwritten quickly during boot, but just in case someone would reach it prior to
+    # booting, we want to have the default value.
+    self.execution_mode = :standalone
   end
 end

data/lib/karafka/setup/config.rb

@@ -14,34 +14,6 @@ module Karafka
     class Config
       extend ::Karafka::Core::Configurable
 
-      # Defaults for kafka settings, that will be overwritten only if not present already
-      KAFKA_DEFAULTS = {
-        # We emit the statistics by default, so all the instrumentation and web-ui work out of
-        # the box, without requiring users to take any extra actions aside from enabling.
-        'statistics.interval.ms': 5_000,
-        'client.software.name': 'karafka',
-        'client.software.version': [
-          "v#{Karafka::VERSION}",
-          "rdkafka-ruby-v#{Rdkafka::VERSION}",
-          "librdkafka-v#{Rdkafka::LIBRDKAFKA_VERSION}"
-        ].join('-')
-      }.freeze
-
-      # Contains settings that should not be used in production but make life easier in dev
-      KAFKA_DEV_DEFAULTS = {
-        # Will create non-existing topics automatically.
-        # Note that the broker needs to be configured with `auto.create.topics.enable=true`
-        # While it is not recommended in prod, it simplifies work in dev
-        'allow.auto.create.topics': 'true',
-        # We refresh the cluster state often as newly created topics in dev may not be detected
-        # fast enough. Fast enough means within reasonable time to provide decent user experience
-        # While it's only a one time thing for new topics, it can still be irritating to have to
-        # restart the process.
-        'topic.metadata.refresh.interval.ms': 5_000
-      }.freeze
-
-      private_constant :KAFKA_DEFAULTS, :KAFKA_DEV_DEFAULTS
-
       # Available settings
 
       # Namespace for Pro version related license management. If you use LGPL, no need to worry
@@ -350,7 +322,6 @@ module Karafka
           Pro::Loader.pre_setup_all(config) if Karafka.pro?
 
           configure(&block)
-          merge_kafka_defaults!(config)
 
           Contracts::Config.new.validate!(config.to_h)
 
@@ -374,26 +345,6 @@ module Karafka
 
         private
 
-        # Propagates the kafka setting defaults unless they are already present
-        # This makes it easier to set some values that users usually don't change but still allows
-        # them to overwrite the whole hash if they want to
-        # @param config [Karafka::Core::Configurable::Node] config of this producer
-        def merge_kafka_defaults!(config)
-          KAFKA_DEFAULTS.each do |key, value|
-            next if config.kafka.key?(key)
-
-            config.kafka[key] = value
-          end
-
-          return if Karafka::App.env.production?
-
-          KAFKA_DEV_DEFAULTS.each do |key, value|
-            next if config.kafka.key?(key)
-
-            config.kafka[key] = value
-          end
-        end
-
         # Sets up all the components that are based on the user configuration
         # @note At the moment it is only WaterDrop
         def configure_components

data/lib/karafka/setup/defaults_injector.rb

@@ -0,0 +1,64 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Setup
+    # Injector that enriches each Kafka cluster with needed defaults. User may use more than one
+    # cluster and define them on a per-topic basis. We use this when we build the final config
+    # per subscription group.
+    module DefaultsInjector
+      # Defaults for kafka settings, that will be overwritten only if not present already
+      CONSUMER_KAFKA_DEFAULTS = {
+        # We emit the statistics by default, so all the instrumentation and web-ui work out of
+        # the box, without requiring users to take any extra actions aside from enabling.
+        'statistics.interval.ms': 5_000,
+        'client.software.name': 'karafka',
+        # Same as librdkafka default, we inject it nonetheless to have it always available as
+        # some features may use this value for computation and it is better to ensure, we do
+        # always have it
+        'max.poll.interval.ms': 300_000,
+        'client.software.version': [
+          "v#{Karafka::VERSION}",
+          "rdkafka-ruby-v#{Rdkafka::VERSION}",
+          "librdkafka-v#{Rdkafka::LIBRDKAFKA_VERSION}"
+        ].join('-')
+      }.freeze
+
+      # Contains settings that should not be used in production but make life easier in dev
+      CONSUMER_KAFKA_DEV_DEFAULTS = {
+        # Will create non-existing topics automatically.
+        # Note that the broker needs to be configured with `auto.create.topics.enable=true`
+        # While it is not recommended in prod, it simplifies work in dev
+        'allow.auto.create.topics': 'true',
+        # We refresh the cluster state often as newly created topics in dev may not be detected
+        # fast enough. Fast enough means within reasonable time to provide decent user experience
+        # While it's only a one time thing for new topics, it can still be irritating to have to
+        # restart the process.
+        'topic.metadata.refresh.interval.ms': 5_000
+      }.freeze
+
+      private_constant :CONSUMER_KAFKA_DEFAULTS, :CONSUMER_KAFKA_DEV_DEFAULTS
+
+      class << self
+        # Propagates the kafka setting defaults unless they are already present for consumer config
+        # This makes it easier to set some values that users usually don't change but still allows
+        # them to overwrite the whole hash if they want to
+        # @param kafka_config [Hash] kafka scoped config
+        def consumer(kafka_config)
+          CONSUMER_KAFKA_DEFAULTS.each do |key, value|
+            next if kafka_config.key?(key)
+
+            kafka_config[key] = value
+          end
+
+          return if Karafka::App.env.production?
+
+          CONSUMER_KAFKA_DEV_DEFAULTS.each do |key, value|
+            next if kafka_config.key?(key)
+
+            kafka_config[key] = value
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/swarm/node.rb

@@ -68,6 +68,8 @@ module Karafka
           monitor.subscribe(liveness_listener)
           monitor.instrument('swarm.node.after_fork', caller: self)
 
+          Karafka::Process.tags.add(:execution_mode, 'mode:swarm')
+          Server.execution_mode = :swarm
           Server.run
 
           @writer.close

data/lib/karafka/version.rb

@@ -3,5 +3,5 @@
 # Main module namespace
 module Karafka
   # Current Karafka version
-  VERSION = '2.4.6'
+  VERSION = '2.4.7'
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: karafka
 version: !ruby/object:Gem::Version
-  version: 2.4.6
+  version: 2.4.7
 platform: ruby
 authors:
 - Maciej Mensfeld
@@ -35,7 +35,7 @@ cert_chain:
   AnG1dJU+yL2BK7vaVytLTstJME5mepSZ46qqIJXMuWob/YPDmVaBF39TDSG9e34s
   msG3BiCqgOgHAnL23+CN3Rt8MsuRfEtoTKpJVcCfoEoNHOkc
   -----END CERTIFICATE-----
-date: 2024-07-22 00:00:00.000000000 Z
+date: 2024-08-01 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: base64
@@ -505,6 +505,7 @@ files:
 - lib/karafka/server.rb
 - lib/karafka/setup/attributes_map.rb
 - lib/karafka/setup/config.rb
+- lib/karafka/setup/defaults_injector.rb
 - lib/karafka/setup/dsl.rb
 - lib/karafka/status.rb
 - lib/karafka/swarm.rb
@@ -524,7 +525,7 @@ files:
 - renovate.json
 homepage: https://karafka.io
 licenses:
-- LGPL-3.0
+- LGPL-3.0-only
 - Commercial
 metadata:
   funding_uri: https://karafka.io/#become-pro