karafka 2.3.2 → 2.3.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: cb47082224d857f3029f9bb8e1b04a35e6b8ed2f7ae75bbe52bf1b778ff56226
-  data.tar.gz: 53d59fd7e140f5b3e9b89dd3e4af28469bc534074110e2e93fae24c59bf81b88
+  metadata.gz: 5e7307013403e4df36017d242a2fc2a56555376af8f2015474a9602ba655343a
+  data.tar.gz: '08807329903078bf5c8014da5250aa2c96bfd170f28e57e8110c018ed480e8f9'
 SHA512:
-  metadata.gz: 00e09a345122ad2facaf8adcbb52fae3ce87374083d9e6785a1f07a74c87e53c6b0b3dd32b82566e846d160569aafc55c614013d8c9f95664612150fb51d07b1
-  data.tar.gz: 283e50a6b3b25579419bdc9b947e5ba802e22a1cd6d0097ab8929c5394d3858461bda15c3e70a0d2d466a8705466ef1d5b24e1a8bfe5ac8e356c64790049c7b0
+  metadata.gz: fdd01cf50e55478e2dbd7bdbdac4c42b7f86c0b55507023cce0e14e61697936ea80ab8fb00d9447ec39f2179d5e59aebc2e763e1202323319122391994954d69
+  data.tar.gz: 379eafa7fb52d35ae003583a98ae03d9cf5ff799c1599d1a2b22966d9f4e5a0fe65dd7a21b743bc81ca153dc73313f3869e1d4ac01e7784df87dafabbd98533e

data/CHANGELOG.md

@@ -1,5 +1,16 @@
 # Karafka framework changelog
 
+## 2.3.4 (2024-04-11)
+- [Fix] Seek consumer group on a topic level is updating only recent partition.
+
+## 2.3.3 (2024-02-26)
+- [Enhancement] Routing based topics allocation for swarm (Pro)
+- [Enhancement] Publish the `-1` shutdown reason status for a non-responding node in swarm.
+- [Enhancement] Allow for using the `distribution` mode for DataDog listener histogram reporting (Aerdayne).
+- [Change] Change `internal.swarm.node_report_timeout` to 60 seconds from 30 seconds to compensate for long pollings.
+- [Fix] Static membership routing evaluation happens too early in swarm.
+- [Fix] Close producer in supervisor prior to forking and warmup to prevent invalid memory states.
+
 ## 2.3.2 (2024-02-16)
 - **[Feature]** Provide swarm capabilities to OSS and Pro.
 - **[Feature]** Provide ability to use complex strategies in DLQ (Pro).

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    karafka (2.3.2)
+    karafka (2.3.4)
       karafka-core (>= 2.3.0, < 2.4.0)
       waterdrop (>= 2.6.12, < 3.0.0)
       zeitwerk (~> 2.3)

data/README.md

@@ -9,13 +9,13 @@
 Karafka is a Ruby and Rails multi-threaded efficient Kafka processing framework that:
 
 - Has a built-in [Web UI](https://karafka.io/docs/Web-UI-Features/) providing a convenient way to monitor and manage Karafka-based applications.
-- Supports parallel processing in [multiple threads](https://karafka.io/docs/Concurrency-and-multithreading) (also for a [single topic partition](https://karafka.io/docs/Pro-Virtual-Partitions) work)
+- Supports parallel processing in [multiple threads](https://karafka.io/docs/Concurrency-and-multithreading) (also for a [single topic partition](https://karafka.io/docs/Pro-Virtual-Partitions) work) and [processes](https://karafka.io/docs/Swarm-Multi-Process).
 - [Automatically integrates](https://karafka.io/docs/Integrating-with-Ruby-on-Rails-and-other-frameworks#integrating-with-ruby-on-rails) with Ruby on Rails
 - Has [ActiveJob backend](https://karafka.io/docs/Active-Job) support (including [ordered jobs](https://karafka.io/docs/Pro-Enhanced-Active-Job#ordered-jobs))
 - Has a seamless [Dead Letter Queue](https://karafka.io/docs/Dead-Letter-Queue/) functionality built-in
 - Supports in-development [code reloading](https://karafka.io/docs/Auto-reload-of-code-changes-in-development)
 - Is powered by [librdkafka](https://github.com/edenhill/librdkafka) (the Apache Kafka C/C++ client library)
-- Has an out-of the box [StatsD/DataDog monitoring](https://karafka.io/docs/Monitoring-and-logging) with a dashboard template.
+- Has an out-of the box [AppSignal](https://karafka.io/docs/Monitoring-and-Logging/#appsignal-metrics-and-error-tracking) and [StatsD/DataDog](https://karafka.io/docs/Monitoring-and-Logging/#datadog-and-statsd-integration) monitoring with dashboard templates.
 
 ```ruby
 # Define what topics you want to consume with which consumers in karafka.rb

data/config/locales/errors.yml

@@ -15,6 +15,9 @@ en:
       pause_with_exponential_backoff_format: needs to be either true or false
       shutdown_timeout_format: needs to be an integer bigger than 0
       max_wait_time_format: needs to be an integer bigger than 0
+      max_wait_time_max_wait_time_vs_swarm_node_report_timeout: >
+        cannot be more than 80% of internal.swarm.node_report_timeout.
+        Decrease max_wait_time or increase node_report_timeout
       kafka_format: needs to be a filled hash
       key_must_be_a_symbol: All keys under the kafka settings scope need to be symbols
       max_timeout_vs_pause_max_timeout: pause_timeout must be less or equal to pause_max_timeout

data/config/locales/pro_errors.yml

@@ -58,6 +58,10 @@ en:
       subscription_group_details.multiplexing_boot_format: 'needs to be an integer equal or more than 1'
       subscription_group_details.multiplexing_boot_not_dynamic: 'needs to be equal to max when not in dynamic mode'
 
+      swarm.active_format: needs to be true
+      swarm.nodes_format: needs to be a range or an array of nodes ids
+      swarm_nodes_with_non_existent_nodes: includes unreachable nodes ids
+
     consumer_group:
       patterns_format: must be an array with hashes
       patterns_missing: needs to be present

data/lib/karafka/admin.rb

@@ -183,7 +183,13 @@ module Karafka
         tpl_base.each do |topic, partitions_with_offsets|
           partitions_with_offsets.each do |partition, offset|
             target = offset.is_a?(Time) ? time_tpl : tpl
-            target.add_topic_and_partitions_with_offsets(topic, [[partition, offset]])
+            # We reverse and uniq to make sure that potentially duplicated references are removed
+            # in such a way that the newest stays
+            target.to_h[topic] ||= []
+            target.to_h[topic] << Rdkafka::Consumer::Partition.new(partition, offset)
+            target.to_h[topic].reverse!
+            target.to_h[topic].uniq!(&:partition)
+            target.to_h[topic].reverse!
           end
         end
 
@@ -219,7 +225,11 @@ module Karafka
                   end
 
                   # Since now we have proper offsets, we can add this to the final tpl for commit
-                  tpl.add_topic_and_partitions_with_offsets(name, [[partition, offset]])
+                  tpl.to_h[name] ||= []
+                  tpl.to_h[name] << Rdkafka::Consumer::Partition.new(partition, offset)
+                  tpl.to_h[name].reverse!
+                  tpl.to_h[name].uniq!(&:partition)
+                  tpl.to_h[name].reverse!
                 end
               end
             end

data/lib/karafka/connection/client.rb

@@ -600,6 +600,13 @@ module Karafka
       # @return [Rdkafka::Consumer]
       def build_consumer
         ::Rdkafka::Config.logger = ::Karafka::App.config.logger
+
+        # We need to refresh the setup of this subscription group in case we started running in a
+        # swarm. The initial configuration for validation comes from the parent node, but it needs
+        # to be altered in case of a static group membership usage for correct mapping of the
+        # group instance id.
+        @subscription_group.refresh
+
         config = ::Rdkafka::Config.new(@subscription_group.kafka)
         config.consumer_rebalance_listener = @rebalance_callback
         # We want to manage the events queue independently from the messages queue. Thanks to that

data/lib/karafka/contracts/config.rb

@@ -162,6 +162,28 @@ module Karafka
 
         [[%i[shutdown_timeout], :shutdown_timeout_vs_max_wait_time]]
       end
+
+      # `internal.swarm.node_report_timeout` should not be close to `max_wait_time` otherwise
+      # there may be a case where node cannot report often enough because it is clogged by waiting
+      # on more data.
+      #
+      # We handle that at a config level to make sure that this is correctly configured.
+      #
+      # We do not validate this in the context of swarm usage (validate only if...) because it is
+      # often that swarm only runs on prod and we do not want to crash it surprisingly.
+      virtual do |data, errors|
+        next unless errors.empty?
+
+        max_wait_time = data.fetch(:max_wait_time)
+        node_report_timeout = data.fetch(:internal)[:swarm][:node_report_timeout] || false
+
+        next unless node_report_timeout
+        # max wait time should be at least 20% smaller than the reporting time to have enough
+        # time for reporting
+        next if max_wait_time < node_report_timeout * 0.8
+
+        [[%i[max_wait_time], :max_wait_time_vs_swarm_node_report_timeout]]
+      end
     end
   end
 end

data/lib/karafka/instrumentation/vendors/datadog/metrics_listener.rb

@@ -14,7 +14,8 @@ module Karafka
           include ::Karafka::Core::Configurable
           extend Forwardable
 
-          def_delegators :config, :client, :rd_kafka_metrics, :namespace, :default_tags
+          def_delegators :config, :client, :rd_kafka_metrics, :namespace,
+                         :default_tags, :distribution_mode
 
           # Value object for storing a single rdkafka metric publishing details
           RdKafkaMetric = Struct.new(:type, :scope, :name, :key_location)
@@ -53,6 +54,13 @@ module Karafka
             RdKafkaMetric.new(:gauge, :topics, 'consumer.lags_delta', 'consumer_lag_stored_d')
           ].freeze
 
+          # Whether histogram metrics should be sent as distributions or histograms.
+          # Distribution metrics are aggregated globally and not agent-side,
+          # providing more accurate percentiles whenever consumers are running on multiple hosts.
+          #
+          # Learn more at https://docs.datadoghq.com/metrics/types/?tab=distribution#metric-types
+          setting :distribution_mode, default: :histogram
+
           configure
 
           # @param block [Proc] configuration block
@@ -169,18 +177,40 @@ module Karafka
           %i[
             count
             gauge
-            histogram
             increment
             decrement
           ].each do |metric_type|
-            class_eval <<~METHODS, __FILE__, __LINE__ + 1
+            class_eval <<~RUBY, __FILE__, __LINE__ + 1
               def #{metric_type}(key, *args)
                 client.#{metric_type}(
                   namespaced_metric(key),
                   *args
                 )
               end
-            METHODS
+            RUBY
+          end
+
+          # Selects the histogram mode configured and uses it to report to DD client
+          # @param key [String] non-namespaced key
+          # @param args [Array] extra arguments to pass to the client
+          def histogram(key, *args)
+            case distribution_mode
+            when :histogram
+              client.histogram(
+                namespaced_metric(key),
+                *args
+              )
+            when :distribution
+              client.distribution(
+                namespaced_metric(key),
+                *args
+              )
+            else
+              raise(
+                ::ArgumentError,
+                'distribution_mode setting value must be either :histogram or :distribution'
+              )
+            end
           end
 
           # Wraps metric name in listener's namespace

data/lib/karafka/pro/routing/features/swarm/config.rb

@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Routing
+      module Features
+        class Swarm < Base
+          # Swarm feature configuration
+          Config = Struct.new(
+            :active,
+            :nodes,
+            keyword_init: true
+          ) do
+            alias_method :active?, :active
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/routing/features/swarm/contracts/topic.rb

@@ -0,0 +1,67 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Routing
+      module Features
+        class Swarm < Base
+          # Namespace for swarm contracts
+          module Contracts
+            # Contract to validate configuration of the swarm feature
+            class Topic < Karafka::Contracts::Base
+              configure do |config|
+                config.error_messages = YAML.safe_load(
+                  File.read(
+                    File.join(Karafka.gem_root, 'config', 'locales', 'pro_errors.yml')
+                  )
+                ).fetch('en').fetch('validations').fetch('topic')
+              end
+
+              nested(:swarm) do
+                required(:active) { |val| val == true }
+
+                required(:nodes) do |val|
+                  val.is_a?(Range) || (
+                    val.is_a?(Array) &&
+                    val.all? { |id| id.is_a?(Integer) }
+                  )
+                end
+              end
+
+              # Make sure that if range is defined it fits number of nodes (except infinity)
+              # As it may be a common error to accidentally define a node that will never be
+              # reached
+              virtual do |data, errors|
+                next unless errors.empty?
+
+                nodes = data[:swarm][:nodes]
+
+                # Defaults
+                next if nodes.first.zero? && nodes.last == Float::INFINITY
+
+                # If our expectation towards which node should run things matches at least one
+                # node, then it's ok
+                next if Karafka::App.config.swarm.nodes.times.any? do |node_id|
+                  nodes.include?(node_id)
+                end
+
+                [[%i[swarm_nodes], :with_non_existent_nodes]]
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/routing/features/swarm/topic.rb

@@ -0,0 +1,54 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Routing
+      module Features
+        class Swarm < Base
+          # Topic swarm API extensions
+          module Topic
+            # Allows defining swarm routing topic settings
+            # @param nodes [Range, Array] range of nodes ids or array with nodes ids for which we
+            #   should run given topic
+            def swarm(nodes: (0...Karafka::App.config.swarm.nodes))
+              @swarm ||= Config.new(active: true, nodes: nodes)
+            end
+
+            # @return [true] swarm setup is always true. May not be in use but is active
+            def swarm?
+              swarm.active?
+            end
+
+            # @return [Boolean] should this topic be active. In the context of swarm it is only
+            #   active when swarm routing setup does not limit nodes on which it should operate
+            def active?
+              node = Karafka::App.config.swarm.node
+
+              return super unless node
+
+              super && swarm.nodes.include?(node.id)
+            end
+
+            # @return [Hash] topic with all its native configuration options plus swarm
+            def to_h
+              super.merge(
+                swarm: swarm.to_h
+              ).freeze
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/routing/features/swarm.rb

@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Routing
+      module Features
+        # Karafka Pro Swarm extensions to the routing
+        # They allow for more granular work assignment in the swarm
+        class Swarm < Base
+        end
+      end
+    end
+  end
+end

data/lib/karafka/routing/subscription_group.rb

@@ -91,6 +91,19 @@ module Karafka
         id
       end
 
+      # Refreshes the configuration of this subscription group if needed based on the execution
+      # context.
+      #
+      # Since the initial routing setup happens in the supervisor, it is inherited by the children.
+      # This causes incomplete assignment of `group.instance.id` which is not expanded with proper
+      # node identifier. This refreshes this if needed when in swarm.
+      def refresh
+        return unless node
+        return unless kafka.key?(:'group.instance.id')
+
+        @kafka = build_kafka
+      end
+
       private
 
       # @return [Hash] kafka settings are a bit special. They are exactly the same for all of the

data/lib/karafka/setup/config.rb

@@ -196,7 +196,7 @@ module Karafka
           setting :liveness_listener, default: Swarm::LivenessListener.new
           # How long should we wait for any info from the node before we consider it hanging at
           # stop it
-          setting :node_report_timeout, default: 30_000
+          setting :node_report_timeout, default: 60_000
           # How long should we wait before restarting a node. This can prevent us from having a
           # case where for some external reason our spawned process would die immediately and we
           # would immediately try to start it back in an endless loop

data/lib/karafka/swarm/manager.rb

@@ -19,6 +19,13 @@ module Karafka
         node_restart_timeout: %i[internal swarm node_restart_timeout]
       )
 
+      # Status we issue when we decide to shutdown unresponsive node
+      # We use -1 because nodes are expected to report 0+ statuses and we can use negative numbers
+      # for non-node based statuses
+      NOT_RESPONDING_SHUTDOWN_STATUS = -1
+
+      private_constant :NOT_RESPONDING_SHUTDOWN_STATUS
+
       # @return [Array<Node>] All nodes that manager manages
       attr_reader :nodes
 
@@ -29,10 +36,10 @@ module Karafka
 
       # Starts all the expected nodes for the first time
       def start
-        pidfd = Pidfd.new(::Process.pid)
+        parent_pid = ::Process.pid
 
         @nodes = Array.new(nodes_count) do |i|
-          start_one Node.new(i, pidfd)
+          start_one Node.new(i, parent_pid)
         end
       end
 
@@ -148,7 +155,12 @@ module Karafka
         return true unless over?(statuses[:control], node_report_timeout)
 
         # Start the stopping procedure if the node stopped reporting frequently enough
-        monitor.instrument('swarm.manager.stopping', caller: self, node: node) do
+        monitor.instrument(
+          'swarm.manager.stopping',
+          caller: self,
+          node: node,
+          status: NOT_RESPONDING_SHUTDOWN_STATUS
+        ) do
           node.stop
           statuses[:stop] = monotonic_now
         end

data/lib/karafka/swarm/node.rb

@@ -30,10 +30,10 @@ module Karafka
       # @param id [Integer] number of the fork. Used for uniqueness setup for group client ids and
       #   other stuff where we need to know a unique reference of the fork in regards to the rest
       #   of them.
-      # @param parent_pidfd [Pidfd] parent pidfd for zombie fencing
-      def initialize(id, parent_pidfd)
+      # @param parent_pid [Integer] parent pid for zombie fencing
+      def initialize(id, parent_pid)
         @id = id
-        @parent_pidfd = parent_pidfd
+        @parent_pidfd = Pidfd.new(parent_pid)
       end
 
       # Starts a new fork and:

data/lib/karafka/swarm/pidfd.rb

@@ -72,17 +72,33 @@ module Karafka
       def alive?
         @pidfd_select ||= [@pidfd_io]
 
-        IO.select(@pidfd_select, nil, nil, 0).nil?
+        if @mutex.owned?
+          return false if @cleaned
+
+          IO.select(@pidfd_select, nil, nil, 0).nil?
+        else
+          @mutex.synchronize do
+            return false if @cleaned
+
+            IO.select(@pidfd_select, nil, nil, 0).nil?
+          end
+        end
       end
 
       # Cleans the zombie process
       # @note This should run **only** on processes that exited, otherwise will wait
       def cleanup
-        return if @cleaned
+        @mutex.synchronize do
+          return if @cleaned
 
-        waitid(P_PIDFD, @pidfd, nil, WEXITED)
+          waitid(P_PIDFD, @pidfd, nil, WEXITED)
 
-        @cleaned = true
+          @pidfd_io.close
+          @pidfd_select = nil
+          @pidfd_io = nil
+          @pidfd = nil
+          @cleaned = true
+        end
       end
 
       # Sends given signal to the process using its pidfd

data/lib/karafka/swarm/supervisor.rb

@@ -30,14 +30,16 @@ module Karafka
 
       # Creates needed number of forks, installs signals and starts supervision
       def run
-        Karafka::App.warmup
-
-        manager.start
-
         # Close producer just in case. While it should not be used, we do not want even a
         # theoretical case since librdkafka is not thread-safe.
+        # We close it prior to forking just to make sure, there is no issue with initialized
+        # producer (should not be initialized but just in case)
         Karafka.producer.close
 
+        Karafka::App.warmup
+
+        manager.start
+
         process.on_sigint { stop }
         process.on_sigquit { stop }
         process.on_sigterm { stop }
@@ -132,8 +134,9 @@ module Karafka
         # Cleanup the process table
         manager.cleanup
 
-        # exit! is not within the instrumentation as it would not trigger due to exit
-        Kernel.exit!(forceful_exit_code)
+        # We do not use `exit!` here similar to regular server because we do not have to worry
+        # about any librdkafka related hanging connections, etc
+        Kernel.exit(forceful_exit_code)
       ensure
         if initialized
           Karafka::App.stopped!

data/lib/karafka/version.rb

@@ -3,5 +3,5 @@
 # Main module namespace
 module Karafka
   # Current Karafka version
-  VERSION = '2.3.2'
+  VERSION = '2.3.4'
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: karafka
 version: !ruby/object:Gem::Version
-  version: 2.3.2
+  version: 2.3.4
 platform: ruby
 authors:
 - Maciej Mensfeld
@@ -35,7 +35,7 @@ cert_chain:
   AnG1dJU+yL2BK7vaVytLTstJME5mepSZ46qqIJXMuWob/YPDmVaBF39TDSG9e34s
   msG3BiCqgOgHAnL23+CN3Rt8MsuRfEtoTKpJVcCfoEoNHOkc
   -----END CERTIFICATE-----
-date: 2024-02-16 00:00:00.000000000 Z
+date: 2024-04-11 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: karafka-core
@@ -381,6 +381,10 @@ files:
 - lib/karafka/pro/routing/features/periodic_job/config.rb
 - lib/karafka/pro/routing/features/periodic_job/contracts/topic.rb
 - lib/karafka/pro/routing/features/periodic_job/topic.rb
+- lib/karafka/pro/routing/features/swarm.rb
+- lib/karafka/pro/routing/features/swarm/config.rb
+- lib/karafka/pro/routing/features/swarm/contracts/topic.rb
+- lib/karafka/pro/routing/features/swarm/topic.rb
 - lib/karafka/pro/routing/features/throttling.rb
 - lib/karafka/pro/routing/features/throttling/config.rb
 - lib/karafka/pro/routing/features/throttling/contracts/topic.rb