karafka 2.4.0.rc1 → 2.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 0d682c1f18dd2c0bba4048462095d2129f7758bcbafb810a52f3e2e1515c5e47
-  data.tar.gz: bd4c87fda52f44b61cbcb579369343b327ce04b2229e0903642a6c5a537072eb
+  metadata.gz: 8111bc347c891837ee1f7cbabd4d98ee1c55f2bb7366e72aa077628cc4052c14
+  data.tar.gz: 9a616eb14fbbddc433e56691540613a728cab6163632981de1c9f21abee8f0b4
 SHA512:
-  metadata.gz: 5ced4aea757aa2bda21ac9a3519344389a7f03a3229b849e3c05fce262494cd3c8f58c1544f1c94fab02559def0f5b1c4c2165933bef6af8559a3b9a19e86299
-  data.tar.gz: c5f4a61cd58554d62fd9ce48a627d8bc4266eb4a244d08ec7ed9ca04006c2e19f113698262d09caa743c8c6896c49666d454fb93b7fdf81d17c3c9ce4a561dd8
+  metadata.gz: 35e52c0f5ffeace79cc58e6ef85918f6ff32c80233419dec83a5b99485cd6f5ac2b163f3d0a73e96b4ac424e27a81a3f8f848d3bed5e461b69eb28399f102d3c
+  data.tar.gz: 544848837b53a5e0e64b9c842f8dadc63a486b784660e37a6f44a67947682c59cd112e6ac509e19de99b7b084402b38d7e00d7fff52a89c9e07172f593b3032e

data/.github/workflows/ci.yml

@@ -75,10 +75,7 @@ jobs:
         ruby:
           - '3.3'
           - '3.2'
-          # We run it against the oldest and the newest of a given major to make sure, that there
-          # are no syntax-sugars that we would use that were introduced down the road
           - '3.1'
-          - '3.1.0'
           - '3.0'
         include:
           - ruby: '3.3'

data/.ruby-version

@@ -1 +1 @@
-3.3.0
+3.3.1

data/CHANGELOG.md

@@ -1,6 +1,6 @@
 # Karafka framework changelog
 
-## 2.4.0 (Unreleased)
+## 2.4.0 (2024-04-26)
 
 This release contains **BREAKING** changes. Make sure to read and apply upgrade notes.
 
@@ -17,6 +17,7 @@ This release contains **BREAKING** changes. Make sure to read and apply upgrade
 - **[Feature]** Introduce ability to use custom message key deserializers.
 - **[Feature]** Introduce ability to use custom message headers deserializers.
 - **[Feature]** Provide `Karafka::Admin::Configs` API for cluster and topics configuration management.
+- [Enhancement] Protect critical `rdkafka` thread executable code sections.
 - [Enhancement] Assign names to internal threads for better debuggability when on `TTIN`.
 - [Enhancement] Provide `log_polling` setting to the `Karafka::Instrumentation::LoggerListener` to silence polling in any non-debug mode.
 - [Enhancement] Provide `metadata#message` to be able to retrieve message from metadata.
@@ -42,6 +43,7 @@ This release contains **BREAKING** changes. Make sure to read and apply upgrade
 - [Enhancement] No longer raise `Karafka::UnsupportedCaseError` for not recognized error types to support dynamic errors reporting.
 - [Change] Do not create new proxy object to Rdkafka with certain low-level operations and re-use existing.
 - [Change] Update `karafka.erb` template with a placeholder for waterdrop and karafka error instrumentation.
+- [Change] Replace `statistics.emitted.error` error type with `callbacks.statistics.error` to align naming conventions.
 - [Fix] Pro Swarm liveness listener can report incorrect failure when dynamic multiplexing scales down.
 - [Fix] K8s liveness listener can report incorrect failure when dynamic multiplexing scales down.
 - [Fix] Fix a case where connection conductor would not be released during manager state changes.

data/Gemfile

@@ -12,7 +12,7 @@ gemspec
 group :integrations do
   gem 'activejob', require: false
   gem 'karafka-testing', '>= 2.4.0.rc1', require: false
-  gem 'karafka-web', '>= 0.9.0.rc1', require: false
+  gem 'karafka-web', '>= 0.9.0.rc3', require: false
   gem 'rspec', require: false
 end
 

data/Gemfile.lock

@@ -1,9 +1,9 @@
 PATH
   remote: .
   specs:
-    karafka (2.4.0.rc1)
-      karafka-core (>= 2.4.0.rc2, < 2.5.0)
-      waterdrop (>= 2.7.0.rc2, < 3.0.0)
+    karafka (2.4.0)
+      karafka-core (>= 2.4.0, < 2.5.0)
+      waterdrop (>= 2.7.0, < 3.0.0)
       zeitwerk (~> 2.3)
 
 GEM
@@ -38,19 +38,19 @@ GEM
       activesupport (>= 6.1)
     i18n (1.14.4)
       concurrent-ruby (~> 1.0)
-    karafka-core (2.4.0.rc2)
-      karafka-rdkafka (>= 0.15.0.rc2, < 0.16.0)
-    karafka-rdkafka (0.15.0.rc2)
+    karafka-core (2.4.0)
+      karafka-rdkafka (>= 0.15.0, < 0.16.0)
+    karafka-rdkafka (0.15.0)
       ffi (~> 1.15)
       mini_portile2 (~> 2.6)
       rake (> 12)
-    karafka-testing (2.4.0.rc1)
+    karafka-testing (2.4.0.rc2)
       karafka (>= 2.4.0.beta1, < 2.5.0)
       waterdrop (>= 2.7.0.rc1)
-    karafka-web (0.9.0.rc2)
+    karafka-web (0.9.0.rc3)
       erubi (~> 1.4)
-      karafka (>= 2.4.0.beta1, < 2.5.0)
-      karafka-core (>= 2.4.0.rc1, < 2.5.0)
+      karafka (>= 2.4.0.rc1, < 2.5.0)
+      karafka-core (>= 2.4.0.rc2, < 2.5.0)
       roda (~> 3.68, >= 3.69)
       tilt (~> 2.0)
     mini_portile2 (2.8.6)
@@ -82,8 +82,8 @@ GEM
     tilt (2.3.0)
     tzinfo (2.0.6)
       concurrent-ruby (~> 1.0)
-    waterdrop (2.7.0.rc2)
-      karafka-core (>= 2.4.0.rc2, < 3.0.0)
+    waterdrop (2.7.0)
+      karafka-core (>= 2.4.0, < 3.0.0)
       zeitwerk (~> 2.3)
     zeitwerk (2.6.13)
 
@@ -97,9 +97,9 @@ DEPENDENCIES
   factory_bot
   karafka!
   karafka-testing (>= 2.4.0.rc1)
-  karafka-web (>= 0.9.0.rc1)
+  karafka-web (>= 0.9.0.rc3)
   rspec
   simplecov
 
 BUNDLED WITH
-   2.5.7
+   2.5.9

data/README.md

@@ -57,8 +57,8 @@ We also maintain many [integration specs](https://github.com/karafka/karafka/tre
 1. Add and install Karafka:
 
 ```bash
-# Make sure to install Karafka 2.3
-bundle add karafka --version ">= 2.3.0"
+# Make sure to install Karafka 2.4
+bundle add karafka --version ">= 2.4.0"
 
 bundle exec karafka install
 ```

data/bin/integrations

@@ -44,7 +44,8 @@ class Scenario
     'shutdown/on_hanging_jobs_and_a_shutdown_spec.rb' => [2].freeze,
     'shutdown/on_hanging_on_shutdown_job_and_a_shutdown_spec.rb' => [2].freeze,
     'shutdown/on_hanging_listener_and_shutdown_spec.rb' => [2].freeze,
-    'swarm/forceful_shutdown_of_hanging_spec.rb' => [2].freeze
+    'swarm/forceful_shutdown_of_hanging_spec.rb' => [2].freeze,
+    'instrumentation/post_errors_instrumentation_error_spec.rb' => [1].freeze
   }.freeze
 
   private_constant :MAX_RUN_TIME, :EXIT_CODES

data/karafka.gemspec

@@ -21,8 +21,8 @@ Gem::Specification.new do |spec|
     without having to focus on things that are not your business domain.
   DESC
 
-  spec.add_dependency 'karafka-core', '>= 2.4.0.rc2', '< 2.5.0'
-  spec.add_dependency 'waterdrop', '>= 2.7.0.rc2', '< 3.0.0'
+  spec.add_dependency 'karafka-core', '>= 2.4.0', '< 2.5.0'
+  spec.add_dependency 'waterdrop', '>= 2.7.0', '< 3.0.0'
   spec.add_dependency 'zeitwerk', '~> 2.3'
 
   spec.required_ruby_version = '>= 3.0.0'

data/lib/karafka/constraints.rb

@@ -15,8 +15,8 @@ module Karafka
         # Skip verification if web is not used at all
         return unless require_version('karafka/web')
 
-        # All good if version higher than 0.9.0.rc1 because we expect 0.9.0.rc1 or higher
-        return if version(Karafka::Web::VERSION) >= version('0.9.0.rc1')
+        # All good if version higher than 0.9.0.rc3 because we expect 0.9.0.rc3 or higher
+        return if version(Karafka::Web::VERSION) >= version('0.9.0.rc3')
 
         # If older web-ui used, we cannot allow it
         raise(

data/lib/karafka/instrumentation/callbacks/error.rb

@@ -6,6 +6,10 @@ module Karafka
     module Callbacks
       # Callback that kicks in when consumer error occurs and is published in a background thread
       class Error
+        include Helpers::ConfigImporter.new(
+          monitor: %i[monitor]
+        )
+
         # @param subscription_group_id [String] id of the current subscription group instance
         # @param consumer_group_id [String] id of the current consumer group
         # @param client_name [String] rdkafka client name
@@ -24,7 +28,7 @@ module Karafka
           # Same as with statistics (mor explanation there)
           return unless @client_name == client_name
 
-          ::Karafka.monitor.instrument(
+          monitor.instrument(
             'error.occurred',
             caller: self,
             subscription_group_id: @subscription_group_id,
@@ -32,6 +36,15 @@ module Karafka
             type: 'librdkafka.error',
             error: error
           )
+        rescue StandardError => e
+          monitor.instrument(
+            'error.occurred',
+            caller: self,
+            subscription_group_id: @subscription_group_id,
+            consumer_group_id: @consumer_group_id,
+            type: 'callbacks.error.error',
+            error: e
+          )
         end
       end
     end

data/lib/karafka/instrumentation/callbacks/oauthbearer_token_refresh.rb

@@ -5,6 +5,10 @@ module Karafka
     module Callbacks
       # Callback that is triggered when oauth token needs to be refreshed.
       class OauthbearerTokenRefresh
+        include Helpers::ConfigImporter.new(
+          monitor: %i[monitor]
+        )
+
         # @param bearer [Rdkafka::Consumer, Rdkafka::Admin] given rdkafka instance. It is needed as
         #   we need to have a reference to call `#oauthbearer_set_token` or
         #   `#oauthbearer_set_token_failure` upon the event.
@@ -17,11 +21,18 @@ module Karafka
         def call(_rd_config, bearer_name)
           return unless @bearer.name == bearer_name
 
-          ::Karafka.monitor.instrument(
+          monitor.instrument(
             'oauthbearer.token_refresh',
             bearer: @bearer,
             caller: self
           )
+        rescue StandardError => e
+          monitor.instrument(
+            'error.occurred',
+            caller: self,
+            type: 'callbacks.oauthbearer_token_refresh.error',
+            error: e
+          )
         end
       end
     end

data/lib/karafka/instrumentation/callbacks/rebalance.rb

@@ -6,6 +6,10 @@ module Karafka
       # Callback that connects to the librdkafka rebalance callback and converts those events into
       # our internal events
       class Rebalance
+        include Helpers::ConfigImporter.new(
+          monitor: %i[monitor]
+        )
+
         # @param subscription_group [Karafka::Routes::SubscriptionGroup] subscription group for
         #   which we want to manage rebalances
         def initialize(subscription_group)
@@ -49,7 +53,7 @@ module Karafka
         # @param name [String] name of the event
         # @param tpl [Rdkafka::Consumer::TopicPartitionList]
         def instrument(name, tpl)
-          ::Karafka.monitor.instrument(
+          monitor.instrument(
             "rebalance.#{name}",
             caller: self,
             # We keep the id references here for backwards compatibility as some of the monitors
@@ -60,6 +64,15 @@ module Karafka
             consumer_group: @subscription_group.consumer_group,
             tpl: tpl
           )
+        rescue StandardError => e
+          monitor.instrument(
+            'error.occurred',
+            caller: self,
+            subscription_group_id: @subscription_group.id,
+            consumer_group_id: @subscription_group.consumer_group.id,
+            type: "callbacks.rebalance.#{name}.error",
+            error: e
+          )
         end
       end
     end

data/lib/karafka/instrumentation/callbacks/statistics.rb

@@ -7,6 +7,10 @@ module Karafka
       # @see `WaterDrop::Instrumentation::Callbacks::Statistics` for details on why we decorate
       #   those statistics
       class Statistics
+        include Helpers::ConfigImporter.new(
+          monitor: %i[monitor]
+        )
+
         # @param subscription_group_id [String] id of the current subscription group
         # @param consumer_group_id [String] id of the current consumer group
         # @param client_name [String] rdkafka client name
@@ -26,7 +30,7 @@ module Karafka
           # all the time.
           return unless @client_name == statistics['name']
 
-          ::Karafka.monitor.instrument(
+          monitor.instrument(
             'statistics.emitted',
             subscription_group_id: @subscription_group_id,
             consumer_group_id: @consumer_group_id,
@@ -36,12 +40,12 @@ module Karafka
         # as otherwise, in case of statistics which run in the main librdkafka thread, any crash
         # will hang the whole process.
         rescue StandardError => e
-          ::Karafka.monitor.instrument(
+          monitor.instrument(
             'error.occurred',
             caller: self,
             subscription_group_id: @subscription_group_id,
             consumer_group_id: @consumer_group_id,
-            type: 'statistics.emitted.error',
+            type: 'callbacks.statistics.error',
             error: e
           )
         end

data/lib/karafka/instrumentation/logger_listener.rb

@@ -313,8 +313,11 @@ module Karafka
         # Those can occur when emitted statistics are consumed by the end user and the processing
         # of statistics fails. The statistics are emitted from librdkafka main loop thread and
         # any errors there crash the whole thread
-        when 'statistics.emitted.error'
-          error "statistics.emitted processing failed due to an error: #{error}"
+        when 'callbacks.statistics.error'
+          error "callbacks.statistics processing failed due to an error: #{error}"
+          error details
+        when 'callbacks.error.error'
+          error "callbacks.error processing failed due to an error: #{error}"
           error details
         # Those will only occur when retries in the client fail and when they did not stop after
         # back-offs

data/lib/karafka/pro/processing/piping/consumer.rb

@@ -36,9 +36,9 @@ module Karafka
           # @note It will NOT deserialize the payload so it is fast
           #
           # @note We assume that there can be different number of partitions in the target topic,
-          #   this is why we use `key` based on the original topic partition number and not the
-          #   partition id itself. This will not utilize partitions beyond the number of partitions
-          #   of original topic, but will accommodate for topics with less partitions.
+          #   this is why we use `key` based on the original topic key and not the partition id.
+          #   This will not utilize partitions beyond the number of partitions of original topic,
+          #   but will accommodate for topics with less partitions.
           def pipe_async(topic:, message:)
             produce_async(
               build_pipe_message(topic: topic, message: message)
@@ -94,20 +94,26 @@ module Karafka
           #
           # @note If you need to alter this, please define the `#enhance_pipe_message` method
           def build_pipe_message(topic:, message:)
-            original_partition = message.partition.to_s
-
             pipe_message = {
               topic: topic,
-              key: original_partition,
               payload: message.raw_payload,
-              headers: message.headers.merge(
+              headers: message.raw_headers.merge(
                 'original_topic' => message.topic,
-                'original_partition' => original_partition,
+                'original_partition' => message.partition.to_s,
                 'original_offset' => message.offset.to_s,
                 'original_consumer_group' => self.topic.consumer_group.id
               )
             }
 
+            # Use a key only if key was provided
+            if message.raw_key
+              pipe_message[:key] = message.raw_key
+            # Otherwise pipe creating a key that will assign it based on the original partition
+            # number
+            else
+              pipe_message[:key] = message.partition.to_s
+            end
+
             # Optional method user can define in consumer to enhance the dlq message hash with
             # some extra details if needed or to replace payload, etc
             if respond_to?(:enhance_pipe_message, true)

data/lib/karafka/setup/attributes_map.rb

@@ -52,7 +52,9 @@ module Karafka
         fetch.wait.max.ms
         group.id
         group.instance.id
+        group.protocol
         group.protocol.type
+        group.remote.assignor
         heartbeat.interval.ms
         interceptors
         internal.termination.signal

data/lib/karafka/templates/karafka.rb.erb

@@ -39,8 +39,8 @@ class KarafkaApp < Karafka::App
   # listen to only what you really need for given environment.
   Karafka.monitor.subscribe(
     Karafka::Instrumentation::LoggerListener.new(
-      # Karafka, when the logger is set to info producers logs each time it polls data from an
-      # internal messages wueue. This can be extensive, so you can turn it off by setting below
+      # Karafka, when the logger is set to info, produces logs each time it polls data from an
+      # internal messages queue. This can be extensive, so you can turn it off by setting below
       # to false.
       log_polling: true
     )

data/lib/karafka/version.rb

@@ -3,5 +3,5 @@
 # Main module namespace
 module Karafka
   # Current Karafka version
-  VERSION = '2.4.0.rc1'
+  VERSION = '2.4.0'
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: karafka
 version: !ruby/object:Gem::Version
-  version: 2.4.0.rc1
+  version: 2.4.0
 platform: ruby
 authors:
 - Maciej Mensfeld
@@ -35,7 +35,7 @@ cert_chain:
   AnG1dJU+yL2BK7vaVytLTstJME5mepSZ46qqIJXMuWob/YPDmVaBF39TDSG9e34s
   msG3BiCqgOgHAnL23+CN3Rt8MsuRfEtoTKpJVcCfoEoNHOkc
   -----END CERTIFICATE-----
-date: 2024-04-18 00:00:00.000000000 Z
+date: 2024-04-26 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: karafka-core
@@ -43,7 +43,7 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 2.4.0.rc2
+        version: 2.4.0
     - - "<"
       - !ruby/object:Gem::Version
         version: 2.5.0
@@ -53,7 +53,7 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 2.4.0.rc2
+        version: 2.4.0
     - - "<"
       - !ruby/object:Gem::Version
         version: 2.5.0
@@ -63,7 +63,7 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 2.7.0.rc2
+        version: 2.7.0
     - - "<"
       - !ruby/object:Gem::Version
         version: 3.0.0
@@ -73,7 +73,7 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 2.7.0.rc2
+        version: 2.7.0
     - - "<"
       - !ruby/object:Gem::Version
         version: 3.0.0
@@ -535,7 +535,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.5.3
+rubygems_version: 3.5.9
 signing_key: 
 specification_version: 4
 summary: Karafka is Ruby and Rails efficient Kafka processing framework.