karafka 2.4.15 → 2.4.17

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 5b49de31147b2a64d8927c91e25e6b607fdf1a8a7280b7cc3ba8c3663cf96b6f
-  data.tar.gz: 4a3d9e439e21b79b9ac6a2c8bf4e76219f970be1f6bb7316feb5c7f2f792271e
+  metadata.gz: a26065fae7fe2d8f878b56e497d3551f3f0277b27ed3356b59fd478f017b4144
+  data.tar.gz: 356d0e078a32f0b22dc7d06d0f0e896d17b55e7a52866acf92933aeb6a611548
 SHA512:
-  metadata.gz: 527a1c169ddc5a0f978f69dc3dbae6d30288997d2cfc95104ad6343d7a468c67afbde4c55051c1e7902d2c161fc1e615f8d703b9dfba123f734b9fbd7ccec5d1
-  data.tar.gz: a895657a8dece0c9bdb65afeeb00bc8df2e03e1957d345bc71ceab9d6cce74e5d0b4afcc251e26dfe230778a05b8742e40505a779695325844c6be630307ad82
+  metadata.gz: 1043bb3ff799cc5294de4936552f4dcc3a78f9aabca275fd35259c87dfdc257cc63a778722d833c8a1f3e9449084e6266e6cd507d396cd92457b8e6078ced631
+  data.tar.gz: 57fe56b280f460be5f3fe2a734587d75ad08e25ae96363da68d5791fe62b6427402de400e6cae2e38660bbcefdb314be45d0299d15c50131a1ebe15f44c23c96

data/.github/workflows/ci.yml

@@ -27,7 +27,7 @@ jobs:
       - name: Set up Ruby
         uses: ruby/setup-ruby@v1
         with:
-          ruby-version: 3.3
+          ruby-version: 3.4
           bundler-cache: true
 
       - name: Install Diffend plugin
@@ -73,12 +73,12 @@ jobs:
       fail-fast: false
       matrix:
         ruby:
-          - '3.4.0-preview1'
+          - '3.4'
           - '3.3'
           - '3.2'
           - '3.1'
         include:
-          - ruby: '3.3'
+          - ruby: '3.4'
             coverage: 'true'
     steps:
       - uses: actions/checkout@v4
@@ -120,7 +120,7 @@ jobs:
       fail-fast: false
       matrix:
         ruby:
-          - '3.4.0-preview1'
+          - '3.4'
           - '3.3'
           - '3.2'
           - '3.1'
@@ -178,7 +178,7 @@ jobs:
       fail-fast: false
       matrix:
         ruby:
-          - '3.4.0-preview1'
+          - '3.4'
           - '3.3'
           - '3.2'
           - '3.1'

data/.ruby-version

@@ -1 +1 @@
-3.3.6
+3.4.1

data/CHANGELOG.md

@@ -1,5 +1,20 @@
 # Karafka Framework Changelog
 
+## 2.4.17 (2025-01-15)
+- [Enhancement] Clean message key and headers when cleaning messages via the cleaner API (Pro).
+- [Enhancement] Allow for setting `metadata: false` in the cleaner API for granular cleaning control (Pro)
+- [Enhancement] Instrument successful transaction via `consumer.consuming.transaction` event (Pro).
+
+## 2.4.16 (2024-12-27)
+- [Enhancement] Improve post-rebalance revocation messages filtering.
+- [Enhancement] Introduce `Consumer#wrap` for connection pooling management and other wrapped operations.
+- [Enhancement] Guard transactional operations from marking beyond assignment ownership under some extreme edge-cases.
+- [Enhancement] Improve VPs work with transactional producers.
+- [Enhancement] Prevent non-transactional operations leakage into transactional managed offset management consumers.
+- [Fix] Prevent transactions from being marked with a non-transactional default producer when automatic offset management and other advanced features are on.
+- [Fix] Fix `kafka_format` `KeyError` that occurs when a non-hash is assigned to the kafka scope of the settings.
+- [Fix] Non cooperative-sticky transactional offset management can refetch reclaimed partitions.
+
 ## 2.4.15 (2024-12-04)
 - [Fix] Assignment tracker current state fetch during a rebalance loop can cause an error on multi CG setup.
 - [Fix] Prevent double post-transaction offset dispatch to Kafka.

data/Gemfile

@@ -12,6 +12,7 @@ gemspec
 group :integrations, :test do
   gem 'fugit', require: false
   gem 'rspec', require: false
+  gem 'stringio'
 end
 
 group :integrations do

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    karafka (2.4.15)
+    karafka (2.4.17)
       base64 (~> 0.2)
       karafka-core (>= 2.4.4, < 2.5.0)
       karafka-rdkafka (>= 0.17.2)
@@ -11,10 +11,10 @@ PATH
 GEM
   remote: https://rubygems.org/
   specs:
-    activejob (8.0.0)
-      activesupport (= 8.0.0)
+    activejob (8.0.1)
+      activesupport (= 8.0.1)
       globalid (>= 0.3.6)
-    activesupport (8.0.0)
+    activesupport (8.0.1)
       base64
       benchmark (>= 0.3)
       bigdecimal
@@ -49,9 +49,10 @@ GEM
       activesupport (>= 6.1)
     i18n (1.14.6)
       concurrent-ruby (~> 1.0)
-    karafka-core (2.4.5)
-      karafka-rdkafka (>= 0.17.6, < 0.18.0)
-    karafka-rdkafka (0.17.6)
+    karafka-core (2.4.8)
+      karafka-rdkafka (>= 0.17.6, < 0.19.0)
+      logger (>= 1.6.0)
+    karafka-rdkafka (0.18.1)
       ffi (~> 1.15)
       mini_portile2 (~> 2.6)
       rake (> 12)
@@ -64,9 +65,9 @@ GEM
       karafka-core (>= 2.4.0, < 2.5.0)
       roda (~> 3.68, >= 3.69)
       tilt (~> 2.0)
-    logger (1.6.1)
+    logger (1.6.3)
     mini_portile2 (2.8.8)
-    minitest (5.25.1)
+    minitest (5.25.4)
     ostruct (0.6.1)
     raabro (1.4.0)
     rack (3.1.8)
@@ -86,22 +87,23 @@ GEM
       diff-lcs (>= 1.2.0, < 2.0)
       rspec-support (~> 3.13.0)
     rspec-support (3.13.1)
-    securerandom (0.3.1)
+    securerandom (0.3.2)
     simplecov (0.22.0)
       docile (~> 1.1)
       simplecov-html (~> 0.11)
       simplecov_json_formatter (~> 0.1)
     simplecov-html (0.12.3)
     simplecov_json_formatter (0.1.4)
+    stringio (3.1.2)
     tilt (2.4.0)
     tzinfo (2.0.6)
       concurrent-ruby (~> 1.0)
-    uri (1.0.0)
-    waterdrop (2.8.0)
+    uri (1.0.2)
+    waterdrop (2.8.1)
       karafka-core (>= 2.4.3, < 3.0.0)
       karafka-rdkafka (>= 0.17.5)
       zeitwerk (~> 2.3)
-    zeitwerk (2.6.18)
+    zeitwerk (2.7.1)
 
 PLATFORMS
   ruby
@@ -118,6 +120,7 @@ DEPENDENCIES
   ostruct
   rspec
   simplecov
+  stringio
 
 BUNDLED WITH
-   2.4.22
+   2.6.2

data/LICENSE

@@ -6,8 +6,13 @@ for license text.
 
 Karafka has also commercial-friendly license, commercial support and commercial components.
 
-All of the commercial components are present in the lib/karafka/pro directory of this repository
-and their usage requires commercial license agreement.
+All of the commercial components are present in the:
+
+- `lib/karafka/pro`
+- `spec/lib/karafka/pro`
+- `spec/integrations/pro`
+
+directories of this repository and their usage requires commercial license agreement.
 
 By sending a pull request to the pro components, you are agreeing to transfer the copyright of your
 code to Maciej Mensfeld.

data/config/locales/errors.yml

@@ -95,6 +95,7 @@ en:
 
     topic:
       kafka: needs to be a hash with kafka scope settings details
+      kafka_format: needs to be a filled hash
       missing: needs to be present
       max_messages_format: 'needs to be an integer bigger than 0'
       max_wait_time_format: 'needs to be an integer bigger than 0'

data/docker-compose.yml

@@ -1,5 +1,3 @@
-version: '2'
-
 services:
   kafka:
     container_name: kafka

data/lib/karafka/base_consumer.rb

@@ -6,10 +6,13 @@ module Karafka
   class BaseConsumer
     # Allow for consumer instance tagging for instrumentation
     include ::Karafka::Core::Taggable
+    include Helpers::ConfigImporter.new(
+      monitor: %i[monitor]
+    )
 
     extend Forwardable
 
-    def_delegators :@coordinator, :topic, :partition, :eofed?
+    def_delegators :@coordinator, :topic, :partition, :eofed?, :seek_offset, :seek_offset=
 
     def_delegators :producer, :produce_async, :produce_sync, :produce_many_async,
                    :produce_many_sync
@@ -39,7 +42,7 @@ module Karafka
     def on_initialized
       handle_initialized
     rescue StandardError => e
-      Karafka.monitor.instrument(
+      monitor.instrument(
         'error.occurred',
         error: e,
         caller: self,
@@ -73,6 +76,23 @@ module Karafka
       handle_before_consume
     end
 
+    # Executes the default wrapping flow
+    #
+    # @private
+    #
+    # @param action [Symbol]
+    # @param block [Proc]
+    def on_wrap(action, &block)
+      handle_wrap(action, &block)
+    rescue StandardError => e
+      monitor.instrument(
+        'error.occurred',
+        error: e,
+        caller: self,
+        type: 'consumer.wrap.error'
+      )
+    end
+
     # Executes the default consumer flow.
     #
     # @private
@@ -85,11 +105,11 @@ module Karafka
     def on_consume
       handle_consume
     rescue StandardError => e
-      Karafka.monitor.instrument(
+      monitor.instrument(
         'error.occurred',
         error: e,
         caller: self,
-        seek_offset: coordinator.seek_offset,
+        seek_offset: seek_offset,
         type: 'consumer.consume.error'
       )
     end
@@ -105,11 +125,11 @@ module Karafka
     def on_after_consume
       handle_after_consume
     rescue StandardError => e
-      Karafka.monitor.instrument(
+      monitor.instrument(
         'error.occurred',
         error: e,
         caller: self,
-        seek_offset: coordinator.seek_offset,
+        seek_offset: seek_offset,
         type: 'consumer.after_consume.error'
       )
 
@@ -125,11 +145,11 @@ module Karafka
     def on_eofed
       handle_eofed
     rescue StandardError => e
-      Karafka.monitor.instrument(
+      monitor.instrument(
         'error.occurred',
         error: e,
         caller: self,
-        seek_offset: coordinator.seek_offset,
+        seek_offset: seek_offset,
         type: 'consumer.eofed.error'
       )
     end
@@ -161,7 +181,7 @@ module Karafka
     def on_revoked
       handle_revoked
     rescue StandardError => e
-      Karafka.monitor.instrument(
+      monitor.instrument(
         'error.occurred',
         error: e,
         caller: self,
@@ -182,7 +202,7 @@ module Karafka
     def on_shutdown
       handle_shutdown
     rescue StandardError => e
-      Karafka.monitor.instrument(
+      monitor.instrument(
         'error.occurred',
         error: e,
         caller: self,
@@ -209,6 +229,40 @@ module Karafka
       raise NotImplementedError, 'Implement this in a subclass'
     end
 
+    # This method can be redefined to build a wrapping API around user code + karafka flow control
+    # code starting from the user code (operations prior to that are not part of this).
+    # The wrapping relates to a single job flow.
+    #
+    # Karafka framework may require user configured "state" like for example a selected
+    # transactional producer that should be used not only by the user but also by the framework.
+    # By using this API user can checkout a producer and return it to the pool.
+    #
+    # @param _action [Symbol] what action are we wrapping. Useful if we want for example to only
+    #   wrap the `:consume` action.
+    # @yield Runs the execution block
+    #
+    # @note User related errors should not leak to this level of execution. This should not be used
+    #   for anything consumption related but only for setting up state that that Karafka code
+    #   may need outside of user code.
+    #
+    # @example Redefine to use a producer from a pool for consume
+    #   def wrap(action)
+    #     # Do not checkout producer for any other actions
+    #     return yield unless action == :consume
+    #
+    #     default_producer = self.producer
+    #
+    #     $producers.with do |producer|
+    #       self.producer = producer
+    #       yield
+    #     end
+    #
+    #     self.producer = default_producer
+    #   end
+    def wrap(_action)
+      yield
+    end
+
     # Method that will be executed when a given topic partition reaches eof without any new
     # incoming messages alongside
     def eofed; end
@@ -255,7 +309,7 @@ module Karafka
       # Indicate, that user took a manual action of pausing
       coordinator.manual_pause if manual_pause
 
-      Karafka.monitor.instrument(
+      monitor.instrument(
         'consumer.consuming.pause',
         caller: self,
         manual: manual_pause,
@@ -291,7 +345,7 @@ module Karafka
     # @note Please note, that if you are seeking to a time offset, getting the offset is blocking
     def seek(offset, manual_seek = true, reset_offset: false)
       coordinator.manual_seek if manual_seek
-      coordinator.seek_offset = nil if reset_offset
+      self.seek_offset = nil if reset_offset
 
       message = Karafka::Messages::Seek.new(
         topic.name,
@@ -299,7 +353,7 @@ module Karafka
         offset
       )
 
-      Karafka.monitor.instrument(
+      monitor.instrument(
         'consumer.consuming.seek',
         caller: self,
         topic: topic.name,
@@ -343,16 +397,16 @@ module Karafka
     # Pauses the processing from the last offset to retry on given message
     # @private
     def retry_after_pause
-      pause(coordinator.seek_offset, nil, false)
+      pause(seek_offset, nil, false)
 
       # Instrumentation needs to run **after** `#pause` invocation because we rely on the states
       # set by `#pause`
-      Karafka.monitor.instrument(
+      monitor.instrument(
         'consumer.consuming.retry',
         caller: self,
         topic: topic.name,
         partition: partition,
-        offset: coordinator.seek_offset,
+        offset: seek_offset,
         timeout: coordinator.pause_tracker.current_timeout,
         attempt: attempt
       )

data/lib/karafka/connection/client.rb

@@ -66,7 +66,7 @@ module Karafka
         @subscription_group = subscription_group
         @buffer = RawMessagesBuffer.new
         @tick_interval = ::Karafka::App.config.internal.tick_interval
-        @rebalance_manager = RebalanceManager.new(@subscription_group.id)
+        @rebalance_manager = RebalanceManager.new(@subscription_group.id, @buffer)
         @rebalance_callback = Instrumentation::Callbacks::Rebalance.new(@subscription_group)
 
         @interval_runner = Helpers::IntervalRunner.new do
@@ -141,7 +141,7 @@ module Karafka
             # Since rebalances do not occur often, we can run events polling as well without
             # any throttling
             events_poll
-            remove_revoked_and_duplicated_messages
+
             break
           end
 
@@ -717,20 +717,6 @@ module Karafka
         consumer
       end
 
-      # We may have a case where in the middle of data polling, we've lost a partition.
-      # In a case like this we should remove all the pre-buffered messages from list partitions as
-      # we are no longer responsible in a given process for processing those messages and they
-      # should have been picked up by a different process.
-      def remove_revoked_and_duplicated_messages
-        @rebalance_manager.lost_partitions.each do |topic, partitions|
-          partitions.each do |partition|
-            @buffer.delete(topic, partition)
-          end
-        end
-
-        @buffer.uniq!
-      end
-
       # @return [Rdkafka::Consumer] librdkafka consumer instance
       def kafka
         return @kafka if @kafka

data/lib/karafka/connection/rebalance_manager.rb

@@ -2,7 +2,9 @@
 
 module Karafka
   module Connection
-    # Manager for tracking changes in the partitions assignment after the assignment is done.
+    # Manager for tracking changes in the partitions assignment after the assignment is done and
+    # for ensuring, that proper buffer related operations that may be impacted by the rebalance
+    # state are applied.
     #
     # We need tracking of those to clean up consumers that will no longer process given partitions
     # as they were taken away.
@@ -30,13 +32,15 @@ module Karafka
       private_constant :EMPTY_ARRAY
 
       # @param subscription_group_id [String] subscription group id
+      # @param buffer [Karafka::Connection::RawMessagesBuffer]
       # @return [RebalanceManager]
-      def initialize(subscription_group_id)
+      def initialize(subscription_group_id, buffer)
         @assigned_partitions = {}
         @revoked_partitions = {}
         @changed = false
         @active = false
         @subscription_group_id = subscription_group_id
+        @buffer = buffer
 
         # Connects itself to the instrumentation pipeline so rebalances can be tracked
         ::Karafka.monitor.subscribe(self)
@@ -64,17 +68,6 @@ module Karafka
         @active
       end
 
-      # We consider as lost only partitions that were taken away and not re-assigned back to us
-      def lost_partitions
-        lost_partitions = {}
-
-        revoked_partitions.each do |topic, partitions|
-          lost_partitions[topic] = partitions - assigned_partitions.fetch(topic, EMPTY_ARRAY)
-        end
-
-        lost_partitions
-      end
-
       # Callback that kicks in inside of rdkafka, when new partitions were assigned.
       #
       # @private
@@ -99,6 +92,24 @@ module Karafka
         @active = true
         @revoked_partitions = event[:tpl].to_h.transform_values { |part| part.map(&:partition) }
         @changed = true
+
+        remove_revoked_and_duplicated_messages
+      end
+
+      private
+
+      # We may have a case where in the middle of data polling, we've lost a partition.
+      # In a case like this we should remove all the pre-buffered messages from list partitions as
+      # we are no longer responsible in a given process for processing those messages and they
+      # should have been picked up by a different process.
+      def remove_revoked_and_duplicated_messages
+        @revoked_partitions.each do |topic, partitions|
+          partitions.each do |partition|
+            @buffer.delete(topic, partition)
+          end
+        end
+
+        @buffer.uniq!
       end
     end
   end

data/lib/karafka/instrumentation/logger_listener.rb

@@ -293,6 +293,9 @@ module Karafka
         when 'consumer.initialized.error'
           error "Consumer initialized error: #{error}"
           error details
+        when 'consumer.wrap.error'
+          error "Consumer wrap failed due to an error: #{error}"
+          error details
         when 'consumer.consume.error'
           error "Consumer consuming error: #{error}"
           error details

data/lib/karafka/instrumentation/notifications.rb

@@ -57,6 +57,7 @@ module Karafka
         consumer.consuming.pause
         consumer.consuming.retry
         consumer.consuming.seek
+        consumer.consuming.transaction
 
         consumer.before_schedule_idle
         consumer.idle
@@ -77,6 +78,9 @@ module Karafka
         consumer.shutting_down
         consumer.shutdown
 
+        consumer.wrap
+        consumer.wrapped
+
         dead_letter_queue.dispatched
 
         filtering.throttled

data/lib/karafka/instrumentation/vendors/datadog/logger_listener.rb

@@ -56,32 +56,7 @@ module Karafka
             consumer = job.executor.topic.consumer
             topic = job.executor.topic.name
 
-            action = case job_type
-                     when 'Periodic'
-                       'tick'
-                     when 'PeriodicNonBlocking'
-                       'tick'
-                     when 'Shutdown'
-                       'shutdown'
-                     when 'Revoked'
-                       'revoked'
-                     when 'RevokedNonBlocking'
-                       'revoked'
-                     when 'Idle'
-                       'idle'
-                     when 'Eofed'
-                       'eofed'
-                     when 'EofedNonBlocking'
-                       'eofed'
-                     when 'ConsumeNonBlocking'
-                       'consume'
-                     when 'Consume'
-                       'consume'
-                     else
-                       raise Errors::UnsupportedCaseError, job_type
-                     end
-
-            current_span.resource = "#{consumer}##{action}"
+            current_span.resource = "#{consumer}##{job.class.action}"
             info "[#{job.id}] #{job_type} job for #{consumer} on #{topic} started"
 
             pop_tags

data/lib/karafka/pro/active_job/consumer.rb

@@ -1,15 +1,7 @@
 # frozen_string_literal: true
 
-# This Karafka component is a Pro component under a commercial license.
-# This Karafka component is NOT licensed under LGPL.
-#
-# All of the commercial components are present in the lib/karafka/pro directory of this
-# repository and their usage requires commercial license agreement.
-#
-# Karafka has also commercial-friendly license, commercial support and commercial components.
-#
-# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
-# your code to Maciej Mensfeld.
+# This code is part of Karafka Pro, a commercial component not licensed under LGPL.
+# See LICENSE for details.
 
 module Karafka
   module Pro

data/lib/karafka/pro/active_job/dispatcher.rb

@@ -1,15 +1,7 @@
 # frozen_string_literal: true
 
-# This Karafka component is a Pro component under a commercial license.
-# This Karafka component is NOT licensed under LGPL.
-#
-# All of the commercial components are present in the lib/karafka/pro directory of this
-# repository and their usage requires commercial license agreement.
-#
-# Karafka has also commercial-friendly license, commercial support and commercial components.
-#
-# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
-# your code to Maciej Mensfeld.
+# This code is part of Karafka Pro, a commercial component not licensed under LGPL.
+# See LICENSE for details.
 
 module Karafka
   module Pro

data/lib/karafka/pro/active_job/job_options_contract.rb

@@ -1,15 +1,7 @@
 # frozen_string_literal: true
 
-# This Karafka component is a Pro component under a commercial license.
-# This Karafka component is NOT licensed under LGPL.
-#
-# All of the commercial components are present in the lib/karafka/pro directory of this
-# repository and their usage requires commercial license agreement.
-#
-# Karafka has also commercial-friendly license, commercial support and commercial components.
-#
-# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
-# your code to Maciej Mensfeld.
+# This code is part of Karafka Pro, a commercial component not licensed under LGPL.
+# See LICENSE for details.
 
 module Karafka
   module Pro

data/lib/karafka/pro/base_consumer.rb

@@ -1,15 +1,7 @@
 # frozen_string_literal: true
 
-# This Karafka component is a Pro component under a commercial license.
-# This Karafka component is NOT licensed under LGPL.
-#
-# All of the commercial components are present in the lib/karafka/pro directory of this
-# repository and their usage requires commercial license agreement.
-#
-# Karafka has also commercial-friendly license, commercial support and commercial components.
-#
-# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
-# your code to Maciej Mensfeld.
+# This code is part of Karafka Pro, a commercial component not licensed under LGPL.
+# See LICENSE for details.
 
 module Karafka
   module Pro

data/lib/karafka/pro/cleaner/errors.rb

@@ -1,15 +1,7 @@
 # frozen_string_literal: true
 
-# This Karafka component is a Pro component under a commercial license.
-# This Karafka component is NOT licensed under LGPL.
-#
-# All of the commercial components are present in the lib/karafka/pro directory of this
-# repository and their usage requires commercial license agreement.
-#
-# Karafka has also commercial-friendly license, commercial support and commercial components.
-#
-# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
-# your code to Maciej Mensfeld.
+# This code is part of Karafka Pro, a commercial component not licensed under LGPL.
+# See LICENSE for details.
 
 module Karafka
   module Pro