karafka 2.4.14 → 2.4.16

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 22f45da117cf90a2ecbec04dbcaf39634b1e1b85e00f521d4880954c65268ecf
-  data.tar.gz: ce5318aaa8f52954a80981662a41ad17ab314e9706a00f2edf3e840604b87b32
+  metadata.gz: 8510d261a0722b764aff2c59e214bb2b87481ce606488e8f1323c4bf29ee4528
+  data.tar.gz: 4574f71706749ec5effa830b9c04a6e3072c880220a08fa33f81648f7d3c79be
 SHA512:
-  metadata.gz: d8ec82f91aea2bdba595fa290feec1ca0b25dbd73cd0eafac6554538d3506c47ad8db42036ed838b199a3016341bf67d6daf4f03dfced3ddcf783850c0dc97bf
-  data.tar.gz: 2b750fce3294dda031f5a97bf5343711dcf52d1dbc00c61e029b602b302a549e9a70bb6ac4e952842e1ad840d7dd7b28115ed72118aad54917497672593de133
+  metadata.gz: 37c43b9634b43e6608fa7f2ab9320d49d00b2582c211095968d3200412029e0301d17556d31be2e24d8401457fd660473cc46adc4d5835de968a2eba9c51fbef
+  data.tar.gz: 15d2e2dbd7ee99ab1c726a47a76959f1e3d877cd723f084550711fd9ba4662a30eb39af26f1354847ab906aaec9580e0cde5f7858b0cda521f5f39844195a3c3

data/.github/workflows/ci.yml

@@ -27,7 +27,7 @@ jobs:
       - name: Set up Ruby
         uses: ruby/setup-ruby@v1
         with:
-          ruby-version: 3.3
+          ruby-version: 3.4
           bundler-cache: true
 
       - name: Install Diffend plugin
@@ -73,12 +73,12 @@ jobs:
       fail-fast: false
       matrix:
         ruby:
-          - '3.4.0-preview1'
+          - '3.4'
           - '3.3'
           - '3.2'
           - '3.1'
         include:
-          - ruby: '3.3'
+          - ruby: '3.4'
             coverage: 'true'
     steps:
       - uses: actions/checkout@v4
@@ -120,7 +120,7 @@ jobs:
       fail-fast: false
       matrix:
         ruby:
-          - '3.4.0-preview1'
+          - '3.4'
           - '3.3'
           - '3.2'
           - '3.1'
@@ -178,7 +178,7 @@ jobs:
       fail-fast: false
       matrix:
         ruby:
-          - '3.4.0-preview1'
+          - '3.4'
           - '3.3'
           - '3.2'
           - '3.1'

data/.ruby-version

@@ -1 +1 @@
-3.3.6
+3.4.1

data/CHANGELOG.md

@@ -1,5 +1,19 @@
 # Karafka Framework Changelog
 
+## 2.4.16 (2024-12-27)
+- [Enhancement] Improve post-rebalance revocation messages filtering.
+- [Enhancement] Introduce `Consumer#wrap` for connection pooling management and other wrapped operations.
+- [Enhancement] Guard transactional operations from marking beyond assignment ownership under some extreme edge-cases.
+- [Enhancement] Improve VPs work with transactional producers.
+- [Enhancement] Prevent non-transactional operations leakage into transactional managed offset management consumers.
+- [Fix] Prevent transactions from being marked with a non-transactional default producer when automatic offset management and other advanced features are on.
+- [Fix] Fix `kafka_format` `KeyError` that occurs when a non-hash is assigned to the kafka scope of the settings.
+- [Fix] Non cooperative-sticky transactional offset management can refetch reclaimed partitions.
+
+## 2.4.15 (2024-12-04)
+- [Fix] Assignment tracker current state fetch during a rebalance loop can cause an error on multi CG setup.
+- [Fix] Prevent double post-transaction offset dispatch to Kafka.
+
 ## 2.4.14 (2024-11-25)
 - [Enhancement] Improve low-level critical error reporting.
 - [Enhancement] Expand Kubernetes Liveness state reporting with critical errors detection.

data/Gemfile

@@ -12,6 +12,7 @@ gemspec
 group :integrations, :test do
   gem 'fugit', require: false
   gem 'rspec', require: false
+  gem 'stringio'
 end
 
 group :integrations do

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    karafka (2.4.14)
+    karafka (2.4.16)
       base64 (~> 0.2)
       karafka-core (>= 2.4.4, < 2.5.0)
       karafka-rdkafka (>= 0.17.2)
@@ -11,10 +11,10 @@ PATH
 GEM
   remote: https://rubygems.org/
   specs:
-    activejob (8.0.0)
-      activesupport (= 8.0.0)
+    activejob (8.0.1)
+      activesupport (= 8.0.1)
       globalid (>= 0.3.6)
-    activesupport (8.0.0)
+    activesupport (8.0.1)
       base64
       benchmark (>= 0.3)
       bigdecimal
@@ -49,27 +49,28 @@ GEM
       activesupport (>= 6.1)
     i18n (1.14.6)
       concurrent-ruby (~> 1.0)
-    karafka-core (2.4.4)
-      karafka-rdkafka (>= 0.15.0, < 0.18.0)
-    karafka-rdkafka (0.17.6)
+    karafka-core (2.4.8)
+      karafka-rdkafka (>= 0.17.6, < 0.19.0)
+      logger (>= 1.6.0)
+    karafka-rdkafka (0.18.1)
       ffi (~> 1.15)
       mini_portile2 (~> 2.6)
       rake (> 12)
     karafka-testing (2.4.6)
       karafka (>= 2.4.0, < 2.5.0)
       waterdrop (>= 2.7.0)
-    karafka-web (0.10.3)
+    karafka-web (0.10.4)
       erubi (~> 1.4)
       karafka (>= 2.4.10, < 2.5.0)
       karafka-core (>= 2.4.0, < 2.5.0)
       roda (~> 3.68, >= 3.69)
       tilt (~> 2.0)
-    logger (1.6.1)
-    mini_portile2 (2.8.7)
-    minitest (5.25.1)
+    logger (1.6.3)
+    mini_portile2 (2.8.8)
+    minitest (5.25.4)
     ostruct (0.6.1)
     raabro (1.4.0)
-    rack (3.1.7)
+    rack (3.1.8)
     rake (13.2.1)
     roda (3.84.0)
       rack
@@ -86,22 +87,23 @@ GEM
       diff-lcs (>= 1.2.0, < 2.0)
       rspec-support (~> 3.13.0)
     rspec-support (3.13.1)
-    securerandom (0.3.1)
+    securerandom (0.3.2)
     simplecov (0.22.0)
       docile (~> 1.1)
       simplecov-html (~> 0.11)
       simplecov_json_formatter (~> 0.1)
     simplecov-html (0.12.3)
     simplecov_json_formatter (0.1.4)
+    stringio (3.1.2)
     tilt (2.4.0)
     tzinfo (2.0.6)
       concurrent-ruby (~> 1.0)
-    uri (1.0.0)
-    waterdrop (2.8.0)
+    uri (1.0.2)
+    waterdrop (2.8.1)
       karafka-core (>= 2.4.3, < 3.0.0)
       karafka-rdkafka (>= 0.17.5)
       zeitwerk (~> 2.3)
-    zeitwerk (2.6.18)
+    zeitwerk (2.7.1)
 
 PLATFORMS
   ruby
@@ -118,6 +120,7 @@ DEPENDENCIES
   ostruct
   rspec
   simplecov
+  stringio
 
 BUNDLED WITH
    2.4.22

data/config/locales/errors.yml

@@ -95,6 +95,7 @@ en:
 
     topic:
       kafka: needs to be a hash with kafka scope settings details
+      kafka_format: needs to be a filled hash
       missing: needs to be present
       max_messages_format: 'needs to be an integer bigger than 0'
       max_wait_time_format: 'needs to be an integer bigger than 0'

data/docker-compose.yml

@@ -1,9 +1,7 @@
-version: '2'
-
 services:
   kafka:
     container_name: kafka
-    image: confluentinc/cp-kafka:7.7.1
+    image: confluentinc/cp-kafka:7.8.0
 
     ports:
       - 9092:9092

data/lib/karafka/base_consumer.rb

@@ -6,6 +6,9 @@ module Karafka
   class BaseConsumer
     # Allow for consumer instance tagging for instrumentation
     include ::Karafka::Core::Taggable
+    include Helpers::ConfigImporter.new(
+      monitor: %i[monitor]
+    )
 
     extend Forwardable
 
@@ -39,7 +42,7 @@ module Karafka
     def on_initialized
       handle_initialized
     rescue StandardError => e
-      Karafka.monitor.instrument(
+      monitor.instrument(
         'error.occurred',
         error: e,
         caller: self,
@@ -73,6 +76,23 @@ module Karafka
       handle_before_consume
     end
 
+    # Executes the default wrapping flow
+    #
+    # @private
+    #
+    # @param action [Symbol]
+    # @param block [Proc]
+    def on_wrap(action, &block)
+      handle_wrap(action, &block)
+    rescue StandardError => e
+      monitor.instrument(
+        'error.occurred',
+        error: e,
+        caller: self,
+        type: 'consumer.wrap.error'
+      )
+    end
+
     # Executes the default consumer flow.
     #
     # @private
@@ -85,7 +105,7 @@ module Karafka
     def on_consume
       handle_consume
     rescue StandardError => e
-      Karafka.monitor.instrument(
+      monitor.instrument(
         'error.occurred',
         error: e,
         caller: self,
@@ -105,7 +125,7 @@ module Karafka
     def on_after_consume
       handle_after_consume
     rescue StandardError => e
-      Karafka.monitor.instrument(
+      monitor.instrument(
         'error.occurred',
         error: e,
         caller: self,
@@ -125,7 +145,7 @@ module Karafka
     def on_eofed
       handle_eofed
     rescue StandardError => e
-      Karafka.monitor.instrument(
+      monitor.instrument(
         'error.occurred',
         error: e,
         caller: self,
@@ -161,7 +181,7 @@ module Karafka
     def on_revoked
       handle_revoked
     rescue StandardError => e
-      Karafka.monitor.instrument(
+      monitor.instrument(
         'error.occurred',
         error: e,
         caller: self,
@@ -182,7 +202,7 @@ module Karafka
     def on_shutdown
       handle_shutdown
     rescue StandardError => e
-      Karafka.monitor.instrument(
+      monitor.instrument(
         'error.occurred',
         error: e,
         caller: self,
@@ -209,6 +229,40 @@ module Karafka
       raise NotImplementedError, 'Implement this in a subclass'
     end
 
+    # This method can be redefined to build a wrapping API around user code + karafka flow control
+    # code starting from the user code (operations prior to that are not part of this).
+    # The wrapping relates to a single job flow.
+    #
+    # Karafka framework may require user configured "state" like for example a selected
+    # transactional producer that should be used not only by the user but also by the framework.
+    # By using this API user can checkout a producer and return it to the pool.
+    #
+    # @param _action [Symbol] what action are we wrapping. Useful if we want for example to only
+    #   wrap the `:consume` action.
+    # @yield Runs the execution block
+    #
+    # @note User related errors should not leak to this level of execution. This should not be used
+    #   for anything consumption related but only for setting up state that that Karafka code
+    #   may need outside of user code.
+    #
+    # @example Redefine to use a producer from a pool for consume
+    #   def wrap(action)
+    #     # Do not checkout producer for any other actions
+    #     return yield unless action == :consume
+    #
+    #     default_producer = self.producer
+    #
+    #     $producers.with do |producer|
+    #       self.producer = producer
+    #       yield
+    #     end
+    #
+    #     self.producer = default_producer
+    #   end
+    def wrap(_action)
+      yield
+    end
+
     # Method that will be executed when a given topic partition reaches eof without any new
     # incoming messages alongside
     def eofed; end
@@ -255,7 +309,7 @@ module Karafka
       # Indicate, that user took a manual action of pausing
       coordinator.manual_pause if manual_pause
 
-      Karafka.monitor.instrument(
+      monitor.instrument(
         'consumer.consuming.pause',
         caller: self,
         manual: manual_pause,
@@ -299,7 +353,7 @@ module Karafka
         offset
       )
 
-      Karafka.monitor.instrument(
+      monitor.instrument(
         'consumer.consuming.seek',
         caller: self,
         topic: topic.name,
@@ -347,7 +401,7 @@ module Karafka
 
       # Instrumentation needs to run **after** `#pause` invocation because we rely on the states
       # set by `#pause`
-      Karafka.monitor.instrument(
+      monitor.instrument(
         'consumer.consuming.retry',
         caller: self,
         topic: topic.name,

data/lib/karafka/connection/client.rb

@@ -66,7 +66,7 @@ module Karafka
         @subscription_group = subscription_group
         @buffer = RawMessagesBuffer.new
         @tick_interval = ::Karafka::App.config.internal.tick_interval
-        @rebalance_manager = RebalanceManager.new(@subscription_group.id)
+        @rebalance_manager = RebalanceManager.new(@subscription_group.id, @buffer)
         @rebalance_callback = Instrumentation::Callbacks::Rebalance.new(@subscription_group)
 
         @interval_runner = Helpers::IntervalRunner.new do
@@ -141,7 +141,7 @@ module Karafka
             # Since rebalances do not occur often, we can run events polling as well without
             # any throttling
             events_poll
-            remove_revoked_and_duplicated_messages
+
             break
           end
 
@@ -717,20 +717,6 @@ module Karafka
         consumer
       end
 
-      # We may have a case where in the middle of data polling, we've lost a partition.
-      # In a case like this we should remove all the pre-buffered messages from list partitions as
-      # we are no longer responsible in a given process for processing those messages and they
-      # should have been picked up by a different process.
-      def remove_revoked_and_duplicated_messages
-        @rebalance_manager.lost_partitions.each do |topic, partitions|
-          partitions.each do |partition|
-            @buffer.delete(topic, partition)
-          end
-        end
-
-        @buffer.uniq!
-      end
-
       # @return [Rdkafka::Consumer] librdkafka consumer instance
       def kafka
         return @kafka if @kafka

data/lib/karafka/connection/rebalance_manager.rb

@@ -2,7 +2,9 @@
 
 module Karafka
   module Connection
-    # Manager for tracking changes in the partitions assignment after the assignment is done.
+    # Manager for tracking changes in the partitions assignment after the assignment is done and
+    # for ensuring, that proper buffer related operations that may be impacted by the rebalance
+    # state are applied.
     #
     # We need tracking of those to clean up consumers that will no longer process given partitions
     # as they were taken away.
@@ -30,13 +32,15 @@ module Karafka
       private_constant :EMPTY_ARRAY
 
       # @param subscription_group_id [String] subscription group id
+      # @param buffer [Karafka::Connection::RawMessagesBuffer]
       # @return [RebalanceManager]
-      def initialize(subscription_group_id)
+      def initialize(subscription_group_id, buffer)
         @assigned_partitions = {}
         @revoked_partitions = {}
         @changed = false
         @active = false
         @subscription_group_id = subscription_group_id
+        @buffer = buffer
 
         # Connects itself to the instrumentation pipeline so rebalances can be tracked
         ::Karafka.monitor.subscribe(self)
@@ -64,17 +68,6 @@ module Karafka
         @active
       end
 
-      # We consider as lost only partitions that were taken away and not re-assigned back to us
-      def lost_partitions
-        lost_partitions = {}
-
-        revoked_partitions.each do |topic, partitions|
-          lost_partitions[topic] = partitions - assigned_partitions.fetch(topic, EMPTY_ARRAY)
-        end
-
-        lost_partitions
-      end
-
       # Callback that kicks in inside of rdkafka, when new partitions were assigned.
       #
       # @private
@@ -99,6 +92,24 @@ module Karafka
         @active = true
         @revoked_partitions = event[:tpl].to_h.transform_values { |part| part.map(&:partition) }
         @changed = true
+
+        remove_revoked_and_duplicated_messages
+      end
+
+      private
+
+      # We may have a case where in the middle of data polling, we've lost a partition.
+      # In a case like this we should remove all the pre-buffered messages from list partitions as
+      # we are no longer responsible in a given process for processing those messages and they
+      # should have been picked up by a different process.
+      def remove_revoked_and_duplicated_messages
+        @revoked_partitions.each do |topic, partitions|
+          partitions.each do |partition|
+            @buffer.delete(topic, partition)
+          end
+        end
+
+        @buffer.uniq!
       end
     end
   end

data/lib/karafka/instrumentation/assignments_tracker.rb

@@ -31,8 +31,13 @@ module Karafka
       def current
         assignments = {}
 
-        @assignments.each do |topic, partitions|
-          assignments[topic] = partitions.dup.freeze
+        # Since the `@assignments` state can change during a rebalance, if we would iterate over
+        # it exactly during state change, we would end up with the following error:
+        #   RuntimeError: can't add a new key into hash during iteration
+        @mutex.synchronize do
+          @assignments.each do |topic, partitions|
+            assignments[topic] = partitions.dup.freeze
+          end
         end
 
         assignments.freeze

data/lib/karafka/instrumentation/logger_listener.rb

@@ -293,6 +293,9 @@ module Karafka
         when 'consumer.initialized.error'
           error "Consumer initialized error: #{error}"
           error details
+        when 'consumer.wrap.error'
+          error "Consumer wrap failed due to an error: #{error}"
+          error details
         when 'consumer.consume.error'
           error "Consumer consuming error: #{error}"
           error details

data/lib/karafka/instrumentation/notifications.rb

@@ -77,6 +77,9 @@ module Karafka
         consumer.shutting_down
         consumer.shutdown
 
+        consumer.wrap
+        consumer.wrapped
+
         dead_letter_queue.dispatched
 
         filtering.throttled

data/lib/karafka/instrumentation/vendors/datadog/logger_listener.rb

@@ -56,32 +56,7 @@ module Karafka
             consumer = job.executor.topic.consumer
             topic = job.executor.topic.name
 
-            action = case job_type
-                     when 'Periodic'
-                       'tick'
-                     when 'PeriodicNonBlocking'
-                       'tick'
-                     when 'Shutdown'
-                       'shutdown'
-                     when 'Revoked'
-                       'revoked'
-                     when 'RevokedNonBlocking'
-                       'revoked'
-                     when 'Idle'
-                       'idle'
-                     when 'Eofed'
-                       'eofed'
-                     when 'EofedNonBlocking'
-                       'eofed'
-                     when 'ConsumeNonBlocking'
-                       'consume'
-                     when 'Consume'
-                       'consume'
-                     else
-                       raise Errors::UnsupportedCaseError, job_type
-                     end
-
-            current_span.resource = "#{consumer}##{action}"
+            current_span.resource = "#{consumer}##{job.class.action}"
             info "[#{job.id}] #{job_type} job for #{consumer} on #{topic} started"
 
             pop_tags

data/lib/karafka/pro/processing/jobs/consume_non_blocking.rb

@@ -26,6 +26,8 @@ module Karafka
         # @note It needs to be working with a proper consumer that will handle the partition
         #   management. This layer of the framework knows nothing about Kafka messages consumption.
         class ConsumeNonBlocking < ::Karafka::Processing::Jobs::Consume
+          self.action = :consume
+
           # Makes this job non-blocking from the start
           # @param args [Array] any arguments accepted by `::Karafka::Processing::Jobs::Consume`
           def initialize(*args)

data/lib/karafka/pro/processing/jobs/eofed_non_blocking.rb

@@ -20,6 +20,8 @@ module Karafka
         # to run this job for extended period of time. Under such scenarios, if we would not use
         # a non-blocking one, we would reach max.poll.interval.ms.
         class EofedNonBlocking < ::Karafka::Processing::Jobs::Eofed
+          self.action = :eofed
+
           # @param args [Array] any arguments accepted by `::Karafka::Processing::Jobs::Eofed`
           def initialize(*args)
             super

data/lib/karafka/pro/processing/jobs/periodic.rb

@@ -18,6 +18,8 @@ module Karafka
         # Job that represents a "ticking" work. Work that we run periodically for the Periodics
         # enabled topics.
         class Periodic < ::Karafka::Processing::Jobs::Base
+          self.action = :tick
+
           # @param executor [Karafka::Pro::Processing::Executor] pro executor that is suppose to
           #   run a given job
           def initialize(executor)

data/lib/karafka/pro/processing/jobs/periodic_non_blocking.rb

@@ -20,6 +20,8 @@ module Karafka
         # to run this job for extended period of time. Under such scenarios, if we would not use
         # a non-blocking one, we would reach max.poll.interval.ms.
         class PeriodicNonBlocking < Periodic
+          self.action = :tick
+
           # @param args [Array] any arguments accepted by `::Karafka::Processing::Jobs::Periodic`
           def initialize(*args)
             super

data/lib/karafka/pro/processing/jobs/revoked_non_blocking.rb

@@ -24,6 +24,8 @@ module Karafka
         # in scenarios where there are more jobs than threads, without this being async we
         # would potentially stop polling
         class RevokedNonBlocking < ::Karafka::Processing::Jobs::Revoked
+          self.action = :revoked
+
           # Makes this job non-blocking from the start
           # @param args [Array] any arguments accepted by `::Karafka::Processing::Jobs::Revoked`
           def initialize(*args)

data/lib/karafka/pro/processing/strategies/aj/lrj_mom_vp.rb

@@ -69,8 +69,8 @@ module Karafka
                 coordinator.revoke
               end
 
-              Karafka.monitor.instrument('consumer.revoke', caller: self)
-              Karafka.monitor.instrument('consumer.revoked', caller: self) do
+              monitor.instrument('consumer.revoke', caller: self)
+              monitor.instrument('consumer.revoked', caller: self) do
                 revoked
               end
             ensure

data/lib/karafka/pro/processing/strategies/default.rb

@@ -54,8 +54,11 @@ module Karafka
           #   already processed but rather at the next one. This applies to both sync and async
           #   versions of this method.
           def mark_as_consumed(message, offset_metadata = @_current_offset_metadata)
+            # If we are inside a transaction than we can just mark as consumed within it
             if @_in_transaction
               mark_in_transaction(message, offset_metadata, true)
+            elsif @_in_transaction_marked
+              mark_in_memory(message)
             else
               # seek offset can be nil only in case `#seek` was invoked with offset reset request
               # In case like this we ignore marking
@@ -63,7 +66,16 @@ module Karafka
               # Ignore earlier offsets than the one we already committed
               return true if coordinator.seek_offset > message.offset
               return false if revoked?
-              return revoked? unless client.mark_as_consumed(message, offset_metadata)
+
+              # If we are not inside a transaction but this is a transactional topic, we mark with
+              # artificially created transaction
+              stored = if producer.transactional?
+                         mark_with_transaction(message, offset_metadata, true)
+                       else
+                         client.mark_as_consumed(message, offset_metadata)
+                       end
+
+              return revoked? unless stored
 
               coordinator.seek_offset = message.offset + 1
             end
@@ -82,6 +94,8 @@ module Karafka
           def mark_as_consumed!(message, offset_metadata = @_current_offset_metadata)
             if @_in_transaction
               mark_in_transaction(message, offset_metadata, false)
+            elsif @_in_transaction_marked
+              mark_in_memory(message)
             else
               # seek offset can be nil only in case `#seek` was invoked with offset reset request
               # In case like this we ignore marking
@@ -90,7 +104,15 @@ module Karafka
               return true if coordinator.seek_offset > message.offset
               return false if revoked?
 
-              return revoked? unless client.mark_as_consumed!(message, offset_metadata)
+              # If we are not inside a transaction but this is a transactional topic, we mark with
+              # artificially created transaction
+              stored = if producer.transactional?
+                         mark_with_transaction(message, offset_metadata, false)
+                       else
+                         client.mark_as_consumed!(message, offset_metadata)
+                       end
+
+              return revoked? unless stored
 
               coordinator.seek_offset = message.offset + 1
             end
@@ -112,7 +134,7 @@ module Karafka
           #   managing multiple producers. If not provided, default producer taken from `#producer`
           #   will be used.
           #
-          # @param block [Proc] code that we want to run in a transaction
+          # @yield code that we want to run in a transaction
           #
           # @note Please note, that if you provide the producer, it will reassign the producer of
           #   the consumer for the transaction time. This means, that in case you would even
@@ -120,7 +142,7 @@ module Karafka
           #   reassigned producer and not the initially used/assigned producer. It is done that
           #   way, so the message producing aliases operate from within transactions and since the
           #   producer in transaction is locked, it will prevent other threads from using it.
-          def transaction(active_producer = producer, &block)
+          def transaction(active_producer = producer)
             default_producer = producer
             self.producer = active_producer
 
@@ -132,8 +154,16 @@ module Karafka
             transaction_started = true
             @_transaction_marked = []
             @_in_transaction = true
+            @_in_transaction_marked = false
 
-            producer.transaction(&block)
+            producer.transaction do
+              yield
+
+              # Ensure this transaction is rolled back if we have lost the ownership of this
+              # transaction. We do it only for transactions that contain offset management as for
+              # producer only, this is not relevant.
+              raise Errors::AssignmentLostError if @_in_transaction_marked && revoked?
+            end
 
             @_in_transaction = false
 
@@ -143,6 +173,11 @@ module Karafka
             # @note We never need to use the blocking `#mark_as_consumed!` here because the offset
             #   anyhow was already stored during the transaction
             #
+            # @note Since the offset could have been already stored in Kafka (could have because
+            #   you can have transactions without marking), we use the `@_in_transaction_marked`
+            #   state to decide if we need to dispatch the offset via client at all
+            #   (if post transaction, then we do not have to)
+            #
             # @note In theory we could only keep reference to the most recent marking and reject
             #   others. We however do not do it for two reasons:
             #   - User may have non standard flow relying on some alternative order and we want to
@@ -152,12 +187,15 @@ module Karafka
             @_transaction_marked.each do |marking|
               marking.pop ? mark_as_consumed(*marking) : mark_as_consumed!(*marking)
             end
+
+            true
           ensure
             self.producer = default_producer
 
             if transaction_started
               @_transaction_marked.clear
               @_in_transaction = false
+              @_in_transaction_marked = false
             end
           end
 
@@ -178,13 +216,60 @@ module Karafka
               offset_metadata
             )
 
+            @_in_transaction_marked = true
             @_transaction_marked ||= []
             @_transaction_marked << [message, offset_metadata, async]
           end
 
+          # @private
+          # @param message [Messages::Message] message we want to commit inside of a transaction
+          # @param offset_metadata [String, nil] offset metadata or nil if none
+          # @param async [Boolean] should we mark in async or sync way (applicable only to post
+          #   transaction state synchronization usage as within transaction it is always sync)
+          # @return [Boolean] false if marking failed otherwise true
+          def mark_with_transaction(message, offset_metadata, async)
+            # This flag is used by VPs to differentiate between user initiated transactions and
+            # post-execution system transactions.
+            @_transaction_internal = true
+
+            transaction do
+              mark_in_transaction(message, offset_metadata, async)
+            end
+
+            true
+          # We handle both cases here because this is a private API for internal usage and we want
+          # the post-user code execution marking with transactional producer to result in a
+          # boolean state of marking for further framework flow. This is a normalization to make it
+          # behave the same way as it would behave with a non-transactional one
+          rescue ::Rdkafka::RdkafkaError, Errors::AssignmentLostError
+            false
+          ensure
+            @_transaction_internal = false
+          end
+
+          # Marks the current state only in memory as the offset marking has already happened
+          # using the producer transaction
+          # @param message [Messages::Message] last successfully processed message.
+          # @return [Boolean] true if all good, false if we lost assignment and no point in marking
+          def mark_in_memory(message)
+            # seek offset can be nil only in case `#seek` was invoked with offset reset request
+            # In case like this we ignore marking
+            return true if coordinator.seek_offset.nil?
+            # Ignore earlier offsets than the one we already committed
+            return true if coordinator.seek_offset > message.offset
+            return false if revoked?
+
+            # If we have already marked this successfully in a transaction that was running
+            # we should not mark it again with the client offset delegation but instead we should
+            # just align the in-memory state
+            coordinator.seek_offset = message.offset + 1
+
+            true
+          end
+
           # No actions needed for the standard flow here
           def handle_before_schedule_consume
-            Karafka.monitor.instrument('consumer.before_schedule_consume', caller: self)
+            monitor.instrument('consumer.before_schedule_consume', caller: self)
 
             nil
           end
@@ -203,8 +288,8 @@ module Karafka
             # This can happen primarily when an LRJ job gets to the internal worker queue and
             # this partition is revoked prior processing.
             unless revoked?
-              Karafka.monitor.instrument('consumer.consume', caller: self)
-              Karafka.monitor.instrument('consumer.consumed', caller: self) do
+              monitor.instrument('consumer.consume', caller: self)
+              monitor.instrument('consumer.consumed', caller: self) do
                 consume
               end
             end
@@ -250,8 +335,8 @@ module Karafka
               coordinator.revoke
             end
 
-            Karafka.monitor.instrument('consumer.revoke', caller: self)
-            Karafka.monitor.instrument('consumer.revoked', caller: self) do
+            monitor.instrument('consumer.revoke', caller: self)
+            monitor.instrument('consumer.revoked', caller: self) do
               revoked
             end
           ensure
@@ -260,15 +345,15 @@ module Karafka
 
           # No action needed for the tick standard flow
           def handle_before_schedule_tick
-            Karafka.monitor.instrument('consumer.before_schedule_tick', caller: self)
+            monitor.instrument('consumer.before_schedule_tick', caller: self)
 
             nil
           end
 
           # Runs the consumer `#tick` method with reporting
           def handle_tick
-            Karafka.monitor.instrument('consumer.tick', caller: self)
-            Karafka.monitor.instrument('consumer.ticked', caller: self) do
+            monitor.instrument('consumer.tick', caller: self)
+            monitor.instrument('consumer.ticked', caller: self) do
               tick
             end
           ensure

data/lib/karafka/pro/processing/strategies/dlq/default.rb

@@ -119,7 +119,7 @@ module Karafka
               )
 
               # Notify about dispatch on the events bus
-              Karafka.monitor.instrument(
+              monitor.instrument(
                 'dead_letter_queue.dispatched',
                 caller: self,
                 message: skippable_message

data/lib/karafka/pro/processing/strategies/ftr/default.rb

@@ -89,7 +89,7 @@ module Karafka
 
                 throttle_message = filter.cursor
 
-                Karafka.monitor.instrument(
+                monitor.instrument(
                   'filtering.seek',
                   caller: self,
                   message: throttle_message
@@ -104,7 +104,7 @@ module Karafka
 
                 throttle_message = filter.cursor
 
-                Karafka.monitor.instrument(
+                monitor.instrument(
                   'filtering.throttled',
                   caller: self,
                   message: throttle_message,

data/lib/karafka/pro/processing/strategies/lrj/default.rb

@@ -72,8 +72,8 @@ module Karafka
                 coordinator.revoke
               end
 
-              Karafka.monitor.instrument('consumer.revoke', caller: self)
-              Karafka.monitor.instrument('consumer.revoked', caller: self) do
+              monitor.instrument('consumer.revoke', caller: self)
+              monitor.instrument('consumer.revoked', caller: self) do
                 revoked
               end
             ensure

data/lib/karafka/pro/processing/strategies/lrj/mom.rb

@@ -64,8 +64,8 @@ module Karafka
                 coordinator.revoke
               end
 
-              Karafka.monitor.instrument('consumer.revoke', caller: self)
-              Karafka.monitor.instrument('consumer.revoked', caller: self) do
+              monitor.instrument('consumer.revoke', caller: self)
+              monitor.instrument('consumer.revoked', caller: self) do
                 revoked
               end
             ensure

data/lib/karafka/pro/processing/strategies/vp/default.rb

@@ -101,6 +101,9 @@ module Karafka
               raise Errors::AssignmentLostError if revoked?
 
               return super if collapsed?
+              # If this is user post-execution transaction (one initiated by the system) we should
+              # delegate to the original implementation that will store the offset via the producer
+              return super if @_transaction_internal
 
               @_transaction_marked << [message, offset_metadata, async]
             end

data/lib/karafka/processing/executor.rb

@@ -81,6 +81,13 @@ module Karafka
         consumer.on_before_consume
       end
 
+      # Runs the wrap/around execution context appropriate for a given action
+      # @param action [Symbol] action execution wrapped with our block
+      # @param block [Proc] execution context
+      def wrap(action, &block)
+        consumer.on_wrap(action, &block)
+      end
+
       # Runs consumer data processing against given batch and handles failures and errors.
       def consume
         # We run the consumer client logic...

data/lib/karafka/processing/jobs/base.rb

@@ -15,6 +15,11 @@ module Karafka
 
         attr_reader :executor
 
+        class << self
+          # @return [Symbol] Job matching appropriate action
+          attr_accessor :action
+        end
+
         # Creates a new job instance
         def initialize
           # All jobs are blocking by default and they can release the lock when blocking operations
@@ -23,6 +28,14 @@ module Karafka
           @status = :pending
         end
 
+        # Runs the wrap/around job hook within which the rest of the flow happens
+        # @param block [Proc] whole user related processing flow
+        # @note We inject the action name so user can decide whether to run custom logic on a
+        #   given action or not.
+        def wrap(&block)
+          executor.wrap(self.class.action, &block)
+        end
+
         # When redefined can run any code prior to the job being scheduled
         # @note This will run in the listener thread and not in the worker
         def before_schedule

data/lib/karafka/processing/jobs/consume.rb

@@ -9,6 +9,8 @@ module Karafka
         # @return [Array<Rdkafka::Consumer::Message>] array with messages
         attr_reader :messages
 
+        self.action = :consume
+
         # @param executor [Karafka::Processing::Executor] executor that is suppose to run a given
         #   job
         # @param messages [Karafka::Messages::Messages] karafka messages batch

data/lib/karafka/processing/jobs/eofed.rb

@@ -5,6 +5,8 @@ module Karafka
     module Jobs
       # Job that runs the eofed operation when we receive eof without messages alongside.
       class Eofed < Base
+        self.action = :eofed
+
         # @param executor [Karafka::Processing::Executor] executor that is suppose to run the job
         # @return [Eofed]
         def initialize(executor)

data/lib/karafka/processing/jobs/idle.rb

@@ -6,6 +6,8 @@ module Karafka
       # Type of job that we may use to run some extra handling that happens without the user
       # related lifecycle event like consumption, revocation, etc.
       class Idle < Base
+        self.action = :idle
+
         # @param executor [Karafka::Processing::Executor] executor that is suppose to run a given
         #   job on an active consumer
         # @return [Shutdown]

data/lib/karafka/processing/jobs/revoked.rb

@@ -5,6 +5,8 @@ module Karafka
     module Jobs
       # Job that runs the revoked operation when we loose a partition on a consumer that lost it.
       class Revoked < Base
+        self.action = :revoked
+
         # @param executor [Karafka::Processing::Executor] executor that is suppose to run the job
         # @return [Revoked]
         def initialize(executor)

data/lib/karafka/processing/jobs/shutdown.rb

@@ -5,6 +5,8 @@ module Karafka
     module Jobs
       # Job that runs on each active consumer upon process shutdown (one job per consumer).
       class Shutdown < Base
+        self.action = :shutdown
+
         # @param executor [Karafka::Processing::Executor] executor that is suppose to run a given
         #   job on an active consumer
         # @return [Shutdown]

data/lib/karafka/processing/strategies/default.rb

@@ -24,7 +24,7 @@ module Karafka
           class_eval <<~RUBY, __FILE__, __LINE__ + 1
             # No actions needed for the standard flow here
             def handle_before_schedule_#{action}
-              Karafka.monitor.instrument('consumer.before_schedule_#{action}', caller: self)
+              monitor.instrument('consumer.before_schedule_#{action}', caller: self)
 
               nil
             end
@@ -35,8 +35,8 @@ module Karafka
         # @note It runs in the listener loop. Should **not** be used for anything heavy or
         #   with any potential errors. Mostly for initialization of states, etc.
         def handle_initialized
-          Karafka.monitor.instrument('consumer.initialize', caller: self)
-          Karafka.monitor.instrument('consumer.initialized', caller: self) do
+          monitor.instrument('consumer.initialize', caller: self)
+          monitor.instrument('consumer.initialized', caller: self) do
             initialized
           end
         end
@@ -115,10 +115,21 @@ module Karafka
           coordinator.pause_tracker.increment
         end
 
+        # Runs the wrapping to execute appropriate action wrapped with the wrapper method code
+        #
+        # @param action [Symbol]
+        # @param block [Proc]
+        def handle_wrap(action, &block)
+          monitor.instrument('consumer.wrap', caller: self)
+          monitor.instrument('consumer.wrapped', caller: self) do
+            wrap(action, &block)
+          end
+        end
+
         # Run the user consumption code
         def handle_consume
-          Karafka.monitor.instrument('consumer.consume', caller: self)
-          Karafka.monitor.instrument('consumer.consumed', caller: self) do
+          monitor.instrument('consumer.consume', caller: self)
+          monitor.instrument('consumer.consumed', caller: self) do
             consume
           end
 
@@ -164,8 +175,8 @@ module Karafka
 
         # Runs the consumer `#eofed` method with reporting
         def handle_eofed
-          Karafka.monitor.instrument('consumer.eof', caller: self)
-          Karafka.monitor.instrument('consumer.eofed', caller: self) do
+          monitor.instrument('consumer.eof', caller: self)
+          monitor.instrument('consumer.eofed', caller: self) do
             eofed
           end
         ensure
@@ -180,8 +191,8 @@ module Karafka
 
           coordinator.revoke
 
-          Karafka.monitor.instrument('consumer.revoke', caller: self)
-          Karafka.monitor.instrument('consumer.revoked', caller: self) do
+          monitor.instrument('consumer.revoke', caller: self)
+          monitor.instrument('consumer.revoked', caller: self) do
             revoked
           end
         ensure
@@ -190,8 +201,8 @@ module Karafka
 
         # Runs the shutdown code
         def handle_shutdown
-          Karafka.monitor.instrument('consumer.shutting_down', caller: self)
-          Karafka.monitor.instrument('consumer.shutdown', caller: self) do
+          monitor.instrument('consumer.shutting_down', caller: self)
+          monitor.instrument('consumer.shutdown', caller: self) do
             shutdown
           end
         ensure

data/lib/karafka/processing/strategies/dlq.rb

@@ -117,7 +117,7 @@ module Karafka
           )
 
           # Notify about dispatch on the events bus
-          Karafka.monitor.instrument(
+          monitor.instrument(
             'dead_letter_queue.dispatched',
             caller: self,
             message: skippable_message

data/lib/karafka/processing/worker.rb

@@ -19,7 +19,8 @@ module Karafka
     class Worker
       include Helpers::Async
       include Helpers::ConfigImporter.new(
-        worker_job_call_wrapper: %i[internal processing worker_job_call_wrapper]
+        worker_job_call_wrapper: %i[internal processing worker_job_call_wrapper],
+        monitor: %i[monitor]
       )
 
       # @return [String] id of this worker
@@ -53,28 +54,30 @@ module Karafka
         instrument_details = { caller: self, job: job, jobs_queue: @jobs_queue }
 
         if job
-          Karafka.monitor.instrument('worker.process', instrument_details)
+          job.wrap do
+            monitor.instrument('worker.process', instrument_details)
 
-          Karafka.monitor.instrument('worker.processed', instrument_details) do
-            job.before_call
+            monitor.instrument('worker.processed', instrument_details) do
+              job.before_call
 
-            # If a job is marked as non blocking, we can run a tick in the job queue and if there
-            # are no other blocking factors, the job queue will be unlocked.
-            # If this does not run, all the things will be blocking and job queue won't allow to
-            # pass it until done.
-            @jobs_queue.tick(job.group_id) if job.non_blocking?
+              # If a job is marked as non blocking, we can run a tick in the job queue and if there
+              # are no other blocking factors, the job queue will be unlocked.
+              # If this does not run, all the things will be blocking and job queue won't allow to
+              # pass it until done.
+              @jobs_queue.tick(job.group_id) if job.non_blocking?
 
-            if @non_wrapped_flow
-              job.call
-            else
-              worker_job_call_wrapper.wrap do
+              if @non_wrapped_flow
                 job.call
+              else
+                worker_job_call_wrapper.wrap do
+                  job.call
+                end
               end
-            end
 
-            job.after_call
+              job.after_call
 
-            true
+              true
+            end
           end
         else
           false
@@ -83,7 +86,7 @@ module Karafka
       # rubocop:disable Lint/RescueException
       rescue Exception => e
         # rubocop:enable Lint/RescueException
-        Karafka.monitor.instrument(
+        monitor.instrument(
           'error.occurred',
           caller: self,
           job: job,
@@ -99,7 +102,7 @@ module Karafka
         end
 
         # Always publish info, that we completed all the work despite its result
-        Karafka.monitor.instrument('worker.completed', instrument_details)
+        monitor.instrument('worker.completed', instrument_details)
       end
     end
   end

data/lib/karafka/routing/builder.rb

@@ -67,6 +67,20 @@ module Karafka
         end
       end
 
+      # Clear out the drawn routes.
+      alias array_clear clear
+      private :array_clear
+
+      # Clear routes and draw them again with the given block. Helpful for testing purposes.
+      # @param block [Proc] block we will evaluate within the builder context
+      def redraw(&block)
+        @mutex.synchronize do
+          @draws.clear
+          array_clear
+        end
+        draw(&block)
+      end
+
       # @return [Array<Karafka::Routing::ConsumerGroup>] only active consumer groups that
       #   we want to use. Since Karafka supports multi-process setup, we need to be able
       #   to pick only those consumer groups that should be active in our given process context
@@ -79,7 +93,7 @@ module Karafka
         @mutex.synchronize do
           @defaults = EMPTY_DEFAULTS
           @draws.clear
-          super
+          array_clear
         end
       end
 

data/lib/karafka/version.rb

@@ -3,5 +3,5 @@
 # Main module namespace
 module Karafka
   # Current Karafka version
-  VERSION = '2.4.14'
+  VERSION = '2.4.16'
 end

metadata

@@ -1,11 +1,10 @@
 --- !ruby/object:Gem::Specification
 name: karafka
 version: !ruby/object:Gem::Version
-  version: 2.4.14
+  version: 2.4.16
 platform: ruby
 authors:
 - Maciej Mensfeld
-autorequire: 
 bindir: bin
 cert_chain:
 - |
@@ -35,7 +34,7 @@ cert_chain:
   i9zWxov0mr44TWegTVeypcWGd/0nxu1+QHVNHJrpqlPBRvwQsUm7fwmRInGpcaB8
   ap8wNYvryYzrzvzUxIVFBVM5PacgkFqRmolCa8I7tdKQN+R1
   -----END CERTIFICATE-----
-date: 2024-11-25 00:00:00.000000000 Z
+date: 2024-12-27 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: base64
@@ -605,7 +604,6 @@ metadata:
   source_code_uri: https://github.com/karafka/karafka
   documentation_uri: https://karafka.io/docs
   rubygems_mfa_required: 'true'
-post_install_message: 
 rdoc_options: []
 require_paths:
 - lib
@@ -620,8 +618,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.5.22
-signing_key: 
+rubygems_version: 3.6.2
 specification_version: 4
 summary: Karafka is Ruby and Rails efficient Kafka processing framework.
 test_files: []