karafka 2.3.0.alpha1 → 2.3.0.alpha2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: de7ea23762cefa19d5f3620e92a39f0030cd8ff78f318c92f30c494d79b78163
-  data.tar.gz: 775cfbd40d181036004dcf72dbcb84394dc8367bed0f6d69812f2324dc179d6f
+  metadata.gz: 07bebe70b6697a90d04154dcbfe5837f5bcfaf934073f91bbfc9e8939d9d1a6c
+  data.tar.gz: 13e41b276eee5142b55eb6908c8b9292bd8f802f470fb4e9bdc1f812dbd50189
 SHA512:
-  metadata.gz: d68a4122a35afad517e4280b94f6f3d7cb3cab94fb37c11729e5e5c7a7aca082a7a272a52ff09a86e2f55ad0e078e234c88be79ce3b730527a2f6e7629ef259c
-  data.tar.gz: aa2ddb108cc39caa8ad5c95a86d07006b5be374647e703414a7761ffd5c333010d7e53b9fd2c42216780e35f00639d8cf80126c291a59d0585424077840cc6b5
+  metadata.gz: fef2cbded4409d951cf7e752f25b7db016052cc5f2a77c54ccf7e8778fbf44edba74cd1b04cf82caf341cf05beda552af5c4cbd994a510c54ad1d3c4e561fd17
+  data.tar.gz: 00bf893d7c6f29e559530585c40ba721ab3a9f3ce5906cd7c5b6948f79725d6d99423444d9ec7e6f8593f9780aec3375e400986ed8478dfab2cbbabb5807a85d

data/CHANGELOG.md

@@ -1,6 +1,7 @@
 # Karafka framework changelog
 
 ## 2.3.0 (Unreleased)
+- **[Feature]** Introduce Exactly-Once Semantics within consumers `#transaction` block (Pro)
 - **[Feature]** Provide ability to multiplex subscription groups (Pro)
 - **[Feature]** Provide `Karafka::Admin::Acl` for Kafka ACL management via the Admin APIs.
 - **[Feature]** Periodic Jobs (Pro)

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    karafka (2.3.0.alpha1)
+    karafka (2.3.0.alpha2)
       karafka-core (>= 2.3.0.alpha1, < 2.4.0)
       waterdrop (>= 2.6.12, < 3.0.0)
       zeitwerk (~> 2.3)
@@ -9,10 +9,10 @@ PATH
 GEM
   remote: https://rubygems.org/
   specs:
-    activejob (7.1.2)
-      activesupport (= 7.1.2)
+    activejob (7.1.3)
+      activesupport (= 7.1.3)
       globalid (>= 0.3.6)
-    activesupport (7.1.2)
+    activesupport (7.1.3)
       base64
       bigdecimal
       concurrent-ruby (~> 1.0, >= 1.0.2)
@@ -25,7 +25,7 @@ GEM
     base64 (0.2.0)
     bigdecimal (3.1.5)
     byebug (11.1.3)
-    concurrent-ruby (1.2.2)
+    concurrent-ruby (1.2.3)
     connection_pool (2.4.1)
     diff-lcs (1.5.0)
     docile (1.4.0)

data/bin/integrations

@@ -28,7 +28,7 @@ ROOT_PATH = Pathname.new(File.expand_path(File.join(File.dirname(__FILE__), '../
 CONCURRENCY = ENV.key?('CI') ? 5 : Etc.nprocessors * 3
 
 # How may bytes do we want to keep from the stdout in the buffer for when we need to print it
-MAX_BUFFER_OUTPUT = 51_200
+MAX_BUFFER_OUTPUT = 102_400
 
 # Abstraction around a single test scenario execution process
 class Scenario

data/lib/karafka/errors.rb

@@ -76,5 +76,9 @@ module Karafka
 
     # Raised when we want to un-pause listener that was not paused
     InvalidListenerPauseError = Class.new(BaseError)
+
+    # Raised in transactions when we attempt to store offset for a partition that we have lost
+    # This does not affect producer only transactions, hence we raise it only on offset storage
+    AssignmentLostError = Class.new(BaseError)
   end
 end

data/lib/karafka/pro/connection/manager.rb

@@ -24,18 +24,9 @@ module Karafka
       # @note Manager operations relate to consumer groups and not subscription groups. Since
       #   cluster operations can cause consumer group wide effects, we always apply only one
       #   change on a consumer group.
-      #
-      # @note Since we collect statistical data from listeners and this happens in a background
-      #   thread, we need to make sure we lock not to have race conditions with expired data
-      #   eviction.
       class Manager < Karafka::Connection::Manager
         include Core::Helpers::Time
 
-        # How long should we keep stale stats before evicting them completely
-        EVICTION_DELAY = 5 * 60 * 1_000
-
-        private_constant :EVICTION_DELAY
-
         # How long should we wait after a rebalance before doing anything on a consumer group
         #
         # @param scale_delay [Integer] How long should we wait before making any changes. Any
@@ -50,7 +41,6 @@ module Karafka
               state: '',
               join_state: '',
               state_age: 0,
-              state_age_sync: monotonic_now,
               changed_at: monotonic_now
             }
           end
@@ -65,6 +55,9 @@ module Karafka
         def register(listeners)
           @listeners = listeners
 
+          # Preload all the keys into the hash so we never add keys to changes but just change them
+          listeners.each { |listener| @changes[listener.subscription_group.id] }
+
           in_sg_families do |first_subscription_group, sg_listeners|
             multiplexing = first_subscription_group.multiplexing
 
@@ -86,25 +79,22 @@ module Karafka
         # @note Please note that while we collect here per subscription group, we use those metrics
         #   collectively on a whole consumer group. This reduces the friction.
         def notice(subscription_group_id, statistics)
-          @mutex.synchronize do
-            times = []
-            # stateage is in microseconds
-            # We monitor broker changes to make sure we do not introduce extra friction
-            times << statistics['brokers'].values.map { |stats| stats['stateage'] }.min / 1_000
-            times << statistics['cgrp']['rebalance_age']
-            times << statistics['cgrp']['stateage']
-
-            # Keep the previous change age for changes that were triggered by us
-            previous_changed_at = @changes[subscription_group_id][:changed_at]
-
-            @changes[subscription_group_id] = {
-              state_age: times.min,
-              changed_at: previous_changed_at,
-              join_state: statistics['cgrp']['join_state'],
-              state: statistics['cgrp']['state'],
-              state_age_sync: monotonic_now
-            }
-          end
+          times = []
+          # stateage is in microseconds
+          # We monitor broker changes to make sure we do not introduce extra friction
+          times << statistics['brokers'].values.map { |stats| stats['stateage'] }.min / 1_000
+          times << statistics['cgrp']['rebalance_age']
+          times << statistics['cgrp']['stateage']
+
+          # Keep the previous change age for changes that were triggered by us
+          previous_changed_at = @changes[subscription_group_id][:changed_at]
+
+          @changes[subscription_group_id].merge!(
+            state_age: times.min,
+            changed_at: previous_changed_at,
+            join_state: statistics['cgrp']['join_state'],
+            state: statistics['cgrp']['state']
+          )
         end
 
         # Shuts down all the listeners when it is time (including moving to quiet) or rescales
@@ -158,8 +148,6 @@ module Karafka
         #
         # We always run scaling down and up because it may be applicable to different CGs
         def rescale
-          evict
-
           scale_down
           scale_up
         end
@@ -232,23 +220,11 @@ module Karafka
           end
         end
 
-        # Removes states that are no longer being reported for stopped/pending listeners
-        def evict
-          @mutex.synchronize do
-            @changes.delete_if do |_, details|
-              monotonic_now - details[:state_age_sync] >= EVICTION_DELAY
-            end
-          end
-        end
-
         # Indicates, that something has changed on a subscription group. We consider every single
         # change we make as a change to the setup as well.
         # @param subscription_group_id [String]
         def touch(subscription_group_id)
-          @mutex.synchronize do
-            @changes[subscription_group_id][:changed_at] = 0
-            @changes[subscription_group_id][:state_age_sync] = monotonic_now
-          end
+          @changes[subscription_group_id][:changed_at] = 0
         end
 
         # @param sg_listeners [Array<Listener>] listeners from one multiplexed sg
@@ -257,17 +233,10 @@ module Karafka
         #   are also stable. This is a strong indicator that no rebalances or other operations are
         #   happening at a given moment.
         def stable?(sg_listeners)
-          # If none of listeners has changes reported it means we did not yet start collecting
-          # metrics about any of them and at least one must be present. We do not consider it
-          # stable in such case as we still are waiting for metrics.
-          return false if sg_listeners.none? do |sg_listener|
-            @changes.key?(sg_listener.subscription_group.id)
-          end
-
           sg_listeners.all? do |sg_listener|
-            # Not all SGs may be started initially or may be stopped, we ignore them here as they
-            # are irrelevant from the point of view of establishing stability
-            next true unless @changes.key?(sg_listener.subscription_group.id)
+            # If a listener is not active, we do not take it into consideration when looking at
+            # the stability data
+            next true unless sg_listener.active?
 
             state = @changes[sg_listener.subscription_group.id]
 

data/lib/karafka/pro/processing/strategies/default.rb

@@ -27,6 +27,21 @@ module Karafka
           # Apply strategy for a non-feature based flow
           FEATURES = %i[].freeze
 
+          # Allows to set offset metadata that will be used with the upcoming marking as consumed
+          # as long as a different offset metadata was not used. After it was used either via
+          # `#mark_as_consumed` or `#mark_as_consumed!` it will be set back to `nil`. It is done
+          # that way to provide the end user with ability to influence metadata on the non-user
+          # initiated markings in complex flows.
+          #
+          # @param offset_metadata [String, nil] metadata we want to store with the upcoming
+          #   marking as consumed
+          #
+          # @note Please be aware, that offset metadata set this way will be passed to any marking
+          #   as consumed even if it was not user initiated. For example in the DLQ flow.
+          def store_offset_metadata(offset_metadata)
+            @_current_offset_metadata = offset_metadata
+          end
+
           # Marks message as consumed in an async way.
           #
           # @param message [Messages::Message] last successfully processed message.
@@ -38,7 +53,7 @@ module Karafka
           #   processing another message. In case like this we do not pause on the message we've
           #   already processed but rather at the next one. This applies to both sync and async
           #   versions of this method.
-          def mark_as_consumed(message, offset_metadata = nil)
+          def mark_as_consumed(message, offset_metadata = @_current_offset_metadata)
             if @_in_transaction
               mark_in_transaction(message, offset_metadata, true)
             else
@@ -54,6 +69,8 @@ module Karafka
             end
 
             true
+          ensure
+            @_current_offset_metadata = nil
           end
 
           # Marks message as consumed in a sync way.
@@ -62,7 +79,7 @@ module Karafka
           # @param offset_metadata [String, nil] offset metadata string or nil if nothing
           # @return [Boolean] true if we were able to mark the offset, false otherwise.
           #   False indicates that we were not able and that we have lost the partition.
-          def mark_as_consumed!(message, offset_metadata = nil)
+          def mark_as_consumed!(message, offset_metadata = @_current_offset_metadata)
             if @_in_transaction
               mark_in_transaction(message, offset_metadata, false)
             else
@@ -79,6 +96,8 @@ module Karafka
             end
 
             true
+          ensure
+            @_current_offset_metadata = nil
           end
 
           # Starts producer transaction, saves the transaction context for transactional marking
@@ -134,6 +153,7 @@ module Karafka
           #   transaction state synchronization usage as within transaction it is always sync)
           def mark_in_transaction(message, offset_metadata, async)
             raise Errors::TransactionRequiredError unless @_in_transaction
+            raise Errors::AssignmentLostError if revoked?
 
             producer.transaction_mark_as_consumed(
               client,

data/lib/karafka/pro/processing/strategies/dlq/default.rb

@@ -35,7 +35,7 @@ module Karafka
             # @see `Strategies::Default#mark_as_consumed` for more details
             # @param message [Messages::Message]
             # @param offset_metadata [String, nil]
-            def mark_as_consumed(message, offset_metadata = nil)
+            def mark_as_consumed(message, offset_metadata = @_current_offset_metadata)
               return super unless retrying?
               return super unless topic.dead_letter_queue.independent?
               return false unless super
@@ -43,6 +43,8 @@ module Karafka
               coordinator.pause_tracker.reset
 
               true
+            ensure
+              @_current_offset_metadata = nil
             end
 
             # Override of the standard `#mark_as_consumed!`. Resets the pause tracker count in case
@@ -51,7 +53,7 @@ module Karafka
             # @see `Strategies::Default#mark_as_consumed!` for more details
             # @param message [Messages::Message]
             # @param offset_metadata [String, nil]
-            def mark_as_consumed!(message, offset_metadata = nil)
+            def mark_as_consumed!(message, offset_metadata = @_current_offset_metadata)
               return super unless retrying?
               return super unless topic.dead_letter_queue.independent?
               return false unless super
@@ -59,6 +61,8 @@ module Karafka
               coordinator.pause_tracker.reset
 
               true
+            ensure
+              @_current_offset_metadata = nil
             end
 
             # When we encounter non-recoverable message, we skip it and go on with our lives

data/lib/karafka/pro/processing/strategies/vp/default.rb

@@ -33,7 +33,7 @@ module Karafka
             # @note This virtual offset management uses a regular default marking API underneath.
             #   We do not alter the "real" marking API, as VPs are just one of many cases we want
             #   to support and we do not want to impact them with collective offsets management
-            def mark_as_consumed(message, offset_metadata = nil)
+            def mark_as_consumed(message, offset_metadata = @_current_offset_metadata)
               if @_in_transaction && !collapsed?
                 mark_in_transaction(message, offset_metadata, true)
               elsif collapsed?
@@ -55,11 +55,13 @@ module Karafka
                   manager.markable? ? super(*manager.markable) : revoked?
                 end
               end
+            ensure
+              @_current_offset_metadata = nil
             end
 
             # @param message [Karafka::Messages::Message] blocking marks message as consumed
             # @param offset_metadata [String, nil]
-            def mark_as_consumed!(message, offset_metadata = nil)
+            def mark_as_consumed!(message, offset_metadata = @_current_offset_metadata)
               if @_in_transaction && !collapsed?
                 mark_in_transaction(message, offset_metadata, false)
               elsif collapsed?
@@ -73,6 +75,8 @@ module Karafka
                   manager.markable? ? super(*manager.markable) : revoked?
                 end
               end
+            ensure
+              @_current_offset_metadata = nil
             end
 
             # Stores the next offset for processing inside of the transaction when collapsed and
@@ -93,6 +97,8 @@ module Karafka
             #   transaction state synchronization usage as within transaction it is always sync)
             def mark_in_transaction(message, offset_metadata, async)
               raise Errors::TransactionRequiredError unless @_in_transaction
+              # Prevent from attempts of offset storage when we no longer own the assignment
+              raise Errors::AssignmentLostError if revoked?
 
               return super if collapsed?
 

data/lib/karafka/version.rb

@@ -3,5 +3,5 @@
 # Main module namespace
 module Karafka
   # Current Karafka version
-  VERSION = '2.3.0.alpha1'
+  VERSION = '2.3.0.alpha2'
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: karafka
 version: !ruby/object:Gem::Version
-  version: 2.3.0.alpha1
+  version: 2.3.0.alpha2
 platform: ruby
 authors:
 - Maciej Mensfeld
@@ -35,7 +35,7 @@ cert_chain:
   AnG1dJU+yL2BK7vaVytLTstJME5mepSZ46qqIJXMuWob/YPDmVaBF39TDSG9e34s
   msG3BiCqgOgHAnL23+CN3Rt8MsuRfEtoTKpJVcCfoEoNHOkc
   -----END CERTIFICATE-----
-date: 2024-01-15 00:00:00.000000000 Z
+date: 2024-01-17 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: karafka-core