karafka 2.3.0.alpha2 → 2.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 07bebe70b6697a90d04154dcbfe5837f5bcfaf934073f91bbfc9e8939d9d1a6c
-  data.tar.gz: 13e41b276eee5142b55eb6908c8b9292bd8f802f470fb4e9bdc1f812dbd50189
+  metadata.gz: bceb8995575bd7c51d4fa6df9905619082080c932b5295749d9562fb53eba2a7
+  data.tar.gz: d16247c466fe08f6a671e48e8b25af7edcf2c32d769a497a46245773a66794de
 SHA512:
-  metadata.gz: fef2cbded4409d951cf7e752f25b7db016052cc5f2a77c54ccf7e8778fbf44edba74cd1b04cf82caf341cf05beda552af5c4cbd994a510c54ad1d3c4e561fd17
-  data.tar.gz: 00bf893d7c6f29e559530585c40ba721ab3a9f3ce5906cd7c5b6948f79725d6d99423444d9ec7e6f8593f9780aec3375e400986ed8478dfab2cbbabb5807a85d
+  metadata.gz: 3367c6bace7ab7a1ed7c6ef04e41eefa40c6fef7baf41d0e8e32bb891e1f2799e1564a6ee36b38735b5d514272c88c62e90bcbb54ffa45f175e80b888837d1e5
+  data.tar.gz: 2d31b87c97ce9deb3d47c0332782ad116dbcf4661a7ea55b35ac04a0e87003a8eb73a5b50f7fc9a21eb1ad288eaafff0528b95133fe58f97ada832526959b628

data/CHANGELOG.md

@@ -1,6 +1,6 @@
 # Karafka framework changelog
 
-## 2.3.0 (Unreleased)
+## 2.3.0 (2024-01-26)
 - **[Feature]** Introduce Exactly-Once Semantics within consumers `#transaction` block (Pro)
 - **[Feature]** Provide ability to multiplex subscription groups (Pro)
 - **[Feature]** Provide `Karafka::Admin::Acl` for Kafka ACL management via the Admin APIs.
@@ -14,6 +14,7 @@
 - [Enhancement] Provide an `:independent` configuration to DLQ allowing to reset pause count track on each marking as consumed when retrying.
 - [Enhancement] Remove no longer needed shutdown patches for `librdkafka` improving multi-sg shutdown times for `cooperative-sticky`.
 - [Enhancement] Allow for parallel closing of connections from independent consumer groups.
+- [Enhancement] Provide recovery flow for cases where DLQ dispatch would fail.
 - [Change] Make `Kubernetes::LivenessListener` not start until Karafka app starts running.
 - [Change] Remove the legacy "inside of topics" way of defining subscription groups names
 - [Change] Update supported instrumentation to report on `#tick`.
@@ -22,8 +23,13 @@
 - [Fix] Make the Iterator `#stop_partition` work with karafka-rdkafka `0.14.6`.
 - [Fix] Ensure Pro components are not loaded during OSS specs execution (not affecting usage).
 - [Fix] Fix invalid action label for consumers in DataDog logger instrumentation.
+- [Fix] Fix a scenario where `Karafka::Admin#seek_consumer_group` would fail because reaching not the coordinator.
 - [Ignore] option --include-consumer-groups not working as intended after removal of "thor"
 
+### Upgrade Notes
+
+Available [here](https://karafka.io/docs/Upgrades-2.3/).
+
 ## 2.2.14 (2023-12-07)
 - **[Feature]** Provide `Karafka::Admin#delete_consumer_group` and `Karafka::Admin#seek_consumer_group`.
 - **[Feature]** Provide `Karafka::App.assignments` that will return real-time assignments tracking.

data/Gemfile

@@ -10,7 +10,7 @@ gemspec
 # They are added here because they are part of the integration suite
 group :integrations do
   gem 'activejob'
-  gem 'karafka-web'
+  gem 'karafka-web', '>= 0.8.0.rc1'
 end
 
 group :test do

data/Gemfile.lock

@@ -1,8 +1,8 @@
 PATH
   remote: .
   specs:
-    karafka (2.3.0.alpha2)
-      karafka-core (>= 2.3.0.alpha1, < 2.4.0)
+    karafka (2.3.0)
+      karafka-core (>= 2.3.0, < 2.4.0)
       waterdrop (>= 2.6.12, < 3.0.0)
       zeitwerk (~> 2.3)
 
@@ -23,7 +23,7 @@ GEM
       mutex_m
       tzinfo (~> 2.0)
     base64 (0.2.0)
-    bigdecimal (3.1.5)
+    bigdecimal (3.1.6)
     byebug (11.1.3)
     concurrent-ruby (1.2.3)
     connection_pool (2.4.1)
@@ -39,24 +39,24 @@ GEM
       activesupport (>= 6.1)
     i18n (1.14.1)
       concurrent-ruby (~> 1.0)
-    karafka-core (2.3.0.alpha1)
-      karafka-rdkafka (>= 0.14.7, < 0.15.0)
-    karafka-rdkafka (0.14.7)
+    karafka-core (2.3.0)
+      karafka-rdkafka (>= 0.14.8, < 0.15.0)
+    karafka-rdkafka (0.14.8)
       ffi (~> 1.15)
       mini_portile2 (~> 2.6)
       rake (> 12)
-    karafka-web (0.7.10)
+    karafka-web (0.8.0.rc1)
       erubi (~> 1.4)
-      karafka (>= 2.2.9, < 3.0.0)
-      karafka-core (>= 2.2.4, < 3.0.0)
+      karafka (>= 2.3.0.rc1, < 2.4.0)
+      karafka-core (>= 2.3.0.rc1, < 2.4.0)
       roda (~> 3.68, >= 3.69)
       tilt (~> 2.0)
     mini_portile2 (2.8.5)
-    minitest (5.20.0)
+    minitest (5.21.2)
     mutex_m (0.2.0)
     rack (3.0.8)
     rake (13.1.0)
-    roda (3.75.0)
+    roda (3.76.0)
       rack
     rspec (3.12.0)
       rspec-core (~> 3.12.0)
@@ -95,9 +95,9 @@ DEPENDENCIES
   byebug
   factory_bot
   karafka!
-  karafka-web
+  karafka-web (>= 0.8.0.rc1)
   rspec
   simplecov
 
 BUNDLED WITH
-   2.5.3
+   2.5.4

data/config/locales/errors.yml

@@ -75,6 +75,7 @@ en:
       dead_letter_queue.topic_format: 'needs to be a string with a Kafka accepted format'
       dead_letter_queue.active_format: needs to be either true or false
       dead_letter_queue.independent_format: needs to be either true or false
+      dead_letter_queue.transactional_format: needs to be either true or false
       active_format: needs to be either true or false
       declaratives.partitions_format: needs to be more or equal to 1
       declaratives.active_format: needs to be true

data/karafka.gemspec

@@ -21,7 +21,7 @@ Gem::Specification.new do |spec|
     without having to focus on things that are not your business domain.
   DESC
 
-  spec.add_dependency 'karafka-core', '>= 2.3.0.alpha1', '< 2.4.0'
+  spec.add_dependency 'karafka-core', '>= 2.3.0', '< 2.4.0'
   spec.add_dependency 'waterdrop', '>= 2.6.12', '< 3.0.0'
   spec.add_dependency 'zeitwerk', '~> 2.3'
 

data/lib/karafka/admin.rb

@@ -191,34 +191,41 @@ module Karafka
         mapped_consumer_group_id = app_config.consumer_mapper.call(consumer_group_id)
         settings = { 'group.id': mapped_consumer_group_id }
 
-        with_consumer(settings) do |consumer|
-          # If we have any time based stuff to resolve, we need to do it prior to commits
-          unless time_tpl.empty?
-            real_offsets = consumer.offsets_for_times(time_tpl)
-
-            real_offsets.to_h.each do |name, results|
-              results.each do |result|
-                raise(Errors::InvalidTimeBasedOffsetError) unless result
-
-                partition = result.partition
-
-                # Negative offset means we're beyond last message and we need to query for the
-                # high watermark offset to get the most recent offset and move there
-                if result.offset.negative?
-                  _, offset = consumer.query_watermark_offsets(name, result.partition)
-                else
-                  # If we get an offset, it means there existed a message close to this time
-                  # location
-                  offset = result.offset
+        # This error can occur when we query a broker that is not a coordinator because something
+        # was changing in the cluster. We should be able to safely restart our seeking request
+        # when this happens without any issues
+        #
+        # We wrap the consumer creation, so we retry with a new consumer instance
+        with_rdkafka_retry(codes: %i[not_coordinator]) do
+          with_consumer(settings) do |consumer|
+            # If we have any time based stuff to resolve, we need to do it prior to commits
+            unless time_tpl.empty?
+              real_offsets = consumer.offsets_for_times(time_tpl)
+
+              real_offsets.to_h.each do |name, results|
+                results.each do |result|
+                  raise(Errors::InvalidTimeBasedOffsetError) unless result
+
+                  partition = result.partition
+
+                  # Negative offset means we're beyond last message and we need to query for the
+                  # high watermark offset to get the most recent offset and move there
+                  if result.offset.negative?
+                    _, offset = consumer.query_watermark_offsets(name, result.partition)
+                  else
+                    # If we get an offset, it means there existed a message close to this time
+                    # location
+                    offset = result.offset
+                  end
+
+                  # Since now we have proper offsets, we can add this to the final tpl for commit
+                  tpl.add_topic_and_partitions_with_offsets(name, [[partition, offset]])
                 end
-
-                # Since now we have proper offsets, we can add this to the final tpl for commit
-                tpl.add_topic_and_partitions_with_offsets(name, [[partition, offset]])
               end
             end
-          end
 
-          consumer.commit(tpl, false)
+            consumer.commit(tpl, false)
+          end
         end
       end
 

data/lib/karafka/base_consumer.rb

@@ -79,10 +79,25 @@ module Karafka
     end
 
     # @private
+    #
     # @note This should not be used by the end users as it is part of the lifecycle of things but
     #   not as part of the public api.
+    #
+    # @note We handle and report errors here because of flows that could fail. For example a DLQ
+    #   flow could fail if it was not able to dispatch the DLQ message. Other "non-user" based
+    #   flows do not interact with external systems and their errors are expected to bubble up
     def on_after_consume
       handle_after_consume
+    rescue StandardError => e
+      Karafka.monitor.instrument(
+        'error.occurred',
+        error: e,
+        caller: self,
+        seek_offset: coordinator.seek_offset,
+        type: 'consumer.after_consume.error'
+      )
+
+      retry_after_pause
     end
 
     # Can be used to run code prior to scheduling of idle execution

data/lib/karafka/constraints.rb

@@ -0,0 +1,49 @@
+# frozen_string_literal: true
+
+module Karafka
+  # Module used to check some constraints that cannot be easily defined by Bundler
+  # At the moment we use it to ensure, that if Karafka is used, it operates with the expected
+  # web ui version and that older versions of Web UI that would not be compatible with the API
+  # changes in karafka are not used.
+  #
+  # We can make Web UI require certain karafka version range, but at the moment we do not have a
+  # strict 1:1 release pattern matching those two.
+  module Constraints
+    class << self
+      # Verifies that optional requirements are met.
+      def verify!
+        # Skip verification if web is not used at all
+        return unless require_version('karafka/web')
+
+        # All good if version higher than 0.7.x because we expect 0.8.0 or higher
+        return if version(Karafka::Web::VERSION) >= version('0.7.100')
+
+        # If older web-ui used, we cannot allow it
+        raise(
+          Errors::DependencyConstraintsError,
+          'karafka-web < 0.8.0 is not compatible with this karafka version'
+        )
+      end
+
+      private
+
+      # Requires given version file from a gem location
+      # @param version_location [String]
+      # @return [Boolean] true if it was required or false if not reachable
+      def require_version(version_location)
+        require "#{version_location}/version"
+
+        true
+      rescue LoadError
+        false
+      end
+
+      # Builds a version object for comparing
+      # @param string [String]
+      # @return [::Gem::Version]
+      def version(string)
+        ::Gem::Version.new(string)
+      end
+    end
+  end
+end

data/lib/karafka/errors.rb

@@ -80,5 +80,12 @@ module Karafka
     # Raised in transactions when we attempt to store offset for a partition that we have lost
     # This does not affect producer only transactions, hence we raise it only on offset storage
     AssignmentLostError = Class.new(BaseError)
+
+    # Raised if optional dependencies like karafka-web are required in a version that is not
+    # supported by the current framework version.
+    #
+    # Because we do not want to require web out of the box and we do not want to lock web with
+    # karafka 1:1, we do such a sanity check
+    DependencyConstraintsError = Class.new(BaseError)
   end
 end

data/lib/karafka/instrumentation/logger_listener.rb

@@ -248,7 +248,10 @@ module Karafka
           error "Consumer on shutdown failed due to an error: #{error}"
           error details
         when 'consumer.tick.error'
-          error "Consumer tick failed due to an error: #{error}"
+          error "Consumer on tick failed due to an error: #{error}"
+          error details
+        when 'consumer.after_consume.error'
+          error "Consumer on after_consume failed due to an error: #{error}"
           error details
         when 'worker.process.error'
           fatal "Worker processing failed due to an error: #{error}"

data/lib/karafka/pro/processing/strategies/default.rb

@@ -107,8 +107,23 @@ module Karafka
           # allow to mark offsets inside of the transaction. If the transaction is initialized
           # only from the consumer, the offset will be stored in a regular fashion.
           #
+          # @param active_producer [WaterDrop::Producer] alternative producer instance we may want
+          #   to use. It is useful when we have connection pool or any other selective engine for
+          #   managing multiple producers. If not provided, default producer taken from `#producer`
+          #   will be used.
+          #
           # @param block [Proc] code that we want to run in a transaction
-          def transaction(&block)
+          #
+          # @note Please note, that if you provide the producer, it will reassign the producer of
+          #   the consumer for the transaction time. This means, that in case you would even
+          #   accidentally refer to `Consumer#producer` from other threads, it will contain the
+          #   reassigned producer and not the initially used/assigned producer. It is done that
+          #   way, so the message producing aliases operate from within transactions and since the
+          #   producer in transaction is locked, it will prevent other threads from using it.
+          def transaction(active_producer = producer, &block)
+            default_producer = producer
+            self.producer = active_producer
+
             transaction_started = false
 
             # Prevent from nested transactions. It would not make any sense
@@ -138,6 +153,8 @@ module Karafka
               marking.pop ? mark_as_consumed(*marking) : mark_as_consumed!(*marking)
             end
           ensure
+            self.producer = default_producer
+
             if transaction_started
               @_transaction_marked.clear
               @_in_transaction = false

data/lib/karafka/pro/processing/strategies/dlq/default.rb

@@ -84,9 +84,7 @@ module Karafka
                 else
                   # We reset the pause to indicate we will now consider it as "ok".
                   coordinator.pause_tracker.reset
-                  skippable_message, = find_skippable_message
-                  dispatch_to_dlq(skippable_message) if dispatch_to_dlq?
-                  mark_as_consumed(skippable_message)
+                  dispatch_if_needed_and_mark_as_consumed
                   pause(coordinator.seek_offset, nil, false)
                 end
               end
@@ -133,6 +131,25 @@ module Karafka
               )
             end
 
+            # Dispatches the message to the DLQ (when needed and when applicable based on settings)
+            #   and marks this message as consumed for non MOM flows.
+            #
+            # If producer is transactional and config allows, uses transaction to do that
+            def dispatch_if_needed_and_mark_as_consumed
+              skippable_message, = find_skippable_message
+
+              dispatch = lambda do
+                dispatch_to_dlq(skippable_message) if dispatch_to_dlq?
+                mark_as_consumed(skippable_message)
+              end
+
+              if dispatch_in_a_transaction?
+                transaction { dispatch.call }
+              else
+                dispatch.call
+              end
+            end
+
             # @param skippable_message [Array<Karafka::Messages::Message>]
             # @return [Hash] dispatch DLQ message
             def build_dlq_message(skippable_message)
@@ -168,6 +185,13 @@ module Karafka
             def dispatch_to_dlq?
               topic.dead_letter_queue.topic
             end
+
+            # @return [Boolean] should we use a transaction to move the data to the DLQ.
+            #   We can do it only when producer is transactional and configuration for DLQ
+            #   transactional dispatches is not set to false.
+            def dispatch_in_a_transaction?
+              producer.transactional? && topic.dead_letter_queue.transactional?
+            end
           end
         end
       end

data/lib/karafka/pro/processing/strategies/dlq/ftr.rb

@@ -46,9 +46,9 @@ module Karafka
                   retry_after_pause
                 else
                   coordinator.pause_tracker.reset
-                  skippable_message, = find_skippable_message
-                  dispatch_to_dlq(skippable_message) if dispatch_to_dlq?
-                  mark_as_consumed(skippable_message)
+
+                  dispatch_if_needed_and_mark_as_consumed
+
                   pause(coordinator.seek_offset, nil, false)
                 end
               end

data/lib/karafka/pro/processing/strategies/dlq/ftr_lrj.rb

@@ -60,9 +60,8 @@ module Karafka
 
                   return resume if revoked?
 
-                  skippable_message, = find_skippable_message
-                  dispatch_to_dlq(skippable_message) if dispatch_to_dlq?
-                  mark_as_consumed(skippable_message)
+                  dispatch_if_needed_and_mark_as_consumed
+
                   pause(coordinator.seek_offset, nil, false)
                 end
               end

data/lib/karafka/pro/processing/strategies/dlq/lrj.rb

@@ -49,9 +49,8 @@ module Karafka
 
                   return resume if revoked?
 
-                  skippable_message, = find_skippable_message
-                  dispatch_to_dlq(skippable_message) if dispatch_to_dlq?
-                  mark_as_consumed(skippable_message)
+                  dispatch_if_needed_and_mark_as_consumed
+
                   pause(coordinator.seek_offset, nil, false)
                 end
               end

data/lib/karafka/routing/features/dead_letter_queue/config.rb

@@ -13,10 +13,13 @@ module Karafka
           :topic,
           # Should retries be handled collectively on a batch or independently per message
           :independent,
+          # Move to DLQ and mark as consumed in transactional mode (if applicable)
+          :transactional,
           keyword_init: true
         ) do
           alias_method :active?, :active
           alias_method :independent?, :independent
+          alias_method :transactional?, :transactional
         end
       end
     end

data/lib/karafka/routing/features/dead_letter_queue/contracts/topic.rb

@@ -20,6 +20,7 @@ module Karafka
               required(:active) { |val| [true, false].include?(val) }
               required(:independent) { |val| [true, false].include?(val) }
               required(:max_retries) { |val| val.is_a?(Integer) && val >= 0 }
+              required(:transactional) { |val| [true, false].include?(val) }
             end
 
             # Validate topic name only if dlq is active

data/lib/karafka/routing/features/dead_letter_queue/topic.rb

@@ -16,17 +16,21 @@ module Karafka
           #   if we do not want to move it anywhere and just skip
           # @param independent [Boolean] needs to be true in order for each marking as consumed
           #   in a retry flow to reset the errors counter
+          # @param transactional [Boolean] if applicable, should transaction be used to move
+          #   given message to the dead-letter topic and mark it as consumed.
           # @return [Config] defined config
           def dead_letter_queue(
             max_retries: DEFAULT_MAX_RETRIES,
             topic: nil,
-            independent: false
+            independent: false,
+            transactional: true
           )
             @dead_letter_queue ||= Config.new(
               active: !topic.nil?,
               max_retries: max_retries,
               topic: topic,
-              independent: independent
+              independent: independent,
+              transactional: transactional
             )
           end
 

data/lib/karafka/version.rb

@@ -3,5 +3,5 @@
 # Main module namespace
 module Karafka
   # Current Karafka version
-  VERSION = '2.3.0.alpha2'
+  VERSION = '2.3.0'
 end

data/lib/karafka.rb

@@ -143,3 +143,5 @@ end
 
 # Load railtie after everything else is ready so we know we can rely on it.
 require 'karafka/railtie'
+
+Karafka::Constraints.verify!

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: karafka
 version: !ruby/object:Gem::Version
-  version: 2.3.0.alpha2
+  version: 2.3.0
 platform: ruby
 authors:
 - Maciej Mensfeld
@@ -35,7 +35,7 @@ cert_chain:
   AnG1dJU+yL2BK7vaVytLTstJME5mepSZ46qqIJXMuWob/YPDmVaBF39TDSG9e34s
   msG3BiCqgOgHAnL23+CN3Rt8MsuRfEtoTKpJVcCfoEoNHOkc
   -----END CERTIFICATE-----
-date: 2024-01-17 00:00:00.000000000 Z
+date: 2024-01-26 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: karafka-core
@@ -43,7 +43,7 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 2.3.0.alpha1
+        version: 2.3.0
     - - "<"
       - !ruby/object:Gem::Version
         version: 2.4.0
@@ -53,7 +53,7 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 2.3.0.alpha1
+        version: 2.3.0
     - - "<"
       - !ruby/object:Gem::Version
         version: 2.4.0
@@ -174,6 +174,7 @@ files:
 - lib/karafka/connection/raw_messages_buffer.rb
 - lib/karafka/connection/rebalance_manager.rb
 - lib/karafka/connection/status.rb
+- lib/karafka/constraints.rb
 - lib/karafka/contracts.rb
 - lib/karafka/contracts/base.rb
 - lib/karafka/contracts/config.rb