karafka 2.1.8 → 2.1.9

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 042f365fb134a24ae360d678590ce798014751c8a23fb267001c920a42aa5324
-  data.tar.gz: ba2950de557a5f6c775577ce392d60ee839184f50b7d9225969c684625c9ecd0
+  metadata.gz: 65296040c91ec5646620f047567c9229b7d9044d41d37e690f9f6685d1199e36
+  data.tar.gz: dedea921e5bc3b9b985861d9b6bcf6c5b5de1a148ef2c1aa72b0d96de44fb623
 SHA512:
-  metadata.gz: 30b5fcd92c348c50482cb84542380ad28b317d47e01efad2d2049cd3ba7872c5f66a7a4fde93d8bfa262d7fcb3745dedbdb89a4fd5e6cdf2288a43606ea2361d
-  data.tar.gz: d9c8ba95b2b71f46a3d35e2f5be634473a7aa9f45d9b5924e1019dcb53c7cea6aca8c594f9f4d3bf85a14176c3cb98a7167b4eb6e1f6f2192059d8040440e4a9
+  metadata.gz: 781e45af707a9818e9166d6c37d8be214e9334703ee598f4ce8924c99ec71283cc4364de6662fbd1bb7454600b94b71e2e32750b58630e3631235e5bf593c3f9
+  data.tar.gz: b86b74c4d7909d504e0b60ae536360412622451a7a9868034dca63fbc3c5e962ba122967aeb8a1491196ce6b879b25eef11497798d861fcf85d3acca40fc8163

data/CHANGELOG.md

@@ -1,5 +1,10 @@
 # Karafka framework changelog
 
+## 2.1.9 (2023-08-06)
+- **[Feature]** Introduce ability to customize pause strategy on a per topic basis (Pro).
+- [Improvement] Disable the extensive messages logging in the default `karafka.rb` template.
+- [Change] Require `waterdrop` `>= 2.6.6` due to extra `LoggerListener` API.
+
 ## 2.1.8 (2023-07-29)
 - [Improvement] Introduce `Karafka::BaseConsumer#used?` method to indicate, that at least one invocation of `#consume` took or will take place. This can be used as a replacement to the non-direct `messages.count` check for shutdown and revocation to ensure, that the consumption took place or is taking place (in case of running LRJ).
 - [Improvement] Make `messages#to_a` return copy of the underlying array to prevent scenarios, where the mutation impacts offset management.

data/Gemfile.lock

@@ -1,10 +1,10 @@
 PATH
   remote: .
   specs:
-    karafka (2.1.8)
+    karafka (2.1.9)
       karafka-core (>= 2.1.1, < 2.2.0)
       thor (>= 0.20)
-      waterdrop (>= 2.6.2, < 3.0.0)
+      waterdrop (>= 2.6.6, < 3.0.0)
       zeitwerk (~> 2.3)
 
 GEM
@@ -72,7 +72,7 @@ GEM
     tilt (2.2.0)
     tzinfo (2.0.6)
       concurrent-ruby (~> 1.0)
-    waterdrop (2.6.5)
+    waterdrop (2.6.6)
       karafka-core (>= 2.1.1, < 3.0.0)
       zeitwerk (~> 2.3)
     zeitwerk (2.6.8)

data/config/locales/pro_errors.yml

@@ -23,6 +23,11 @@ en:
       delaying.delay_format: 'needs to be equal or more than 0 and an integer'
       delaying.active_format: 'needs to be boolean'
 
+      pause_timeout_format: needs to be an integer bigger than 0
+      pause_max_timeout_format: needs to be an integer bigger than 0
+      pause_with_exponential_backoff_format: needs to be either true or false
+      pause_timeout_max_timeout_vs_pause_max_timeout: pause_timeout must be less or equal to pause_max_timeout
+
     config:
       encryption.active_format: 'needs to be either true or false'
       encryption.public_key_invalid: 'is not a valid public RSA key'

data/karafka.gemspec

@@ -23,7 +23,7 @@ Gem::Specification.new do |spec|
 
   spec.add_dependency 'karafka-core', '>= 2.1.1', '< 2.2.0'
   spec.add_dependency 'thor', '>= 0.20'
-  spec.add_dependency 'waterdrop', '>= 2.6.2', '< 3.0.0'
+  spec.add_dependency 'waterdrop', '>= 2.6.6', '< 3.0.0'
   spec.add_dependency 'zeitwerk', '~> 2.3'
 
   if $PROGRAM_NAME.end_with?('gem')

data/lib/karafka/connection/listener.rb

@@ -192,7 +192,7 @@ module Karafka
       # Resumes processing of partitions that were paused due to an error.
       def resume_paused_partitions
         @coordinators.resume do |topic, partition|
-          @client.resume(topic, partition)
+          @client.resume(topic.name, partition)
         end
       end
 

data/lib/karafka/connection/pauses_manager.rb

@@ -14,20 +14,20 @@ module Karafka
 
       # Creates or fetches pause tracker of a given topic partition.
       #
-      # @param topic [String] topic name
+      # @param topic [::Karafka::Routing::Topic] topic
       # @param partition [Integer] partition number
       # @return [Karafka::TimeTrackers::Pause] pause tracker instance
       def fetch(topic, partition)
         @pauses[topic][partition] ||= TimeTrackers::Pause.new(
-          timeout: Karafka::App.config.pause_timeout,
-          max_timeout: Karafka::App.config.pause_max_timeout,
-          exponential_backoff: Karafka::App.config.pause_with_exponential_backoff
+          timeout: topic.pause_timeout,
+          max_timeout: topic.pause_max_timeout,
+          exponential_backoff: topic.pause_with_exponential_backoff
         )
       end
 
       # Resumes processing of partitions for which pause time has ended.
       #
-      # @yieldparam [String] topic name
+      # @yieldparam [Karafka::Routing::Topic] topic
       # @yieldparam [Integer] partition number
       def resume
         @pauses.each do |topic, partitions|

data/lib/karafka/pro/routing/features/pausing/contract.rb

@@ -0,0 +1,48 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Routing
+      module Features
+        class Pausing < Base
+          # Contract to make sure, that the pause settings on a per topic basis are as expected
+          class Contract < Contracts::Base
+            configure do |config|
+              config.error_messages = YAML.safe_load(
+                File.read(
+                  File.join(Karafka.gem_root, 'config', 'locales', 'pro_errors.yml')
+                )
+              ).fetch('en').fetch('validations').fetch('topic')
+
+              required(:pause_timeout) { |val| val.is_a?(Integer) && val.positive? }
+              required(:pause_max_timeout) { |val| val.is_a?(Integer) && val.positive? }
+              required(:pause_with_exponential_backoff) { |val| [true, false].include?(val) }
+
+              virtual do |data, errors|
+                next unless errors.empty?
+
+                pause_timeout = data.fetch(:pause_timeout)
+                pause_max_timeout = data.fetch(:pause_max_timeout)
+
+                next if pause_timeout <= pause_max_timeout
+
+                [[%i[pause_timeout], :max_timeout_vs_pause_max_timeout]]
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/routing/features/pausing/topic.rb

@@ -0,0 +1,44 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Routing
+      module Features
+        class Pausing < Base
+          # Expansion allowing for a per topic pause strategy definitions
+          module Topic
+            # Allows for per-topic pausing strategy setting
+            #
+            # @param timeout [Integer] how long should we wait upon processing error (milliseconds)
+            # @param max_timeout [Integer] what is the max timeout in case of an exponential
+            #   backoff (milliseconds)
+            # @param with_exponential_backoff [Boolean] should we use exponential backoff
+            #
+            # @note We do not construct here the nested config like we do with other routing
+            #   features, because this feature operates on the OSS layer by injection of values
+            #   and a nested config is not needed.
+            def pause(timeout: nil, max_timeout: nil, with_exponential_backoff: nil)
+              self.pause_timeout = timeout if timeout
+              self.pause_max_timeout = max_timeout if max_timeout
+
+              return unless with_exponential_backoff
+
+              self.pause_with_exponential_backoff = with_exponential_backoff
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/routing/features/pausing.rb

@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Routing
+      module Features
+        # Feature allowing for a per-route reconfiguration of the pausing strategy
+        # It can be useful when different topics should have different backoff policies
+        class Pausing < Base
+        end
+      end
+    end
+  end
+end

data/lib/karafka/processing/coordinators_buffer.rb

@@ -17,14 +17,18 @@ module Karafka
         @topics = topics
       end
 
-      # @param topic [String] topic name
+      # @param topic_name [String] topic name
       # @param partition [Integer] partition number
-      def find_or_create(topic, partition)
-        @coordinators[topic][partition] ||= @coordinator_class.new(
-          @topics.find(topic),
-          partition,
-          @pauses_manager.fetch(topic, partition)
-        )
+      def find_or_create(topic_name, partition)
+        @coordinators[topic_name][partition] ||= begin
+          routing_topic = @topics.find(topic_name)
+
+          @coordinator_class.new(
+            routing_topic,
+            partition,
+            @pauses_manager.fetch(routing_topic, partition)
+          )
+        end
       end
 
       # Resumes processing of partitions for which pause time has ended.
@@ -35,16 +39,16 @@ module Karafka
         @pauses_manager.resume(&block)
       end
 
-      # @param topic [String] topic name
+      # @param topic_name [String] topic name
       # @param partition [Integer] partition number
-      def revoke(topic, partition)
-        return unless @coordinators[topic].key?(partition)
+      def revoke(topic_name, partition)
+        return unless @coordinators[topic_name].key?(partition)
 
         # The fact that we delete here does not change the fact that the executor still holds the
         # reference to this coordinator. We delete it here, as we will no longer process any
         # new stuff with it and we may need a new coordinator if we regain this partition, but the
         # coordinator may still be in use
-        @coordinators[topic].delete(partition).revoke
+        @coordinators[topic_name].delete(partition).revoke
       end
 
       # Clears coordinators and re-created the pauses manager

data/lib/karafka/routing/topic.rb

@@ -18,6 +18,9 @@ module Karafka
         max_wait_time
         initial_offset
         consumer_persistence
+        pause_timeout
+        pause_max_timeout
+        pause_with_exponential_backoff
       ].freeze
 
       private_constant :INHERITABLE_ATTRIBUTES

data/lib/karafka/templates/karafka.rb.erb

@@ -43,7 +43,13 @@ class KarafkaApp < Karafka::App
   # This logger prints the producer development info using the Karafka logger.
   # It is similar to the consumer logger listener but producer oriented.
   Karafka.producer.monitor.subscribe(
-    WaterDrop::Instrumentation::LoggerListener.new(Karafka.logger)
+    WaterDrop::Instrumentation::LoggerListener.new(
+      # Log producer operations using the Karafka logger
+      Karafka.logger,
+      # If you set this to true, logs will contain each message details
+      # Please note, that this can be extensive
+      log_messages: false
+    )
   )
 
   routes.draw do

data/lib/karafka/version.rb

@@ -3,5 +3,5 @@
 # Main module namespace
 module Karafka
   # Current Karafka version
-  VERSION = '2.1.8'
+  VERSION = '2.1.9'
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: karafka
 version: !ruby/object:Gem::Version
-  version: 2.1.8
+  version: 2.1.9
 platform: ruby
 authors:
 - Maciej Mensfeld
@@ -35,7 +35,7 @@ cert_chain:
   Qf04B9ceLUaC4fPVEz10FyobjaFoY4i32xRto3XnrzeAgfEe4swLq8bQsR3w/EF3
   MGU0FeSV2Yj7Xc2x/7BzLK8xQn5l7Yy75iPF+KP3vVmDHnNl
   -----END CERTIFICATE-----
-date: 2023-07-29 00:00:00.000000000 Z
+date: 2023-08-06 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: karafka-core
@@ -77,7 +77,7 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 2.6.2
+        version: 2.6.6
     - - "<"
       - !ruby/object:Gem::Version
         version: 3.0.0
@@ -87,7 +87,7 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 2.6.2
+        version: 2.6.6
     - - "<"
       - !ruby/object:Gem::Version
         version: 3.0.0
@@ -314,6 +314,9 @@ files:
 - lib/karafka/pro/routing/features/long_running_job/config.rb
 - lib/karafka/pro/routing/features/long_running_job/contract.rb
 - lib/karafka/pro/routing/features/long_running_job/topic.rb
+- lib/karafka/pro/routing/features/pausing.rb
+- lib/karafka/pro/routing/features/pausing/contract.rb
+- lib/karafka/pro/routing/features/pausing/topic.rb
 - lib/karafka/pro/routing/features/throttling.rb
 - lib/karafka/pro/routing/features/throttling/config.rb
 - lib/karafka/pro/routing/features/throttling/contract.rb