karafka 2.0.15 → 2.0.16

data/lib/karafka/pro/routing/features/virtual_partitions/config.rb

@@ -0,0 +1,30 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Routing
+      module Features
+        class VirtualPartitions < Base
+          # Config for virtual partitions
+          Config = Struct.new(
+            :active,
+            :partitioner,
+            :max_partitions,
+            keyword_init: true
+          ) { alias_method :active?, :active }
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/routing/features/virtual_partitions/contract.rb

@@ -0,0 +1,69 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Routing
+      module Features
+        class VirtualPartitions < Base
+          # Rules around virtual partitions
+          class Contract < Contracts::Base
+            configure do |config|
+              config.error_messages = YAML.safe_load(
+                File.read(
+                  File.join(Karafka.gem_root, 'config', 'errors.yml')
+                )
+              ).fetch('en').fetch('validations').fetch('pro_topic')
+            end
+
+            nested(:virtual_partitions) do
+              required(:active) { |val| [true, false].include?(val) }
+              required(:partitioner) { |val| val.nil? || val.respond_to?(:call) }
+              required(:max_partitions) { |val| val.is_a?(Integer) && val >= 1 }
+            end
+
+            # When virtual partitions are defined, partitioner needs to respond to `#call` and it
+            # cannot be nil
+            virtual do |data, errors|
+              next unless errors.empty?
+
+              virtual_partitions = data[:virtual_partitions]
+
+              next unless virtual_partitions[:active]
+              next if virtual_partitions[:partitioner].respond_to?(:call)
+
+              [[%i[virtual_partitions partitioner], :respond_to_call]]
+            end
+
+            # Make sure that manual offset management is not used together with Virtual Partitions
+            # This would not make any sense as there would be edge cases related to skipping
+            # messages even if there were errors.
+            virtual do |data, errors|
+              next unless errors.empty?
+
+              virtual_partitions = data[:virtual_partitions]
+              manual_offset_management = data[:manual_offset_management]
+              active_job = data[:active_job]
+
+              next unless virtual_partitions[:active]
+              next unless manual_offset_management[:active]
+              next if active_job[:active]
+
+              [[%i[manual_offset_management], :not_with_virtual_partitions]]
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/routing/features/virtual_partitions/topic.rb

@@ -0,0 +1,56 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Routing
+      module Features
+        class VirtualPartitions < Base
+          # Topic extensions to be able to manage virtual partitions feature
+          module Topic
+            # @param max_partitions [Integer] max number of virtual partitions that can come out of
+            #   the single distribution flow. When set to more than the Karafka threading, will
+            #   create more work than workers. When less, can ensure we have spare resources to
+            #   process other things in parallel.
+            # @param partitioner [nil, #call] nil or callable partitioner
+            # @return [VirtualPartitions] method that allows to set the virtual partitions details
+            #   during the routing configuration and then allows to retrieve it
+            def virtual_partitions(
+              max_partitions: Karafka::App.config.concurrency,
+              partitioner: nil
+            )
+              @virtual_partitions ||= Config.new(
+                active: !partitioner.nil?,
+                max_partitions: max_partitions,
+                partitioner: partitioner
+              )
+            end
+
+            # @return [Boolean] are virtual partitions enabled for given topic
+            def virtual_partitions?
+              virtual_partitions.active?
+            end
+
+            # @return [Hash] topic with all its native configuration options plus manual offset
+            #   management namespace settings
+            def to_h
+              super.merge(
+                virtual_partitions: virtual_partitions.to_h
+              ).freeze
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/routing/features/virtual_partitions.rb

@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Routing
+      module Features
+        # Virtual Partitions feature config and DSL namespace.
+        #
+        # Virtual Partitions allow you to parallelize the processing of data from a single
+        # partition. This can drastically increase throughput when IO operations are involved.
+        class VirtualPartitions < Base
+        end
+      end
+    end
+  end
+end

data/lib/karafka/processing/coordinator.rb

@@ -63,7 +63,7 @@ module Karafka
 
           # This should never happen. If it does, something is heavily out of sync. Please reach
           # out to us if you encounter this
-          raise Karafka::Errors::InvalidCoordinatorState, 'Was zero before decrementation'
+          raise Karafka::Errors::InvalidCoordinatorStateError, 'Was zero before decrementation'
         end
       end
 

data/lib/karafka/processing/executor.rb

@@ -114,10 +114,16 @@ module Karafka
       # @return [Object] cached consumer instance
       def consumer
         @consumer ||= begin
+          strategy = ::Karafka::App.config.internal.processing.strategy_selector.find(@topic)
+
           consumer = @topic.consumer_class.new
+          # We use singleton class as the same consumer class may be used to process different
+          # topics with different settings
+          consumer.singleton_class.include(strategy)
           consumer.topic = @topic
           consumer.client = @client
           consumer.producer = ::Karafka::App.producer
+
           consumer
         end
       end

data/lib/karafka/processing/strategies/aj_dlq_mom.rb

@@ -0,0 +1,44 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Processing
+    module Strategies
+      # ActiveJob strategy to cooperate with the DLQ.
+      #
+      # While AJ is uses MOM by default because it delegates the offset management to the AJ
+      # consumer. With DLQ however there is an extra case for skipping broken jobs with offset
+      # marking due to ordered processing.
+      module AjDlqMom
+        include DlqMom
+
+        # Apply strategy when only when using AJ with MOM and DLQ
+        FEATURES = %i[
+          active_job
+          dead_letter_queue
+          manual_offset_management
+        ].freeze
+
+        # How should we post-finalize consumption.
+        def handle_after_consume
+          return if revoked?
+
+          if coordinator.success?
+            # Do NOT commit offsets, they are comitted after each job in the AJ consumer.
+            coordinator.pause_tracker.reset
+          elsif coordinator.pause_tracker.attempt <= topic.dead_letter_queue.max_retries
+            pause(coordinator.seek_offset)
+          else
+            coordinator.pause_tracker.reset
+            skippable_message = find_skippable_message
+            dispatch_to_dlq(skippable_message)
+            # We can commit the offset here because we know that we skip it "forever" and
+            # since AJ consumer commits the offset after each job, we also know that the
+            # previous job was successful
+            mark_as_consumed(skippable_message)
+            pause(coordinator.seek_offset)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/processing/strategies/aj_mom.rb

@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Processing
+    module Strategies
+      # ActiveJob enabled
+      # Manual offset management enabled
+      #
+      # This is the default AJ strategy since AJ cannot be used without MOM
+      module AjMom
+        include Mom
+
+        # Apply strategy when only when using AJ with MOM
+        FEATURES = %i[
+          active_job
+          manual_offset_management
+        ].freeze
+      end
+    end
+  end
+end

data/lib/karafka/processing/strategies/base.rb

@@ -0,0 +1,37 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Processing
+    # Our processing patterns differ depending on various features configurations
+    # In this namespace we collect strategies for particular feature combinations to simplify the
+    # design. Based on features combinations we can then select handling strategy for a given case.
+    #
+    # @note The lack of common code here is intentional. It would get complex if there would be
+    #   any type of composition, so each strategy is expected to be self-sufficient
+    module Strategies
+      # Base strategy that should be included in each strategy, just to ensure the API
+      module Base
+        # What should happen before jobs are enqueued
+        # @note This runs from the listener thread, not recommended to put anything slow here
+        def handle_before_enqueue
+          raise NotImplementedError, 'Implement in a subclass'
+        end
+
+        # What should happen before we kick in the processing
+        def handle_before_consume
+          raise NotImplementedError, 'Implement in a subclass'
+        end
+
+        # Post-consumption handling
+        def handle_after_consume
+          raise NotImplementedError, 'Implement in a subclass'
+        end
+
+        # Revocation handling
+        def handle_revoked
+          raise NotImplementedError, 'Implement in a subclass'
+        end
+      end
+    end
+  end
+end

data/lib/karafka/processing/strategies/default.rb

@@ -0,0 +1,52 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Processing
+    module Strategies
+      # No features enabled.
+      # No manual offset management
+      # No long running jobs
+      # Nothing. Just standard, automatic flow
+      module Default
+        include Base
+
+        # Apply strategy for a non-feature based flow
+        FEATURES = %i[].freeze
+
+        # No actions needed for the standard flow here
+        def handle_before_enqueue
+          nil
+        end
+
+        # Increment number of attempts
+        def handle_before_consume
+          coordinator.pause_tracker.increment
+        end
+
+        # Standard flow marks work as consumed and moves on if everything went ok.
+        # If there was a processing error, we will pause and continue from the next message
+        # (next that is +1 from the last one that was successfully marked as consumed)
+        def handle_after_consume
+          return if revoked?
+
+          if coordinator.success?
+            coordinator.pause_tracker.reset
+
+            mark_as_consumed(messages.last)
+          else
+            pause(coordinator.seek_offset)
+          end
+        end
+
+        # We need to always un-pause the processing in case we have lost a given partition.
+        # Otherwise the underlying librdkafka would not know we may want to continue processing and
+        # the pause could in theory last forever
+        def handle_revoked
+          resume
+
+          coordinator.revoke
+        end
+      end
+    end
+  end
+end

data/lib/karafka/processing/strategies/dlq.rb

@@ -0,0 +1,77 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Processing
+    module Strategies
+      # When using dead letter queue, processing won't stop after defined number of retries
+      # upon encountering non-critical errors but the messages that error will be moved to a
+      # separate topic with their payload and metadata, so they can be handled differently.
+      module Dlq
+        include Default
+
+        # Apply strategy when only dead letter queue is turned on
+        FEATURES = %i[
+          dead_letter_queue
+        ].freeze
+
+        # When manual offset management is on, we do not mark anything as consumed automatically
+        # and we rely on the user to figure things out
+        def handle_after_consume
+          return if revoked?
+
+          if coordinator.success?
+            coordinator.pause_tracker.reset
+
+            mark_as_consumed(messages.last)
+          elsif coordinator.pause_tracker.attempt <= topic.dead_letter_queue.max_retries
+            pause(coordinator.seek_offset)
+          # If we've reached number of retries that we could, we need to skip the first message
+          # that was not marked as consumed, pause and continue, while also moving this message
+          # to the dead topic
+          else
+            # We reset the pause to indicate we will now consider it as "ok".
+            coordinator.pause_tracker.reset
+
+            skippable_message = find_skippable_message
+
+            # Send skippable message to the dql topic
+            dispatch_to_dlq(skippable_message)
+
+            # We mark the broken message as consumed and move on
+            mark_as_consumed(skippable_message)
+
+            return if revoked?
+
+            # We pause to backoff once just in case.
+            pause(coordinator.seek_offset)
+          end
+        end
+
+        # Finds the message we want to skip
+        # @private
+        def find_skippable_message
+          skippable_message = messages.find { |message| message.offset == coordinator.seek_offset }
+          skippable_message || raise(Errors::SkipMessageNotFoundError, topic.name)
+        end
+
+        # Moves the broken message into a separate queue defined via the settings
+        # @private
+        # @param skippable_message [Karafka::Messages::Message] message we are skipping that also
+        #   should go to the dlq topic
+        def dispatch_to_dlq(skippable_message)
+          producer.produce_async(
+            topic: topic.dead_letter_queue.topic,
+            payload: skippable_message.raw_payload
+          )
+
+          # Notify about dispatch on the events bus
+          Karafka.monitor.instrument(
+            'dead_letter_queue.dispatched',
+            caller: self,
+            message: skippable_message
+          )
+        end
+      end
+    end
+  end
+end

data/lib/karafka/processing/strategies/dlq_mom.rb

@@ -0,0 +1,42 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Processing
+    module Strategies
+      # Same as pure dead letter queue but we do not marked failed message as consumed
+      module DlqMom
+        include Dlq
+
+        # Apply strategy when dlq is on with manual offset management
+        FEATURES = %i[
+          dead_letter_queue
+          manual_offset_management
+        ].freeze
+
+        # When manual offset management is on, we do not mark anything as consumed automatically
+        # and we rely on the user to figure things out
+        def handle_after_consume
+          return if revoked?
+
+          if coordinator.success?
+            coordinator.pause_tracker.reset
+          elsif coordinator.pause_tracker.attempt <= topic.dead_letter_queue.max_retries
+            pause(coordinator.seek_offset)
+          # If we've reached number of retries that we could, we need to skip the first message
+          # that was not marked as consumed, pause and continue, while also moving this message
+          # to the dead topic
+          else
+            # We reset the pause to indicate we will now consider it as "ok".
+            coordinator.pause_tracker.reset
+
+            skippable_message = find_skippable_message
+            dispatch_to_dlq(skippable_message)
+
+            # We pause to backoff once just in case.
+            pause(coordinator.seek_offset)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/processing/strategies/mom.rb

@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Processing
+    module Strategies
+      # When using manual offset management, we do not mark as consumed after successful processing
+      module Mom
+        include Default
+
+        # Apply strategy when only manual offset management is turned on
+        FEATURES = %i[
+          manual_offset_management
+        ].freeze
+
+        # When manual offset management is on, we do not mark anything as consumed automatically
+        # and we rely on the user to figure things out
+        def handle_after_consume
+          return if revoked?
+
+          if coordinator.success?
+            coordinator.pause_tracker.reset
+          else
+            pause(coordinator.seek_offset)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/processing/strategy_selector.rb

@@ -0,0 +1,30 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Processing
+    # Selector of appropriate processing strategy matching topic combinations
+    class StrategySelector
+      def initialize
+        # We load them once for performance reasons not to do too many lookups
+        @available_strategies = Strategies
+                                .constants
+                                .delete_if { |k| k == :Base }
+                                .map { |k| Strategies.const_get(k) }
+      end
+
+      # @param topic [Karafka::Routing::Topic] topic with settings based on which we find strategy
+      # @return [Module] module with proper strategy
+      def find(topic)
+        feature_set = [
+          topic.active_job? ? :active_job : nil,
+          topic.manual_offset_management? ? :manual_offset_management : nil,
+          topic.dead_letter_queue? ? :dead_letter_queue : nil
+        ].compact
+
+        @available_strategies.find do |strategy|
+          strategy::FEATURES.sort == feature_set.sort
+        end || raise(Errors::StrategyNotFoundError, topic.name)
+      end
+    end
+  end
+end

data/lib/karafka/railtie.rb

@@ -81,15 +81,16 @@ if rails
 
           Rails.application.reloader.reload!
         end
+      end
 
-        ::Karafka::App.monitor.subscribe('worker.completed') do
-          # Skip in case someone is using Rails without ActiveRecord
-          next unless Object.const_defined?('ActiveRecord::Base')
-
-          # Always release the connection after processing is done. Otherwise thread may hang
-          # blocking the reload and further processing
-          # @see https://github.com/rails/rails/issues/44183
-          ActiveRecord::Base.connection_pool.release_connection
+      initializer 'karafka.release_active_record_connections' do
+        ActiveSupport.on_load(:active_record) do
+          ::Karafka::App.monitor.subscribe('worker.completed') do
+            # Always release the connection after processing is done. Otherwise thread may hang
+            # blocking the reload and further processing
+            # @see https://github.com/rails/rails/issues/44183
+            ActiveRecord::Base.clear_active_connections!
+          end
         end
       end
 

data/lib/karafka/routing/builder.rb

@@ -33,7 +33,13 @@ module Karafka
         instance_eval(&block)
 
         each do |consumer_group|
+          # Validate consumer group settings
           Contracts::ConsumerGroup.new.validate!(consumer_group.to_h)
+
+          # and then its topics settings
+          consumer_group.topics.each do |topic|
+            Contracts::Topic.new.validate!(topic.to_h)
+          end
         end
       end
 

data/lib/karafka/routing/features/active_job/builder.rb

@@ -0,0 +1,33 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Routing
+    module Features
+      class ActiveJob < Base
+        # Routing extensions for ActiveJob
+        module Builder
+          # This method simplifies routes definition for ActiveJob topics / queues by
+          # auto-injecting the consumer class
+          #
+          # @param name [String, Symbol] name of the topic where ActiveJobs jobs should go
+          # @param block [Proc] block that we can use for some extra configuration
+          def active_job_topic(name, &block)
+            topic(name) do
+              consumer App.config.internal.active_job.consumer_class
+              active_job true
+
+              # This is handled by our custom ActiveJob consumer
+              # Without this, default behaviour would cause messages to skip upon shutdown as the
+              # offset would be committed for the last message
+              manual_offset_management true
+
+              next unless block
+
+              instance_eval(&block)
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/routing/features/active_job/config.rb

@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Routing
+    module Features
+      class ActiveJob < Base
+        # Config for ActiveJob usage
+        Config = Struct.new(
+          :active,
+          keyword_init: true
+        ) { alias_method :active?, :active }
+      end
+    end
+  end
+end