karafka 2.0.14 → 2.0.16

data/lib/karafka/processing/strategies/mom.rb

@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Processing
+    module Strategies
+      # When using manual offset management, we do not mark as consumed after successful processing
+      module Mom
+        include Default
+
+        # Apply strategy when only manual offset management is turned on
+        FEATURES = %i[
+          manual_offset_management
+        ].freeze
+
+        # When manual offset management is on, we do not mark anything as consumed automatically
+        # and we rely on the user to figure things out
+        def handle_after_consume
+          return if revoked?
+
+          if coordinator.success?
+            coordinator.pause_tracker.reset
+          else
+            pause(coordinator.seek_offset)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/processing/strategy_selector.rb

@@ -0,0 +1,30 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Processing
+    # Selector of appropriate processing strategy matching topic combinations
+    class StrategySelector
+      def initialize
+        # We load them once for performance reasons not to do too many lookups
+        @available_strategies = Strategies
+                                .constants
+                                .delete_if { |k| k == :Base }
+                                .map { |k| Strategies.const_get(k) }
+      end
+
+      # @param topic [Karafka::Routing::Topic] topic with settings based on which we find strategy
+      # @return [Module] module with proper strategy
+      def find(topic)
+        feature_set = [
+          topic.active_job? ? :active_job : nil,
+          topic.manual_offset_management? ? :manual_offset_management : nil,
+          topic.dead_letter_queue? ? :dead_letter_queue : nil
+        ].compact
+
+        @available_strategies.find do |strategy|
+          strategy::FEATURES.sort == feature_set.sort
+        end || raise(Errors::StrategyNotFoundError, topic.name)
+      end
+    end
+  end
+end

data/lib/karafka/railtie.rb

@@ -81,15 +81,16 @@ if rails
 
           Rails.application.reloader.reload!
         end
+      end
 
-        ::Karafka::App.monitor.subscribe('worker.completed') do
-          # Skip in case someone is using Rails without ActiveRecord
-          next unless Object.const_defined?('ActiveRecord::Base')
-
-          # Always release the connection after processing is done. Otherwise thread may hang
-          # blocking the reload and further processing
-          # @see https://github.com/rails/rails/issues/44183
-          ActiveRecord::Base.connection_pool.release_connection
+      initializer 'karafka.release_active_record_connections' do
+        ActiveSupport.on_load(:active_record) do
+          ::Karafka::App.monitor.subscribe('worker.completed') do
+            # Always release the connection after processing is done. Otherwise thread may hang
+            # blocking the reload and further processing
+            # @see https://github.com/rails/rails/issues/44183
+            ActiveRecord::Base.clear_active_connections!
+          end
         end
       end
 

data/lib/karafka/routing/builder.rb

@@ -33,7 +33,13 @@ module Karafka
         instance_eval(&block)
 
         each do |consumer_group|
+          # Validate consumer group settings
           Contracts::ConsumerGroup.new.validate!(consumer_group.to_h)
+
+          # and then its topics settings
+          consumer_group.topics.each do |topic|
+            Contracts::Topic.new.validate!(topic.to_h)
+          end
         end
       end
 

data/lib/karafka/routing/features/active_job/builder.rb

@@ -0,0 +1,33 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Routing
+    module Features
+      class ActiveJob < Base
+        # Routing extensions for ActiveJob
+        module Builder
+          # This method simplifies routes definition for ActiveJob topics / queues by
+          # auto-injecting the consumer class
+          #
+          # @param name [String, Symbol] name of the topic where ActiveJobs jobs should go
+          # @param block [Proc] block that we can use for some extra configuration
+          def active_job_topic(name, &block)
+            topic(name) do
+              consumer App.config.internal.active_job.consumer_class
+              active_job true
+
+              # This is handled by our custom ActiveJob consumer
+              # Without this, default behaviour would cause messages to skip upon shutdown as the
+              # offset would be committed for the last message
+              manual_offset_management true
+
+              next unless block
+
+              instance_eval(&block)
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/routing/features/active_job/config.rb

@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Routing
+    module Features
+      class ActiveJob < Base
+        # Config for ActiveJob usage
+        Config = Struct.new(
+          :active,
+          keyword_init: true
+        ) { alias_method :active?, :active }
+      end
+    end
+  end
+end

data/lib/karafka/routing/features/active_job/contract.rb

@@ -0,0 +1,41 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Routing
+    module Features
+      class ActiveJob < Base
+        # Rules around using ActiveJob routing - basically you need to have ActiveJob available
+        # in order to be able to use active job routing
+        class Contract < Contracts::Base
+          configure do |config|
+            config.error_messages = YAML.safe_load(
+              File.read(
+                File.join(Karafka.gem_root, 'config', 'errors.yml')
+              )
+            ).fetch('en').fetch('validations').fetch('topic')
+          end
+
+          virtual do |data, errors|
+            next unless errors.empty?
+            next unless data[:active_job][:active]
+            # One should not define active job jobs without ActiveJob being available for usage
+            next if Object.const_defined?('ActiveJob::Base')
+
+            [[%i[consumer], :active_job_missing]]
+          end
+
+          # ActiveJob needs to always run with manual offset management
+          # Automatic offset management cannot work with ActiveJob. Otherwise we could mark as
+          # consumed jobs that did not run because of shutdown.
+          virtual do |data, errors|
+            next unless errors.empty?
+            next unless data[:active_job][:active]
+            next if data[:manual_offset_management][:active]
+
+            [[%i[manual_offset_management], :must_be_enabled]]
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/routing/features/active_job/topic.rb

@@ -0,0 +1,33 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Routing
+    module Features
+      class ActiveJob < Base
+        # Topic extensions to be able to check if given topic is ActiveJob topic
+        module Topic
+          # @param active [Boolean] should this topic be considered one working with ActiveJob
+          #
+          # @note Since this feature supports only one setting (active), we can use the old API
+          # where the boolean would be an argument
+          def active_job(active = false)
+            @active_job ||= Config.new(active: active)
+          end
+
+          # @return [Boolean] is this an ActiveJob topic
+          def active_job?
+            active_job.active?
+          end
+
+          # @return [Hash] topic with all its native configuration options plus active job
+          #   namespace settings
+          def to_h
+            super.merge(
+              active_job: active_job.to_h
+            ).freeze
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/routing/features/active_job.rb

@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Routing
+    module Features
+      # Active-Job related components
+      # @note We can load it always, despite someone not using ActiveJob as it just adds a method
+      #   to the routing, without actually breaking anything.
+      class ActiveJob < Base
+      end
+    end
+  end
+end

data/lib/karafka/routing/features/base/expander.rb

@@ -0,0 +1,53 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Routing
+    module Features
+      class Base
+        # Routing builder expander that injects feature related drawing operations into it
+        class Expander < Module
+          # @param scope [Module] feature scope in which contract and other things should be
+          # @return [Expander] builder expander instance
+          def initialize(scope)
+            super()
+            @scope = scope
+          end
+
+          # Builds anonymous module that alters how `#draw` behaves allowing the feature contracts
+          # to run.
+          # @param mod [::Karafka::Routing::Builder] builder we will prepend to
+          def prepended(mod)
+            super
+
+            mod.prepend(prepended_module)
+          end
+
+          private
+
+          # @return [Module] builds an anonymous module with `#draw` that will alter the builder
+          #   `#draw` allowing to run feature context aware code.
+          def prepended_module
+            scope = @scope
+
+            Module.new do
+              # Runs validations related to this feature on a topic
+              #
+              # @param block [Proc] routing defining block
+              define_method :draw do |&block|
+                result = super(&block)
+
+                each do |consumer_group|
+                  consumer_group.topics.each do |topic|
+                    scope::Contract.new.validate!(topic.to_h)
+                  end
+                end
+
+                result
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/routing/features/base.rb

@@ -0,0 +1,34 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Routing
+    # Namespace for all the topic related features we support
+    #
+    # @note Not all the Karafka features need to be defined here as only those that have routing
+    #   or other extensions need to be here. That is why we keep (for now) features under the
+    #   routing namespace.
+    module Features
+      # Base for all the features
+      class Base
+        class << self
+          # Extends topic and builder with given feature API
+          def activate
+            Topic.prepend(self::Topic) if const_defined?('Topic')
+            Proxy.prepend(self::Builder) if const_defined?('Builder')
+            Builder.prepend(self::Builder) if const_defined?('Builder')
+            Builder.prepend(Base::Expander.new(self)) if const_defined?('Contract')
+          end
+
+          # Loads all the features and activates them
+          def load_all
+            ObjectSpace
+              .each_object(Class)
+              .select { |klass| klass < self }
+              .sort_by(&:to_s)
+              .each(&:activate)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/routing/features/dead_letter_queue/config.rb

@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Routing
+    module Features
+      class DeadLetterQueue < Base
+        # Config for dead letter queue feature
+        Config = Struct.new(
+          :active,
+          # We add skip variants but in regular we support only `:one`
+          :max_retries,
+          # To what topic the skipped messages should be moved
+          :topic,
+          keyword_init: true
+        ) { alias_method :active?, :active }
+      end
+    end
+  end
+end

data/lib/karafka/routing/features/dead_letter_queue/contract.rb

@@ -0,0 +1,40 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Routing
+    module Features
+      class DeadLetterQueue < Base
+        # Rules around dead letter queue settings
+        class Contract < Contracts::Base
+          configure do |config|
+            config.error_messages = YAML.safe_load(
+              File.read(
+                File.join(Karafka.gem_root, 'config', 'errors.yml')
+              )
+            ).fetch('en').fetch('validations').fetch('topic')
+          end
+
+          nested :dead_letter_queue do
+            required(:active) { |val| [true, false].include?(val) }
+            required(:max_retries) { |val| val.is_a?(Integer) && val >= 0 }
+          end
+
+          # Validate topic name only if dlq is active
+          virtual do |data, errors|
+            next unless errors.empty?
+
+            dead_letter_queue = data[:dead_letter_queue]
+
+            next unless dead_letter_queue[:active]
+
+            topic = dead_letter_queue[:topic]
+
+            next if topic.is_a?(String) && Contracts::TOPIC_REGEXP.match?(topic)
+
+            [[%i[dead_letter_queue topic], :format]]
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/routing/features/dead_letter_queue/topic.rb

@@ -0,0 +1,40 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Routing
+    module Features
+      class DeadLetterQueue < Base
+        # DQL topic extensions
+        module Topic
+          # After how many retries should be move data to DLQ
+          DEFAULT_MAX_RETRIES = 3
+
+          private_constant :DEFAULT_MAX_RETRIES
+
+          # @param max_retries [Integer] after how many retries should we move data to dlq
+          # @param topic [String] where the messages should be moved if failing
+          # @return [Config] defined config
+          def dead_letter_queue(max_retries: DEFAULT_MAX_RETRIES, topic: nil)
+            @dead_letter_queue ||= Config.new(
+              active: !topic.nil?,
+              max_retries: max_retries,
+              topic: topic
+            )
+          end
+
+          # @return [Boolean] is the dlq active or not
+          def dead_letter_queue?
+            dead_letter_queue.active?
+          end
+
+          # @return [Hash] topic with all its native configuration options plus dlq settings
+          def to_h
+            super.merge(
+              dead_letter_queue: dead_letter_queue.to_h
+            ).freeze
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/routing/features/dead_letter_queue.rb

@@ -0,0 +1,16 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Routing
+    module Features
+      # This feature allows to continue processing when encountering errors.
+      # After certain number of retries, given messages will be moved to alternative topic,
+      # unclogging processing.
+      #
+      # @note This feature has an expanded version in the Pro mode. We do not use a new feature
+      #   injection in Pro (topic settings)
+      class DeadLetterQueue < Base
+      end
+    end
+  end
+end

data/lib/karafka/routing/features/manual_offset_management/config.rb

@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Routing
+    module Features
+      class ManualOffsetManagement < Base
+        # Config for manual offset management feature
+        Config = Struct.new(
+          :active,
+          keyword_init: true
+        ) { alias_method :active?, :active }
+      end
+    end
+  end
+end

data/lib/karafka/routing/features/manual_offset_management/contract.rb

@@ -0,0 +1,24 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Routing
+    module Features
+      class ManualOffsetManagement < Base
+        # Rules around manual offset management settings
+        class Contract < Contracts::Base
+          configure do |config|
+            config.error_messages = YAML.safe_load(
+              File.read(
+                File.join(Karafka.gem_root, 'config', 'errors.yml')
+              )
+            ).fetch('en').fetch('validations').fetch('topic')
+          end
+
+          nested :manual_offset_management do
+            required(:active) { |val| [true, false].include?(val) }
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/routing/features/manual_offset_management/topic.rb

@@ -0,0 +1,35 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Routing
+    module Features
+      class ManualOffsetManagement < Base
+        # Topic extensions to be able to manage manual offset management settings
+        module Topic
+          # @param active [Boolean] should we stop managing the offset in Karafka and make the user
+          #   responsible for marking messages as consumed.
+          # @return [Config] defined config
+          #
+          # @note Since this feature supports only one setting (active), we can use the old API
+          # where the boolean would be an argument
+          def manual_offset_management(active = false)
+            @manual_offset_management ||= Config.new(active: active)
+          end
+
+          # @return [Boolean] is manual offset management enabled for a given topic
+          def manual_offset_management?
+            manual_offset_management.active?
+          end
+
+          # @return [Hash] topic with all its native configuration options plus manual offset
+          #   management namespace settings
+          def to_h
+            super.merge(
+              manual_offset_management: manual_offset_management.to_h
+            ).freeze
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/routing/features/manual_offset_management.rb

@@ -0,0 +1,18 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Routing
+    module Features
+      # All the things needed to be able to manage manual offset management from the routing
+      # perspective.
+      #
+      # Manual offset management allows users to completely disable automatic management of the
+      # offset. This can be used for implementing long-living window operations and other things
+      # where we do not want to commit the offset with each batch.
+      #
+      # Not all the Karafka and Karafka Pro features may be compatible with this feature being on.
+      class ManualOffsetManagement < Base
+      end
+    end
+  end
+end

data/lib/karafka/routing/topic.rb

@@ -6,7 +6,7 @@ module Karafka
     # It belongs to a consumer group as from 0.6 all the topics can work in the same consumer group
     # It is a part of Karafka's DSL.
     class Topic
-      attr_reader :id, :name, :consumer_group, :tags
+      attr_reader :id, :name, :consumer_group
       attr_writer :consumer
       attr_accessor :subscription_group
 
@@ -14,7 +14,6 @@ module Karafka
       INHERITABLE_ATTRIBUTES = %i[
         kafka
         deserializer
-        manual_offset_management
         max_messages
         max_wait_time
         initial_offset
@@ -32,7 +31,6 @@ module Karafka
         #   Karafka 0.6 we can handle multiple Kafka instances with the same process and we can
         #   have same topic name across multiple consumer groups
         @id = "#{consumer_group.id}_#{@name}"
-        @tags = []
       end
 
       INHERITABLE_ATTRIBUTES.each do |attribute|
@@ -77,11 +75,6 @@ module Karafka
         consumer
       end
 
-      # @return [Boolean] true if this topic offset is handled by the end user
-      def manual_offset_management?
-        manual_offset_management
-      end
-
       # @return [Hash] hash with all the topic attributes
       # @note This is being used when we validate the consumer_group and its topics
       def to_h
@@ -94,8 +87,7 @@ module Karafka
           name: name,
           consumer: consumer,
           consumer_group_id: consumer_group.id,
-          subscription_group: subscription_group,
-          tags: tags
+          subscription_group: subscription_group
         ).freeze
       end
     end

data/lib/karafka/server.rb

@@ -74,7 +74,7 @@ module Karafka
       #   please start a separate thread to do so.
       def stop
         # Initialize the stopping process only if Karafka was running
-        return unless Karafka::App.running?
+        return if Karafka::App.stopping? || Karafka::App.stopped?
 
         Karafka::App.stop!
 
@@ -120,7 +120,9 @@ module Karafka
         # exit! is not within the instrumentation as it would not trigger due to exit
         Kernel.exit!(FORCEFUL_EXIT_CODE)
       ensure
-        Karafka::App.stopped!
+        # We need to check if it wasn't an early exit to make sure that only on stop invocation
+        # can change the status after everything is closed
+        Karafka::App.stopped! if timeout
       end
 
       private

data/lib/karafka/setup/attributes_map.rb

@@ -128,6 +128,7 @@ module Karafka
         ssl.key.pem
         ssl.keystore.location
         ssl.keystore.password
+        ssl.providers
         ssl.sigalgs.list
         ssl_ca
         ssl_certificate
@@ -259,6 +260,7 @@ module Karafka
         ssl.key.pem
         ssl.keystore.location
         ssl.keystore.password
+        ssl.providers
         ssl.sigalgs.list
         ssl_ca
         ssl_certificate
@@ -327,6 +329,9 @@ module Karafka
             end
           end
 
+          # This can be removed when 0.13 librdkafka is released
+          attributes[:producer].delete_if { |val| val == 'allow.auto.create.topics' }
+
           attributes.transform_values!(&:sort)
           attributes.each_value { |vals| vals.map!(&:to_sym) }
           attributes

data/lib/karafka/setup/config.rb

@@ -68,8 +68,6 @@ module Karafka
       # option [String] should we start with the earliest possible offset or latest
       # This will set the `auto.offset.reset` value unless present in the kafka scope
       setting :initial_offset, default: 'earliest'
-      # option [Boolean] should we leave offset management to the user
-      setting :manual_offset_management, default: false
       # options max_messages [Integer] how many messages do we want to fetch from Kafka in one go
       setting :max_messages, default: 100
       # option [Integer] number of milliseconds we can wait while fetching data
@@ -80,9 +78,9 @@ module Karafka
       setting :shutdown_timeout, default: 60_000
       # option [Integer] number of threads in which we want to do parallel processing
       setting :concurrency, default: 5
-      # option [Integer] how long should we wait upon processing error
+      # option [Integer] how long should we wait upon processing error (milliseconds)
       setting :pause_timeout, default: 1_000
-      # option [Integer] what is the max timeout in case of an exponential backoff
+      # option [Integer] what is the max timeout in case of an exponential backoff (milliseconds)
       setting :pause_max_timeout, default: 30_000
       # option [Boolean] should we use exponential backoff
       setting :pause_with_exponential_backoff, default: true
@@ -120,6 +118,8 @@ module Karafka
           setting :coordinator_class, default: Processing::Coordinator
           # option partitioner_class [Class] partitioner we use against a batch of data
           setting :partitioner_class, default: Processing::Partitioner
+          # option strategy_selector [Object] processing strategy selector to be used
+          setting :strategy_selector, default: Processing::StrategySelector.new
         end
 
         # Karafka components for ActiveJob