karafka 2.0.15 → 2.0.16

data/lib/karafka/routing/features/active_job/contract.rb

@@ -0,0 +1,41 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Routing
+    module Features
+      class ActiveJob < Base
+        # Rules around using ActiveJob routing - basically you need to have ActiveJob available
+        # in order to be able to use active job routing
+        class Contract < Contracts::Base
+          configure do |config|
+            config.error_messages = YAML.safe_load(
+              File.read(
+                File.join(Karafka.gem_root, 'config', 'errors.yml')
+              )
+            ).fetch('en').fetch('validations').fetch('topic')
+          end
+
+          virtual do |data, errors|
+            next unless errors.empty?
+            next unless data[:active_job][:active]
+            # One should not define active job jobs without ActiveJob being available for usage
+            next if Object.const_defined?('ActiveJob::Base')
+
+            [[%i[consumer], :active_job_missing]]
+          end
+
+          # ActiveJob needs to always run with manual offset management
+          # Automatic offset management cannot work with ActiveJob. Otherwise we could mark as
+          # consumed jobs that did not run because of shutdown.
+          virtual do |data, errors|
+            next unless errors.empty?
+            next unless data[:active_job][:active]
+            next if data[:manual_offset_management][:active]
+
+            [[%i[manual_offset_management], :must_be_enabled]]
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/routing/features/active_job/topic.rb

@@ -0,0 +1,33 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Routing
+    module Features
+      class ActiveJob < Base
+        # Topic extensions to be able to check if given topic is ActiveJob topic
+        module Topic
+          # @param active [Boolean] should this topic be considered one working with ActiveJob
+          #
+          # @note Since this feature supports only one setting (active), we can use the old API
+          # where the boolean would be an argument
+          def active_job(active = false)
+            @active_job ||= Config.new(active: active)
+          end
+
+          # @return [Boolean] is this an ActiveJob topic
+          def active_job?
+            active_job.active?
+          end
+
+          # @return [Hash] topic with all its native configuration options plus active job
+          #   namespace settings
+          def to_h
+            super.merge(
+              active_job: active_job.to_h
+            ).freeze
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/routing/features/active_job.rb

@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Routing
+    module Features
+      # Active-Job related components
+      # @note We can load it always, despite someone not using ActiveJob as it just adds a method
+      #   to the routing, without actually breaking anything.
+      class ActiveJob < Base
+      end
+    end
+  end
+end

data/lib/karafka/routing/features/base/expander.rb

@@ -0,0 +1,53 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Routing
+    module Features
+      class Base
+        # Routing builder expander that injects feature related drawing operations into it
+        class Expander < Module
+          # @param scope [Module] feature scope in which contract and other things should be
+          # @return [Expander] builder expander instance
+          def initialize(scope)
+            super()
+            @scope = scope
+          end
+
+          # Builds anonymous module that alters how `#draw` behaves allowing the feature contracts
+          # to run.
+          # @param mod [::Karafka::Routing::Builder] builder we will prepend to
+          def prepended(mod)
+            super
+
+            mod.prepend(prepended_module)
+          end
+
+          private
+
+          # @return [Module] builds an anonymous module with `#draw` that will alter the builder
+          #   `#draw` allowing to run feature context aware code.
+          def prepended_module
+            scope = @scope
+
+            Module.new do
+              # Runs validations related to this feature on a topic
+              #
+              # @param block [Proc] routing defining block
+              define_method :draw do |&block|
+                result = super(&block)
+
+                each do |consumer_group|
+                  consumer_group.topics.each do |topic|
+                    scope::Contract.new.validate!(topic.to_h)
+                  end
+                end
+
+                result
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/routing/features/base.rb

@@ -0,0 +1,34 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Routing
+    # Namespace for all the topic related features we support
+    #
+    # @note Not all the Karafka features need to be defined here as only those that have routing
+    #   or other extensions need to be here. That is why we keep (for now) features under the
+    #   routing namespace.
+    module Features
+      # Base for all the features
+      class Base
+        class << self
+          # Extends topic and builder with given feature API
+          def activate
+            Topic.prepend(self::Topic) if const_defined?('Topic')
+            Proxy.prepend(self::Builder) if const_defined?('Builder')
+            Builder.prepend(self::Builder) if const_defined?('Builder')
+            Builder.prepend(Base::Expander.new(self)) if const_defined?('Contract')
+          end
+
+          # Loads all the features and activates them
+          def load_all
+            ObjectSpace
+              .each_object(Class)
+              .select { |klass| klass < self }
+              .sort_by(&:to_s)
+              .each(&:activate)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/routing/features/dead_letter_queue/config.rb

@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Routing
+    module Features
+      class DeadLetterQueue < Base
+        # Config for dead letter queue feature
+        Config = Struct.new(
+          :active,
+          # We add skip variants but in regular we support only `:one`
+          :max_retries,
+          # To what topic the skipped messages should be moved
+          :topic,
+          keyword_init: true
+        ) { alias_method :active?, :active }
+      end
+    end
+  end
+end

data/lib/karafka/routing/features/dead_letter_queue/contract.rb

@@ -0,0 +1,40 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Routing
+    module Features
+      class DeadLetterQueue < Base
+        # Rules around dead letter queue settings
+        class Contract < Contracts::Base
+          configure do |config|
+            config.error_messages = YAML.safe_load(
+              File.read(
+                File.join(Karafka.gem_root, 'config', 'errors.yml')
+              )
+            ).fetch('en').fetch('validations').fetch('topic')
+          end
+
+          nested :dead_letter_queue do
+            required(:active) { |val| [true, false].include?(val) }
+            required(:max_retries) { |val| val.is_a?(Integer) && val >= 0 }
+          end
+
+          # Validate topic name only if dlq is active
+          virtual do |data, errors|
+            next unless errors.empty?
+
+            dead_letter_queue = data[:dead_letter_queue]
+
+            next unless dead_letter_queue[:active]
+
+            topic = dead_letter_queue[:topic]
+
+            next if topic.is_a?(String) && Contracts::TOPIC_REGEXP.match?(topic)
+
+            [[%i[dead_letter_queue topic], :format]]
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/routing/features/dead_letter_queue/topic.rb

@@ -0,0 +1,40 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Routing
+    module Features
+      class DeadLetterQueue < Base
+        # DQL topic extensions
+        module Topic
+          # After how many retries should be move data to DLQ
+          DEFAULT_MAX_RETRIES = 3
+
+          private_constant :DEFAULT_MAX_RETRIES
+
+          # @param max_retries [Integer] after how many retries should we move data to dlq
+          # @param topic [String] where the messages should be moved if failing
+          # @return [Config] defined config
+          def dead_letter_queue(max_retries: DEFAULT_MAX_RETRIES, topic: nil)
+            @dead_letter_queue ||= Config.new(
+              active: !topic.nil?,
+              max_retries: max_retries,
+              topic: topic
+            )
+          end
+
+          # @return [Boolean] is the dlq active or not
+          def dead_letter_queue?
+            dead_letter_queue.active?
+          end
+
+          # @return [Hash] topic with all its native configuration options plus dlq settings
+          def to_h
+            super.merge(
+              dead_letter_queue: dead_letter_queue.to_h
+            ).freeze
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/routing/features/dead_letter_queue.rb

@@ -0,0 +1,16 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Routing
+    module Features
+      # This feature allows to continue processing when encountering errors.
+      # After certain number of retries, given messages will be moved to alternative topic,
+      # unclogging processing.
+      #
+      # @note This feature has an expanded version in the Pro mode. We do not use a new feature
+      #   injection in Pro (topic settings)
+      class DeadLetterQueue < Base
+      end
+    end
+  end
+end

data/lib/karafka/routing/features/manual_offset_management/config.rb

@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Routing
+    module Features
+      class ManualOffsetManagement < Base
+        # Config for manual offset management feature
+        Config = Struct.new(
+          :active,
+          keyword_init: true
+        ) { alias_method :active?, :active }
+      end
+    end
+  end
+end

data/lib/karafka/routing/features/manual_offset_management/contract.rb

@@ -0,0 +1,24 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Routing
+    module Features
+      class ManualOffsetManagement < Base
+        # Rules around manual offset management settings
+        class Contract < Contracts::Base
+          configure do |config|
+            config.error_messages = YAML.safe_load(
+              File.read(
+                File.join(Karafka.gem_root, 'config', 'errors.yml')
+              )
+            ).fetch('en').fetch('validations').fetch('topic')
+          end
+
+          nested :manual_offset_management do
+            required(:active) { |val| [true, false].include?(val) }
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/routing/features/manual_offset_management/topic.rb

@@ -0,0 +1,35 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Routing
+    module Features
+      class ManualOffsetManagement < Base
+        # Topic extensions to be able to manage manual offset management settings
+        module Topic
+          # @param active [Boolean] should we stop managing the offset in Karafka and make the user
+          #   responsible for marking messages as consumed.
+          # @return [Config] defined config
+          #
+          # @note Since this feature supports only one setting (active), we can use the old API
+          # where the boolean would be an argument
+          def manual_offset_management(active = false)
+            @manual_offset_management ||= Config.new(active: active)
+          end
+
+          # @return [Boolean] is manual offset management enabled for a given topic
+          def manual_offset_management?
+            manual_offset_management.active?
+          end
+
+          # @return [Hash] topic with all its native configuration options plus manual offset
+          #   management namespace settings
+          def to_h
+            super.merge(
+              manual_offset_management: manual_offset_management.to_h
+            ).freeze
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/routing/features/manual_offset_management.rb

@@ -0,0 +1,18 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Routing
+    module Features
+      # All the things needed to be able to manage manual offset management from the routing
+      # perspective.
+      #
+      # Manual offset management allows users to completely disable automatic management of the
+      # offset. This can be used for implementing long-living window operations and other things
+      # where we do not want to commit the offset with each batch.
+      #
+      # Not all the Karafka and Karafka Pro features may be compatible with this feature being on.
+      class ManualOffsetManagement < Base
+      end
+    end
+  end
+end

data/lib/karafka/routing/topic.rb

@@ -6,7 +6,7 @@ module Karafka
     # It belongs to a consumer group as from 0.6 all the topics can work in the same consumer group
     # It is a part of Karafka's DSL.
     class Topic
-      attr_reader :id, :name, :consumer_group, :tags
+      attr_reader :id, :name, :consumer_group
       attr_writer :consumer
       attr_accessor :subscription_group
 
@@ -14,7 +14,6 @@ module Karafka
       INHERITABLE_ATTRIBUTES = %i[
         kafka
         deserializer
-        manual_offset_management
         max_messages
         max_wait_time
         initial_offset
@@ -32,7 +31,6 @@ module Karafka
         #   Karafka 0.6 we can handle multiple Kafka instances with the same process and we can
         #   have same topic name across multiple consumer groups
         @id = "#{consumer_group.id}_#{@name}"
-        @tags = []
       end
 
       INHERITABLE_ATTRIBUTES.each do |attribute|
@@ -77,11 +75,6 @@ module Karafka
         consumer
       end
 
-      # @return [Boolean] true if this topic offset is handled by the end user
-      def manual_offset_management?
-        manual_offset_management
-      end
-
       # @return [Hash] hash with all the topic attributes
       # @note This is being used when we validate the consumer_group and its topics
       def to_h
@@ -94,8 +87,7 @@ module Karafka
           name: name,
           consumer: consumer,
           consumer_group_id: consumer_group.id,
-          subscription_group: subscription_group,
-          tags: tags
+          subscription_group: subscription_group
         ).freeze
       end
     end

data/lib/karafka/server.rb

@@ -74,7 +74,7 @@ module Karafka
       #   please start a separate thread to do so.
       def stop
         # Initialize the stopping process only if Karafka was running
-        return unless Karafka::App.running?
+        return if Karafka::App.stopping? || Karafka::App.stopped?
 
         Karafka::App.stop!
 
@@ -120,7 +120,9 @@ module Karafka
         # exit! is not within the instrumentation as it would not trigger due to exit
         Kernel.exit!(FORCEFUL_EXIT_CODE)
       ensure
-        Karafka::App.stopped!
+        # We need to check if it wasn't an early exit to make sure that only on stop invocation
+        # can change the status after everything is closed
+        Karafka::App.stopped! if timeout
       end
 
       private

data/lib/karafka/setup/attributes_map.rb

@@ -128,6 +128,7 @@ module Karafka
         ssl.key.pem
         ssl.keystore.location
         ssl.keystore.password
+        ssl.providers
         ssl.sigalgs.list
         ssl_ca
         ssl_certificate
@@ -259,6 +260,7 @@ module Karafka
         ssl.key.pem
         ssl.keystore.location
         ssl.keystore.password
+        ssl.providers
         ssl.sigalgs.list
         ssl_ca
         ssl_certificate
@@ -327,6 +329,9 @@ module Karafka
             end
           end
 
+          # This can be removed when 0.13 librdkafka is released
+          attributes[:producer].delete_if { |val| val == 'allow.auto.create.topics' }
+
           attributes.transform_values!(&:sort)
           attributes.each_value { |vals| vals.map!(&:to_sym) }
           attributes

data/lib/karafka/setup/config.rb

@@ -68,8 +68,6 @@ module Karafka
       # option [String] should we start with the earliest possible offset or latest
       # This will set the `auto.offset.reset` value unless present in the kafka scope
       setting :initial_offset, default: 'earliest'
-      # option [Boolean] should we leave offset management to the user
-      setting :manual_offset_management, default: false
       # options max_messages [Integer] how many messages do we want to fetch from Kafka in one go
       setting :max_messages, default: 100
       # option [Integer] number of milliseconds we can wait while fetching data
@@ -80,9 +78,9 @@ module Karafka
       setting :shutdown_timeout, default: 60_000
       # option [Integer] number of threads in which we want to do parallel processing
       setting :concurrency, default: 5
-      # option [Integer] how long should we wait upon processing error
+      # option [Integer] how long should we wait upon processing error (milliseconds)
       setting :pause_timeout, default: 1_000
-      # option [Integer] what is the max timeout in case of an exponential backoff
+      # option [Integer] what is the max timeout in case of an exponential backoff (milliseconds)
       setting :pause_max_timeout, default: 30_000
       # option [Boolean] should we use exponential backoff
       setting :pause_with_exponential_backoff, default: true
@@ -120,6 +118,8 @@ module Karafka
           setting :coordinator_class, default: Processing::Coordinator
           # option partitioner_class [Class] partitioner we use against a batch of data
           setting :partitioner_class, default: Processing::Partitioner
+          # option strategy_selector [Object] processing strategy selector to be used
+          setting :strategy_selector, default: Processing::StrategySelector.new
         end
 
         # Karafka components for ActiveJob

data/lib/karafka/time_trackers/pause.rb

@@ -10,7 +10,7 @@ module Karafka
     #   code here, as this is not a frequently used tracker. It is active only once per batch in
     #   case of long-running-jobs and upon errors.
     class Pause < Base
-      attr_reader :count
+      attr_reader :attempt
 
       # @param timeout [Integer] how long should we wait when anything went wrong (in ms)
       # @param max_timeout [Integer, nil] if exponential is on, what is the max value we can reach
@@ -19,26 +19,29 @@ module Karafka
       #   timeout value
       # @return [Karafka::TimeTrackers::Pause]
       # @example
-      #   pause = Karafka::TimeTrackers::Pause.new(timeout: 1000)
+      #   options = { timeout: 1000, max_timeout: 1000, exponential_backoff: false }
+      #   pause = Karafka::TimeTrackers::Pause.new(**options)
       #   pause.expired? #=> true
       #   pause.paused? #=> false
       #   pause.pause
+      #   pause.increment
       #   sleep(1.1)
       #   pause.paused? #=> true
       #   pause.expired? #=> true
-      #   pause.count #=> 1
+      #   pause.attempt #=> 1
       #   pause.pause
-      #   pause.count #=> 1
+      #   pause.increment
+      #   pause.attempt #=> 2
       #   pause.paused? #=> true
       #   pause.expired? #=> false
       #   pause.resume
-      #   pause.count #=> 2
+      #   pause.attempt #=> 2
       #   pause.paused? #=> false
       #   pause.reset
-      #   pause.count #=> 0
+      #   pause.attempt #=> 0
       def initialize(timeout:, max_timeout:, exponential_backoff:)
         @started_at = nil
-        @count = 0
+        @attempt = 0
         @timeout = timeout
         @max_timeout = max_timeout
         @exponential_backoff = exponential_backoff
@@ -47,7 +50,7 @@ module Karafka
       end
 
       # Pauses the processing from now till the end of the interval (backoff or non-backoff)
-      # and records the count.
+      # and records the attempt.
       # @param timeout [Integer] timeout value in milliseconds that overwrites the default timeout
       # @note Providing this value can be useful when we explicitly want to pause for a certain
       #   period of time, outside of any regular pausing logic
@@ -55,7 +58,13 @@ module Karafka
         @mutex.synchronize do
           @started_at = now
           @ends_at = @started_at + timeout
-          @count += 1
+        end
+      end
+
+      # Increments the number of attempt by 1
+      def increment
+        @mutex.synchronize do
+          @attempt += 1
         end
       end
 
@@ -88,10 +97,10 @@ module Karafka
         end
       end
 
-      # Resets the pause counter.
+      # Resets the pause attempt count.
       def reset
         @mutex.synchronize do
-          @count = 0
+          @attempt = 0
         end
       end
 
@@ -100,7 +109,7 @@ module Karafka
       # Computers the exponential backoff
       # @return [Integer] backoff in milliseconds
       def backoff_interval
-        backoff_factor = @exponential_backoff ? 2**@count : 1
+        backoff_factor = @exponential_backoff ? 2**@attempt : 1
 
         timeout = backoff_factor * @timeout
 

data/lib/karafka/version.rb

@@ -3,5 +3,5 @@
 # Main module namespace
 module Karafka
   # Current Karafka version
-  VERSION = '2.0.15'
+  VERSION = '2.0.16'
 end

data/lib/karafka.rb

@@ -86,17 +86,13 @@ end
 loader = Zeitwerk::Loader.for_gem
 # Do not load Rails extensions by default, this will be handled by Railtie if they are needed
 loader.ignore(Karafka.gem_root.join('lib/active_job'))
-
-begin
-  require 'karafka-license'
-rescue LoadError
-  # Do not load pro components if we cannot load the license
-  # This is a preliminary check so autoload works as expected
-  # Later on the licenser will make sure to setup all the needed components anyhow
-  loader.ignore(Karafka.gem_root.join('lib/karafka/pro'))
-end
-
+# Do not load pro components as they will be loaded if needed and allowed
+loader.ignore(Karafka.core_root.join('pro/'))
 # Do not load vendors instrumentation components. Those need to be required manually if needed
-loader.ignore(Karafka.gem_root.join('lib/karafka/instrumentation/vendors'))
+loader.ignore(Karafka.core_root.join('instrumentation/vendors'))
 loader.setup
 loader.eager_load
+
+# This will load features but since Pro are not loaded automatically, they will not be visible
+# nor included here
+::Karafka::Routing::Features::Base.load_all