karafka 2.1.13 → 2.2.0

data/lib/karafka/pro/routing/features/patterns/topics.rb

@@ -0,0 +1,53 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Routing
+      module Features
+        class Patterns < Base
+          # Patterns feature topic extensions
+          module Topics
+            # Finds topic by its name in a more extensive way than the regular. Regular uses the
+            # pre-existing topics definitions. This extension also runs the expansion based on
+            # defined routing patterns (if any)
+            #
+            # If topic does not exist, it will try to run discovery in case there are patterns
+            # defined that would match it.
+            # This allows us to support lookups for newly appearing topics based on their regexp
+            # patterns.
+            #
+            # @param topic_name [String] topic name
+            # @return [Karafka::Routing::Topic]
+            # @raise [Karafka::Errors::TopicNotFoundError] this should never happen. If you see it,
+            #   please create an issue.
+            #
+            # @note This method should not be used in context of finding multiple missing topics in
+            #   loops because it catches exceptions and attempts to expand routes. If this is used
+            #   in a loop for lookups on thousands of topics with detector expansion, this may
+            #   be slow. It should be used in the context where newly discovered topics are found
+            #   and should by design match a pattern. For quick lookups on batches of topics, it
+            #   is recommended to use a custom built lookup with conditional expander.
+            def find(topic_name)
+              super
+            rescue Karafka::Errors::TopicNotFoundError
+              Detector.new.expand(self, topic_name)
+
+              super
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/routing/features/patterns.rb

@@ -0,0 +1,33 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Routing
+      module Features
+        # Dynamic topics builder feature.
+        #
+        # Allows you to define patterns in routes that would then automatically subscribe and
+        # start consuming new topics.
+        #
+        # This feature works by injecting a topic that represents a regexp subscription (matcher)
+        # that at the same time holds the builder block for full config of a newly detected topic.
+        #
+        # We inject a virtual topic to hold settings but also to be able to run validations
+        # during boot to ensure consistency of the pattern base setup.
+        class Patterns < Base
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/routing/features/pausing/contracts/topic.rb

@@ -0,0 +1,51 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Routing
+      module Features
+        class Pausing < Base
+          # Namespace for pausing feature
+          module Contracts
+            # Contract to make sure, that the pause settings on a per topic basis are as expected
+            class Topic < Karafka::Contracts::Base
+              configure do |config|
+                config.error_messages = YAML.safe_load(
+                  File.read(
+                    File.join(Karafka.gem_root, 'config', 'locales', 'pro_errors.yml')
+                  )
+                ).fetch('en').fetch('validations').fetch('topic')
+
+                required(:pause_timeout) { |val| val.is_a?(Integer) && val.positive? }
+                required(:pause_max_timeout) { |val| val.is_a?(Integer) && val.positive? }
+                required(:pause_with_exponential_backoff) { |val| [true, false].include?(val) }
+
+                virtual do |data, errors|
+                  next unless errors.empty?
+
+                  pause_timeout = data.fetch(:pause_timeout)
+                  pause_max_timeout = data.fetch(:pause_max_timeout)
+
+                  next if pause_timeout <= pause_max_timeout
+
+                  [[%i[pause_timeout], :max_timeout_vs_pause_max_timeout]]
+                end
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/routing/features/throttling/contracts/topic.rb

@@ -0,0 +1,44 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Routing
+      module Features
+        class Throttling < Base
+          # Namespace for throttling contracts
+          module Contracts
+            # Rules around throttling settings
+            class Topic < Karafka::Contracts::Base
+              configure do |config|
+                config.error_messages = YAML.safe_load(
+                  File.read(
+                    File.join(Karafka.gem_root, 'config', 'locales', 'pro_errors.yml')
+                  )
+                ).fetch('en').fetch('validations').fetch('topic')
+              end
+
+              nested(:throttling) do
+                required(:active) { |val| [true, false].include?(val) }
+                required(:interval) { |val| val.is_a?(Integer) && val.positive? }
+                required(:limit) do |val|
+                  (val.is_a?(Integer) || val == Float::INFINITY) && val.positive?
+                end
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/routing/features/virtual_partitions/contracts/topic.rb

@@ -0,0 +1,55 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Routing
+      module Features
+        class VirtualPartitions < Base
+          # Namespace for VP contracts
+          module Contracts
+            # Rules around virtual partitions
+            class Topic < Karafka::Contracts::Base
+              configure do |config|
+                config.error_messages = YAML.safe_load(
+                  File.read(
+                    File.join(Karafka.gem_root, 'config', 'locales', 'pro_errors.yml')
+                  )
+                ).fetch('en').fetch('validations').fetch('topic')
+              end
+
+              nested(:virtual_partitions) do
+                required(:active) { |val| [true, false].include?(val) }
+                required(:partitioner) { |val| val.nil? || val.respond_to?(:call) }
+                required(:max_partitions) { |val| val.is_a?(Integer) && val >= 1 }
+              end
+
+              # When virtual partitions are defined, partitioner needs to respond to `#call` and it
+              # cannot be nil
+              virtual do |data, errors|
+                next unless errors.empty?
+
+                virtual_partitions = data[:virtual_partitions]
+
+                next unless virtual_partitions[:active]
+                next if virtual_partitions[:partitioner].respond_to?(:call)
+
+                [[%i[virtual_partitions partitioner], :respond_to_call]]
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/routing/consumer_group.rb

@@ -55,7 +55,7 @@ module Karafka
       def subscription_group=(name = SubscriptionGroup.id, &block)
         # We cast it here, so the routing supports symbol based but that's anyhow later on
         # validated as a string
-        @current_subscription_group_id = name
+        @current_subscription_group_id = name.to_s
 
         Proxy.new(self, &block)
 

data/lib/karafka/routing/features/active_job/contracts/topic.rb

@@ -0,0 +1,44 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Routing
+    module Features
+      class ActiveJob < Base
+        # This feature validation contracts
+        module Contracts
+          # Rules around using ActiveJob routing - basically you need to have ActiveJob available
+          # in order to be able to use active job routing
+          class Topic < Karafka::Contracts::Base
+            configure do |config|
+              config.error_messages = YAML.safe_load(
+                File.read(
+                  File.join(Karafka.gem_root, 'config', 'locales', 'errors.yml')
+                )
+              ).fetch('en').fetch('validations').fetch('topic')
+            end
+
+            virtual do |data, errors|
+              next unless errors.empty?
+              next unless data[:active_job][:active]
+              # One should not define active job jobs without ActiveJob being available for usage
+              next if Object.const_defined?('ActiveJob::Base')
+
+              [[%i[consumer], :active_job_missing]]
+            end
+
+            # ActiveJob needs to always run with manual offset management
+            # Automatic offset management cannot work with ActiveJob. Otherwise we could mark as
+            # consumed jobs that did not run because of shutdown.
+            virtual do |data, errors|
+              next unless errors.empty?
+              next unless data[:active_job][:active]
+              next if data[:manual_offset_management][:active]
+
+              [[%i[manual_offset_management], :must_be_enabled]]
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/routing/features/active_job/proxy.rb

@@ -0,0 +1,14 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Routing
+    module Features
+      class ActiveJob < Base
+        # Routing proxy extensions for ActiveJob
+        module Proxy
+          include Builder
+        end
+      end
+    end
+  end
+end

data/lib/karafka/routing/features/base/expander.rb

@@ -30,15 +30,21 @@ module Karafka
             scope = @scope
 
             Module.new do
-              # Runs validations related to this feature on a topic
+              # Runs validations related to this feature on a routing resources
               #
               # @param block [Proc] routing defining block
               define_method :draw do |&block|
                 result = super(&block)
 
                 each do |consumer_group|
+                  if scope::Contracts.const_defined?('ConsumerGroup', false)
+                    scope::Contracts::ConsumerGroup.new.validate!(consumer_group.to_h)
+                  end
+
+                  next unless scope::Contracts.const_defined?('Topic', false)
+
                   consumer_group.topics.each do |topic|
-                    scope::Contract.new.validate!(topic.to_h)
+                    scope::Contracts::Topic.new.validate!(topic.to_h)
                   end
                 end
 

data/lib/karafka/routing/features/base.rb

@@ -14,9 +14,11 @@ module Karafka
           # Extends topic and builder with given feature API
           def activate
             Topic.prepend(self::Topic) if const_defined?('Topic', false)
-            Proxy.prepend(self::Builder) if const_defined?('Builder', false)
+            Topics.prepend(self::Topics) if const_defined?('Topics', false)
+            ConsumerGroup.prepend(self::ConsumerGroup) if const_defined?('ConsumerGroup', false)
+            Proxy.prepend(self::Proxy) if const_defined?('Proxy', false)
             Builder.prepend(self::Builder) if const_defined?('Builder', false)
-            Builder.prepend(Base::Expander.new(self)) if const_defined?('Contract', false)
+            Builder.prepend(Base::Expander.new(self)) if const_defined?('Contracts', false)
           end
 
           # Loads all the features and activates them

data/lib/karafka/routing/features/dead_letter_queue/contracts/topic.rb

@@ -0,0 +1,46 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Routing
+    module Features
+      class DeadLetterQueue < Base
+        # This feature validation contracts
+        module Contracts
+          # Rules around dead letter queue settings
+          class Topic < Karafka::Contracts::Base
+            configure do |config|
+              config.error_messages = YAML.safe_load(
+                File.read(
+                  File.join(Karafka.gem_root, 'config', 'locales', 'errors.yml')
+                )
+              ).fetch('en').fetch('validations').fetch('topic')
+            end
+
+            nested :dead_letter_queue do
+              required(:active) { |val| [true, false].include?(val) }
+              required(:max_retries) { |val| val.is_a?(Integer) && val >= 0 }
+            end
+
+            # Validate topic name only if dlq is active
+            virtual do |data, errors|
+              next unless errors.empty?
+
+              dead_letter_queue = data[:dead_letter_queue]
+
+              next unless dead_letter_queue[:active]
+
+              topic = dead_letter_queue[:topic]
+              topic_regexp = ::Karafka::Contracts::TOPIC_REGEXP
+
+              # When topic is set to false, it means we just want to skip dispatch on DLQ
+              next if topic == false
+              next if topic.is_a?(String) && topic_regexp.match?(topic)
+
+              [[%i[dead_letter_queue topic], :format]]
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/routing/features/declaratives/contracts/topic.rb

@@ -0,0 +1,33 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Routing
+    module Features
+      class Declaratives < Base
+        # This feature validation contracts
+        module Contracts
+          # Basic validation of the Kafka expected config details
+          class Topic < Karafka::Contracts::Base
+            configure do |config|
+              config.error_messages = YAML.safe_load(
+                File.read(
+                  File.join(Karafka.gem_root, 'config', 'locales', 'errors.yml')
+                )
+              ).fetch('en').fetch('validations').fetch('topic')
+            end
+
+            nested :declaratives do
+              required(:active) { |val| [true, false].include?(val) }
+              required(:partitions) { |val| val.is_a?(Integer) && val.positive? }
+              required(:replication_factor) { |val| val.is_a?(Integer) && val.positive? }
+              required(:details) do |val|
+                val.is_a?(Hash) &&
+                  val.keys.all? { |key| key.is_a?(Symbol) }
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/routing/features/manual_offset_management/contracts/topic.rb

@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Routing
+    module Features
+      class ManualOffsetManagement < Base
+        # This feature validation contracts
+        module Contracts
+          # Rules around manual offset management settings
+          class Topic < Karafka::Contracts::Base
+            configure do |config|
+              config.error_messages = YAML.safe_load(
+                File.read(
+                  File.join(Karafka.gem_root, 'config', 'locales', 'errors.yml')
+                )
+              ).fetch('en').fetch('validations').fetch('topic')
+            end
+
+            nested :manual_offset_management do
+              required(:active) { |val| [true, false].include?(val) }
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/routing/router.rb

@@ -7,16 +7,6 @@ module Karafka
     # @note Since Kafka does not provide namespaces or modules for topics, they all have "flat"
     #  structure so all the routes are being stored in a single level array
     module Router
-      # Find a proper topic based on full topic id
-      # @param topic_id [String] proper topic id (already mapped, etc) for which we want to find
-      #   routing topic
-      # @return [Karafka::Routing::Topic] proper route details
-      # @raise [Karafka::Topic::NonMatchingTopicError] raised if topic name does not match
-      #   any route defined by user using routes.draw
-      def find(topic_id)
-        find_by(id: topic_id) || raise(Errors::NonMatchingRouteError, topic_id)
-      end
-
       # Finds first reference of a given topic based on provided lookup attribute
       # @param lookup [Hash<Symbol, String>] hash with attribute - value key pairs
       # @return [Karafka::Routing::Topic, nil] proper route details or nil if not found
@@ -46,7 +36,6 @@ module Karafka
         find_by(name: name) || Topic.new(name, ConsumerGroup.new(name))
       end
 
-      module_function :find
       module_function :find_by
       module_function :find_or_initialize_by_name
     end

data/lib/karafka/routing/subscription_group.rb

@@ -61,6 +61,15 @@ module Karafka
         Karafka::App.config.internal.routing.activity_manager.active?(:subscription_groups, name)
       end
 
+      # @return [Array<String>] names of topics to which we should subscribe.
+      #
+      # @note Most of the time it should not include inactive topics but in case of pattern
+      #   matching the matcher topics become inactive down the road, hence we filter out so
+      #   they are later removed.
+      def subscriptions
+        topics.select(&:active?).map(&:subscription_name)
+      end
+
       private
 
       # @return [Hash] kafka settings are a bit special. They are exactly the same for all of the

data/lib/karafka/routing/topic.rb

@@ -52,6 +52,11 @@ module Karafka
         end
       end
 
+      # @return [String] name of subscription that will go to librdkafka
+      def subscription_name
+        name
+      end
+
       # @return [Class] consumer class that we should use
       def consumer
         if consumer_persistence

data/lib/karafka/server.rb

@@ -32,8 +32,8 @@ module Karafka
         # embedded
         # We cannot validate this during the start because config needs to be populated and routes
         # need to be defined.
-        Contracts::ServerCliOptions.new.validate!(
-          Karafka::App.config.internal.routing.activity_manager.to_h
+        config.internal.cli.contract.validate!(
+          config.internal.routing.activity_manager.to_h
         )
 
         process.on_sigint { stop }
@@ -82,7 +82,7 @@ module Karafka
 
         Karafka::App.stop!
 
-        timeout = Karafka::App.config.shutdown_timeout
+        timeout = config.shutdown_timeout
 
         # We check from time to time (for the timeout period) if all the threads finished
         # their work and if so, we can just return and normal shutdown process will take place
@@ -148,9 +148,14 @@ module Karafka
 
       private
 
+      # @return [Karafka::Core::Configurable::Node] root config node
+      def config
+        Karafka::App.config
+      end
+
       # @return [Karafka::Process] process wrapper instance used to catch system signal calls
       def process
-        Karafka::App.config.internal.process
+        config.internal.process
       end
     end
   end

data/lib/karafka/setup/config.rb

@@ -105,6 +105,44 @@ module Karafka
       # @see https://github.com/edenhill/librdkafka/blob/master/CONFIGURATION.md
       setting :kafka, default: {}
 
+      # Admin specific settings.
+      #
+      # Since admin operations are often specific, they may require specific librdkafka settings
+      # or other settings that are unique to admin.
+      setting :admin do
+        # Specific kafka settings that are tuned to operate within the Admin.
+        #
+        # Please do not change them unless you know what you are doing as their misconfiguration
+        # may cause Admin API to misbehave
+        # option [Hash] extra changes to the default root kafka settings
+        setting :kafka, default: {
+          # We want to know when there is no more data not to end up with an endless loop
+          'enable.partition.eof': true,
+          # Do not publish statistics from admin as they are not relevant
+          'statistics.interval.ms': 0,
+          # Fetch at most 5 MBs when using admin
+          'fetch.message.max.bytes': 5 * 1_048_576,
+          # Do not commit offset automatically, this prevents offset tracking for operations
+          # involving a consumer instance
+          'enable.auto.commit': false,
+          # Make sure that topic metadata lookups do not create topics accidentally
+          'allow.auto.create.topics': false
+        }
+
+        # option [String] default name for the admin consumer group. Please note, that this is a
+        # subject to be remapped by the consumer mapper as any other consumer group in the routes
+        setting :group_id, default: 'karafka_admin'
+
+        # option max_wait_time [Integer] We wait only for this amount of time before raising error
+        # as we intercept this error and retry after checking that the operation was finished or
+        # failed using external factor.
+        setting :max_wait_time, default: 1_000
+
+        # How many times should be try. 1 000 ms x 60 => 60 seconds wait in total and then we give
+        # up on pending operations
+        setting :max_attempts, default: 60
+      end
+
       # Namespace for internal settings that should not be modified directly
       setting :internal do
         # option status [Karafka::Status] app status
@@ -114,6 +152,13 @@ module Karafka
         #   instances
         setting :process, default: Process.new
 
+        # Namespace for CLI related settings
+        setting :cli do
+          # option contract [Object] cli setup validation contract (in the context of options and
+          # topics)
+          setting :contract, default: Contracts::ServerCliOptions.new
+        end
+
         setting :routing do
           # option builder [Karafka::Routing::Builder] builder instance
           setting :builder, default: Routing::Builder.new

data/lib/karafka/version.rb

@@ -3,5 +3,5 @@
 # Main module namespace
 module Karafka
   # Current Karafka version
-  VERSION = '2.1.13'
+  VERSION = '2.2.0'
 end