karafka 2.1.12 → 2.2.0

data/lib/karafka/routing/features/dead_letter_queue/contracts/topic.rb

@@ -0,0 +1,46 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Routing
+    module Features
+      class DeadLetterQueue < Base
+        # This feature validation contracts
+        module Contracts
+          # Rules around dead letter queue settings
+          class Topic < Karafka::Contracts::Base
+            configure do |config|
+              config.error_messages = YAML.safe_load(
+                File.read(
+                  File.join(Karafka.gem_root, 'config', 'locales', 'errors.yml')
+                )
+              ).fetch('en').fetch('validations').fetch('topic')
+            end
+
+            nested :dead_letter_queue do
+              required(:active) { |val| [true, false].include?(val) }
+              required(:max_retries) { |val| val.is_a?(Integer) && val >= 0 }
+            end
+
+            # Validate topic name only if dlq is active
+            virtual do |data, errors|
+              next unless errors.empty?
+
+              dead_letter_queue = data[:dead_letter_queue]
+
+              next unless dead_letter_queue[:active]
+
+              topic = dead_letter_queue[:topic]
+              topic_regexp = ::Karafka::Contracts::TOPIC_REGEXP
+
+              # When topic is set to false, it means we just want to skip dispatch on DLQ
+              next if topic == false
+              next if topic.is_a?(String) && topic_regexp.match?(topic)
+
+              [[%i[dead_letter_queue topic], :format]]
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/routing/features/declaratives/contracts/topic.rb

@@ -0,0 +1,33 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Routing
+    module Features
+      class Declaratives < Base
+        # This feature validation contracts
+        module Contracts
+          # Basic validation of the Kafka expected config details
+          class Topic < Karafka::Contracts::Base
+            configure do |config|
+              config.error_messages = YAML.safe_load(
+                File.read(
+                  File.join(Karafka.gem_root, 'config', 'locales', 'errors.yml')
+                )
+              ).fetch('en').fetch('validations').fetch('topic')
+            end
+
+            nested :declaratives do
+              required(:active) { |val| [true, false].include?(val) }
+              required(:partitions) { |val| val.is_a?(Integer) && val.positive? }
+              required(:replication_factor) { |val| val.is_a?(Integer) && val.positive? }
+              required(:details) do |val|
+                val.is_a?(Hash) &&
+                  val.keys.all? { |key| key.is_a?(Symbol) }
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/routing/features/manual_offset_management/contracts/topic.rb

@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Routing
+    module Features
+      class ManualOffsetManagement < Base
+        # This feature validation contracts
+        module Contracts
+          # Rules around manual offset management settings
+          class Topic < Karafka::Contracts::Base
+            configure do |config|
+              config.error_messages = YAML.safe_load(
+                File.read(
+                  File.join(Karafka.gem_root, 'config', 'locales', 'errors.yml')
+                )
+              ).fetch('en').fetch('validations').fetch('topic')
+            end
+
+            nested :manual_offset_management do
+              required(:active) { |val| [true, false].include?(val) }
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/routing/router.rb

@@ -7,16 +7,6 @@ module Karafka
     # @note Since Kafka does not provide namespaces or modules for topics, they all have "flat"
     #  structure so all the routes are being stored in a single level array
     module Router
-      # Find a proper topic based on full topic id
-      # @param topic_id [String] proper topic id (already mapped, etc) for which we want to find
-      #   routing topic
-      # @return [Karafka::Routing::Topic] proper route details
-      # @raise [Karafka::Topic::NonMatchingTopicError] raised if topic name does not match
-      #   any route defined by user using routes.draw
-      def find(topic_id)
-        find_by(id: topic_id) || raise(Errors::NonMatchingRouteError, topic_id)
-      end
-
       # Finds first reference of a given topic based on provided lookup attribute
       # @param lookup [Hash<Symbol, String>] hash with attribute - value key pairs
       # @return [Karafka::Routing::Topic, nil] proper route details or nil if not found
@@ -46,7 +36,6 @@ module Karafka
         find_by(name: name) || Topic.new(name, ConsumerGroup.new(name))
       end
 
-      module_function :find
       module_function :find_by
       module_function :find_or_initialize_by_name
     end

data/lib/karafka/routing/subscription_group.rb

@@ -61,6 +61,15 @@ module Karafka
         Karafka::App.config.internal.routing.activity_manager.active?(:subscription_groups, name)
       end
 
+      # @return [Array<String>] names of topics to which we should subscribe.
+      #
+      # @note Most of the time it should not include inactive topics but in case of pattern
+      #   matching the matcher topics become inactive down the road, hence we filter out so
+      #   they are later removed.
+      def subscriptions
+        topics.select(&:active?).map(&:subscription_name)
+      end
+
       private
 
       # @return [Hash] kafka settings are a bit special. They are exactly the same for all of the

data/lib/karafka/routing/topic.rb

@@ -52,6 +52,11 @@ module Karafka
         end
       end
 
+      # @return [String] name of subscription that will go to librdkafka
+      def subscription_name
+        name
+      end
+
       # @return [Class] consumer class that we should use
       def consumer
         if consumer_persistence

data/lib/karafka/server.rb

@@ -32,8 +32,8 @@ module Karafka
         # embedded
         # We cannot validate this during the start because config needs to be populated and routes
         # need to be defined.
-        Contracts::ServerCliOptions.new.validate!(
-          Karafka::App.config.internal.routing.activity_manager.to_h
+        config.internal.cli.contract.validate!(
+          config.internal.routing.activity_manager.to_h
         )
 
         process.on_sigint { stop }
@@ -82,7 +82,7 @@ module Karafka
 
         Karafka::App.stop!
 
-        timeout = Karafka::App.config.shutdown_timeout
+        timeout = config.shutdown_timeout
 
         # We check from time to time (for the timeout period) if all the threads finished
         # their work and if so, we can just return and normal shutdown process will take place
@@ -148,9 +148,14 @@ module Karafka
 
       private
 
+      # @return [Karafka::Core::Configurable::Node] root config node
+      def config
+        Karafka::App.config
+      end
+
       # @return [Karafka::Process] process wrapper instance used to catch system signal calls
       def process
-        Karafka::App.config.internal.process
+        config.internal.process
       end
     end
   end

data/lib/karafka/setup/config.rb

@@ -105,6 +105,44 @@ module Karafka
       # @see https://github.com/edenhill/librdkafka/blob/master/CONFIGURATION.md
       setting :kafka, default: {}
 
+      # Admin specific settings.
+      #
+      # Since admin operations are often specific, they may require specific librdkafka settings
+      # or other settings that are unique to admin.
+      setting :admin do
+        # Specific kafka settings that are tuned to operate within the Admin.
+        #
+        # Please do not change them unless you know what you are doing as their misconfiguration
+        # may cause Admin API to misbehave
+        # option [Hash] extra changes to the default root kafka settings
+        setting :kafka, default: {
+          # We want to know when there is no more data not to end up with an endless loop
+          'enable.partition.eof': true,
+          # Do not publish statistics from admin as they are not relevant
+          'statistics.interval.ms': 0,
+          # Fetch at most 5 MBs when using admin
+          'fetch.message.max.bytes': 5 * 1_048_576,
+          # Do not commit offset automatically, this prevents offset tracking for operations
+          # involving a consumer instance
+          'enable.auto.commit': false,
+          # Make sure that topic metadata lookups do not create topics accidentally
+          'allow.auto.create.topics': false
+        }
+
+        # option [String] default name for the admin consumer group. Please note, that this is a
+        # subject to be remapped by the consumer mapper as any other consumer group in the routes
+        setting :group_id, default: 'karafka_admin'
+
+        # option max_wait_time [Integer] We wait only for this amount of time before raising error
+        # as we intercept this error and retry after checking that the operation was finished or
+        # failed using external factor.
+        setting :max_wait_time, default: 1_000
+
+        # How many times should be try. 1 000 ms x 60 => 60 seconds wait in total and then we give
+        # up on pending operations
+        setting :max_attempts, default: 60
+      end
+
       # Namespace for internal settings that should not be modified directly
       setting :internal do
         # option status [Karafka::Status] app status
@@ -114,6 +152,13 @@ module Karafka
         #   instances
         setting :process, default: Process.new
 
+        # Namespace for CLI related settings
+        setting :cli do
+          # option contract [Object] cli setup validation contract (in the context of options and
+          # topics)
+          setting :contract, default: Contracts::ServerCliOptions.new
+        end
+
         setting :routing do
           # option builder [Karafka::Routing::Builder] builder instance
           setting :builder, default: Routing::Builder.new

data/lib/karafka/version.rb

@@ -3,5 +3,5 @@
 # Main module namespace
 module Karafka
   # Current Karafka version
-  VERSION = '2.1.12'
+  VERSION = '2.2.0'
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: karafka
 version: !ruby/object:Gem::Version
-  version: 2.1.12
+  version: 2.2.0
 platform: ruby
 authors:
 - Maciej Mensfeld
@@ -35,7 +35,7 @@ cert_chain:
   AnG1dJU+yL2BK7vaVytLTstJME5mepSZ46qqIJXMuWob/YPDmVaBF39TDSG9e34s
   msG3BiCqgOgHAnL23+CN3Rt8MsuRfEtoTKpJVcCfoEoNHOkc
   -----END CERTIFICATE-----
-date: 2023-08-25 00:00:00.000000000 Z
+date: 2023-09-01 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: karafka-core
@@ -141,6 +141,7 @@ files:
 - bin/create_token
 - bin/integrations
 - bin/karafka
+- bin/record_rss
 - bin/rspecs
 - bin/scenario
 - bin/stress_many
@@ -219,6 +220,12 @@ files:
 - lib/karafka/pro/active_job/consumer.rb
 - lib/karafka/pro/active_job/dispatcher.rb
 - lib/karafka/pro/active_job/job_options_contract.rb
+- lib/karafka/pro/cleaner.rb
+- lib/karafka/pro/cleaner/errors.rb
+- lib/karafka/pro/cleaner/messages/message.rb
+- lib/karafka/pro/cleaner/messages/messages.rb
+- lib/karafka/pro/contracts/base.rb
+- lib/karafka/pro/contracts/server_cli_options.rb
 - lib/karafka/pro/encryption.rb
 - lib/karafka/pro/encryption/cipher.rb
 - lib/karafka/pro/encryption/contracts/config.rb
@@ -296,35 +303,49 @@ files:
 - lib/karafka/pro/processing/strategies/vp/default.rb
 - lib/karafka/pro/processing/strategy_selector.rb
 - lib/karafka/pro/processing/virtual_offset_manager.rb
+- lib/karafka/pro/routing/features/active_job.rb
+- lib/karafka/pro/routing/features/active_job/builder.rb
 - lib/karafka/pro/routing/features/base.rb
 - lib/karafka/pro/routing/features/dead_letter_queue.rb
-- lib/karafka/pro/routing/features/dead_letter_queue/contract.rb
+- lib/karafka/pro/routing/features/dead_letter_queue/contracts/topic.rb
 - lib/karafka/pro/routing/features/delaying.rb
 - lib/karafka/pro/routing/features/delaying/config.rb
-- lib/karafka/pro/routing/features/delaying/contract.rb
+- lib/karafka/pro/routing/features/delaying/contracts/topic.rb
 - lib/karafka/pro/routing/features/delaying/topic.rb
 - lib/karafka/pro/routing/features/expiring.rb
 - lib/karafka/pro/routing/features/expiring/config.rb
-- lib/karafka/pro/routing/features/expiring/contract.rb
+- lib/karafka/pro/routing/features/expiring/contracts/topic.rb
 - lib/karafka/pro/routing/features/expiring/topic.rb
 - lib/karafka/pro/routing/features/filtering.rb
 - lib/karafka/pro/routing/features/filtering/config.rb
-- lib/karafka/pro/routing/features/filtering/contract.rb
+- lib/karafka/pro/routing/features/filtering/contracts/topic.rb
 - lib/karafka/pro/routing/features/filtering/topic.rb
 - lib/karafka/pro/routing/features/long_running_job.rb
 - lib/karafka/pro/routing/features/long_running_job/config.rb
-- lib/karafka/pro/routing/features/long_running_job/contract.rb
+- lib/karafka/pro/routing/features/long_running_job/contracts/topic.rb
 - lib/karafka/pro/routing/features/long_running_job/topic.rb
+- lib/karafka/pro/routing/features/patterns.rb
+- lib/karafka/pro/routing/features/patterns/builder.rb
+- lib/karafka/pro/routing/features/patterns/config.rb
+- lib/karafka/pro/routing/features/patterns/consumer_group.rb
+- lib/karafka/pro/routing/features/patterns/contracts/consumer_group.rb
+- lib/karafka/pro/routing/features/patterns/contracts/pattern.rb
+- lib/karafka/pro/routing/features/patterns/contracts/topic.rb
+- lib/karafka/pro/routing/features/patterns/detector.rb
+- lib/karafka/pro/routing/features/patterns/pattern.rb
+- lib/karafka/pro/routing/features/patterns/patterns.rb
+- lib/karafka/pro/routing/features/patterns/topic.rb
+- lib/karafka/pro/routing/features/patterns/topics.rb
 - lib/karafka/pro/routing/features/pausing.rb
-- lib/karafka/pro/routing/features/pausing/contract.rb
+- lib/karafka/pro/routing/features/pausing/contracts/topic.rb
 - lib/karafka/pro/routing/features/pausing/topic.rb
 - lib/karafka/pro/routing/features/throttling.rb
 - lib/karafka/pro/routing/features/throttling/config.rb
-- lib/karafka/pro/routing/features/throttling/contract.rb
+- lib/karafka/pro/routing/features/throttling/contracts/topic.rb
 - lib/karafka/pro/routing/features/throttling/topic.rb
 - lib/karafka/pro/routing/features/virtual_partitions.rb
 - lib/karafka/pro/routing/features/virtual_partitions/config.rb
-- lib/karafka/pro/routing/features/virtual_partitions/contract.rb
+- lib/karafka/pro/routing/features/virtual_partitions/contracts/topic.rb
 - lib/karafka/pro/routing/features/virtual_partitions/topic.rb
 - lib/karafka/process.rb
 - lib/karafka/processing/coordinator.rb
@@ -359,21 +380,22 @@ files:
 - lib/karafka/routing/features/active_job.rb
 - lib/karafka/routing/features/active_job/builder.rb
 - lib/karafka/routing/features/active_job/config.rb
-- lib/karafka/routing/features/active_job/contract.rb
+- lib/karafka/routing/features/active_job/contracts/topic.rb
+- lib/karafka/routing/features/active_job/proxy.rb
 - lib/karafka/routing/features/active_job/topic.rb
 - lib/karafka/routing/features/base.rb
 - lib/karafka/routing/features/base/expander.rb
 - lib/karafka/routing/features/dead_letter_queue.rb
 - lib/karafka/routing/features/dead_letter_queue/config.rb
-- lib/karafka/routing/features/dead_letter_queue/contract.rb
+- lib/karafka/routing/features/dead_letter_queue/contracts/topic.rb
 - lib/karafka/routing/features/dead_letter_queue/topic.rb
 - lib/karafka/routing/features/declaratives.rb
 - lib/karafka/routing/features/declaratives/config.rb
-- lib/karafka/routing/features/declaratives/contract.rb
+- lib/karafka/routing/features/declaratives/contracts/topic.rb
 - lib/karafka/routing/features/declaratives/topic.rb
 - lib/karafka/routing/features/manual_offset_management.rb
 - lib/karafka/routing/features/manual_offset_management/config.rb
-- lib/karafka/routing/features/manual_offset_management/contract.rb
+- lib/karafka/routing/features/manual_offset_management/contracts/topic.rb
 - lib/karafka/routing/features/manual_offset_management/topic.rb
 - lib/karafka/routing/proxy.rb
 - lib/karafka/routing/router.rb
@@ -424,7 +446,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.4.10
+rubygems_version: 3.4.19
 signing_key: 
 specification_version: 4
 summary: Karafka is Ruby and Rails efficient Kafka processing framework.

data/lib/karafka/pro/routing/features/dead_letter_queue/contract.rb

@@ -1,50 +0,0 @@
-# frozen_string_literal: true
-
-# This Karafka component is a Pro component under a commercial license.
-# This Karafka component is NOT licensed under LGPL.
-#
-# All of the commercial components are present in the lib/karafka/pro directory of this
-# repository and their usage requires commercial license agreement.
-#
-# Karafka has also commercial-friendly license, commercial support and commercial components.
-#
-# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
-# your code to Maciej Mensfeld.
-
-module Karafka
-  module Pro
-    module Routing
-      module Features
-        class DeadLetterQueue < Base
-          # Extended rules for dead letter queue settings
-          class Contract < Contracts::Base
-            configure do |config|
-              config.error_messages = YAML.safe_load(
-                File.read(
-                  File.join(Karafka.gem_root, 'config', 'locales', 'pro_errors.yml')
-                )
-              ).fetch('en').fetch('validations').fetch('topic')
-            end
-
-            # Make sure that when we use virtual partitions with DLQ, at least one retry is set
-            # We cannot use VP with DLQ without retries as we in order to provide ordering
-            # warranties on errors with VP, we need to collapse the VPs concurrency and retry
-            # without any indeterministic work
-            virtual do |data, errors|
-              next unless errors.empty?
-
-              dead_letter_queue = data[:dead_letter_queue]
-              virtual_partitions = data[:virtual_partitions]
-
-              next unless dead_letter_queue[:active]
-              next unless virtual_partitions[:active]
-              next if dead_letter_queue[:max_retries].positive?
-
-              [[%i[dead_letter_queue], :with_virtual_partitions]]
-            end
-          end
-        end
-      end
-    end
-  end
-end

data/lib/karafka/pro/routing/features/expiring/contract.rb

@@ -1,38 +0,0 @@
-# frozen_string_literal: true
-
-# This Karafka component is a Pro component under a commercial license.
-# This Karafka component is NOT licensed under LGPL.
-#
-# All of the commercial components are present in the lib/karafka/pro directory of this
-# repository and their usage requires commercial license agreement.
-#
-# Karafka has also commercial-friendly license, commercial support and commercial components.
-#
-# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
-# your code to Maciej Mensfeld.
-
-module Karafka
-  module Pro
-    module Routing
-      module Features
-        class Expiring < Base
-          # Contract to validate configuration of the expiring feature
-          class Contract < Contracts::Base
-            configure do |config|
-              config.error_messages = YAML.safe_load(
-                File.read(
-                  File.join(Karafka.gem_root, 'config', 'locales', 'pro_errors.yml')
-                )
-              ).fetch('en').fetch('validations').fetch('topic')
-            end
-
-            nested(:expiring) do
-              required(:active) { |val| [true, false].include?(val) }
-              required(:ttl) { |val| val.nil? || (val.is_a?(Integer) && val.positive?) }
-            end
-          end
-        end
-      end
-    end
-  end
-end

data/lib/karafka/pro/routing/features/pausing/contract.rb

@@ -1,48 +0,0 @@
-# frozen_string_literal: true
-
-# This Karafka component is a Pro component under a commercial license.
-# This Karafka component is NOT licensed under LGPL.
-#
-# All of the commercial components are present in the lib/karafka/pro directory of this
-# repository and their usage requires commercial license agreement.
-#
-# Karafka has also commercial-friendly license, commercial support and commercial components.
-#
-# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
-# your code to Maciej Mensfeld.
-
-module Karafka
-  module Pro
-    module Routing
-      module Features
-        class Pausing < Base
-          # Contract to make sure, that the pause settings on a per topic basis are as expected
-          class Contract < Contracts::Base
-            configure do |config|
-              config.error_messages = YAML.safe_load(
-                File.read(
-                  File.join(Karafka.gem_root, 'config', 'locales', 'pro_errors.yml')
-                )
-              ).fetch('en').fetch('validations').fetch('topic')
-
-              required(:pause_timeout) { |val| val.is_a?(Integer) && val.positive? }
-              required(:pause_max_timeout) { |val| val.is_a?(Integer) && val.positive? }
-              required(:pause_with_exponential_backoff) { |val| [true, false].include?(val) }
-
-              virtual do |data, errors|
-                next unless errors.empty?
-
-                pause_timeout = data.fetch(:pause_timeout)
-                pause_max_timeout = data.fetch(:pause_max_timeout)
-
-                next if pause_timeout <= pause_max_timeout
-
-                [[%i[pause_timeout], :max_timeout_vs_pause_max_timeout]]
-              end
-            end
-          end
-        end
-      end
-    end
-  end
-end

data/lib/karafka/pro/routing/features/throttling/contract.rb

@@ -1,41 +0,0 @@
-# frozen_string_literal: true
-
-# This Karafka component is a Pro component under a commercial license.
-# This Karafka component is NOT licensed under LGPL.
-#
-# All of the commercial components are present in the lib/karafka/pro directory of this
-# repository and their usage requires commercial license agreement.
-#
-# Karafka has also commercial-friendly license, commercial support and commercial components.
-#
-# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
-# your code to Maciej Mensfeld.
-
-module Karafka
-  module Pro
-    module Routing
-      module Features
-        class Throttling < Base
-          # Rules around throttling settings
-          class Contract < Contracts::Base
-            configure do |config|
-              config.error_messages = YAML.safe_load(
-                File.read(
-                  File.join(Karafka.gem_root, 'config', 'locales', 'pro_errors.yml')
-                )
-              ).fetch('en').fetch('validations').fetch('topic')
-            end
-
-            nested(:throttling) do
-              required(:active) { |val| [true, false].include?(val) }
-              required(:interval) { |val| val.is_a?(Integer) && val.positive? }
-              required(:limit) do |val|
-                (val.is_a?(Integer) || val == Float::INFINITY) && val.positive?
-              end
-            end
-          end
-        end
-      end
-    end
-  end
-end

data/lib/karafka/pro/routing/features/virtual_partitions/contract.rb

@@ -1,52 +0,0 @@
-# frozen_string_literal: true
-
-# This Karafka component is a Pro component under a commercial license.
-# This Karafka component is NOT licensed under LGPL.
-#
-# All of the commercial components are present in the lib/karafka/pro directory of this
-# repository and their usage requires commercial license agreement.
-#
-# Karafka has also commercial-friendly license, commercial support and commercial components.
-#
-# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
-# your code to Maciej Mensfeld.
-
-module Karafka
-  module Pro
-    module Routing
-      module Features
-        class VirtualPartitions < Base
-          # Rules around virtual partitions
-          class Contract < Contracts::Base
-            configure do |config|
-              config.error_messages = YAML.safe_load(
-                File.read(
-                  File.join(Karafka.gem_root, 'config', 'locales', 'pro_errors.yml')
-                )
-              ).fetch('en').fetch('validations').fetch('topic')
-            end
-
-            nested(:virtual_partitions) do
-              required(:active) { |val| [true, false].include?(val) }
-              required(:partitioner) { |val| val.nil? || val.respond_to?(:call) }
-              required(:max_partitions) { |val| val.is_a?(Integer) && val >= 1 }
-            end
-
-            # When virtual partitions are defined, partitioner needs to respond to `#call` and it
-            # cannot be nil
-            virtual do |data, errors|
-              next unless errors.empty?
-
-              virtual_partitions = data[:virtual_partitions]
-
-              next unless virtual_partitions[:active]
-              next if virtual_partitions[:partitioner].respond_to?(:call)
-
-              [[%i[virtual_partitions partitioner], :respond_to_call]]
-            end
-          end
-        end
-      end
-    end
-  end
-end