karafka 2.1.12 → 2.2.0

data/lib/karafka/pro/contracts/server_cli_options.rb

@@ -0,0 +1,111 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Contracts
+      # Contract for validating correctness of the server cli command options.
+      # It differs slightly from the OSS one because it is aware of the routing patterns
+      class ServerCliOptions < ::Karafka::Contracts::ServerCliOptions
+        configure do |config|
+          config.error_messages = YAML.safe_load(
+            File.read(
+              File.join(Karafka.gem_root, 'config', 'locales', 'errors.yml')
+            )
+          ).fetch('en').fetch('validations').fetch('server_cli_options')
+        end
+
+        %i[
+          include
+          exclude
+        ].each do |action|
+          optional(:"#{action}_consumer_groups") { |cg| cg.is_a?(Array) }
+          optional(:"#{action}_subscription_groups") { |sg| sg.is_a?(Array) }
+          optional(:"#{action}_topics") { |topics| topics.is_a?(Array) }
+
+          virtual do |data, errors|
+            next unless errors.empty?
+
+            value = data.fetch(:"#{action}_consumer_groups")
+
+            # If there were no consumer_groups declared in the server cli, it means that we will
+            # run all of them and no need to validate them here at all
+            next if value.empty?
+            next if (value - Karafka::App.consumer_groups.map(&:name)).empty?
+
+            # Found unknown consumer groups
+            [[[:"#{action}_consumer_groups"], :consumer_groups_inclusion]]
+          end
+
+          virtual do |data, errors|
+            next unless errors.empty?
+
+            value = data.fetch(:"#{action}_subscription_groups")
+
+            # If there were no subscription_groups declared in the server cli, it means that we
+            # will run all of them and no need to validate them here at all
+            next if value.empty?
+
+            subscription_groups = Karafka::App
+                                  .consumer_groups
+                                  .map(&:subscription_groups)
+                                  .flatten
+                                  .map(&:name)
+
+            next if (value - subscription_groups).empty?
+
+            # Found unknown subscription groups
+            [[[:"#{action}_subscription_groups"], :subscription_groups_inclusion]]
+          end
+
+          virtual do |data, errors|
+            next unless errors.empty?
+
+            value = data.fetch(:"#{action}_topics")
+
+            # If there were no topics declared in the server cli, it means that we will
+            # run all of them and no need to validate them here at all
+            next if value.empty?
+
+            topics = Karafka::App
+                     .consumer_groups
+                     .map(&:subscription_groups)
+                     .flatten
+                     .map(&:topics)
+                     .map { |gtopics| gtopics.map(&:name) }
+                     .flatten
+
+            next if (value - topics).empty?
+
+            # If there are any patterns defined, we cannot report on topics inclusions because
+            # topics may be added during boot or runtime. We go with simple assumption:
+            # if there are patterns defined, we do not check the inclusions at all
+            next unless Karafka::App.consumer_groups.map(&:patterns).flatten.empty?
+
+            # Found unknown topics
+            [[[:"#{action}_topics"], :topics_inclusion]]
+          end
+        end
+
+        # Makes sure we have anything to subscribe to when we start the server
+        virtual do |_, errors|
+          next unless errors.empty?
+
+          next unless Karafka::App.subscription_groups.empty?
+
+          [[%i[include_topics], :topics_missing]]
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/encryption/errors.rb

@@ -16,8 +16,11 @@ module Karafka
     module Encryption
       # Encryption related errors
       module Errors
+        # Base for all the encryption errors
+        BaseError = Class.new(::Karafka::Errors::BaseError)
+
         # Raised when we have encountered encryption key with version we do not have
-        PrivateKeyNotFound = Class.new(::Karafka::Errors::BaseError)
+        PrivateKeyNotFound = Class.new(BaseError)
       end
     end
   end

data/lib/karafka/pro/loader.rb

@@ -66,10 +66,12 @@ module Karafka
 
         private
 
-        # @return [Array<Module>] extra non-routing related pro features
+        # @return [Array<Module>] extra non-routing related pro features and routing components
+        #   that need to have some special configuration stuff injected into config, etc
         def features
           [
-            Encryption
+            Encryption,
+            Cleaner
           ]
         end
 
@@ -78,6 +80,8 @@ module Karafka
         def reconfigure(config)
           icfg = config.internal
 
+          icfg.cli.contract = Contracts::ServerCliOptions.new
+
           icfg.processing.coordinator_class = Processing::Coordinator
           icfg.processing.partitioner_class = Processing::Partitioner
           icfg.processing.scheduler = Processing::Scheduler.new

data/lib/karafka/pro/processing/strategies/dlq/default.rb

@@ -74,6 +74,12 @@ module Karafka
             # @param skippable_message [Array<Karafka::Messages::Message>] message we want to
             #   dispatch to DLQ
             def dispatch_to_dlq(skippable_message)
+              # DLQ should never try to dispatch a message that was cleaned. It message was
+              # cleaned, we will not have all the needed data. If you see this error, it means
+              # that your processing flow is not as expected and you have cleaned message that
+              # should not be cleaned as it should go to the DLQ
+              raise(Cleaner::Errors::MessageCleanedError) if skippable_message.cleaned?
+
               producer.produce_async(
                 build_dlq_message(
                   skippable_message

data/lib/karafka/pro/routing/features/active_job/builder.rb

@@ -0,0 +1,45 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Routing
+      module Features
+        class ActiveJob < Base
+          # Pro ActiveJob builder expansions
+          module Builder
+            # This method simplifies routes definition for ActiveJob patterns / queues by
+            # auto-injecting the consumer class and other things needed
+            #
+            # @param regexp_or_name [String, Symbol, Regexp] pattern name or regexp to use
+            #   auto-generated regexp names
+            # @param regexp [Regexp, nil] activejob regexp pattern or nil when regexp is provided
+            #   as the first argument
+            # @param block [Proc] block that we can use for some extra configuration
+            def active_job_pattern(regexp_or_name, regexp = nil, &block)
+              pattern(regexp_or_name, regexp) do
+                consumer App.config.internal.active_job.consumer_class
+                active_job true
+                manual_offset_management true
+
+                next unless block
+
+                instance_eval(&block)
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/routing/features/active_job.rb

@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Routing
+      module Features
+        # Small extension to make ActiveJob work with pattern matching.
+        # Since our `#active_job_topic` is just a topic wrapper, we can introduce a similar
+        # `#active_job_pattern` to align with pattern building
+        class ActiveJob < Base
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/routing/features/dead_letter_queue/contracts/topic.rb

@@ -0,0 +1,53 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Routing
+      module Features
+        class DeadLetterQueue < Base
+          # Namespace for DLQ contracts
+          module Contracts
+            # Extended rules for dead letter queue settings
+            class Topic < Karafka::Contracts::Base
+              configure do |config|
+                config.error_messages = YAML.safe_load(
+                  File.read(
+                    File.join(Karafka.gem_root, 'config', 'locales', 'pro_errors.yml')
+                  )
+                ).fetch('en').fetch('validations').fetch('topic')
+              end
+
+              # Make sure that when we use virtual partitions with DLQ, at least one retry is set
+              # We cannot use VP with DLQ without retries as we in order to provide ordering
+              # warranties on errors with VP, we need to collapse the VPs concurrency and retry
+              # without any indeterministic work
+              virtual do |data, errors|
+                next unless errors.empty?
+
+                dead_letter_queue = data[:dead_letter_queue]
+                virtual_partitions = data[:virtual_partitions]
+
+                next unless dead_letter_queue[:active]
+                next unless virtual_partitions[:active]
+                next if dead_letter_queue[:max_retries].positive?
+
+                [[%i[dead_letter_queue], :with_virtual_partitions]]
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/routing/features/delaying/contracts/topic.rb

@@ -0,0 +1,41 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Routing
+      module Features
+        class Delaying < Base
+          # Namespace for delaying feature contracts
+          module Contracts
+            # Contract to validate configuration of the expiring feature
+            class Topic < Karafka::Contracts::Base
+              configure do |config|
+                config.error_messages = YAML.safe_load(
+                  File.read(
+                    File.join(Karafka.gem_root, 'config', 'locales', 'pro_errors.yml')
+                  )
+                ).fetch('en').fetch('validations').fetch('topic')
+              end
+
+              nested(:delaying) do
+                required(:active) { |val| [true, false].include?(val) }
+                required(:delay) { |val| val.nil? || (val.is_a?(Integer) && val.positive?) }
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/routing/features/expiring/contracts/topic.rb

@@ -0,0 +1,41 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Routing
+      module Features
+        class Expiring < Base
+          # Namespace for expiring messages contracts
+          module Contracts
+            # Contract to validate configuration of the expiring feature
+            class Topic < Karafka::Contracts::Base
+              configure do |config|
+                config.error_messages = YAML.safe_load(
+                  File.read(
+                    File.join(Karafka.gem_root, 'config', 'locales', 'pro_errors.yml')
+                  )
+                ).fetch('en').fetch('validations').fetch('topic')
+              end
+
+              nested(:expiring) do
+                required(:active) { |val| [true, false].include?(val) }
+                required(:ttl) { |val| val.nil? || (val.is_a?(Integer) && val.positive?) }
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/routing/features/filtering/contracts/topic.rb

@@ -0,0 +1,44 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Routing
+      module Features
+        class Filtering < Base
+          # Namespace for filtering feature contracts
+          module Contracts
+            # Contract to validate configuration of the filtering feature
+            class Topic < Karafka::Contracts::Base
+              configure do |config|
+                config.error_messages = YAML.safe_load(
+                  File.read(
+                    File.join(Karafka.gem_root, 'config', 'locales', 'pro_errors.yml')
+                  )
+                ).fetch('en').fetch('validations').fetch('topic')
+              end
+
+              nested(:filtering) do
+                required(:active) { |val| [true, false].include?(val) }
+
+                required(:factories) do |val|
+                  val.is_a?(Array) && val.all? { |factory| factory.respond_to?(:call) }
+                end
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/routing/features/long_running_job/{contract.rb → contracts/topic.rb}

@@ -16,18 +16,21 @@ module Karafka
     module Routing
       module Features
         class LongRunningJob < Base
-          # Rules around long-running job settings
-          class Contract < Contracts::Base
-            configure do |config|
-              config.error_messages = YAML.safe_load(
-                File.read(
-                  File.join(Karafka.gem_root, 'config', 'locales', 'pro_errors.yml')
-                )
-              ).fetch('en').fetch('validations').fetch('topic')
-            end
+          # Namespace for LRJ contracts
+          module Contracts
+            # Rules around long-running job settings
+            class Topic < Karafka::Contracts::Base
+              configure do |config|
+                config.error_messages = YAML.safe_load(
+                  File.read(
+                    File.join(Karafka.gem_root, 'config', 'locales', 'pro_errors.yml')
+                  )
+                ).fetch('en').fetch('validations').fetch('topic')
+              end
 
-            nested(:long_running_job) do
-              required(:active) { |val| [true, false].include?(val) }
+              nested(:long_running_job) do
+                required(:active) { |val| [true, false].include?(val) }
+              end
             end
           end
         end

data/lib/karafka/pro/routing/features/{filtering/contract.rb → patterns/builder.rb}

@@ -15,22 +15,19 @@ module Karafka
   module Pro
     module Routing
       module Features
-        class Filtering < Base
-          # Contract to validate configuration of the filtering feature
-          class Contract < Contracts::Base
-            configure do |config|
-              config.error_messages = YAML.safe_load(
-                File.read(
-                  File.join(Karafka.gem_root, 'config', 'locales', 'pro_errors.yml')
-                )
-              ).fetch('en').fetch('validations').fetch('topic')
-            end
-
-            nested(:filtering) do
-              required(:active) { |val| [true, false].include?(val) }
-
-              required(:factories) do |val|
-                val.is_a?(Array) && val.all? { |factory| factory.respond_to?(:call) }
+        class Patterns < Base
+          # Expansions for the routing builder
+          module Builder
+            # Allows us to define the simple routing pattern matching
+            #
+            # @param regexp_or_name [Symbol, String, Regexp] name of the pattern or regexp for
+            #   automatic-based named patterns
+            # @param regexp [Regexp, nil] nil if we use auto-generated name based on the regexp or
+            #   the regexp if we used named patterns
+            # @param block [Proc]
+            def pattern(regexp_or_name, regexp = nil, &block)
+              consumer_group('app') do
+                pattern(regexp_or_name, regexp, &block)
               end
             end
           end

data/lib/karafka/pro/routing/features/patterns/config.rb

@@ -0,0 +1,54 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Routing
+      module Features
+        # Holds pattern info reference
+        # Type is set to:
+        #   `:regular` - in case patterns are not used and topic is just a regular existing topic
+        #                matched directly based on the name
+        #   `:discovered` - in case it is a real topic on which we started to listed
+        #   `:matcher` - represents a regular expression used by librdkafka
+        class Patterns < Base
+          # Config for pattern based topic
+          # Only pattern related topics are active in this context
+          Config = Struct.new(
+            :active,
+            :type,
+            :pattern,
+            keyword_init: true
+          ) do
+            alias_method :active?, :active
+
+            # @return [Boolean] is this a matcher topic
+            def matcher?
+              type == :matcher
+            end
+
+            # @return [Boolean] is this a discovered topic
+            def discovered?
+              type == :discovered
+            end
+
+            # @return [Boolean] is this a regular topic
+            def regular?
+              type == :regular
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/routing/features/patterns/consumer_group.rb

@@ -0,0 +1,68 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Routing
+      module Features
+        class Patterns < Base
+          # Expansion of the consumer groups routing component to work with patterns
+          module ConsumerGroup
+            # @param args [Object] whatever consumer group accepts
+            def initialize(*args)
+              super
+              @patterns = Patterns.new([])
+            end
+
+            # @return [::Karafka::Pro::Routing::Features::Patterns::Patterns] created patterns
+            def patterns
+              @patterns
+            end
+
+            # Creates the pattern for topic matching with appropriate virtual topic
+            # @param regexp_or_name [Symbol, String, Regexp] name of the pattern or regexp for
+            #   automatic-based named patterns
+            # @param regexp [Regexp, nil] nil if we use auto-generated name based on the regexp or
+            #   the regexp if we used named patterns
+            # @param block [Proc] appropriate underlying topic settings
+            def pattern=(regexp_or_name, regexp = nil, &block)
+              # This code allows us to have a nice nameless (automatic-named) patterns that do not
+              # have to be explicitly named. However if someone wants to use names for exclusions
+              # it can be done by providing both
+              if regexp_or_name.is_a?(Regexp)
+                name = nil
+                regexp = regexp_or_name
+              else
+                name = regexp_or_name
+              end
+
+              pattern = Pattern.new(name, regexp, block)
+              virtual_topic = public_send(:topic=, pattern.name, &block)
+              # Indicate the nature of this topic (matcher)
+              virtual_topic.patterns(active: true, type: :matcher, pattern: pattern)
+              pattern.topic = virtual_topic
+              @patterns << pattern
+            end
+
+            # @return [Hash] consumer group with patterns injected
+            def to_h
+              super.merge(
+                patterns: patterns.map(&:to_h)
+              ).freeze
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/routing/features/patterns/contracts/consumer_group.rb

@@ -0,0 +1,62 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Routing
+      module Features
+        class Patterns < Base
+          module Contracts
+            # Contract to validate configuration of the filtering feature
+            class ConsumerGroup < Karafka::Contracts::Base
+              configure do |config|
+                config.error_messages = YAML.safe_load(
+                  File.read(
+                    File.join(Karafka.gem_root, 'config', 'locales', 'pro_errors.yml')
+                  )
+                ).fetch('en').fetch('validations').fetch('consumer_group')
+
+                required(:patterns) { |val| val.is_a?(Array) && val.all? { |el| el.is_a?(Hash) } }
+
+                virtual do |data, errors|
+                  next unless errors.empty?
+
+                  validator = Pattern.new
+
+                  data[:patterns].each do |pattern|
+                    validator.validate!(pattern)
+                  end
+
+                  nil
+                end
+
+                # Make sure, that there are no same regular expressions with different names
+                # in a single consumer group
+                virtual do |data, errors|
+                  next unless errors.empty?
+
+                  regexp_strings = data[:patterns].map { |pattern| pattern.fetch(:regexp_string) }
+
+                  next if regexp_strings.empty?
+                  next if regexp_strings.uniq.size == regexp_strings.size
+
+                  [[%i[patterns], :regexps_not_unique]]
+                end
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end