karafka 2.1.13 → 2.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 262c2bbfa4fb09c4038ce559c71baa6c40b861497dd7ef3b1915ba6b8aa47652
-  data.tar.gz: a34d5fad3bdbd1a58a0938f33c8fd28a5df72cb89293d7835c41cfce09c9e736
+  metadata.gz: 6ca6426dc8527aac122a1d9ebcaccf33a4eff1608133b210e68e66d3a5f5c2c7
+  data.tar.gz: 85233cc04e591e5a96d53b3b9d3fe203b4b56bbde24c564ffa3663daeb3673c6
 SHA512:
-  metadata.gz: 2fd4672fef274f5913b543d9b3a91aef8a8aba59a23e457a61f455cf6095ea8176c6f8e799f65602c7ef0bb15add27a960df70cd99913fe6b5340df9cfff8c31
-  data.tar.gz: 4b890531f2a783c72573bcc2e915cc70b85e09c5e43de76b979fc34ea732f51a2c8ec4f64e91c95293337973103c7eeee89e424d27fa35dc708f935364fa9db3
+  metadata.gz: e89b72adccdb6a622d571ab9c580191e725cdb98d88f5d86f59d9e4a9900eff74fa438b3838cfe104f7259c0c4cee0dc60f7fb6d16cf14b59ff9229170bae504
+  data.tar.gz: a8ba4b92e1002c6d1112837e1b96706208ba0d55fa39feaadaa4f04f4c616095eb406d72bffa50bd346fb1daff7ca45951efd2237a71c25a7539be5a4397a781

data/CHANGELOG.md

@@ -1,5 +1,37 @@
 # Karafka framework changelog
 
+## 2.2.0 (2023-09-01)
+- **[Feature]** Introduce dynamic topic subscriptions based on patterns [Pro].
+- [Enhancement] Allow for `Karafka::Admin` setup reconfiguration via `config.admin` scope.
+- [Enhancement] Make sure that consumer group used by `Karafka::Admin` obeys the `ConsumerMapper` setup.
+- [Fix] Fix a case where subscription group would not accept a symbol name.
+
+### Upgrade notes
+
+As always, please make sure you have upgraded to the most recent version of `2.1` before upgrading to `2.2`.
+
+If you are not using Kafka ACLs, there is no action you need to take.
+
+If you are using Kafka ACLs and you've set up permissions for `karafka_admin` group, please note that this name has now been changed and is subject to [Consumer Name Mapping](https://karafka.io/docs/Consumer-mappers/).
+
+That means you must ensure that the new consumer group that by default equals `CLIENT_ID_karafka_admin` has appropriate permissions. Please note that the Web UI also uses this group.
+
+`Karafka::Admin` now has its own set of configuration options available, and you can find more details about that [here](https://karafka.io/docs/Topics-management-and-administration/#configuration).
+
+If you want to maintain the `2.1` behavior, that is `karafka_admin` admin group, we recommend introducing this case inside your consumer mapper. Assuming you use the default one, the code will look as follows:
+
+```ruby
+  class MyMapper
+    def call(raw_consumer_group_name)
+      # If group is the admin one, use as it was in 2.1
+      return 'karafka_admin' if raw_consumer_group_name == 'karafka_admin'
+
+      # Otherwise use default karafka strategy for the rest
+      "#{Karafka::App.config.client_id}_#{raw_consumer_group_name}"
+    end
+  end
+```
+
 ## 2.1.13 (2023-08-28)
 - **[Feature]** Introduce Cleaning API for much better memory management for iterative data processing [Pro].
 - [Enhancement] Automatically free message resources after processed for ActiveJob jobs [Pro]
@@ -15,7 +47,10 @@
 
 ## 2.1.10 (2023-08-21)
 - [Enhancement] Introduce `connection.client.rebalance_callback` event for instrumentation of rebalances.
+- [Enhancement] Introduce new `runner.before_call` monitor event.
 - [Refactor] Introduce low level commands proxy to handle deviation in how we want to run certain commands and how rdkafka-ruby runs that by design.
+- [Change] No longer validate excluded topics routing presence if patterns any as it does not match pattern subscriptions where you can exclude things that could be subscribed in the future.
+- [Fix] do not report negative lag stored in the DD listener.
 - [Fix] Do not report lags in the DD listener for cases where the assignment is not workable.
 - [Fix] Do not report negative lags in the DD listener.
 - [Fix] Extremely fast shutdown after boot in specs can cause process not to stop.

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    karafka (2.1.13)
+    karafka (2.2.0)
       karafka-core (>= 2.1.1, < 2.2.0)
       thor (>= 0.20)
       waterdrop (>= 2.6.6, < 3.0.0)

data/config/locales/errors.yml

@@ -35,6 +35,10 @@ en:
       key_must_be_a_symbol: All keys under the kafka settings scope need to be symbols
       max_timeout_vs_pause_max_timeout: pause_timeout must be less or equal to pause_max_timeout
       shutdown_timeout_vs_max_wait_time: shutdown_timeout must be more than max_wait_time
+      admin.kafka_format: needs to be a hash
+      admin.group_id_format: 'needs to be a string with a Kafka accepted format'
+      admin.max_wait_time_format: 'needs to be an integer bigger than 0'
+      admin.max_attempts_format: 'needs to be an integer bigger than 0'
 
     server_cli_options:
       missing: needs to be present

data/config/locales/pro_errors.yml

@@ -28,6 +28,20 @@ en:
       pause_with_exponential_backoff_format: needs to be either true or false
       pause_timeout_max_timeout_vs_pause_max_timeout: pause_timeout must be less or equal to pause_max_timeout
 
+      patterns.active_format: 'needs to be boolean'
+      patterns.type_format: 'needs to be :matcher, :discovered or :regular'
+
+    consumer_group:
+      patterns_format: must be an array with hashes
+      patterns_missing: needs to be present
+      patterns_regexps_not_unique: 'must be unique within consumer group'
+
+    pattern:
+      regexp_format: must be a regular expression
+      name_format: 'needs to be a string with a Kafka accepted format'
+      regexp_string_format: 'needs to be a string and start with ^'
+      missing: needs to be present
+
     config:
       encryption.active_format: 'needs to be either true or false'
       encryption.public_key_invalid: 'is not a valid public RSA key'
@@ -37,3 +51,6 @@ en:
       encryption.version_format: must be a non-empty string
       encryption.public_key_format: 'is not a valid public RSA key'
       encryption.private_keys_invalid: 'contains an invalid private RSA key string'
+
+      patterns.ttl_format: needs to be an integer bigger than 0
+      patterns.ttl_missing: needs to be present

data/lib/karafka/admin.rb

@@ -7,32 +7,9 @@ module Karafka
   #   Since admin actions are not performed that often, that should be ok.
   #
   # @note It always uses the primary defined cluster and does not support multi-cluster work.
-  #   If you need this, just replace the cluster info for the time you use this
+  #   Cluster on which operations are performed can be changed via `admin.kafka` config, however
+  #   there is no multi-cluster runtime support.
   module Admin
-    # We wait only for this amount of time before raising error as we intercept this error and
-    # retry after checking that the operation was finished or failed using external factor.
-    MAX_WAIT_TIMEOUT = 1
-
-    # How many times should be try. 1 x 60 => 60 seconds wait in total
-    MAX_ATTEMPTS = 60
-
-    # Defaults for config
-    CONFIG_DEFAULTS = {
-      'group.id': 'karafka_admin',
-      # We want to know when there is no more data not to end up with an endless loop
-      'enable.partition.eof': true,
-      'statistics.interval.ms': 0,
-      # Fetch at most 5 MBs when using admin
-      'fetch.message.max.bytes': 5 * 1_048_576,
-      # Do not commit offset automatically, this prevents offset tracking for operations involving
-      # a consumer instance
-      'enable.auto.commit': false,
-      # Make sure that topic metadata lookups do not create topics accidentally
-      'allow.auto.create.topics': false
-    }.freeze
-
-    private_constant :CONFIG_DEFAULTS, :MAX_WAIT_TIMEOUT, :MAX_ATTEMPTS
-
     class << self
       # Allows us to read messages from the topic
       #
@@ -123,7 +100,7 @@ module Karafka
           handler = admin.create_topic(name, partitions, replication_factor, topic_config)
 
           with_re_wait(
-            -> { handler.wait(max_wait_timeout: MAX_WAIT_TIMEOUT) },
+            -> { handler.wait(max_wait_timeout: app_config.admin.max_wait_time) },
             -> { topics_names.include?(name) }
           )
         end
@@ -137,7 +114,7 @@ module Karafka
           handler = admin.delete_topic(name)
 
           with_re_wait(
-            -> { handler.wait(max_wait_timeout: MAX_WAIT_TIMEOUT) },
+            -> { handler.wait(max_wait_timeout: app_config.admin.max_wait_time) },
             -> { !topics_names.include?(name) }
           )
         end
@@ -152,7 +129,7 @@ module Karafka
           handler = admin.create_partitions(name, partitions)
 
           with_re_wait(
-            -> { handler.wait(max_wait_timeout: MAX_WAIT_TIMEOUT) },
+            -> { handler.wait(max_wait_timeout: app_config.admin.max_wait_time) },
             -> { topic(name).fetch(:partition_count) >= partitions }
           )
         end
@@ -242,7 +219,7 @@ module Karafka
       rescue Rdkafka::AbstractHandle::WaitTimeoutError
         return if breaker.call
 
-        retry if attempt <= MAX_ATTEMPTS
+        retry if attempt <= app_config.admin.max_attempts
 
         raise
       end
@@ -251,12 +228,18 @@ module Karafka
       # @param settings [Hash] extra settings for config (if needed)
       # @return [::Rdkafka::Config] rdkafka config
       def config(type, settings)
-        config_hash = Karafka::Setup::AttributesMap.public_send(
-          type,
-          Karafka::App.config.kafka.dup.merge(CONFIG_DEFAULTS).merge!(settings)
+        group_id = app_config.consumer_mapper.call(
+          app_config.admin.group_id
         )
 
-        ::Rdkafka::Config.new(config_hash)
+        app_config
+          .kafka
+          .then(&:dup)
+          .merge(app_config.admin.kafka)
+          .merge!(settings)
+          .tap { |config| config[:'group.id'] = group_id }
+          .then { |config| Karafka::Setup::AttributesMap.public_send(type, config) }
+          .then { |config| ::Rdkafka::Config.new(config) }
       end
 
       # Resolves the offset if offset is in a time format. Otherwise returns the offset without
@@ -281,6 +264,11 @@ module Karafka
           offset
         end
       end
+
+      # @return [Karafka::Core::Configurable::Node] root node config
+      def app_config
+        ::Karafka::App.config
+      end
     end
   end
 end

data/lib/karafka/connection/client.rb

@@ -510,7 +510,7 @@ module Karafka
 
         # Subscription needs to happen after we assigned the rebalance callbacks just in case of
         # a race condition
-        consumer.subscribe(*@subscription_group.topics.map(&:name))
+        consumer.subscribe(*@subscription_group.subscriptions)
         consumer
       end
 

data/lib/karafka/contracts/config.rb

@@ -34,6 +34,14 @@ module Karafka
       required(:max_wait_time) { |val| val.is_a?(Integer) && val.positive? }
       required(:kafka) { |val| val.is_a?(Hash) && !val.empty? }
 
+      nested(:admin) do
+        # Can be empty because inherits values from the root kafka
+        required(:kafka) { |val| val.is_a?(Hash) }
+        required(:group_id) { |val| val.is_a?(String) && Contracts::TOPIC_REGEXP.match?(val) }
+        required(:max_wait_time) { |val| val.is_a?(Integer) && val.positive? }
+        required(:max_attempts) { |val| val.is_a?(Integer) && val.positive? }
+      end
+
       # We validate internals just to be sure, that they are present and working
       nested(:internal) do
         required(:status) { |val| !val.nil? }
@@ -74,6 +82,7 @@ module Karafka
         end
       end
 
+      # Ensure all root kafka keys are symbols
       virtual do |data, errors|
         next unless errors.empty?
 
@@ -88,6 +97,21 @@ module Karafka
         detected_errors
       end
 
+      # Ensure all admin kafka keys are symbols
+      virtual do |data, errors|
+        next unless errors.empty?
+
+        detected_errors = []
+
+        data.fetch(:admin).fetch(:kafka).each_key do |key|
+          next if key.is_a?(Symbol)
+
+          detected_errors << [[:admin, :kafka, key], :key_must_be_a_symbol]
+        end
+
+        detected_errors
+      end
+
       virtual do |data, errors|
         next unless errors.empty?
 

data/lib/karafka/pro/contracts/base.rb

@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    # Pro contracts that aim to replace or complement the general framework contracts
+    module Contracts
+      # Base for all the Pro contracts
+      class Base < ::Karafka::Contracts::Base
+      end
+    end
+  end
+end

data/lib/karafka/pro/contracts/server_cli_options.rb

@@ -0,0 +1,111 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Contracts
+      # Contract for validating correctness of the server cli command options.
+      # It differs slightly from the OSS one because it is aware of the routing patterns
+      class ServerCliOptions < ::Karafka::Contracts::ServerCliOptions
+        configure do |config|
+          config.error_messages = YAML.safe_load(
+            File.read(
+              File.join(Karafka.gem_root, 'config', 'locales', 'errors.yml')
+            )
+          ).fetch('en').fetch('validations').fetch('server_cli_options')
+        end
+
+        %i[
+          include
+          exclude
+        ].each do |action|
+          optional(:"#{action}_consumer_groups") { |cg| cg.is_a?(Array) }
+          optional(:"#{action}_subscription_groups") { |sg| sg.is_a?(Array) }
+          optional(:"#{action}_topics") { |topics| topics.is_a?(Array) }
+
+          virtual do |data, errors|
+            next unless errors.empty?
+
+            value = data.fetch(:"#{action}_consumer_groups")
+
+            # If there were no consumer_groups declared in the server cli, it means that we will
+            # run all of them and no need to validate them here at all
+            next if value.empty?
+            next if (value - Karafka::App.consumer_groups.map(&:name)).empty?
+
+            # Found unknown consumer groups
+            [[[:"#{action}_consumer_groups"], :consumer_groups_inclusion]]
+          end
+
+          virtual do |data, errors|
+            next unless errors.empty?
+
+            value = data.fetch(:"#{action}_subscription_groups")
+
+            # If there were no subscription_groups declared in the server cli, it means that we
+            # will run all of them and no need to validate them here at all
+            next if value.empty?
+
+            subscription_groups = Karafka::App
+                                  .consumer_groups
+                                  .map(&:subscription_groups)
+                                  .flatten
+                                  .map(&:name)
+
+            next if (value - subscription_groups).empty?
+
+            # Found unknown subscription groups
+            [[[:"#{action}_subscription_groups"], :subscription_groups_inclusion]]
+          end
+
+          virtual do |data, errors|
+            next unless errors.empty?
+
+            value = data.fetch(:"#{action}_topics")
+
+            # If there were no topics declared in the server cli, it means that we will
+            # run all of them and no need to validate them here at all
+            next if value.empty?
+
+            topics = Karafka::App
+                     .consumer_groups
+                     .map(&:subscription_groups)
+                     .flatten
+                     .map(&:topics)
+                     .map { |gtopics| gtopics.map(&:name) }
+                     .flatten
+
+            next if (value - topics).empty?
+
+            # If there are any patterns defined, we cannot report on topics inclusions because
+            # topics may be added during boot or runtime. We go with simple assumption:
+            # if there are patterns defined, we do not check the inclusions at all
+            next unless Karafka::App.consumer_groups.map(&:patterns).flatten.empty?
+
+            # Found unknown topics
+            [[[:"#{action}_topics"], :topics_inclusion]]
+          end
+        end
+
+        # Makes sure we have anything to subscribe to when we start the server
+        virtual do |_, errors|
+          next unless errors.empty?
+
+          next unless Karafka::App.subscription_groups.empty?
+
+          [[%i[include_topics], :topics_missing]]
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/loader.rb

@@ -66,7 +66,8 @@ module Karafka
 
         private
 
-        # @return [Array<Module>] extra non-routing related pro features
+        # @return [Array<Module>] extra non-routing related pro features and routing components
+        #   that need to have some special configuration stuff injected into config, etc
         def features
           [
             Encryption,
@@ -79,6 +80,8 @@ module Karafka
         def reconfigure(config)
           icfg = config.internal
 
+          icfg.cli.contract = Contracts::ServerCliOptions.new
+
           icfg.processing.coordinator_class = Processing::Coordinator
           icfg.processing.partitioner_class = Processing::Partitioner
           icfg.processing.scheduler = Processing::Scheduler.new

data/lib/karafka/pro/routing/features/active_job/builder.rb

@@ -0,0 +1,45 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Routing
+      module Features
+        class ActiveJob < Base
+          # Pro ActiveJob builder expansions
+          module Builder
+            # This method simplifies routes definition for ActiveJob patterns / queues by
+            # auto-injecting the consumer class and other things needed
+            #
+            # @param regexp_or_name [String, Symbol, Regexp] pattern name or regexp to use
+            #   auto-generated regexp names
+            # @param regexp [Regexp, nil] activejob regexp pattern or nil when regexp is provided
+            #   as the first argument
+            # @param block [Proc] block that we can use for some extra configuration
+            def active_job_pattern(regexp_or_name, regexp = nil, &block)
+              pattern(regexp_or_name, regexp) do
+                consumer App.config.internal.active_job.consumer_class
+                active_job true
+                manual_offset_management true
+
+                next unless block
+
+                instance_eval(&block)
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/routing/features/active_job.rb

@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Routing
+      module Features
+        # Small extension to make ActiveJob work with pattern matching.
+        # Since our `#active_job_topic` is just a topic wrapper, we can introduce a similar
+        # `#active_job_pattern` to align with pattern building
+        class ActiveJob < Base
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/routing/features/dead_letter_queue/contracts/topic.rb

@@ -0,0 +1,53 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Routing
+      module Features
+        class DeadLetterQueue < Base
+          # Namespace for DLQ contracts
+          module Contracts
+            # Extended rules for dead letter queue settings
+            class Topic < Karafka::Contracts::Base
+              configure do |config|
+                config.error_messages = YAML.safe_load(
+                  File.read(
+                    File.join(Karafka.gem_root, 'config', 'locales', 'pro_errors.yml')
+                  )
+                ).fetch('en').fetch('validations').fetch('topic')
+              end
+
+              # Make sure that when we use virtual partitions with DLQ, at least one retry is set
+              # We cannot use VP with DLQ without retries as we in order to provide ordering
+              # warranties on errors with VP, we need to collapse the VPs concurrency and retry
+              # without any indeterministic work
+              virtual do |data, errors|
+                next unless errors.empty?
+
+                dead_letter_queue = data[:dead_letter_queue]
+                virtual_partitions = data[:virtual_partitions]
+
+                next unless dead_letter_queue[:active]
+                next unless virtual_partitions[:active]
+                next if dead_letter_queue[:max_retries].positive?
+
+                [[%i[dead_letter_queue], :with_virtual_partitions]]
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/routing/features/delaying/contracts/topic.rb

@@ -0,0 +1,41 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Routing
+      module Features
+        class Delaying < Base
+          # Namespace for delaying feature contracts
+          module Contracts
+            # Contract to validate configuration of the expiring feature
+            class Topic < Karafka::Contracts::Base
+              configure do |config|
+                config.error_messages = YAML.safe_load(
+                  File.read(
+                    File.join(Karafka.gem_root, 'config', 'locales', 'pro_errors.yml')
+                  )
+                ).fetch('en').fetch('validations').fetch('topic')
+              end
+
+              nested(:delaying) do
+                required(:active) { |val| [true, false].include?(val) }
+                required(:delay) { |val| val.nil? || (val.is_a?(Integer) && val.positive?) }
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/routing/features/expiring/contracts/topic.rb

@@ -0,0 +1,41 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Routing
+      module Features
+        class Expiring < Base
+          # Namespace for expiring messages contracts
+          module Contracts
+            # Contract to validate configuration of the expiring feature
+            class Topic < Karafka::Contracts::Base
+              configure do |config|
+                config.error_messages = YAML.safe_load(
+                  File.read(
+                    File.join(Karafka.gem_root, 'config', 'locales', 'pro_errors.yml')
+                  )
+                ).fetch('en').fetch('validations').fetch('topic')
+              end
+
+              nested(:expiring) do
+                required(:active) { |val| [true, false].include?(val) }
+                required(:ttl) { |val| val.nil? || (val.is_a?(Integer) && val.positive?) }
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end