karafka 2.0.21 → 2.0.22

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 9de6cb2d19b24258eb60548b383e664dc035f1f48ffc7077ca5f238b3a85aa3e
-  data.tar.gz: 706d4e81a74b1aad9d1f9b13a67d168a7ef3c45088b89ec710d29abb3a767f11
+  metadata.gz: 6a096a44a1a2988ff394215b8d63f6f2e33b2a3e5100f68f75fcf33eeda4b490
+  data.tar.gz: 2af0875b550f37ef9ea47b2a3011a3ed3e535adc3f6632ab10510d91bc69590c
 SHA512:
-  metadata.gz: a22be6ff489f9f7cce39e0e4d6d374a70680124842afeeb962e33eaa8c6cd3db8cbcd6f3251543653c98777d025b6d9b0385f4580d92fd4b807e835a68296a43
-  data.tar.gz: 466d9723427771594701071e5a27a3a1a53b6b4efce9ffa6b241f5261c9a887136b2a8a4a90b35e216b2af01539336bde5377f4edc1ab99837087dd751b830a8
+  metadata.gz: a81cf3305482e0e925f3a5aa48d978585766826e0990081d74f5f56bc535ae0a8a6be3a4ec986e6a7702d70d0d207ca9acc71b7aeb3909557b44518413ff9bc6
+  data.tar.gz: 6c66140b258135367a120a5dac4eb8a08f4dc94e24fb3a714b32ec6ad909bc9f85bf5e160aefe5f5d307c9d554c6ecab61cbdebf315ce6953476303e9783122c

data/.github/workflows/ci.yml

@@ -101,6 +101,9 @@ jobs:
       - name: Install package dependencies
         run: "[ -e $APT_DEPS ] || sudo apt-get install -y --no-install-recommends $APT_DEPS"
 
+      - name: Remove libzstd-dev to check no supported compressions
+        run: sudo apt-get -y remove libzstd-dev
+
       - name: Start Kafka with docker-compose
         run: |
           docker-compose up -d
@@ -124,5 +127,8 @@ jobs:
       - name: Run integration tests
         env:
           KARAFKA_PRO_LICENSE_TOKEN: ${{ secrets.KARAFKA_PRO_LICENSE_TOKEN }}
+          KARAFKA_PRO_USERNAME: ${{ secrets.KARAFKA_PRO_USERNAME }}
+          KARAFKA_PRO_PASSWORD: ${{ secrets.KARAFKA_PRO_PASSWORD }}
+          KARAFKA_PRO_VERSION: ${{ secrets.KARAFKA_PRO_VERSION }}
           GITHUB_COVERAGE: ${{matrix.coverage}}
         run: bin/integrations

data/CHANGELOG.md

@@ -1,5 +1,19 @@
 # Karafka framework changelog
 
+## 2.0.22 (2022-12-02)
+- [Improvement] Load Pro components upon Karafka require so they can be altered prior to setup.
+- [Improvement] Do not run LRJ jobs that were added to the jobs queue but were revoked meanwhile.
+- [Improvement] Allow running particular named subscription groups similar to consumer groups.
+- [Improvement] Allow running particular topics similar to consumer groups.
+- [Improvement] Raise configuration error when trying to run Karafka with options leading to no subscriptions.
+- [Fix] Fix `karafka info` subscription groups count reporting as it was misleading.
+- [Fix] Allow for defining subscription groups with symbols similar to consumer groups and topics to align the API.
+- [Fix] Do not allow for an explicit `nil` as a `subscription_group` block argument.
+- [Fix] Fix instability in subscription groups static members ids when using `--consumer_groups` CLI flag.
+- [Fix] Fix a case in routing, where anonymous subscription group could not be used inside of a consumer group.
+- [Fix] Fix a case where shutdown prior to listeners build would crash the server initialization.
+- [Fix] Duplicated logs in development environment for Rails when logger set to `$stdout`.
+
 ## 20.0.21 (2022-11-25)
 - [Improvement] Make revocation jobs for LRJ topics non-blocking to prevent blocking polling when someone uses non-revocation aware LRJ jobs and revocation happens.
 

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    karafka (2.0.21)
+    karafka (2.0.22)
       karafka-core (>= 2.0.4, < 3.0.0)
       rdkafka (>= 0.12)
       thor (>= 0.20)

data/README.md

@@ -13,7 +13,7 @@ Karafka is a Ruby and Rails multi-threaded efficient Kafka processing framework
 - Supports parallel processing in [multiple threads](https://karafka.io/docs/Concurrency-and-multithreading) (also for a [single topic partition](https://karafka.io/docs/Pro-Virtual-Partitions) work)
 - [Automatically integrates](https://karafka.io/docs/Integrating-with-Ruby-on-Rails-and-other-frameworks#integrating-with-ruby-on-rails) with Ruby on Rails
 - Has [ActiveJob backend](https://karafka.io/docs/Active-Job) support (including [ordered jobs](https://karafka.io/docs/Pro-Enhanced-Active-Job#ordered-jobs))
-- Has a seamless [Dead Letter Queue](karafka.io/docs/Dead-Letter-Queue/) functionality built-in
+- Has a seamless [Dead Letter Queue](https://karafka.io/docs/Dead-Letter-Queue/) functionality built-in
 - Supports in-development [code reloading](https://karafka.io/docs/Auto-reload-of-code-changes-in-development)
 - Is powered by [librdkafka](https://github.com/edenhill/librdkafka) (the Apache Kafka C/C++ client library)
 - Has an out-of the box [StatsD/DataDog monitoring](https://karafka.io/docs/Monitoring-and-logging) with a dashboard template.

data/bin/integrations

@@ -74,8 +74,8 @@ class Scenario
   def type
     scenario_dir = File.dirname(@path)
 
-    return :poro if scenario_dir.end_with?('_poro')
-    return :pristine if scenario_dir.end_with?('_pristine')
+    return :poro if scenario_dir.include?('_poro')
+    return :pristine if scenario_dir.include?('_pristine')
 
     :regular
   end

data/bin/rspecs

@@ -1,4 +1,6 @@
 #!/usr/bin/env bash
 
+set -e
+
 SPECS_TYPE=regular bundle exec rspec --tag ~type:pro
 SPECS_TYPE=pro bundle exec rspec --tag type:pro

data/config/errors.yml

@@ -25,7 +25,10 @@ en:
 
     server_cli_options:
       missing: needs to be present
-      consumer_groups_inclusion: Unknown consumer group
+      consumer_groups_inclusion: Unknown consumer group name
+      subscription_groups_inclusion: Unknown subscription group name
+      topics_inclusion: Unknown topic name
+      topics_missing: No topics to subscribe to
 
     topic:
       missing: needs to be present
@@ -34,7 +37,7 @@ en:
       consumer_format: needs to be present
       id_format: 'needs to be a string with a Kafka accepted format'
       initial_offset_format: needs to be either earliest or latest
-      subscription_group_format: must be nil or a non-empty string
+      subscription_group_format: must be a non-empty string
       manual_offset_management.active_format: needs to be either true or false
       consumer_active_job_missing: ActiveJob needs to be available
       manual_offset_management_must_be_enabled: cannot be disabled for ActiveJob topics

data/lib/karafka/app.rb

@@ -16,9 +16,19 @@ module Karafka
 
       # @return [Hash] active subscription groups grouped based on consumer group in a hash
       def subscription_groups
+        # We first build all the subscription groups, so they all get the same position, despite
+        # later narrowing that. It allows us to maintain same position number for static members
+        # even when we want to run subset of consumer groups or subscription groups
+        #
+        # We then narrow this to active consumer groups from which we select active subscription
+        # groups.
         consumer_groups
-          .active
-          .map { |consumer_group| [consumer_group, consumer_group.subscription_groups] }
+          .map { |cg| [cg, cg.subscription_groups] }
+          .select { |cg, _| cg.active? }
+          .select { |_, sgs| sgs.delete_if { |sg| !sg.active? } }
+          .delete_if { |_, sgs| sgs.empty? }
+          .each { |_, sgs| sgs.each { |sg| sg.topics.delete_if { |top| !top.active? } } }
+          .each { |_, sgs| sgs.delete_if { |sg| sg.topics.empty? } }
           .to_h
       end
 

data/lib/karafka/base_consumer.rb

@@ -62,14 +62,8 @@ module Karafka
     #   that may not yet kick in when error occurs. That way we pause always on the last processed
     #   message.
     def on_consume
-      Karafka.monitor.instrument('consumer.consumed', caller: self) do
-        consume
-      end
-
-      coordinator.consumption(self).success!
+      handle_consume
     rescue StandardError => e
-      coordinator.consumption(self).failure!(e)
-
       Karafka.monitor.instrument(
         'error.occurred',
         error: e,
@@ -77,9 +71,6 @@ module Karafka
         seek_offset: coordinator.seek_offset,
         type: 'consumer.consume.error'
       )
-    ensure
-      # We need to decrease number of jobs that this coordinator coordinates as it has finished
-      coordinator.decrement
     end
 
     # @private

data/lib/karafka/cli/info.rb

@@ -37,7 +37,8 @@ module Karafka
           "Karafka version: #{Karafka::VERSION}#{postfix}",
           "Ruby version: #{RUBY_DESCRIPTION}",
           "Rdkafka version: #{::Rdkafka::VERSION}",
-          "Subscription groups count: #{Karafka::App.subscription_groups.size}",
+          "Consumer groups count: #{Karafka::App.consumer_groups.size}",
+          "Subscription groups count: #{Karafka::App.subscription_groups.values.flatten.size}",
           "Workers count: #{Karafka::App.config.concurrency}",
           "Application client id: #{config.client_id}",
           "Boot file: #{Karafka.boot_file}",

data/lib/karafka/cli/server.rb

@@ -9,18 +9,19 @@ module Karafka
 
       desc 'Start the Karafka server (short-cut alias: "s")'
       option aliases: 's'
-      option :consumer_groups, type: :array, default: nil, aliases: :g
+      option :consumer_groups, type: :array, default: [], aliases: :g
+      option :subscription_groups, type: :array, default: []
+      option :topics, type: :array, default: []
 
       # Start the Karafka server
       def call
         # Print our banner and info in the dev mode
         print_marketing_info if Karafka::App.env.development?
 
-        Contracts::ServerCliOptions.new.validate!(cli.options)
-
-        # We assign active topics on a server level, as only server is expected to listen on
-        # part of the topics
-        Karafka::Server.consumer_groups = cli.options[:consumer_groups]
+        active_routing_config = Karafka::App.config.internal.routing.active
+        active_routing_config.consumer_groups = cli.options[:consumer_groups]
+        active_routing_config.subscription_groups = cli.options[:subscription_groups]
+        active_routing_config.topics = cli.options[:topics]
 
         Karafka::Server.run
       end

data/lib/karafka/contracts/server_cli_options.rb

@@ -12,7 +12,9 @@ module Karafka
         ).fetch('en').fetch('validations').fetch('server_cli_options')
       end
 
-      optional(:consumer_groups) { |cg| cg.is_a?(Array) && !cg.empty? }
+      optional(:consumer_groups) { |cg| cg.is_a?(Array) }
+      optional(:subscription_groups) { |sg| sg.is_a?(Array) }
+      optional(:topics) { |topics| topics.is_a?(Array) }
 
       virtual do |data, errors|
         next unless errors.empty?
@@ -22,11 +24,66 @@ module Karafka
 
         # If there were no consumer_groups declared in the server cli, it means that we will
         # run all of them and no need to validate them here at all
-        next if value.nil?
-        next if (value - Karafka::App.config.internal.routing.builder.map(&:name)).empty?
+        next if value.empty?
+        next if (value - Karafka::App.consumer_groups.map(&:name)).empty?
 
+        # Found unknown consumer groups
         [[%i[consumer_groups], :consumer_groups_inclusion]]
       end
+
+      virtual do |data, errors|
+        next unless errors.empty?
+        next unless data.key?(:subscription_groups)
+
+        value = data.fetch(:subscription_groups)
+
+        # If there were no subscription_groups declared in the server cli, it means that we will
+        # run all of them and no need to validate them here at all
+        next if value.empty?
+
+        subscription_groups = Karafka::App
+                              .consumer_groups
+                              .map(&:subscription_groups)
+                              .flatten
+                              .map(&:name)
+
+        next if (value - subscription_groups).empty?
+
+        # Found unknown subscription groups
+        [[%i[subscription_groups], :subscription_groups_inclusion]]
+      end
+
+      virtual do |data, errors|
+        next unless errors.empty?
+        next unless data.key?(:topics)
+
+        value = data.fetch(:topics)
+
+        # If there were no topics declared in the server cli, it means that we will
+        # run all of them and no need to validate them here at all
+        next if value.empty?
+
+        topics = Karafka::App
+                 .consumer_groups
+                 .map(&:subscription_groups)
+                 .flatten
+                 .map(&:topics)
+                 .map { |gtopics| gtopics.map(&:name) }
+                 .flatten
+
+        next if (value - topics).empty?
+
+        # Found unknown topics
+        [[%i[topics], :topics_inclusion]]
+      end
+
+      # Makes sure we have anything to subscribe to when we start the server
+      virtual do |_, errors|
+        next unless errors.empty?
+        next unless Karafka::App.subscription_groups.empty?
+
+        [[%i[topics], :topics_missing]]
+      end
     end
   end
 end

data/lib/karafka/contracts/topic.rb

@@ -20,7 +20,7 @@ module Karafka
       required(:initial_offset) { |val| %w[earliest latest].include?(val) }
       required(:max_wait_time) { |val| val.is_a?(Integer) && val >= 10 }
       required(:name) { |val| val.is_a?(String) && Contracts::TOPIC_REGEXP.match?(val) }
-      required(:subscription_group) { |val| val.nil? || (val.is_a?(String) && !val.empty?) }
+      required(:subscription_group) { |val| val.is_a?(String) && !val.empty? }
 
       virtual do |data, errors|
         next unless errors.empty?

data/lib/karafka/licenser.rb

@@ -8,68 +8,71 @@ module Karafka
 
     private_constant :PUBLIC_KEY_LOCATION
 
-    # Tries to prepare license and verifies it
-    #
-    # @param license_config [Karafka::Core::Configurable::Node] config related to the licensing
-    def prepare_and_verify(license_config)
-      prepare(license_config)
-      verify(license_config)
-    end
-
-    private
+    class << self
+      # Tries to load the license and yields if successful
+      def detect
+        # If required, do not require again
+        require('karafka-license') unless const_defined?('::Karafka::License')
 
-    # @param license_config [Karafka::Core::Configurable::Node] config related to the licensing
-    def prepare(license_config)
-      # If there is token, no action needed
-      # We support a case where someone would put the token in instead of using one from the
-      # license. That's in case there are limitations to using external package sources, etc
-      return if license_config.token
+        yield
 
-      begin
-        license_config.token || require('karafka-license')
+        true
       rescue LoadError
-        return
+        false
       end
 
-      license_config.token = Karafka::License.token
-    end
-
-    # Check license and setup license details (if needed)
-    # @param license_config [Karafka::Core::Configurable::Node] config related to the licensing
-    def verify(license_config)
-      # If no license, it will just run LGPL components without anything extra
-      return unless license_config.token
+      # Tries to prepare license and verifies it
+      #
+      # @param license_config [Karafka::Core::Configurable::Node] config related to the licensing
+      def prepare_and_verify(license_config)
+        # If license is not loaded, nothing to do
+        return unless const_defined?('::Karafka::License')
 
-      public_key = OpenSSL::PKey::RSA.new(File.read(PUBLIC_KEY_LOCATION))
+        prepare(license_config)
+        verify(license_config)
+      end
 
-      # We gsub and strip in case someone copy-pasted it as a multi line string
-      formatted_token = license_config.token.strip.delete("\n").delete(' ')
-      decoded_token = Base64.decode64(formatted_token)
+      private
 
-      begin
-        data = public_key.public_decrypt(decoded_token)
-      rescue OpenSSL::OpenSSLError
-        data = nil
+      # @param license_config [Karafka::Core::Configurable::Node] config related to the licensing
+      def prepare(license_config)
+        license_config.token = Karafka::License.token
       end
 
-      details = data ? JSON.parse(data) : raise_invalid_license_token(license_config)
+      # Check license and setup license details (if needed)
+      # @param license_config [Karafka::Core::Configurable::Node] config related to the licensing
+      def verify(license_config)
+        public_key = OpenSSL::PKey::RSA.new(File.read(PUBLIC_KEY_LOCATION))
 
-      license_config.entity = details.fetch('entity')
-    end
+        # We gsub and strip in case someone copy-pasted it as a multi line string
+        formatted_token = license_config.token.strip.delete("\n").delete(' ')
+        decoded_token = Base64.decode64(formatted_token)
 
-    # Raises an error with info, that used token is invalid
-    # @param license_config [Karafka::Core::Configurable::Node]
-    def raise_invalid_license_token(license_config)
-      # We set it to false so `Karafka.pro?` method behaves as expected
-      license_config.token = false
-
-      raise(
-        Errors::InvalidLicenseTokenError,
-        <<~MSG.tr("\n", ' ')
-          License key you provided is invalid.
-          Please reach us at contact@karafka.io or visit https://karafka.io to obtain a valid one.
-        MSG
-      )
+        begin
+          data = public_key.public_decrypt(decoded_token)
+        rescue OpenSSL::OpenSSLError
+          data = nil
+        end
+
+        details = data ? JSON.parse(data) : raise_invalid_license_token(license_config)
+
+        license_config.entity = details.fetch('entity')
+      end
+
+      # Raises an error with info, that used token is invalid
+      # @param license_config [Karafka::Core::Configurable::Node]
+      def raise_invalid_license_token(license_config)
+        # We set it to false so `Karafka.pro?` method behaves as expected
+        license_config.token = false
+
+        raise(
+          Errors::InvalidLicenseTokenError,
+          <<~MSG.tr("\n", ' ')
+            License key you provided is invalid.
+            Please reach us at contact@karafka.io or visit https://karafka.io to obtain a valid one.
+          MSG
+        )
+      end
     end
   end
 end

data/lib/karafka/pro/loader.rb

@@ -45,8 +45,6 @@ module Karafka
         # @param config [Karafka::Core::Configurable::Node] app config that we can alter with pro
         #   components
         def setup(config)
-          require_all
-
           reconfigure(config)
 
           load_topic_features

data/lib/karafka/pro/processing/strategies/default.rb

@@ -22,6 +22,7 @@ module Karafka
         # Nothing. Just standard, automatic flow
         module Default
           include Base
+          include ::Karafka::Processing::Strategies::Default
 
           # Apply strategy for a non-feature based flow
           FEATURES = %i[].freeze
@@ -39,6 +40,31 @@ module Karafka
             end
           end
 
+          # Run the user consumption code
+          def handle_consume
+            # We should not run the work at all on a partition that was revoked
+            # This can happen primarily when an LRJ job gets to the internal worker queue and
+            # this partition is revoked prior processing.
+            unless revoked?
+              Karafka.monitor.instrument('consumer.consumed', caller: self) do
+                consume
+              end
+            end
+
+            # Mark job as successful
+            coordinator.consumption(self).success!
+          rescue StandardError => e
+            # If failed, mark as failed
+            coordinator.consumption(self).failure!(e)
+
+            # Re-raise so reported in the consumer
+            raise e
+          ensure
+            # We need to decrease number of jobs that this coordinator coordinates as it has
+            # finished
+            coordinator.decrement
+          end
+
           # Standard flow without any features
           def handle_after_consume
             coordinator.on_finished do |last_group_message|

data/lib/karafka/processing/jobs_queue.rb

@@ -100,7 +100,7 @@ module Karafka
           return if @queue.closed?
 
           @queue.close
-          @semaphores.values.each(&:close)
+          @semaphores.each_value(&:close)
         end
       end
 

data/lib/karafka/processing/strategies/base.rb

@@ -22,6 +22,11 @@ module Karafka
           raise NotImplementedError, 'Implement in a subclass'
         end
 
+        # What should happen in the processing
+        def handle_consume
+          raise NotImplementedError, 'Implement in a subclass'
+        end
+
         # Post-consumption handling
         def handle_after_consume
           raise NotImplementedError, 'Implement in a subclass'

data/lib/karafka/processing/strategies/default.rb

@@ -23,6 +23,25 @@ module Karafka
           coordinator.pause_tracker.increment
         end
 
+        # Run the user consumption code
+        def handle_consume
+          Karafka.monitor.instrument('consumer.consumed', caller: self) do
+            consume
+          end
+
+          # Mark job as successful
+          coordinator.consumption(self).success!
+        rescue StandardError => e
+          # If failed, mark as failed
+          coordinator.consumption(self).failure!(e)
+
+          # Re-raise so reported in the consumer
+          raise e
+        ensure
+          # We need to decrease number of jobs that this coordinator coordinates as it has finished
+          coordinator.decrement
+        end
+
         # Standard flow marks work as consumed and moves on if everything went ok.
         # If there was a processing error, we will pause and continue from the next message
         # (next that is +1 from the last one that was successfully marked as consumed)

data/lib/karafka/railtie.rb

@@ -45,6 +45,9 @@ if rails
 
         next unless Rails.env.development?
         next unless ENV.key?('KARAFKA_CLI')
+        # If we are already publishing to STDOUT, no need to add it again.
+        # If added again, would print stuff twice
+        next if ActiveSupport::Logger.logger_outputs_to?(Rails.logger, $stdout)
 
         logger = ActiveSupport::Logger.new($stdout)
         # Inherit the logger level from Rails, otherwise would always run with the debug level

data/lib/karafka/routing/builder.rb

@@ -82,7 +82,7 @@ module Karafka
       # @param block [Proc] further topics definitions
       def subscription_group(subscription_group_name = SecureRandom.uuid, &block)
         consumer_group('app') do
-          target.public_send(:subscription_group=, subscription_group_name, &block)
+          target.public_send(:subscription_group=, subscription_group_name.to_s, &block)
         end
       end
 

data/lib/karafka/routing/consumer_group.rb

@@ -31,7 +31,10 @@ module Karafka
 
       # @return [Boolean] true if this consumer group should be active in our current process
       def active?
-        Karafka::Server.consumer_groups.include?(name)
+        cgs = Karafka::App.config.internal.routing.active.consumer_groups
+
+        # When empty it means no groups were specified, hence all should be used
+        cgs.empty? || cgs.include?(name)
       end
 
       # Builds a topic representation inside of a current consumer group route
@@ -50,9 +53,9 @@ module Karafka
 
       # Assigns the current subscription group id based on the defined one and allows for further
       # topic definition
-      # @param name [String, Symbol]
+      # @param name [String, Symbol] name of the current subscription group
       # @param block [Proc] block that may include topics definitions
-      def subscription_group=(name, &block)
+      def subscription_group=(name = SecureRandom.uuid, &block)
         # We cast it here, so the routing supports symbol based but that's anyhow later on
         # validated as a string
         @current_subscription_group_id = name

data/lib/karafka/routing/subscription_group.rb

@@ -8,7 +8,7 @@ module Karafka
     # @note One subscription group will always belong to one consumer group, but one consumer
     #   group can have multiple subscription groups.
     class SubscriptionGroup
-      attr_reader :id, :topics, :kafka
+      attr_reader :id, :name, :topics, :kafka
 
       # @param position [Integer] position of this subscription group in all the subscriptions
       #   groups array. We need to have this value for sake of static group memberships, where
@@ -16,7 +16,8 @@ module Karafka
       # @param topics [Karafka::Routing::Topics] all the topics that share the same key settings
       # @return [SubscriptionGroup] built subscription group
       def initialize(position, topics)
-        @id = "#{topics.first.subscription_group}_#{position}"
+        @name = topics.first.subscription_group
+        @id = "#{@name}_#{position}"
         @position = position
         @topics = topics
         @kafka = build_kafka
@@ -38,6 +39,14 @@ module Karafka
         @topics.first.max_wait_time
       end
 
+      # @return [Boolean] is this subscription group one of active once
+      def active?
+        sgs = Karafka::App.config.internal.routing.active.subscription_groups
+
+        # When empty it means no groups were specified, hence all should be used
+        sgs.empty? || sgs.include?(name)
+      end
+
       private
 
       # @return [Hash] kafka settings are a bit special. They are exactly the same for all of the

data/lib/karafka/routing/topic.rb

@@ -75,6 +75,14 @@ module Karafka
         consumer
       end
 
+      # @return [Boolean] should this topic be in use
+      def active?
+        topics = Karafka::App.config.internal.routing.active.topics
+
+        # When empty it means no topics were specified, hence all should be used
+        topics.empty? || topics.include?(name)
+      end
+
       # @return [Hash] hash with all the topic attributes
       # @note This is being used when we validate the consumer_group and its topics
       def to_h

data/lib/karafka/routing/topics.rb

@@ -23,6 +23,14 @@ module Karafka
         @accumulator.each(&block)
       end
 
+      # Allows us to remove elements from the topics
+      #
+      # Block to decide what to delete
+      # @param block [Proc]
+      def delete_if(&block)
+        @accumulator.delete_if(&block)
+      end
+
       # Finds topic by its name
       #
       # @param topic_name [String] topic name

data/lib/karafka/server.rb

@@ -20,11 +20,19 @@ module Karafka
       # Set of workers
       attr_accessor :workers
 
-      # Writer for list of consumer groups that we want to consume in our current process context
-      attr_writer :consumer_groups
-
       # Method which runs app
       def run
+        self.listeners = []
+        self.workers = []
+
+        # We need to validate this prior to running because it may be executed also from the
+        # embedded
+        # We cannot validate this during the start because config needs to be populated and routes
+        # need to be defined.
+        Contracts::ServerCliOptions.new.validate!(
+          Karafka::App.config.internal.routing.active.to_h
+        )
+
         process.on_sigint { stop }
         process.on_sigquit { stop }
         process.on_sigterm { stop }
@@ -49,13 +57,6 @@ module Karafka
         raise e
       end
 
-      # @return [Array<String>] array with names of consumer groups that should be consumed in a
-      #   current server context
-      def consumer_groups
-        # If not specified, a server will listen on all the topics
-        @consumer_groups ||= Karafka::App.consumer_groups.map(&:name).freeze
-      end
-
       # Starts Karafka with a supervision
       # @note We don't need to sleep because Karafka::Fetcher is locking and waiting to
       # finish loop (and it won't happen until we explicitly want to stop)

data/lib/karafka/setup/config.rb

@@ -107,6 +107,14 @@ module Karafka
           # option subscription_groups_builder [Routing::SubscriptionGroupsBuilder] subscription
           #   group builder
           setting :subscription_groups_builder, default: Routing::SubscriptionGroupsBuilder.new
+
+          # Internally assigned list of limits on routings active for the current process
+          # This should be overwritten by the CLI command
+          setting :active do
+            setting :consumer_groups, default: [].freeze
+            setting :subscription_groups, default: [].freeze
+            setting :topics, default: [].freeze
+          end
         end
 
         setting :processing do
@@ -142,16 +150,18 @@ module Karafka
         # Configuring method
         # @param block [Proc] block we want to execute with the config instance
         def setup(&block)
+          # Will prepare and verify license if present
+          Licenser.prepare_and_verify(config.license)
+          # Will configure all the pro components
+          # This needs to happen before end user configuration as the end user may overwrite some
+          # of the pro defaults with custom components
+          Pro::Loader.setup(config) if Karafka.pro?
+
           configure(&block)
           merge_kafka_defaults!(config)
 
           Contracts::Config.new.validate!(config.to_h)
 
-          licenser = Licenser.new
-
-          # Tries to load our license gem and if present will try to load the correct license
-          licenser.prepare_and_verify(config.license)
-
           configure_components
 
           Karafka::App.initialized!
@@ -188,12 +198,6 @@ module Karafka
             producer_config.kafka = AttributesMap.producer(config.kafka.dup)
             producer_config.logger = config.logger
           end
-
-          return unless Karafka.pro?
-
-          # Runs the pro loader that includes all the pro components
-          require 'karafka/pro/loader'
-          Pro::Loader.setup(config)
         end
       end
     end

data/lib/karafka/version.rb

@@ -3,5 +3,5 @@
 # Main module namespace
 module Karafka
   # Current Karafka version
-  VERSION = '2.0.21'
+  VERSION = '2.0.22'
 end

data/lib/karafka.rb

@@ -100,5 +100,14 @@ loader.eager_load
 # nor included here
 ::Karafka::Routing::Features::Base.load_all
 
+# We need to detect and require (not setup) Pro components during the gem load, because we need
+# to make pro components available in case anyone wants to use them as a base to their own
+# custom components. Otherwise inheritance would not work.
+Karafka::Licenser.detect do
+  require 'karafka/pro/loader'
+
+  Karafka::Pro::Loader.require_all
+end
+
 # Load railtie after everything else is ready so we know we can rely on it.
 require 'karafka/railtie'

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: karafka
 version: !ruby/object:Gem::Version
-  version: 2.0.21
+  version: 2.0.22
 platform: ruby
 authors:
 - Maciej Mensfeld
@@ -35,7 +35,7 @@ cert_chain:
   Qf04B9ceLUaC4fPVEz10FyobjaFoY4i32xRto3XnrzeAgfEe4swLq8bQsR3w/EF3
   MGU0FeSV2Yj7Xc2x/7BzLK8xQn5l7Yy75iPF+KP3vVmDHnNl
   -----END CERTIFICATE-----
-date: 2022-11-25 00:00:00.000000000 Z
+date: 2022-12-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: karafka-core