karafka 2.0.20 → 2.0.22

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: c3b857c4c930396d4cac4682e4350c4c5fdc2888128dfade3f06e21a02e90c29
-  data.tar.gz: 4f8fdb66df24164ec65886d5b4c4ba766dda9c7d4d23a7a301920a8adb21e16e
+  metadata.gz: 6a096a44a1a2988ff394215b8d63f6f2e33b2a3e5100f68f75fcf33eeda4b490
+  data.tar.gz: 2af0875b550f37ef9ea47b2a3011a3ed3e535adc3f6632ab10510d91bc69590c
 SHA512:
-  metadata.gz: d4ed9d036f7dae85b3ce2f9c45de64eed8a8da110f0a826baf100ec9c90dc7e7dbaf5c74f89b472eb89ee7d3bdc9e265fee259621d93e21c972776fa3f682cc0
-  data.tar.gz: aed5fcd7447be757cd2ca641d1d4f6029db11296ac6b4012b4a02047126f27be4aef8d42da700f736c4f70eba2b0aff252a4ccf332fb6c8b88e6a911e3432171
+  metadata.gz: a81cf3305482e0e925f3a5aa48d978585766826e0990081d74f5f56bc535ae0a8a6be3a4ec986e6a7702d70d0d207ca9acc71b7aeb3909557b44518413ff9bc6
+  data.tar.gz: 6c66140b258135367a120a5dac4eb8a08f4dc94e24fb3a714b32ec6ad909bc9f85bf5e160aefe5f5d307c9d554c6ecab61cbdebf315ce6953476303e9783122c

data/.github/workflows/ci.yml

@@ -101,6 +101,9 @@ jobs:
       - name: Install package dependencies
         run: "[ -e $APT_DEPS ] || sudo apt-get install -y --no-install-recommends $APT_DEPS"
 
+      - name: Remove libzstd-dev to check no supported compressions
+        run: sudo apt-get -y remove libzstd-dev
+
       - name: Start Kafka with docker-compose
         run: |
           docker-compose up -d
@@ -124,5 +127,8 @@ jobs:
       - name: Run integration tests
         env:
           KARAFKA_PRO_LICENSE_TOKEN: ${{ secrets.KARAFKA_PRO_LICENSE_TOKEN }}
+          KARAFKA_PRO_USERNAME: ${{ secrets.KARAFKA_PRO_USERNAME }}
+          KARAFKA_PRO_PASSWORD: ${{ secrets.KARAFKA_PRO_PASSWORD }}
+          KARAFKA_PRO_VERSION: ${{ secrets.KARAFKA_PRO_VERSION }}
           GITHUB_COVERAGE: ${{matrix.coverage}}
         run: bin/integrations

data/CHANGELOG.md

@@ -1,5 +1,22 @@
 # Karafka framework changelog
 
+## 2.0.22 (2022-12-02)
+- [Improvement] Load Pro components upon Karafka require so they can be altered prior to setup.
+- [Improvement] Do not run LRJ jobs that were added to the jobs queue but were revoked meanwhile.
+- [Improvement] Allow running particular named subscription groups similar to consumer groups.
+- [Improvement] Allow running particular topics similar to consumer groups.
+- [Improvement] Raise configuration error when trying to run Karafka with options leading to no subscriptions.
+- [Fix] Fix `karafka info` subscription groups count reporting as it was misleading.
+- [Fix] Allow for defining subscription groups with symbols similar to consumer groups and topics to align the API.
+- [Fix] Do not allow for an explicit `nil` as a `subscription_group` block argument.
+- [Fix] Fix instability in subscription groups static members ids when using `--consumer_groups` CLI flag.
+- [Fix] Fix a case in routing, where anonymous subscription group could not be used inside of a consumer group.
+- [Fix] Fix a case where shutdown prior to listeners build would crash the server initialization.
+- [Fix] Duplicated logs in development environment for Rails when logger set to `$stdout`.
+
+## 20.0.21 (2022-11-25)
+- [Improvement] Make revocation jobs for LRJ topics non-blocking to prevent blocking polling when someone uses non-revocation aware LRJ jobs and revocation happens.
+
 ## 2.0.20 (2022-11-24)
 - [Improvement] Support `group.instance.id` assignment (static group membership) for a case where a single consumer group has multiple subscription groups (#1173).
 

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    karafka (2.0.20)
+    karafka (2.0.22)
       karafka-core (>= 2.0.4, < 3.0.0)
       rdkafka (>= 0.12)
       thor (>= 0.20)
@@ -79,4 +79,4 @@ DEPENDENCIES
   simplecov
 
 BUNDLED WITH
-   2.3.24
+   2.3.26

data/README.md

@@ -13,7 +13,7 @@ Karafka is a Ruby and Rails multi-threaded efficient Kafka processing framework
 - Supports parallel processing in [multiple threads](https://karafka.io/docs/Concurrency-and-multithreading) (also for a [single topic partition](https://karafka.io/docs/Pro-Virtual-Partitions) work)
 - [Automatically integrates](https://karafka.io/docs/Integrating-with-Ruby-on-Rails-and-other-frameworks#integrating-with-ruby-on-rails) with Ruby on Rails
 - Has [ActiveJob backend](https://karafka.io/docs/Active-Job) support (including [ordered jobs](https://karafka.io/docs/Pro-Enhanced-Active-Job#ordered-jobs))
-- Has a seamless [Dead Letter Queue](karafka.io/docs/Dead-Letter-Queue/) functionality built-in
+- Has a seamless [Dead Letter Queue](https://karafka.io/docs/Dead-Letter-Queue/) functionality built-in
 - Supports in-development [code reloading](https://karafka.io/docs/Auto-reload-of-code-changes-in-development)
 - Is powered by [librdkafka](https://github.com/edenhill/librdkafka) (the Apache Kafka C/C++ client library)
 - Has an out-of the box [StatsD/DataDog monitoring](https://karafka.io/docs/Monitoring-and-logging) with a dashboard template.

data/bin/integrations

@@ -74,8 +74,8 @@ class Scenario
   def type
     scenario_dir = File.dirname(@path)
 
-    return :poro if scenario_dir.end_with?('_poro')
-    return :pristine if scenario_dir.end_with?('_pristine')
+    return :poro if scenario_dir.include?('_poro')
+    return :pristine if scenario_dir.include?('_pristine')
 
     :regular
   end

data/bin/rspecs

@@ -1,4 +1,6 @@
 #!/usr/bin/env bash
 
+set -e
+
 SPECS_TYPE=regular bundle exec rspec --tag ~type:pro
 SPECS_TYPE=pro bundle exec rspec --tag type:pro

data/config/errors.yml

@@ -25,7 +25,10 @@ en:
 
     server_cli_options:
       missing: needs to be present
-      consumer_groups_inclusion: Unknown consumer group
+      consumer_groups_inclusion: Unknown consumer group name
+      subscription_groups_inclusion: Unknown subscription group name
+      topics_inclusion: Unknown topic name
+      topics_missing: No topics to subscribe to
 
     topic:
       missing: needs to be present
@@ -34,7 +37,7 @@ en:
       consumer_format: needs to be present
       id_format: 'needs to be a string with a Kafka accepted format'
       initial_offset_format: needs to be either earliest or latest
-      subscription_group_format: must be nil or a non-empty string
+      subscription_group_format: must be a non-empty string
       manual_offset_management.active_format: needs to be either true or false
       consumer_active_job_missing: ActiveJob needs to be available
       manual_offset_management_must_be_enabled: cannot be disabled for ActiveJob topics

data/karafka.gemspec

@@ -12,9 +12,14 @@ Gem::Specification.new do |spec|
   spec.authors     = ['Maciej Mensfeld']
   spec.email       = %w[contact@karafka.io]
   spec.homepage    = 'https://karafka.io'
-  spec.summary     = 'Efficient Kafka processing framework for Ruby and Rails'
-  spec.description = 'Framework used to simplify Apache Kafka based Ruby applications development'
   spec.licenses    = ['LGPL-3.0', 'Commercial']
+  spec.summary     = 'Karafka is Ruby and Rails efficient Kafka processing framework.'
+  spec.description = <<-DESC
+    Karafka is Ruby and Rails efficient Kafka processing framework.
+
+    Karafka allows you to capture everything that happens in your systems in large scale,
+    without having to focus on things that are not your business domain.
+  DESC
 
   spec.add_dependency 'karafka-core', '>= 2.0.4', '< 3.0.0'
   spec.add_dependency 'rdkafka', '>= 0.12'

data/lib/karafka/app.rb

@@ -16,9 +16,19 @@ module Karafka
 
       # @return [Hash] active subscription groups grouped based on consumer group in a hash
       def subscription_groups
+        # We first build all the subscription groups, so they all get the same position, despite
+        # later narrowing that. It allows us to maintain same position number for static members
+        # even when we want to run subset of consumer groups or subscription groups
+        #
+        # We then narrow this to active consumer groups from which we select active subscription
+        # groups.
         consumer_groups
-          .active
-          .map { |consumer_group| [consumer_group, consumer_group.subscription_groups] }
+          .map { |cg| [cg, cg.subscription_groups] }
+          .select { |cg, _| cg.active? }
+          .select { |_, sgs| sgs.delete_if { |sg| !sg.active? } }
+          .delete_if { |_, sgs| sgs.empty? }
+          .each { |_, sgs| sgs.each { |sg| sg.topics.delete_if { |top| !top.active? } } }
+          .each { |_, sgs| sgs.delete_if { |sg| sg.topics.empty? } }
           .to_h
       end
 

data/lib/karafka/base_consumer.rb

@@ -62,14 +62,8 @@ module Karafka
     #   that may not yet kick in when error occurs. That way we pause always on the last processed
     #   message.
     def on_consume
-      Karafka.monitor.instrument('consumer.consumed', caller: self) do
-        consume
-      end
-
-      coordinator.consumption(self).success!
+      handle_consume
     rescue StandardError => e
-      coordinator.consumption(self).failure!(e)
-
       Karafka.monitor.instrument(
         'error.occurred',
         error: e,
@@ -77,9 +71,6 @@ module Karafka
         seek_offset: coordinator.seek_offset,
         type: 'consumer.consume.error'
       )
-    ensure
-      # We need to decrease number of jobs that this coordinator coordinates as it has finished
-      coordinator.decrement
     end
 
     # @private

data/lib/karafka/cli/info.rb

@@ -37,7 +37,8 @@ module Karafka
           "Karafka version: #{Karafka::VERSION}#{postfix}",
           "Ruby version: #{RUBY_DESCRIPTION}",
           "Rdkafka version: #{::Rdkafka::VERSION}",
-          "Subscription groups count: #{Karafka::App.subscription_groups.size}",
+          "Consumer groups count: #{Karafka::App.consumer_groups.size}",
+          "Subscription groups count: #{Karafka::App.subscription_groups.values.flatten.size}",
           "Workers count: #{Karafka::App.config.concurrency}",
           "Application client id: #{config.client_id}",
           "Boot file: #{Karafka.boot_file}",

data/lib/karafka/cli/server.rb

@@ -9,18 +9,19 @@ module Karafka
 
       desc 'Start the Karafka server (short-cut alias: "s")'
       option aliases: 's'
-      option :consumer_groups, type: :array, default: nil, aliases: :g
+      option :consumer_groups, type: :array, default: [], aliases: :g
+      option :subscription_groups, type: :array, default: []
+      option :topics, type: :array, default: []
 
       # Start the Karafka server
       def call
         # Print our banner and info in the dev mode
         print_marketing_info if Karafka::App.env.development?
 
-        Contracts::ServerCliOptions.new.validate!(cli.options)
-
-        # We assign active topics on a server level, as only server is expected to listen on
-        # part of the topics
-        Karafka::Server.consumer_groups = cli.options[:consumer_groups]
+        active_routing_config = Karafka::App.config.internal.routing.active
+        active_routing_config.consumer_groups = cli.options[:consumer_groups]
+        active_routing_config.subscription_groups = cli.options[:subscription_groups]
+        active_routing_config.topics = cli.options[:topics]
 
         Karafka::Server.run
       end

data/lib/karafka/contracts/server_cli_options.rb

@@ -12,7 +12,9 @@ module Karafka
         ).fetch('en').fetch('validations').fetch('server_cli_options')
       end
 
-      optional(:consumer_groups) { |cg| cg.is_a?(Array) && !cg.empty? }
+      optional(:consumer_groups) { |cg| cg.is_a?(Array) }
+      optional(:subscription_groups) { |sg| sg.is_a?(Array) }
+      optional(:topics) { |topics| topics.is_a?(Array) }
 
       virtual do |data, errors|
         next unless errors.empty?
@@ -22,11 +24,66 @@ module Karafka
 
         # If there were no consumer_groups declared in the server cli, it means that we will
         # run all of them and no need to validate them here at all
-        next if value.nil?
-        next if (value - Karafka::App.config.internal.routing.builder.map(&:name)).empty?
+        next if value.empty?
+        next if (value - Karafka::App.consumer_groups.map(&:name)).empty?
 
+        # Found unknown consumer groups
         [[%i[consumer_groups], :consumer_groups_inclusion]]
       end
+
+      virtual do |data, errors|
+        next unless errors.empty?
+        next unless data.key?(:subscription_groups)
+
+        value = data.fetch(:subscription_groups)
+
+        # If there were no subscription_groups declared in the server cli, it means that we will
+        # run all of them and no need to validate them here at all
+        next if value.empty?
+
+        subscription_groups = Karafka::App
+                              .consumer_groups
+                              .map(&:subscription_groups)
+                              .flatten
+                              .map(&:name)
+
+        next if (value - subscription_groups).empty?
+
+        # Found unknown subscription groups
+        [[%i[subscription_groups], :subscription_groups_inclusion]]
+      end
+
+      virtual do |data, errors|
+        next unless errors.empty?
+        next unless data.key?(:topics)
+
+        value = data.fetch(:topics)
+
+        # If there were no topics declared in the server cli, it means that we will
+        # run all of them and no need to validate them here at all
+        next if value.empty?
+
+        topics = Karafka::App
+                 .consumer_groups
+                 .map(&:subscription_groups)
+                 .flatten
+                 .map(&:topics)
+                 .map { |gtopics| gtopics.map(&:name) }
+                 .flatten
+
+        next if (value - topics).empty?
+
+        # Found unknown topics
+        [[%i[topics], :topics_inclusion]]
+      end
+
+      # Makes sure we have anything to subscribe to when we start the server
+      virtual do |_, errors|
+        next unless errors.empty?
+        next unless Karafka::App.subscription_groups.empty?
+
+        [[%i[topics], :topics_missing]]
+      end
     end
   end
 end

data/lib/karafka/contracts/topic.rb

@@ -20,7 +20,7 @@ module Karafka
       required(:initial_offset) { |val| %w[earliest latest].include?(val) }
       required(:max_wait_time) { |val| val.is_a?(Integer) && val >= 10 }
       required(:name) { |val| val.is_a?(String) && Contracts::TOPIC_REGEXP.match?(val) }
-      required(:subscription_group) { |val| val.nil? || (val.is_a?(String) && !val.empty?) }
+      required(:subscription_group) { |val| val.is_a?(String) && !val.empty? }
 
       virtual do |data, errors|
         next unless errors.empty?

data/lib/karafka/licenser.rb

@@ -8,68 +8,71 @@ module Karafka
 
     private_constant :PUBLIC_KEY_LOCATION
 
-    # Tries to prepare license and verifies it
-    #
-    # @param license_config [Karafka::Core::Configurable::Node] config related to the licensing
-    def prepare_and_verify(license_config)
-      prepare(license_config)
-      verify(license_config)
-    end
-
-    private
+    class << self
+      # Tries to load the license and yields if successful
+      def detect
+        # If required, do not require again
+        require('karafka-license') unless const_defined?('::Karafka::License')
 
-    # @param license_config [Karafka::Core::Configurable::Node] config related to the licensing
-    def prepare(license_config)
-      # If there is token, no action needed
-      # We support a case where someone would put the token in instead of using one from the
-      # license. That's in case there are limitations to using external package sources, etc
-      return if license_config.token
+        yield
 
-      begin
-        license_config.token || require('karafka-license')
+        true
       rescue LoadError
-        return
+        false
       end
 
-      license_config.token = Karafka::License.token
-    end
-
-    # Check license and setup license details (if needed)
-    # @param license_config [Karafka::Core::Configurable::Node] config related to the licensing
-    def verify(license_config)
-      # If no license, it will just run LGPL components without anything extra
-      return unless license_config.token
+      # Tries to prepare license and verifies it
+      #
+      # @param license_config [Karafka::Core::Configurable::Node] config related to the licensing
+      def prepare_and_verify(license_config)
+        # If license is not loaded, nothing to do
+        return unless const_defined?('::Karafka::License')
 
-      public_key = OpenSSL::PKey::RSA.new(File.read(PUBLIC_KEY_LOCATION))
+        prepare(license_config)
+        verify(license_config)
+      end
 
-      # We gsub and strip in case someone copy-pasted it as a multi line string
-      formatted_token = license_config.token.strip.delete("\n").delete(' ')
-      decoded_token = Base64.decode64(formatted_token)
+      private
 
-      begin
-        data = public_key.public_decrypt(decoded_token)
-      rescue OpenSSL::OpenSSLError
-        data = nil
+      # @param license_config [Karafka::Core::Configurable::Node] config related to the licensing
+      def prepare(license_config)
+        license_config.token = Karafka::License.token
       end
 
-      details = data ? JSON.parse(data) : raise_invalid_license_token(license_config)
+      # Check license and setup license details (if needed)
+      # @param license_config [Karafka::Core::Configurable::Node] config related to the licensing
+      def verify(license_config)
+        public_key = OpenSSL::PKey::RSA.new(File.read(PUBLIC_KEY_LOCATION))
 
-      license_config.entity = details.fetch('entity')
-    end
+        # We gsub and strip in case someone copy-pasted it as a multi line string
+        formatted_token = license_config.token.strip.delete("\n").delete(' ')
+        decoded_token = Base64.decode64(formatted_token)
 
-    # Raises an error with info, that used token is invalid
-    # @param license_config [Karafka::Core::Configurable::Node]
-    def raise_invalid_license_token(license_config)
-      # We set it to false so `Karafka.pro?` method behaves as expected
-      license_config.token = false
-
-      raise(
-        Errors::InvalidLicenseTokenError,
-        <<~MSG.tr("\n", ' ')
-          License key you provided is invalid.
-          Please reach us at contact@karafka.io or visit https://karafka.io to obtain a valid one.
-        MSG
-      )
+        begin
+          data = public_key.public_decrypt(decoded_token)
+        rescue OpenSSL::OpenSSLError
+          data = nil
+        end
+
+        details = data ? JSON.parse(data) : raise_invalid_license_token(license_config)
+
+        license_config.entity = details.fetch('entity')
+      end
+
+      # Raises an error with info, that used token is invalid
+      # @param license_config [Karafka::Core::Configurable::Node]
+      def raise_invalid_license_token(license_config)
+        # We set it to false so `Karafka.pro?` method behaves as expected
+        license_config.token = false
+
+        raise(
+          Errors::InvalidLicenseTokenError,
+          <<~MSG.tr("\n", ' ')
+            License key you provided is invalid.
+            Please reach us at contact@karafka.io or visit https://karafka.io to obtain a valid one.
+          MSG
+        )
+      end
     end
   end
 end

data/lib/karafka/pro/loader.rb

@@ -45,8 +45,6 @@ module Karafka
         # @param config [Karafka::Core::Configurable::Node] app config that we can alter with pro
         #   components
         def setup(config)
-          require_all
-
           reconfigure(config)
 
           load_topic_features

data/lib/karafka/pro/processing/jobs/consume_non_blocking.rb

@@ -18,8 +18,7 @@ module Karafka
       # Pro jobs
       module Jobs
         # The main job type in a non-blocking variant.
-        # This variant works "like" the regular consumption but pauses the partition for as long
-        # as it is needed until a job is done.
+        # This variant works "like" the regular consumption but does not block the queue.
         #
         # It can be useful when having long lasting jobs that would exceed `max.poll.interval`
         # if would block.

data/lib/karafka/pro/processing/jobs/revoked_non_blocking.rb

@@ -0,0 +1,37 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    # Pro components related to processing part of Karafka
+    module Processing
+      # Pro jobs
+      module Jobs
+        # The revoked job type in a non-blocking variant.
+        # This variant works "like" the regular revoked but does not block the queue.
+        #
+        # It can be useful when having long lasting jobs that would exceed `max.poll.interval`
+        # in scenarios where there are more jobs than threads, without this being async we
+        # would potentially stop polling
+        class RevokedNonBlocking < ::Karafka::Processing::Jobs::Revoked
+          # Makes this job non-blocking from the start
+          # @param args [Array] any arguments accepted by `::Karafka::Processing::Jobs::Revoked`
+          def initialize(*args)
+            super
+            @non_blocking = true
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/processing/jobs_builder.rb

@@ -28,6 +28,18 @@ module Karafka
             super
           end
         end
+
+        # @param executor [Karafka::Processing::Executor]
+        # @return [Karafka::Processing::Jobs::Revoked] revocation job for non LRJ
+        # @return [Karafka::Processing::Jobs::RevokedNonBlocking] revocation job that is
+        #   non-blocking, so when revocation job is scheduled for LRJ it also will not block
+        def revoked(executor)
+          if executor.topic.long_running_job?
+            Jobs::RevokedNonBlocking.new(executor)
+          else
+            super
+          end
+        end
       end
     end
   end

data/lib/karafka/pro/processing/strategies/default.rb

@@ -22,6 +22,7 @@ module Karafka
         # Nothing. Just standard, automatic flow
         module Default
           include Base
+          include ::Karafka::Processing::Strategies::Default
 
           # Apply strategy for a non-feature based flow
           FEATURES = %i[].freeze
@@ -39,6 +40,31 @@ module Karafka
             end
           end
 
+          # Run the user consumption code
+          def handle_consume
+            # We should not run the work at all on a partition that was revoked
+            # This can happen primarily when an LRJ job gets to the internal worker queue and
+            # this partition is revoked prior processing.
+            unless revoked?
+              Karafka.monitor.instrument('consumer.consumed', caller: self) do
+                consume
+              end
+            end
+
+            # Mark job as successful
+            coordinator.consumption(self).success!
+          rescue StandardError => e
+            # If failed, mark as failed
+            coordinator.consumption(self).failure!(e)
+
+            # Re-raise so reported in the consumer
+            raise e
+          ensure
+            # We need to decrease number of jobs that this coordinator coordinates as it has
+            # finished
+            coordinator.decrement
+          end
+
           # Standard flow without any features
           def handle_after_consume
             coordinator.on_finished do |last_group_message|

data/lib/karafka/processing/jobs_queue.rb

@@ -100,7 +100,7 @@ module Karafka
           return if @queue.closed?
 
           @queue.close
-          @semaphores.values.each(&:close)
+          @semaphores.each_value(&:close)
         end
       end
 

data/lib/karafka/processing/strategies/base.rb

@@ -22,6 +22,11 @@ module Karafka
           raise NotImplementedError, 'Implement in a subclass'
         end
 
+        # What should happen in the processing
+        def handle_consume
+          raise NotImplementedError, 'Implement in a subclass'
+        end
+
         # Post-consumption handling
         def handle_after_consume
           raise NotImplementedError, 'Implement in a subclass'

data/lib/karafka/processing/strategies/default.rb

@@ -23,6 +23,25 @@ module Karafka
           coordinator.pause_tracker.increment
         end
 
+        # Run the user consumption code
+        def handle_consume
+          Karafka.monitor.instrument('consumer.consumed', caller: self) do
+            consume
+          end
+
+          # Mark job as successful
+          coordinator.consumption(self).success!
+        rescue StandardError => e
+          # If failed, mark as failed
+          coordinator.consumption(self).failure!(e)
+
+          # Re-raise so reported in the consumer
+          raise e
+        ensure
+          # We need to decrease number of jobs that this coordinator coordinates as it has finished
+          coordinator.decrement
+        end
+
         # Standard flow marks work as consumed and moves on if everything went ok.
         # If there was a processing error, we will pause and continue from the next message
         # (next that is +1 from the last one that was successfully marked as consumed)

data/lib/karafka/railtie.rb

@@ -45,6 +45,9 @@ if rails
 
         next unless Rails.env.development?
         next unless ENV.key?('KARAFKA_CLI')
+        # If we are already publishing to STDOUT, no need to add it again.
+        # If added again, would print stuff twice
+        next if ActiveSupport::Logger.logger_outputs_to?(Rails.logger, $stdout)
 
         logger = ActiveSupport::Logger.new($stdout)
         # Inherit the logger level from Rails, otherwise would always run with the debug level

data/lib/karafka/routing/builder.rb

@@ -82,7 +82,7 @@ module Karafka
       # @param block [Proc] further topics definitions
       def subscription_group(subscription_group_name = SecureRandom.uuid, &block)
         consumer_group('app') do
-          target.public_send(:subscription_group=, subscription_group_name, &block)
+          target.public_send(:subscription_group=, subscription_group_name.to_s, &block)
         end
       end
 

data/lib/karafka/routing/consumer_group.rb

@@ -31,7 +31,10 @@ module Karafka
 
       # @return [Boolean] true if this consumer group should be active in our current process
       def active?
-        Karafka::Server.consumer_groups.include?(name)
+        cgs = Karafka::App.config.internal.routing.active.consumer_groups
+
+        # When empty it means no groups were specified, hence all should be used
+        cgs.empty? || cgs.include?(name)
       end
 
       # Builds a topic representation inside of a current consumer group route
@@ -50,9 +53,9 @@ module Karafka
 
       # Assigns the current subscription group id based on the defined one and allows for further
       # topic definition
-      # @param name [String, Symbol]
+      # @param name [String, Symbol] name of the current subscription group
       # @param block [Proc] block that may include topics definitions
-      def subscription_group=(name, &block)
+      def subscription_group=(name = SecureRandom.uuid, &block)
         # We cast it here, so the routing supports symbol based but that's anyhow later on
         # validated as a string
         @current_subscription_group_id = name

data/lib/karafka/routing/subscription_group.rb

@@ -8,7 +8,7 @@ module Karafka
     # @note One subscription group will always belong to one consumer group, but one consumer
     #   group can have multiple subscription groups.
     class SubscriptionGroup
-      attr_reader :id, :topics, :kafka
+      attr_reader :id, :name, :topics, :kafka
 
       # @param position [Integer] position of this subscription group in all the subscriptions
       #   groups array. We need to have this value for sake of static group memberships, where
@@ -16,7 +16,8 @@ module Karafka
       # @param topics [Karafka::Routing::Topics] all the topics that share the same key settings
       # @return [SubscriptionGroup] built subscription group
       def initialize(position, topics)
-        @id = "#{topics.first.subscription_group}_#{position}"
+        @name = topics.first.subscription_group
+        @id = "#{@name}_#{position}"
         @position = position
         @topics = topics
         @kafka = build_kafka
@@ -38,6 +39,14 @@ module Karafka
         @topics.first.max_wait_time
       end
 
+      # @return [Boolean] is this subscription group one of active once
+      def active?
+        sgs = Karafka::App.config.internal.routing.active.subscription_groups
+
+        # When empty it means no groups were specified, hence all should be used
+        sgs.empty? || sgs.include?(name)
+      end
+
       private
 
       # @return [Hash] kafka settings are a bit special. They are exactly the same for all of the

data/lib/karafka/routing/topic.rb

@@ -75,6 +75,14 @@ module Karafka
         consumer
       end
 
+      # @return [Boolean] should this topic be in use
+      def active?
+        topics = Karafka::App.config.internal.routing.active.topics
+
+        # When empty it means no topics were specified, hence all should be used
+        topics.empty? || topics.include?(name)
+      end
+
       # @return [Hash] hash with all the topic attributes
       # @note This is being used when we validate the consumer_group and its topics
       def to_h

data/lib/karafka/routing/topics.rb

@@ -23,6 +23,14 @@ module Karafka
         @accumulator.each(&block)
       end
 
+      # Allows us to remove elements from the topics
+      #
+      # Block to decide what to delete
+      # @param block [Proc]
+      def delete_if(&block)
+        @accumulator.delete_if(&block)
+      end
+
       # Finds topic by its name
       #
       # @param topic_name [String] topic name

data/lib/karafka/server.rb

@@ -20,11 +20,19 @@ module Karafka
       # Set of workers
       attr_accessor :workers
 
-      # Writer for list of consumer groups that we want to consume in our current process context
-      attr_writer :consumer_groups
-
       # Method which runs app
       def run
+        self.listeners = []
+        self.workers = []
+
+        # We need to validate this prior to running because it may be executed also from the
+        # embedded
+        # We cannot validate this during the start because config needs to be populated and routes
+        # need to be defined.
+        Contracts::ServerCliOptions.new.validate!(
+          Karafka::App.config.internal.routing.active.to_h
+        )
+
         process.on_sigint { stop }
         process.on_sigquit { stop }
         process.on_sigterm { stop }
@@ -49,13 +57,6 @@ module Karafka
         raise e
       end
 
-      # @return [Array<String>] array with names of consumer groups that should be consumed in a
-      #   current server context
-      def consumer_groups
-        # If not specified, a server will listen on all the topics
-        @consumer_groups ||= Karafka::App.consumer_groups.map(&:name).freeze
-      end
-
       # Starts Karafka with a supervision
       # @note We don't need to sleep because Karafka::Fetcher is locking and waiting to
       # finish loop (and it won't happen until we explicitly want to stop)

data/lib/karafka/setup/config.rb

@@ -107,6 +107,14 @@ module Karafka
           # option subscription_groups_builder [Routing::SubscriptionGroupsBuilder] subscription
           #   group builder
           setting :subscription_groups_builder, default: Routing::SubscriptionGroupsBuilder.new
+
+          # Internally assigned list of limits on routings active for the current process
+          # This should be overwritten by the CLI command
+          setting :active do
+            setting :consumer_groups, default: [].freeze
+            setting :subscription_groups, default: [].freeze
+            setting :topics, default: [].freeze
+          end
         end
 
         setting :processing do
@@ -142,16 +150,18 @@ module Karafka
         # Configuring method
         # @param block [Proc] block we want to execute with the config instance
         def setup(&block)
+          # Will prepare and verify license if present
+          Licenser.prepare_and_verify(config.license)
+          # Will configure all the pro components
+          # This needs to happen before end user configuration as the end user may overwrite some
+          # of the pro defaults with custom components
+          Pro::Loader.setup(config) if Karafka.pro?
+
           configure(&block)
           merge_kafka_defaults!(config)
 
           Contracts::Config.new.validate!(config.to_h)
 
-          licenser = Licenser.new
-
-          # Tries to load our license gem and if present will try to load the correct license
-          licenser.prepare_and_verify(config.license)
-
           configure_components
 
           Karafka::App.initialized!
@@ -188,12 +198,6 @@ module Karafka
             producer_config.kafka = AttributesMap.producer(config.kafka.dup)
             producer_config.logger = config.logger
           end
-
-          return unless Karafka.pro?
-
-          # Runs the pro loader that includes all the pro components
-          require 'karafka/pro/loader'
-          Pro::Loader.setup(config)
         end
       end
     end

data/lib/karafka/version.rb

@@ -3,5 +3,5 @@
 # Main module namespace
 module Karafka
   # Current Karafka version
-  VERSION = '2.0.20'
+  VERSION = '2.0.22'
 end

data/lib/karafka.rb

@@ -100,5 +100,14 @@ loader.eager_load
 # nor included here
 ::Karafka::Routing::Features::Base.load_all
 
+# We need to detect and require (not setup) Pro components during the gem load, because we need
+# to make pro components available in case anyone wants to use them as a base to their own
+# custom components. Otherwise inheritance would not work.
+Karafka::Licenser.detect do
+  require 'karafka/pro/loader'
+
+  Karafka::Pro::Loader.require_all
+end
+
 # Load railtie after everything else is ready so we know we can rely on it.
 require 'karafka/railtie'

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: karafka
 version: !ruby/object:Gem::Version
-  version: 2.0.20
+  version: 2.0.22
 platform: ruby
 authors:
 - Maciej Mensfeld
@@ -35,7 +35,7 @@ cert_chain:
   Qf04B9ceLUaC4fPVEz10FyobjaFoY4i32xRto3XnrzeAgfEe4swLq8bQsR3w/EF3
   MGU0FeSV2Yj7Xc2x/7BzLK8xQn5l7Yy75iPF+KP3vVmDHnNl
   -----END CERTIFICATE-----
-date: 2022-11-24 00:00:00.000000000 Z
+date: 2022-12-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: karafka-core
@@ -119,7 +119,11 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '2.3'
-description: Framework used to simplify Apache Kafka based Ruby applications development
+description: |2
+      Karafka is Ruby and Rails efficient Kafka processing framework.
+
+      Karafka allows you to capture everything that happens in your systems in large scale,
+      without having to focus on things that are not your business domain.
 email:
 - contact@karafka.io
 executables:
@@ -227,6 +231,7 @@ files:
 - lib/karafka/pro/performance_tracker.rb
 - lib/karafka/pro/processing/coordinator.rb
 - lib/karafka/pro/processing/jobs/consume_non_blocking.rb
+- lib/karafka/pro/processing/jobs/revoked_non_blocking.rb
 - lib/karafka/pro/processing/jobs_builder.rb
 - lib/karafka/pro/processing/partitioner.rb
 - lib/karafka/pro/processing/scheduler.rb
@@ -353,5 +358,5 @@ requirements: []
 rubygems_version: 3.3.7
 signing_key: 
 specification_version: 4
-summary: Efficient Kafka processing framework for Ruby and Rails
+summary: Karafka is Ruby and Rails efficient Kafka processing framework.
 test_files: []