karafka 2.0.29 → 2.0.30

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 3a258a78f3df80b4cda4f315f56ef4f7fb48003f63d0d58140c189b01b3f4875
-  data.tar.gz: c267cb96bb8a972a2255f4a1298adde63678be4502b4ec8573af9c8a940435b4
+  metadata.gz: 2c983b3fbf07db3e91f3c051a8b4de4b2d88619877fbf66a4e434e38bf67c96b
+  data.tar.gz: babb3c513e6bb3fd0610363e3bbd9e7fcf07b1820f02fdd80ccacc8a109aab08
 SHA512:
-  metadata.gz: beed9b0cbc83249d0bf227edb859142888d748dab6a972bf58c2d7e38ad578292544948bd9f2f25edbeebe6b37d1f0d20d47fc5e15d3d12420964447d414c54a
-  data.tar.gz: 7da728f75e5540d339887d2fb3bee4787016bdcd16a56f29b4f9c0b3e9f4ff45f1d6f0e843d25bf3e7a31e05ea3a320e174a005a149f76156618dc70327e71de
+  metadata.gz: ea000c13a834ffba97c1837fa1f19dabbf78dbe8958354a8200bb5d2c0940852343521e6740113e07890f97a7e6edd108d8595a66688c4c14dbf313e86ec715f
+  data.tar.gz: 81ae980ef7950c6cae7d1bea2751d1826e5bbf4c1384be4736af895977fb7ff38f70c24a9b7b248cb18744e042ffb0ba0dc154171c29dcc73262733d87c31ea1

data/CHANGELOG.md

@@ -1,5 +1,13 @@
 # Karafka framework changelog
 
+## 2.0.30 (2022-01-31)
+- [Improvement] Alias `--consumer-groups` with `--include-consumer-groups`
+- [Improvement] Alias `--subscription-groups` with `--include-subscription-groups`
+- [Improvement] Alias `--topics` with `--include-topics`
+- [Improvement] Introduce `--exclude-consumer-groups` for ability to exclude certain consumer groups from running
+- [Improvement] Introduce `--exclude-subscription-groups` for ability to exclude certain subscription groups from running
+- [Improvement] Introduce `--exclude-topics` for ability to exclude certain topics from running
+
 ## 2.0.29 (2023-01-30)
 - [Improvement] Make sure, that the `Karafka#producer` instance has the `LoggerListener` enabled in the install template, so Karafka by default prints both consumer and producer info.
 - [Improvement] Extract the code loading capabilities of Karafka console from the executable, so web can use it to provide CLI commands.

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    karafka (2.0.29)
+    karafka (2.0.30)
       karafka-core (>= 2.0.9, < 3.0.0)
       thor (>= 0.20)
       waterdrop (>= 2.4.10, < 3.0.0)
@@ -25,7 +25,7 @@ GEM
     factory_bot (6.2.1)
       activesupport (>= 5.0.0)
     ffi (1.15.5)
-    globalid (1.0.1)
+    globalid (1.1.0)
       activesupport (>= 5.0)
     i18n (1.12.0)
       concurrent-ruby (~> 1.0)
@@ -59,7 +59,7 @@ GEM
     simplecov-html (0.12.3)
     simplecov_json_formatter (0.1.4)
     thor (1.2.1)
-    tzinfo (2.0.5)
+    tzinfo (2.0.6)
       concurrent-ruby (~> 1.0)
     waterdrop (2.4.10)
       karafka-core (>= 2.0.9, < 3.0.0)

data/README.md

@@ -10,6 +10,7 @@
 
 Karafka is a Ruby and Rails multi-threaded efficient Kafka processing framework that:
 
+- Has a built-in [Web UI](https://karafka.io/docs/Web-UI-Features/) providing a convenient way to monitor and manage Karafka-based applications.
 - Supports parallel processing in [multiple threads](https://karafka.io/docs/Concurrency-and-multithreading) (also for a [single topic partition](https://karafka.io/docs/Pro-Virtual-Partitions) work)
 - [Automatically integrates](https://karafka.io/docs/Integrating-with-Ruby-on-Rails-and-other-frameworks#integrating-with-ruby-on-rails) with Ruby on Rails
 - Has [ActiveJob backend](https://karafka.io/docs/Active-Job) support (including [ordered jobs](https://karafka.io/docs/Pro-Enhanced-Active-Job#ordered-jobs))
@@ -83,7 +84,7 @@ bundle exec karafka server
 
 ## Want to Upgrade? LGPL is not for you? Want to help?
 
-I also sell Karafka Pro subscriptions. It includes a commercial-friendly license, priority support, architecture consultations, and high throughput data processing-related features (virtual partitions, long-running jobs, and more).
+I also sell Karafka Pro subscriptions. It includes a commercial-friendly license, priority support, architecture consultations, enhanced Web UI and high throughput data processing-related features (virtual partitions, long-running jobs, and more).
 
 **20%** of the income will be distributed back to other OSS projects that Karafka uses under the hood.
 

data/lib/karafka/cli/server.rb

@@ -7,27 +7,73 @@ module Karafka
     class Server < Base
       include Helpers::Colorize
 
+      # Types of things we can include / exclude from the routing via the CLI options
+      SUPPORTED_TYPES = ::Karafka::Routing::ActivityManager::SUPPORTED_TYPES
+
+      private_constant :SUPPORTED_TYPES
+
       desc 'Start the Karafka server (short-cut alias: "s")'
+
       option aliases: 's'
+
+      # Thor does not work well with many aliases combinations, hence we remap the aliases
+      # by ourselves in the code
       option :consumer_groups, type: :array, default: [], aliases: :g
       option :subscription_groups, type: :array, default: []
       option :topics, type: :array, default: []
 
+      %i[
+        include
+        exclude
+      ].each do |action|
+        SUPPORTED_TYPES.each do |type|
+          option(
+            "#{action}_#{type}",
+            type: :array,
+            default: []
+          )
+        end
+      end
+
       # Start the Karafka server
       def call
         # Print our banner and info in the dev mode
         print_marketing_info if Karafka::App.env.development?
 
-        active_routing_config = Karafka::App.config.internal.routing.active
-        active_routing_config.consumer_groups = cli.options[:consumer_groups]
-        active_routing_config.subscription_groups = cli.options[:subscription_groups]
-        active_routing_config.topics = cli.options[:topics]
+        register_inclusions(cli)
+        register_exclusions(cli)
 
         Karafka::Server.run
       end
 
       private
 
+      # Registers things we want to include (if defined)
+      # @param cli [Karafka::Cli] Thor cli handler
+      def register_inclusions(cli)
+        activities = ::Karafka::App.config.internal.routing.activity_manager
+
+        SUPPORTED_TYPES.each do |type|
+          v1 = cli.options[type] || []
+          v2 = cli.options[:"include_#{type}"] || []
+          names = v1 + v2
+
+          names.each { |name| activities.include(type, name) }
+        end
+      end
+
+      # Registers things we want to exclude (if defined)
+      # @param cli [Karafka::Cli] Thor cli handler
+      def register_exclusions(cli)
+        activities = ::Karafka::App.config.internal.routing.activity_manager
+
+        activities.class::SUPPORTED_TYPES.each do |type|
+          names = cli.options[:"exclude_#{type}"] || []
+
+          names.each { |name| activities.exclude(type, name) }
+        end
+      end
+
       # Prints marketing info
       def print_marketing_info
         Karafka.logger.info Info::BANNER

data/lib/karafka/contracts/server_cli_options.rb

@@ -12,69 +12,71 @@ module Karafka
         ).fetch('en').fetch('validations').fetch('server_cli_options')
       end
 
-      optional(:consumer_groups) { |cg| cg.is_a?(Array) }
-      optional(:subscription_groups) { |sg| sg.is_a?(Array) }
-      optional(:topics) { |topics| topics.is_a?(Array) }
+      %i[
+        include
+        exclude
+      ].each do |action|
+        optional(:"#{action}_consumer_groups") { |cg| cg.is_a?(Array) }
+        optional(:"#{action}_subscription_groups") { |sg| sg.is_a?(Array) }
+        optional(:"#{action}_topics") { |topics| topics.is_a?(Array) }
 
-      virtual do |data, errors|
-        next unless errors.empty?
-        next unless data.key?(:consumer_groups)
+        virtual do |data, errors|
+          next unless errors.empty?
 
-        value = data.fetch(:consumer_groups)
+          value = data.fetch(:"#{action}_consumer_groups")
 
-        # If there were no consumer_groups declared in the server cli, it means that we will
-        # run all of them and no need to validate them here at all
-        next if value.empty?
-        next if (value - Karafka::App.consumer_groups.map(&:name)).empty?
+          # If there were no consumer_groups declared in the server cli, it means that we will
+          # run all of them and no need to validate them here at all
+          next if value.empty?
+          next if (value - Karafka::App.consumer_groups.map(&:name)).empty?
 
-        # Found unknown consumer groups
-        [[%i[consumer_groups], :consumer_groups_inclusion]]
-      end
+          # Found unknown consumer groups
+          [[[:"#{action}_consumer_groups"], :consumer_groups_inclusion]]
+        end
 
-      virtual do |data, errors|
-        next unless errors.empty?
-        next unless data.key?(:subscription_groups)
+        virtual do |data, errors|
+          next unless errors.empty?
 
-        value = data.fetch(:subscription_groups)
+          value = data.fetch(:"#{action}_subscription_groups")
 
-        # If there were no subscription_groups declared in the server cli, it means that we will
-        # run all of them and no need to validate them here at all
-        next if value.empty?
+          # If there were no subscription_groups declared in the server cli, it means that we will
+          # run all of them and no need to validate them here at all
+          next if value.empty?
 
-        subscription_groups = Karafka::App
-                              .consumer_groups
-                              .map(&:subscription_groups)
-                              .flatten
-                              .map(&:name)
+          subscription_groups = Karafka::App
+                                .consumer_groups
+                                .map(&:subscription_groups)
+                                .flatten
+                                .map(&:name)
 
-        next if (value - subscription_groups).empty?
+          next if (value - subscription_groups).empty?
 
-        # Found unknown subscription groups
-        [[%i[subscription_groups], :subscription_groups_inclusion]]
-      end
+          # Found unknown subscription groups
+          [[[:"#{action}_subscription_groups"], :subscription_groups_inclusion]]
+        end
 
-      virtual do |data, errors|
-        next unless errors.empty?
-        next unless data.key?(:topics)
+        virtual do |data, errors|
+          next unless errors.empty?
 
-        value = data.fetch(:topics)
+          value = data.fetch(:"#{action}_topics")
 
-        # If there were no topics declared in the server cli, it means that we will
-        # run all of them and no need to validate them here at all
-        next if value.empty?
+          # If there were no topics declared in the server cli, it means that we will
+          # run all of them and no need to validate them here at all
+          next if value.empty?
 
-        topics = Karafka::App
-                 .consumer_groups
-                 .map(&:subscription_groups)
-                 .flatten
-                 .map(&:topics)
-                 .map { |gtopics| gtopics.map(&:name) }
-                 .flatten
+          topics = Karafka::App
+                   .consumer_groups
+                   .map(&:subscription_groups)
+                   .flatten
+                   .map(&:topics)
+                   .map { |gtopics| gtopics.map(&:name) }
+                   .flatten
 
-        next if (value - topics).empty?
+          next if (value - topics).empty?
 
-        # Found unknown topics
-        [[%i[topics], :topics_inclusion]]
+          # Found unknown topics
+          [[[:"#{action}_topics"], :topics_inclusion]]
+        end
       end
 
       # Makes sure we have anything to subscribe to when we start the server
@@ -83,7 +85,7 @@ module Karafka
 
         next unless Karafka::App.subscription_groups.empty?
 
-        [[%i[topics], :topics_missing]]
+        [[%i[include_topics], :topics_missing]]
       end
     end
   end

data/lib/karafka/routing/activity_manager.rb

@@ -0,0 +1,84 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Routing
+    # Allows us to get track of which consumer groups, subscription groups and topics are enabled
+    # or disabled via CLI
+    class ActivityManager
+      # Supported types of inclusions and exclusions
+      SUPPORTED_TYPES = %i[
+        consumer_groups
+        subscription_groups
+        topics
+      ].freeze
+
+      def initialize
+        @included = Hash.new { |h, k| h[k] = [] }
+        @excluded = Hash.new { |h, k| h[k] = [] }
+      end
+
+      # Adds resource to included/active
+      # @param type [Symbol] type for inclusion
+      # @param name [String] name of the element
+      def include(type, name)
+        validate!(type)
+
+        @included[type] << name
+      end
+
+      # Adds resource to excluded
+      # @param type [Symbol] type for inclusion
+      # @param name [String] name of the element
+      def exclude(type, name)
+        validate!(type)
+
+        @excluded[type] << name
+      end
+
+      # @param type [Symbol] type for inclusion
+      # @param name [String] name of the element
+      # @return [Boolean] is the given resource active or not
+      def active?(type, name)
+        validate!(type)
+
+        included = @included[type]
+        excluded = @excluded[type]
+
+        # If nothing defined, all active by default
+        return true if included.empty? && excluded.empty?
+        # Inclusion supersedes exclusion in case someone wrote both
+        return true if !included.empty? && included.include?(name)
+
+        # If there are exclusions but our is not excluded and no inclusions or included, it's ok
+        !excluded.empty? &&
+          !excluded.include?(name) &&
+          (included.empty? || included.include?(name))
+      end
+
+      # @return [Hash] accumulated data in a hash for validations
+      def to_h
+        (
+          SUPPORTED_TYPES.map { |type| ["include_#{type}".to_sym, @included[type]] } +
+          SUPPORTED_TYPES.map { |type| ["exclude_#{type}".to_sym, @excluded[type]] }
+        ).to_h
+      end
+
+      # Clears the manager
+      def clear
+        @included.clear
+        @excluded.clear
+      end
+
+      private
+
+      # Checks if the type we want to register is supported
+      #
+      # @param type [Symbol] type for inclusion
+      def validate!(type)
+        return if SUPPORTED_TYPES.include?(type)
+
+        raise(::Karafka::Errors::UnsupportedCaseError, type)
+      end
+    end
+  end
+end

data/lib/karafka/routing/consumer_group.rb

@@ -31,10 +31,7 @@ module Karafka
 
       # @return [Boolean] true if this consumer group should be active in our current process
       def active?
-        cgs = Karafka::App.config.internal.routing.active.consumer_groups
-
-        # When empty it means no groups were specified, hence all should be used
-        cgs.empty? || cgs.include?(name)
+        Karafka::App.config.internal.routing.activity_manager.active?(:consumer_groups, name)
       end
 
       # Builds a topic representation inside of a current consumer group route

data/lib/karafka/routing/subscription_group.rb

@@ -58,10 +58,7 @@ module Karafka
 
       # @return [Boolean] is this subscription group one of active once
       def active?
-        sgs = Karafka::App.config.internal.routing.active.subscription_groups
-
-        # When empty it means no groups were specified, hence all should be used
-        sgs.empty? || sgs.include?(name)
+        Karafka::App.config.internal.routing.activity_manager.active?(:subscription_groups, name)
       end
 
       private

data/lib/karafka/routing/topic.rb

@@ -87,10 +87,7 @@ module Karafka
         # Never active if disabled via routing
         return false unless @active
 
-        topics = Karafka::App.config.internal.routing.active.topics
-
-        # When empty it means no topics were specified, hence all should be used
-        topics.empty? || topics.include?(name)
+        Karafka::App.config.internal.routing.activity_manager.active?(:topics, name)
       end
 
       # @return [Hash] hash with all the topic attributes

data/lib/karafka/server.rb

@@ -33,7 +33,7 @@ module Karafka
         # We cannot validate this during the start because config needs to be populated and routes
         # need to be defined.
         Contracts::ServerCliOptions.new.validate!(
-          Karafka::App.config.internal.routing.active.to_h
+          Karafka::App.config.internal.routing.activity_manager.to_h
         )
 
         process.on_sigint { stop }

data/lib/karafka/setup/config.rb

@@ -111,12 +111,8 @@ module Karafka
           setting :subscription_groups_builder, default: Routing::SubscriptionGroupsBuilder.new
 
           # Internally assigned list of limits on routings active for the current process
-          # This should be overwritten by the CLI command
-          setting :active do
-            setting :consumer_groups, default: [].freeze
-            setting :subscription_groups, default: [].freeze
-            setting :topics, default: [].freeze
-          end
+          # This can be altered by the CLI command
+          setting :activity_manager, default: Routing::ActivityManager.new
         end
 
         setting :processing do

data/lib/karafka/version.rb

@@ -3,5 +3,5 @@
 # Main module namespace
 module Karafka
   # Current Karafka version
-  VERSION = '2.0.29'
+  VERSION = '2.0.30'
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: karafka
 version: !ruby/object:Gem::Version
-  version: 2.0.29
+  version: 2.0.30
 platform: ruby
 authors:
 - Maciej Mensfeld
@@ -35,7 +35,7 @@ cert_chain:
   Qf04B9ceLUaC4fPVEz10FyobjaFoY4i32xRto3XnrzeAgfEe4swLq8bQsR3w/EF3
   MGU0FeSV2Yj7Xc2x/7BzLK8xQn5l7Yy75iPF+KP3vVmDHnNl
   -----END CERTIFICATE-----
-date: 2023-01-30 00:00:00.000000000 Z
+date: 2023-01-31 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: karafka-core
@@ -289,6 +289,7 @@ files:
 - lib/karafka/processing/worker.rb
 - lib/karafka/processing/workers_batch.rb
 - lib/karafka/railtie.rb
+- lib/karafka/routing/activity_manager.rb
 - lib/karafka/routing/builder.rb
 - lib/karafka/routing/consumer_group.rb
 - lib/karafka/routing/consumer_mapper.rb