karafka 1.4.12 → 2.2.10

data/lib/karafka/cli/server.rb

@@ -2,69 +2,126 @@
 
 module Karafka
   # Karafka framework Cli
-  class Cli < Thor
+  class Cli
     # Server Karafka Cli action
     class Server < Base
-      # Server config settings contract
-      CONTRACT = Contracts::ServerCliOptions.new.freeze
+      include Helpers::Colorize
 
-      private_constant :CONTRACT
+      # Types of things we can include / exclude from the routing via the CLI options
+      SUPPORTED_TYPES = ::Karafka::Routing::ActivityManager::SUPPORTED_TYPES
 
-      desc 'Start the Karafka server (short-cut alias: "s")'
-      option aliases: 's'
-      option :daemon, default: false, type: :boolean, aliases: :d
-      option :pid, default: 'tmp/pids/karafka', type: :string, aliases: :p
-      option :consumer_groups, type: :array, default: nil, aliases: :g
+      private_constant :SUPPORTED_TYPES
 
-      # Start the Karafka server
-      def call
-        cli.info
+      desc 'Starts the Karafka server (short-cut alias: "s")'
 
-        validate!
+      aliases :s
 
-        if cli.options[:daemon]
-          FileUtils.mkdir_p File.dirname(cli.options[:pid])
-          daemonize
-        end
+      option(
+        :consumer_groups,
+        'Runs server only with specified consumer groups',
+        Array,
+        %w[
+          -g
+          --consumer_groups
+          --include_consumer_groups
+        ]
+      )
+
+      option(
+        :subscription_groups,
+        'Runs server only with specified subscription groups',
+        Array,
+        %w[
+          --subscription_groups
+          --include_subscription_groups
+        ]
+      )
+
+      option(
+        :topics,
+        'Runs server only with specified topics',
+        Array,
+        %w[
+          --topics
+          --include_topics
+        ]
+      )
+
+      option(
+        :exclude_consumer_groups,
+        'Runs server without specified consumer groups',
+        Array,
+        %w[
+          --exclude_consumer_groups
+        ]
+      )
+
+      option(
+        :exclude_subscription_groups,
+        'Runs server without specified subscription groups',
+        Array,
+        %w[
+          --exclude_subscription_groups
+        ]
+      )
+
+      option(
+        :exclude_topics,
+        'Runs server without specified topics',
+        Array,
+        %w[
+          --exclude_topics
+        ]
+      )
+
+      # Start the Karafka server
+      def call
+        # Print our banner and info in the dev mode
+        print_marketing_info if Karafka::App.env.development?
 
-        # We assign active topics on a server level, as only server is expected to listen on
-        # part of the topics
-        Karafka::Server.consumer_groups = cli.options[:consumer_groups]
+        register_inclusions
+        register_exclusions
 
         Karafka::Server.run
       end
 
       private
 
-      # Checks the server cli configuration
-      # options validations in terms of app setup (topics, pid existence, etc)
-      def validate!
-        result = CONTRACT.call(cli.options)
-        return if result.success?
+      # Registers things we want to include (if defined)
+      def register_inclusions
+        activities = ::Karafka::App.config.internal.routing.activity_manager
+
+        SUPPORTED_TYPES.each do |type|
+          names = options[type] || []
 
-        raise Errors::InvalidConfigurationError, result.errors.to_h
+          names.each { |name| activities.include(type, name) }
+        end
       end
 
-      # Detaches current process into background and writes its pidfile
-      def daemonize
-        ::Process.daemon(true)
-        File.open(
-          cli.options[:pid],
-          'w'
-        ) { |file| file.write(::Process.pid) }
-
-        # Remove pidfile on stop, just before the server instance is going to be GCed
-        # We want to delay the moment in which the pidfile is removed as much as we can,
-        # so instead of removing it after the server stops running, we rely on the gc moment
-        # when this object gets removed (it is a bit later), so it is closer to the actual
-        # system process end. We do that, so monitoring and deployment tools that rely on a pid
-        # won't alarm or start new system process up until the current one is finished
-        ObjectSpace.define_finalizer(self, proc { send(:clean) })
+      # Registers things we want to exclude (if defined)
+      def register_exclusions
+        activities = ::Karafka::App.config.internal.routing.activity_manager
+
+        activities.class::SUPPORTED_TYPES.each do |type|
+          names = options[:"exclude_#{type}"] || []
+
+          names.each { |name| activities.exclude(type, name) }
+        end
       end
 
-      # Removes a pidfile (if exist)
-      def clean
-        FileUtils.rm_f(cli.options[:pid]) if cli.options[:pid]
+      # Prints marketing info
+      def print_marketing_info
+        Karafka.logger.info Info::BANNER
+
+        if Karafka.pro?
+          Karafka.logger.info(
+            green('Thank you for using Karafka Pro!')
+          )
+        else
+          Karafka.logger.info(
+            red('Upgrade to Karafka Pro for more features and support: https://karafka.io')
+          )
+        end
       end
     end
   end

data/lib/karafka/cli/topics.rb

@@ -0,0 +1,146 @@
+# frozen_string_literal: true
+
+module Karafka
+  class Cli
+    # CLI actions related to Kafka cluster topics management
+    class Topics < Base
+      include Helpers::Colorize
+
+      desc 'Allows for the topics management (create, delete, reset, repartition)'
+      # @param action [String] action we want to take
+      def call(action = 'missing')
+        case action
+        when 'create'
+          create
+        when 'delete'
+          delete
+        when 'reset'
+          reset
+        when 'repartition'
+          repartition
+        when 'migrate'
+          migrate
+        else
+          raise ::ArgumentError, "Invalid topics action: #{action}"
+        end
+      end
+
+      private
+
+      # Creates topics based on the routing setup and configuration
+      def create
+        declaratives_routing_topics.each do |topic|
+          name = topic.name
+
+          if existing_topics_names.include?(name)
+            puts "#{yellow('Skipping')} because topic #{name} already exists."
+          else
+            puts "Creating topic #{name}..."
+            Admin.create_topic(
+              name,
+              topic.declaratives.partitions,
+              topic.declaratives.replication_factor,
+              topic.declaratives.details
+            )
+            puts "#{green('Created')} topic #{name}."
+          end
+        end
+      end
+
+      # Deletes routing based topics
+      def delete
+        declaratives_routing_topics.each do |topic|
+          name = topic.name
+
+          if existing_topics_names.include?(name)
+            puts "Deleting topic #{name}..."
+            Admin.delete_topic(name)
+            puts "#{green('Deleted')} topic #{name}."
+          else
+            puts "#{yellow('Skipping')} because topic #{name} does not exist."
+          end
+        end
+      end
+
+      # Deletes routing based topics and re-creates them
+      def reset
+        delete
+
+        # We need to invalidate the metadata cache, otherwise we will think, that the topic
+        # already exists
+        @existing_topics = nil
+
+        create
+      end
+
+      # Creates missing topics and aligns the partitions count
+      def migrate
+        create
+
+        @existing_topics = nil
+
+        repartition
+      end
+
+      # Increases number of partitions on topics that have less partitions than defined
+      # Will **not** create topics if missing.
+      def repartition
+        existing_partitions = existing_topics.map do |topic|
+          [topic.fetch(:topic_name), topic.fetch(:partition_count)]
+        end.to_h
+
+        declaratives_routing_topics.each do |topic|
+          name = topic.name
+
+          desired_count = topic.config.partitions
+          existing_count = existing_partitions.fetch(name, false)
+
+          if existing_count && existing_count < desired_count
+            puts "Increasing number of partitions to #{desired_count} on topic #{name}..."
+            Admin.create_partitions(name, desired_count)
+            change = desired_count - existing_count
+            puts "#{green('Created')} #{change} additional partitions on topic #{name}."
+          elsif existing_count
+            puts "#{yellow('Skipping')} because topic #{name} has #{existing_count} partitions."
+          else
+            puts "#{yellow('Skipping')} because topic #{name} does not exist."
+          end
+        end
+      end
+
+      # @return [Array<Karafka::Routing::Topic>] all available topics that can be managed
+      # @note If topic is defined in multiple consumer groups, first config will be used. This
+      #   means, that this CLI will not work for simultaneous management of multiple clusters from
+      #   a single CLI command execution flow.
+      def declaratives_routing_topics
+        return @declaratives_routing_topics if @declaratives_routing_topics
+
+        collected_topics = {}
+        default_servers = Karafka::App.config.kafka[:'bootstrap.servers']
+
+        App.consumer_groups.each do |consumer_group|
+          consumer_group.topics.each do |topic|
+            # Skip topics that were explicitly disabled from management
+            next unless topic.declaratives.active?
+            # If bootstrap servers are different, consider this a different cluster
+            next unless default_servers == topic.kafka[:'bootstrap.servers']
+
+            collected_topics[topic.name] ||= topic
+          end
+        end
+
+        @declaratives_routing_topics = collected_topics.values
+      end
+
+      # @return [Array<Hash>] existing topics details
+      def existing_topics
+        @existing_topics ||= Admin.cluster_info.topics
+      end
+
+      # @return [Array<String>] names of already existing topics
+      def existing_topics_names
+        existing_topics.map { |topic| topic.fetch(:topic_name) }
+      end
+    end
+  end
+end

data/lib/karafka/cli.rb

@@ -2,47 +2,44 @@
 
 module Karafka
   # Karafka framework Cli
+  #
   # If you want to add/modify command that belongs to CLI, please review all commands
   # available in cli/ directory inside Karafka source code.
-  #
-  # @note Whole Cli is built using Thor
-  # @see https://github.com/erikhuda/thor
-  class Cli < Thor
-    package_name 'Karafka'
+  class Cli
+    class << self
+      # Starts the CLI
+      def start
+        # Command we want to run, like install, server, etc
+        command_name = ARGV[0]
+        # Action for action-based commands like topics migrate
+        action = ARGV[1].to_s.start_with?('-') ? false : ARGV[1]
 
-    default_task :missingno
+        command = commands.find { |cmd| cmd.names.include?(command_name) }
 
-    class << self
-      # Loads all Cli commands into Thor framework
-      # This method should be executed before we run Karafka::Cli.start, otherwise we won't
-      # have any Cli commands available
-      def prepare
-        cli_commands.each do |action|
-          action.bind_to(self)
-        end
-      end
+        if command
+          # Only actionable commands require command as an argument
+          args = action ? [action] : []
 
-      # When there is a CLI crash, exit
-      # @return [true]
-      def exit_on_failure?
-        true
+          command.new.call(*args)
+        else
+          raise(
+            Karafka::Errors::UnrecognizedCommandError,
+            "Unrecognized command \"#{command_name}\""
+          )
+        end
       end
 
       private
 
-      # @return [Array<Class>] Array with Cli action classes that can be used as commands
-      def cli_commands
-        constants
-          .map! { |object| const_get(object) }
-          .keep_if do |object|
-            object.instance_of?(Class) && (object < Cli::Base)
-          end
+      # @return [Array<Class>] command classes
+      def commands
+        Base.commands
       end
     end
   end
 end
 
-# This is kinda trick - since we don't have a autoload and other magic stuff
+# This is kinda tricky - since we don't have an autoload and other magic stuff
 # like Rails does, so instead this method allows us to replace currently running
 # console with a new one via Kernel.exec. It will start console with new code loaded
 # Yes, we know that it is not turbo fast, however it is turbo convenient and small