karafka 2.3.3 → 2.4.0.beta2

data/lib/karafka/cli/topics/align.rb

@@ -0,0 +1,109 @@
+# frozen_string_literal: true
+
+module Karafka
+  class Cli
+    class Topics < Cli::Base
+      # Aligns configuration of all the declarative topics that exist based on the declarative
+      # topics definitions.
+      #
+      # Takes into consideration already existing settings, so will only align what is needed.
+      #
+      # Keep in mind, this is NOT transactional. Kafka topic changes are not transactional so
+      # it is highly recommended to test it before running in prod.
+      #
+      # @note This command does NOT repartition and does NOT create new topics. It only aligns
+      #   configuration of existing topics.
+      class Align < Base
+        # @return [Boolean] true if there were any changes applied, otherwise false
+        def call
+          if candidate_topics.empty?
+            puts "#{yellow('Skipping')} because no declarative topics exist."
+
+            return false
+          end
+
+          resources_to_migrate = build_resources_to_migrate
+
+          if resources_to_migrate.empty?
+            puts "#{yellow('Skipping')} because there are no configurations to align."
+
+            return false
+          end
+
+          names = resources_to_migrate.map(&:name).join(', ')
+          puts "Updating configuration of the following topics: #{names}"
+          Karafka::Admin::Configs.alter(resources_to_migrate)
+          puts "#{green('Updated')} all requested topics configuration."
+
+          true
+        end
+
+        private
+
+        # Selects topics that exist and potentially may have config to align
+        #
+        # @return [Set<Karafka::Routing::Topic>]
+        def candidate_topics
+          return @candidate_topics if @candidate_topics
+
+          @candidate_topics = Set.new
+
+          # First lets only operate on topics that do exist
+          declaratives_routing_topics.each do |topic|
+            unless existing_topics_names.include?(topic.name)
+              puts "#{yellow('Skipping')} because topic #{topic.name} does not exist."
+              next
+            end
+
+            @candidate_topics << topic
+          end
+
+          @candidate_topics
+        end
+
+        # Iterates over configs of all the candidate topics and prepares alignment resources for
+        # a single request to Kafka
+        # @return [Array<Karafka::Admin::Configs::Resource>] all topics with config change requests
+        def build_resources_to_migrate
+          # We build non-fetched topics resources representations for further altering
+          resources = candidate_topics.map do |topic|
+            Admin::Configs::Resource.new(type: :topic, name: topic.name)
+          end
+
+          resources_to_migrate = Set.new
+
+          # We fetch all the configurations for all the topics
+          Admin::Configs.describe(resources).each do |topic_with_configs|
+            t_candidate = candidate_topics.find do |candidate|
+              candidate.name == topic_with_configs.name
+            end
+
+            change_resource = resources.find do |resource|
+              resource.name == topic_with_configs.name
+            end
+
+            # librdkafka returns us all the results as strings, so we need to align our config
+            # representation so we can compare those
+            desired_configs = t_candidate.declaratives.details.dup
+            desired_configs.transform_values!(&:to_s)
+            desired_configs.transform_keys!(&:to_s)
+
+            topic_with_configs.configs.each do |config|
+              next unless desired_configs.key?(config.name)
+
+              desired_config = desired_configs.fetch(config.name)
+
+              # Do not migrate if existing and desired values are the same
+              next if desired_config == config.value
+
+              change_resource.set(config.name, desired_config)
+              resources_to_migrate << change_resource
+            end
+          end
+
+          resources_to_migrate.to_a
+        end
+      end
+    end
+  end
+end

data/lib/karafka/cli/topics/base.rb

@@ -0,0 +1,66 @@
+# frozen_string_literal: true
+
+module Karafka
+  class Cli
+    class Topics < Cli::Base
+      # Base class for all the topics related operations
+      class Base
+        include Helpers::Colorize
+        include Helpers::ConfigImporter.new(
+          kafka_config: %i[kafka]
+        )
+
+        private
+
+        # @return [Array<Karafka::Routing::Topic>] all available topics that can be managed
+        # @note If topic is defined in multiple consumer groups, first config will be used. This
+        #   means, that this CLI will not work for simultaneous management of multiple clusters
+        #   from a single CLI command execution flow.
+        def declaratives_routing_topics
+          return @declaratives_routing_topics if @declaratives_routing_topics
+
+          collected_topics = {}
+          default_servers = kafka_config[:'bootstrap.servers']
+
+          App.consumer_groups.each do |consumer_group|
+            consumer_group.topics.each do |topic|
+              # Skip topics that were explicitly disabled from management
+              next unless topic.declaratives.active?
+              # If bootstrap servers are different, consider this a different cluster
+              next unless default_servers == topic.kafka[:'bootstrap.servers']
+
+              collected_topics[topic.name] ||= topic
+            end
+          end
+
+          @declaratives_routing_topics = collected_topics.values
+        end
+
+        # @return [Array<Hash>] existing topics details
+        def existing_topics
+          @existing_topics ||= Admin.cluster_info.topics
+        end
+
+        # @return [Array<String>] names of already existing topics
+        def existing_topics_names
+          existing_topics.map { |topic| topic.fetch(:topic_name) }
+        end
+
+        # Waits with a message, that we are waiting on topics
+        # This is not doing much, just waiting as there are some cases that it takes a bit of time
+        # for Kafka to actually propagate new topics knowledge across the cluster. We give it that
+        # bit of time just in case.
+        def wait
+          print 'Waiting for the topics to synchronize in the cluster'
+
+          5.times do
+            sleep(1)
+            print '.'
+          end
+
+          puts
+        end
+      end
+    end
+  end
+end

data/lib/karafka/cli/topics/create.rb

@@ -0,0 +1,35 @@
+# frozen_string_literal: true
+
+module Karafka
+  class Cli
+    class Topics < Cli::Base
+      # Creates topics based on the routing setup and configuration
+      class Create < Base
+        # @return [Boolean] true if any topic was created, otherwise false
+        def call
+          any_created = false
+
+          declaratives_routing_topics.each do |topic|
+            name = topic.name
+
+            if existing_topics_names.include?(name)
+              puts "#{yellow('Skipping')} because topic #{name} already exists."
+            else
+              puts "Creating topic #{name}..."
+              Admin.create_topic(
+                name,
+                topic.declaratives.partitions,
+                topic.declaratives.replication_factor,
+                topic.declaratives.details
+              )
+              puts "#{green('Created')} topic #{name}."
+              any_created = true
+            end
+          end
+
+          any_created
+        end
+      end
+    end
+  end
+end

data/lib/karafka/cli/topics/delete.rb

@@ -0,0 +1,30 @@
+# frozen_string_literal: true
+
+module Karafka
+  class Cli
+    class Topics < Cli::Base
+      # Deletes routing based topics
+      class Delete < Base
+        # @return [Boolean] true if any topic was deleted, otherwise false
+        def call
+          any_deleted = false
+
+          declaratives_routing_topics.each do |topic|
+            name = topic.name
+
+            if existing_topics_names.include?(name)
+              puts "Deleting topic #{name}..."
+              Admin.delete_topic(name)
+              puts "#{green('Deleted')} topic #{name}."
+              any_deleted = true
+            else
+              puts "#{yellow('Skipping')} because topic #{name} does not exist."
+            end
+          end
+
+          any_deleted
+        end
+      end
+    end
+  end
+end

data/lib/karafka/cli/topics/migrate.rb

@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+
+module Karafka
+  class Cli
+    class Topics < Cli::Base
+      # Creates missing topics and aligns the partitions count
+      class Migrate < Base
+        # Runs the migration
+        # @return [Boolean] true if there were any changes applied
+        def call
+          any_changes = false
+
+          if Topics::Create.new.call
+            any_changes = true
+            wait
+          end
+
+          if Topics::Repartition.new.call
+            any_changes = true
+            wait
+          end
+
+          # No need to wait after the last one
+          any_changes = true if Topics::Align.new.call
+
+          any_changes
+        end
+      end
+    end
+  end
+end

data/lib/karafka/cli/topics/plan.rb

@@ -0,0 +1,169 @@
+# frozen_string_literal: true
+
+module Karafka
+  class Cli
+    class Topics < Cli::Base
+      # Plans the migration process and prints what changes are going to be applied if migration
+      # would to be executed
+      class Plan < Base
+        # Figures out scope of changes that need to happen
+        # @return [Boolean] true if running migrate would change anything, false otherwise
+        def call
+          # If no changes at all, just print and stop
+          if topics_to_create.empty? && topics_to_repartition.empty? && topics_to_alter.empty?
+            puts "Karafka will #{yellow('not')} perform any actions. No changes needed."
+
+            return false
+          end
+
+          unless topics_to_create.empty?
+            puts 'Following topics will be created:'
+            puts
+
+            topics_to_create.each do |topic|
+              puts "  #{green('+')} #{topic.name}:"
+              puts "    #{green('+')} partitions: \"#{topic.declaratives.partitions}\""
+
+              topic.declaratives.details.each do |name, value|
+                puts "    #{green('+')} #{name}: \"#{value}\""
+              end
+
+              puts
+            end
+          end
+
+          unless topics_to_repartition.empty?
+            puts 'Following topics will be repartitioned:'
+            puts
+
+            topics_to_repartition.each do |topic, partitions|
+              from = partitions
+              to = topic.declaratives.partitions
+
+              puts "  #{yellow('~')} #{topic.name}:"
+              puts "    #{yellow('~')} partitions: \"#{red(from)}\" #{grey('=>')} \"#{green(to)}\""
+              puts
+            end
+          end
+
+          unless topics_to_alter.empty?
+            puts 'Following topics will have configuration changes:'
+            puts
+
+            topics_to_alter.each do |topic, configs|
+              puts "  #{yellow('~')} #{topic.name}:"
+
+              configs.each do |name, changes|
+                from = changes.fetch(:from)
+                to = changes.fetch(:to)
+                action = changes.fetch(:action)
+                type = action == :change ? yellow('~') : green('+')
+                puts "    #{type} #{name}: \"#{red(from)}\" #{grey('=>')} \"#{green(to)}\""
+              end
+
+              puts
+            end
+          end
+
+          true
+        end
+
+        private
+
+        # @return [Array<Karafka::Routing::Topic>] topics that will be created
+        def topics_to_create
+          return @topics_to_create if @topics_to_create
+
+          @topics_to_create = declaratives_routing_topics.reject do |topic|
+            existing_topics_names.include?(topic.name)
+          end
+
+          @topics_to_create
+        end
+
+        # @return [Array<Array<Karafka::Routing::Topic, Integer>>] array with topics that will
+        #   be repartitioned and current number of partitions
+        def topics_to_repartition
+          return @topics_to_repartition if @topics_to_repartition
+
+          @topics_to_repartition = []
+
+          declaratives_routing_topics.each do |declarative_topic|
+            existing_topic = existing_topics.find do |topic|
+              topic.fetch(:topic_name) == declarative_topic.name
+            end
+
+            next unless existing_topic
+
+            existing_partitions = existing_topic.fetch(:partition_count)
+
+            next if declarative_topic.declaratives.partitions == existing_partitions
+
+            @topics_to_repartition << [declarative_topic, existing_partitions]
+          end
+
+          @topics_to_repartition
+        end
+
+        # @return [Hash] Hash where keys are topics to alter and values are configs that will
+        #   be altered.
+        def topics_to_alter
+          return @topics_to_alter if @topics_to_alter
+
+          topics_to_check = []
+
+          declaratives_routing_topics.each do |declarative_topic|
+            next if declarative_topic.declaratives.details.empty?
+            next unless existing_topics_names.include?(declarative_topic.name)
+
+            topics_to_check << [
+              declarative_topic,
+              Admin::Configs::Resource.new(type: :topic, name: declarative_topic.name)
+            ]
+          end
+
+          @topics_to_alter = {}
+
+          return @topics_to_alter if topics_to_check.empty?
+
+          Admin::Configs.describe(topics_to_check.map(&:last)).each.with_index do |topic_c, index|
+            declarative = topics_to_check[index].first
+            declarative_config = declarative.declaratives.details.dup
+            declarative_config.transform_keys!(&:to_s)
+            declarative_config.transform_values!(&:to_s)
+
+            # We only apply additive/in-place changes so we start from our config
+            declarative_config.each do |declarative_name, declarative_value|
+              @topics_to_alter[declarative] ||= {}
+
+              @topics_to_alter[declarative][declarative_name] ||= {
+                from: '',
+                to: declarative_value,
+                action: :add
+              }
+
+              scoped = @topics_to_alter[declarative][declarative_name]
+
+              topic_c.configs.each do |config|
+                next unless declarative_name == config.name
+
+                scoped[:action] = :change
+                scoped[:from] = config.value
+              end
+
+              # Remove change definitions that would migrate to the same value as present
+              @topics_to_alter[declarative].delete_if do |_name, details|
+                details[:from] == details[:to]
+              end
+            end
+
+            # Remove topics without any changes
+            @topics_to_alter.delete_if { |_name, configs| configs.empty? }
+          end
+
+          @topics_to_alter
+        end
+      end
+    end
+  end
+end

data/lib/karafka/cli/topics/repartition.rb

@@ -0,0 +1,41 @@
+# frozen_string_literal: true
+
+module Karafka
+  class Cli
+    class Topics < Cli::Base
+      # Increases number of partitions on topics that have less partitions than defined
+      # Will **not** create topics if missing.
+      class Repartition < Base
+        # @return [Boolean] true if anything was repartitioned, otherwise false
+        def call
+          any_repartitioned = false
+
+          existing_partitions = existing_topics.map do |topic|
+            [topic.fetch(:topic_name), topic.fetch(:partition_count)]
+          end.to_h
+
+          declaratives_routing_topics.each do |topic|
+            name = topic.name
+
+            desired_count = topic.config.partitions
+            existing_count = existing_partitions.fetch(name, false)
+
+            if existing_count && existing_count < desired_count
+              puts "Increasing number of partitions to #{desired_count} on topic #{name}..."
+              Admin.create_partitions(name, desired_count)
+              change = desired_count - existing_count
+              puts "#{green('Created')} #{change} additional partitions on topic #{name}."
+              any_repartitioned = true
+            elsif existing_count
+              puts "#{yellow('Skipping')} because topic #{name} has #{existing_count} partitions."
+            else
+              puts "#{yellow('Skipping')} because topic #{name} does not exist."
+            end
+          end
+
+          any_repartitioned
+        end
+      end
+    end
+  end
+end

data/lib/karafka/cli/topics/reset.rb

@@ -0,0 +1,18 @@
+# frozen_string_literal: true
+
+module Karafka
+  class Cli
+    class Topics < Cli::Base
+      # Deletes routing based topics and re-creates them
+      class Reset < Base
+        # @return [true] since it is a reset, always changes so `true` always
+        def call
+          Delete.new.call && wait
+          Create.new.call
+
+          true
+        end
+      end
+    end
+  end
+end

data/lib/karafka/cli/topics.rb

@@ -5,142 +5,32 @@ module Karafka
     # CLI actions related to Kafka cluster topics management
     class Topics < Base
       include Helpers::Colorize
+      include Helpers::ConfigImporter.new(
+        kafka_config: %i[kafka]
+      )
 
-      desc 'Allows for the topics management (create, delete, reset, repartition)'
+      desc 'Allows for the topics management'
       # @param action [String] action we want to take
       def call(action = 'missing')
         case action
         when 'create'
-          create
+          Topics::Create.new.call
         when 'delete'
-          delete
+          Topics::Delete.new.call
         when 'reset'
-          reset
+          Topics::Reset.new.call
         when 'repartition'
-          repartition
+          Topics::Repartition.new.call
         when 'migrate'
-          migrate
+          Topics::Migrate.new.call
+        when 'align'
+          Topics::Align.new.call
+        when 'plan'
+          Topics::Plan.new.call
         else
           raise ::ArgumentError, "Invalid topics action: #{action}"
         end
       end
-
-      private
-
-      # Creates topics based on the routing setup and configuration
-      def create
-        declaratives_routing_topics.each do |topic|
-          name = topic.name
-
-          if existing_topics_names.include?(name)
-            puts "#{yellow('Skipping')} because topic #{name} already exists."
-          else
-            puts "Creating topic #{name}..."
-            Admin.create_topic(
-              name,
-              topic.declaratives.partitions,
-              topic.declaratives.replication_factor,
-              topic.declaratives.details
-            )
-            puts "#{green('Created')} topic #{name}."
-          end
-        end
-      end
-
-      # Deletes routing based topics
-      def delete
-        declaratives_routing_topics.each do |topic|
-          name = topic.name
-
-          if existing_topics_names.include?(name)
-            puts "Deleting topic #{name}..."
-            Admin.delete_topic(name)
-            puts "#{green('Deleted')} topic #{name}."
-          else
-            puts "#{yellow('Skipping')} because topic #{name} does not exist."
-          end
-        end
-      end
-
-      # Deletes routing based topics and re-creates them
-      def reset
-        delete
-
-        # We need to invalidate the metadata cache, otherwise we will think, that the topic
-        # already exists
-        @existing_topics = nil
-
-        create
-      end
-
-      # Creates missing topics and aligns the partitions count
-      def migrate
-        create
-
-        @existing_topics = nil
-
-        repartition
-      end
-
-      # Increases number of partitions on topics that have less partitions than defined
-      # Will **not** create topics if missing.
-      def repartition
-        existing_partitions = existing_topics.map do |topic|
-          [topic.fetch(:topic_name), topic.fetch(:partition_count)]
-        end.to_h
-
-        declaratives_routing_topics.each do |topic|
-          name = topic.name
-
-          desired_count = topic.config.partitions
-          existing_count = existing_partitions.fetch(name, false)
-
-          if existing_count && existing_count < desired_count
-            puts "Increasing number of partitions to #{desired_count} on topic #{name}..."
-            Admin.create_partitions(name, desired_count)
-            change = desired_count - existing_count
-            puts "#{green('Created')} #{change} additional partitions on topic #{name}."
-          elsif existing_count
-            puts "#{yellow('Skipping')} because topic #{name} has #{existing_count} partitions."
-          else
-            puts "#{yellow('Skipping')} because topic #{name} does not exist."
-          end
-        end
-      end
-
-      # @return [Array<Karafka::Routing::Topic>] all available topics that can be managed
-      # @note If topic is defined in multiple consumer groups, first config will be used. This
-      #   means, that this CLI will not work for simultaneous management of multiple clusters from
-      #   a single CLI command execution flow.
-      def declaratives_routing_topics
-        return @declaratives_routing_topics if @declaratives_routing_topics
-
-        collected_topics = {}
-        default_servers = Karafka::App.config.kafka[:'bootstrap.servers']
-
-        App.consumer_groups.each do |consumer_group|
-          consumer_group.topics.each do |topic|
-            # Skip topics that were explicitly disabled from management
-            next unless topic.declaratives.active?
-            # If bootstrap servers are different, consider this a different cluster
-            next unless default_servers == topic.kafka[:'bootstrap.servers']
-
-            collected_topics[topic.name] ||= topic
-          end
-        end
-
-        @declaratives_routing_topics = collected_topics.values
-      end
-
-      # @return [Array<Hash>] existing topics details
-      def existing_topics
-        @existing_topics ||= Admin.cluster_info.topics
-      end
-
-      # @return [Array<String>] names of already existing topics
-      def existing_topics_names
-        existing_topics.map { |topic| topic.fetch(:topic_name) }
-      end
     end
   end
 end