kafkat 0.0.9 → 0.0.10

checksums.yaml

@@ -1,15 +1,15 @@
 ---
 !binary "U0hBMQ==":
   metadata.gz: !binary |-
-    ZjAwMzc4N2FlYjVhYmYzZTRkOGRlYWIxZWRkZWY1ZjM4NjlkMWE0Mw==
+    MTY0MTdhYjQ3YjAzNWExNWJjMzMyYThmOTY5ZWQ5NmQyOTcyMGM4Yw==
   data.tar.gz: !binary |-
-    Mzk4ZjIxNzhmZWZjN2QzZjZiODk4ODA3NmZmYWY5N2Y0Yzg2MmNiNQ==
+    MTkzM2UzODgxNTIzMmJiYTAwNDQ2NmI5ZmVjYWZkYTdiYjNkZDk4Ng==
 SHA512:
   metadata.gz: !binary |-
-    YTMwNzhiNTliZTU1ZmY0OWYxYTY2NGIzZWEwZGI4ZDQ3ODQyNDU4Njg1ZTgw
-    ZmM4NGZiYjNlMWFkYjIxMDIwNjFhZWIwN2ZjYWRhNGE0MWE1NmU1NDI2Njhl
-    NWIwNjE0YTQ5YWRhNTNmZjFlM2ZjMDk5N2FhNDM2MWY4Nzc3YTE=
+    N2U0ZTMyMjQ4MmI3M2QzMDkyNTM4ZWFjOWJmMTNhNmIxNjlkM2NjMzBkYTli
+    YmY3N2Y1ZTk5OWIwZjZlZjZhMzNjOGE2MmE4NDg2ZTc4NzFjYTYzZmZlZWNi
+    YzFmNzFjZGU5N2Y5Y2UyMGY5NDhiNmEyZWU5NjdiNzViMjVlNjU=
   data.tar.gz: !binary |-
-    NzZiZmZjNjlmNmRjMzBhYjRhZDVmOTdmZDU5MTE1YWFlOTZmODNmZDY5YmU2
-    ZTFkNmM2YjVhMGNkM2FjMWI4ZmY4MTNlYjZiMDlkY2UyNzU2ZWQwMTY4MmVk
-    ZGNhNGM2OGQxMWVmNGQ5YzMwOWU3MGYyMzc1MmE4ZDdjYWRiNzc=
+    YjRiMzllYmZjNDBiMmYxNjRiNzExMzlkZGM2ZWQwOTA0NmFiMGRkZTQzZjMz
+    ZmRlMGM5OTczMTcwYzI4MmJiYmE1ZGYwMDZlNzhhMTI3ZmQ2YThkN2E4OTdj
+    ZmUzN2QyZDMxNDk5ODE1ZWIwNTdjZmMyNWFkMTYzYWRjOWJmZWY=

data/Gemfile.lock

@@ -1,12 +1,12 @@
 PATH
   remote: .
   specs:
-    kafkat (0.0.7)
+    kafkat (0.0.10)
       colored (~> 1.2)
-      highline (~> 1.6.21)
-      retryable (~> 1.3.5)
+      highline (~> 1.6, >= 1.6.21)
+      retryable (~> 1.3, >= 1.3.5)
       trollop (~> 2.0)
-      zk (~> 1.9.4)
+      zk (~> 1.9, >= 1.9.4)
 
 GEM
   remote: https://rubygems.org/
@@ -23,7 +23,7 @@ GEM
     zk (1.9.4)
       logging (~> 1.8.2)
       zookeeper (~> 1.4.0)
-    zookeeper (1.4.8)
+    zookeeper (1.4.9)
 
 PLATFORMS
   ruby

data/README.md

@@ -34,23 +34,24 @@ Kafkat searches for this file in two places, `~/.kafkatcfg` and `/etc/kafkatcfg`
 
 ```
 $ kafkat
-kafkat 0.0.1: Simplified command-line administration for Kafka brokers
+kafkat 0.0.10: Simplified command-line administration for Kafka brokers
 usage: kafkat [command] [options]
 
 Here's a list of supported commands:
 
-  brokers                                             Print available brokers from Zookeeper.
-  clean-indexes                                       Delete untruncated Kafka log indexes from the filesystem.
-  controller                                          Print the current controller.
-  elect-leaders [topic]                               Begin election of the preferred leaders.
-  partitions [topic]                                  Print partitions by topic.
-  partitions [topic] --under-replicated               Print partitions by topic (only under-replicated).
-  partitions [topic] --unavailable                    Print partitions by topic (only unavailable).
-  reassign [topic] [--brokers <ids>] [--replicas <n>] Begin reassignment of partitions.
-  resign-rewrite <broker id>                          Forcibly rewrite leaderships to exclude a broker.
-  resign-rewrite <broker id> --force                  Same as above but proceed if there are no available ISRs.
-  shutdown <broker id>                                Gracefully remove leaderships from a broker (requires JMX).
-  topics                                              Print all topics.
+  brokers                                                             Print available brokers from Zookeeper.
+  clean-indexes                                                       Delete untruncated Kafka log indexes from the filesystem.
+  controller                                                          Print the current controller.
+  elect-leaders [topic]                                               Begin election of the preferred leaders.
+  partitions [topic]                                                  Print partitions by topic.
+  partitions [topic] --under-replicated                               Print partitions by topic (only under-replicated).
+  partitions [topic] --unavailable                                    Print partitions by topic (only unavailable).
+  reassign [topic] [--brokers <ids>] [--replicas <n>]                 Begin reassignment of partitions.
+  resign-rewrite <broker id>                                          Forcibly rewrite leaderships to exclude a broker.
+  resign-rewrite <broker id> --force                                  Same as above but proceed if there are no available ISRs.
+  set-replication-factor [topic] [--newrf <n>] [--brokers id[,id]]    Set the replication factor of
+  shutdown <broker id>                                                Gracefully remove leaderships from a broker (requires JMX).
+  topics                                                              Print all topics.
   
 ```
 

data/kafkat.gemspec

@@ -18,9 +18,9 @@ Gem::Specification.new do |s|
   s.test_files   = s.files.grep(%r{^(test|spec|features)/})
   s.require_path = 'lib'
 
-  s.add_runtime_dependency 'zk', '~> 1.9.4'
+  s.add_runtime_dependency 'zk', '~> 1.9', '>= 1.9.4'
   s.add_runtime_dependency 'trollop', '~> 2.0'
-  s.add_runtime_dependency 'highline', '~> 1.6.21'
-  s.add_runtime_dependency 'retryable', '~> 1.3.5'
+  s.add_runtime_dependency 'highline', '~> 1.6', '>= 1.6.21'
+  s.add_runtime_dependency 'retryable', '~> 1.3', '>= 1.3.5'
   s.add_runtime_dependency 'colored', '~> 1.2'
 end

data/lib/kafkat/cli.rb

@@ -44,8 +44,8 @@ module Kafkat
       Command.all.values.sort_by(&:command_name).each do |klass|
         klass.usages.each do |usage|
           format, description = usage[0], usage[1]
-          padding_length = 52 - format.length
-          padding = " " * padding_length
+          padding_length = 68 - format.length
+          padding = " " * padding_length unless padding_length < 0
           print "  #{format}#{padding}#{description}\n"
         end
       end

data/lib/kafkat/command/elect-leaders.rb

@@ -22,7 +22,7 @@ module Kafkat
           print "\nBeginning.\n"
           result = admin.elect_leaders!(partitions)
           print "Started.\n"
-        rescue Admin::ExecutionFailedError
+        rescue Interface::Admin::ExecutionFailedError
           print result
         end
       end

data/lib/kafkat/command/reassign.rb

@@ -42,7 +42,7 @@ module Kafkat
           topic_replica_count = replica_count || t.partitions[0].replicas.size
 
           if topic_replica_count > broker_count
-            print "ERROR: Replication factor is larger than brokers.\n"
+            print "ERROR: Replication factor (#{topic_replica_count}) is larger than brokers (#{broker_count}).\n"
             exit 1
           end
 
@@ -80,7 +80,7 @@ module Kafkat
           print "\nBeginning.\n"
           result = admin.reassign!(assignments)
           print "Started.\n"
-        rescue Admin::ExecutionFailedError
+        rescue Interface::Admin::ExecutionFailedError
           print result
         end
       end

data/lib/kafkat/command/set-replication-factor.rb

@@ -0,0 +1,173 @@
+module Kafkat
+  module Command
+
+    #
+    # Command to set the replication factor (RF) of a topic.
+    # The command accepts the topic name, the new desired replication factor,
+    # and, in case of an increase of the replication factor, a list of broker ids.
+    #
+    # When reducing the RF, a new partition assigment will be generated by
+    # removing the last partition replica of every replica set. The leader partition
+    # will not be removed from the replica set to prevent a leader election.
+    #
+    # When increasing the RF, a new partition assignment will be generated by allocating
+    # a new replica to every replica set. The new replica will be assigned to the provided
+    # broker list in a round robin fashion.
+    # If no broker id is specified on the command line, all brokers will be used.
+    #
+    #
+    class SetReplicationFactor < Base
+      register_as 'set-replication-factor'
+
+      usage 'set-replication-factor [topic] [--newrf <n>] [--brokers id[,id]]',
+            'Set the replication factor of'
+
+      def run
+        topic_name = ARGV.shift unless ARGV[0] && ARGV[0].start_with?('--')
+
+        all_brokers = zookeeper.get_brokers
+        topics = topic_name && zookeeper.get_topics([topic_name])
+        topics ||= zookeeper.get_topics
+
+        opts = Trollop.options do
+          opt :brokers, "the comma-separated list of broker the new partitions must be assigned to", type: :string
+          opt :newrf, "the new replication factor", type: :integer, required: true
+        end
+
+        broker_ids = opts[:brokers] && opts[:brokers].split(',').map(&:to_i)
+        new_rf = opts[:newrf]
+
+        if new_rf < 1
+          puts "ERROR: replication factor is smaller than 1"
+          exit 1
+        end
+
+        broker_ids ||= zookeeper.get_brokers.values.map(&:id)
+
+        all_brokers_id = all_brokers.values.map(&:id)
+        broker_ids.each do |id|
+          if !all_brokers_id.include?(id)
+            puts "ERROR: Broker #{id} is not currently active.\n"
+            exit 1
+          end
+        end
+
+        broker_count = broker_ids.size
+        if new_rf > broker_count
+          puts "ERROR: Replication factor is larger than number of brokers.\n"
+          exit 1
+        end
+
+        assignments = []
+        topics.each do |_, t|
+          current_rf = t.partitions[0].replicas.size
+          if new_rf < current_rf
+            warn_reduce_brokers if opts[:brokers]
+            assignments += reduce_rf(t, current_rf, new_rf)
+          elsif new_rf > current_rf
+            assignments += increase_rf(t, current_rf, new_rf, broker_ids)
+          end
+        end
+
+        # ****************
+        if assignments.empty?
+          puts "No partition reassignment required"
+        else
+          print "This operation executes the following assignments:\n\n"
+          print_assignment_header
+          assignments.each { |a| print_assignment(a) }
+          print "\n"
+
+          return unless agree("Proceed (y/n)?")
+
+          result = nil
+          begin
+            print "\nBeginning.\n"
+            result = admin.reassign!(assignments)
+            print "Started.\n"
+          rescue Admin::ExecutionFailedError
+            print result
+          end
+        end
+      end
+
+
+      #
+      # For every partition, remove the last replica from the replica list.
+      # If the last replica is the leader, then the previous replica is removed instead.
+      #
+      def reduce_rf(topic, current_rf, new_rf)
+        delta_rf = current_rf - new_rf
+        if current_rf == 1
+          raise 'Current replication factor if 1. Cannot reduce further.'
+        end
+        unless delta_rf > 0
+          raise "New replication factor (#{new_rf}) must be smaller than current replication factor (#{current_rf})"
+        end
+        assignments = []
+        topic.partitions.map do |p|
+          new_replicas = p.replicas
+
+          (0...delta_rf).each do |_|
+            (0...new_replicas.size).each do |i|
+              if new_replicas[new_replicas.size-1-i] != p.leader
+                new_replicas.delete_at(new_replicas.size-1-i)
+                break
+              end
+            end
+          end
+
+          if new_replicas.size != new_rf
+            raise 'Unexpected state'
+          end
+          assignments << Assignment.new(topic.name, p.id, new_replicas)
+        end
+        assignments
+      end
+
+
+      #
+      # For every partition, filter out the brokers that already have a replica for this partition,
+      # then pick (new_rf - current_rf) brokers and assign them new replicas.
+      #
+      # The count of new replicas assigned to the brokers is maintained in order to uniformly assign new replicas.
+      #
+      def increase_rf(topic, current_rf, new_rf, brokers)
+        unless new_rf > current_rf
+          raise 'New replication factor must be greater than the current replication factor'
+        end
+
+        delta_rf = new_rf - current_rf
+        if delta_rf > brokers.size
+          raise "#{delta_rf} new replicas requested for topic #{p.topic_name} but only #{brokers.size} brokers available"
+        end
+
+        broker_counts = brokers.map { |b| {:id => b, :count => 0} }
+
+        assignments = []
+        topic.partitions.map do |p|
+          existing_replicas = p.replicas
+          pick_from = broker_counts.reject { |b| existing_replicas.include?(b[:id]) }
+          if delta_rf > pick_from.size
+            raise "Cannot create #{delta_rf} new replicas for partition #{p.topic_name}.#{p.id}, not enough brokers"
+          end
+          new_replicas = pick_from.sort { |a, b| a[:count] <=> b[:count] }[0...delta_rf]
+          new_replicas.each { |b| b[:count] += 1 }
+
+          final_replicas = existing_replicas + new_replicas.map { |b| b[:id] }
+
+          assignments << Assignment.new(topic.name, p.id, final_replicas)
+        end
+        assignments
+      end
+
+      def warn_reduce_brokers
+        return if @did_warn_reduce_brokers
+        puts "When reducing the replication factor the list of specified brokers is ignored."
+        puts "Once the replication factor is set, you can use the reassign command."
+        @did_warn_reduce_brokers = true
+      end
+
+    end
+  end
+end

data/lib/kafkat/command/shutdown.rb

@@ -21,7 +21,7 @@ module Kafkat
           print "\nBeginning shutdown.\n"
           result = admin.shutdown!(broker_id)
           print "Started.\n"
-        rescue Admin::ExecutionFailedError
+        rescue Interface::Admin::ExecutionFailedError
           print result
         end
       end

data/lib/kafkat/version.rb

@@ -1,3 +1,3 @@
 module Kafkat
-  VERSION = '0.0.9'
+  VERSION = '0.0.10'
 end

metadata

@@ -1,20 +1,23 @@
 --- !ruby/object:Gem::Specification
 name: kafkat
 version: !ruby/object:Gem::Version
-  version: 0.0.9
+  version: 0.0.10
 platform: ruby
 authors:
 - Nelson Gauthier
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-08-14 00:00:00.000000000 Z
+date: 2015-02-01 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: zk
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.9'
+    - - ! '>='
       - !ruby/object:Gem::Version
         version: 1.9.4
   type: :runtime
@@ -22,6 +25,9 @@ dependencies:
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.9'
+    - - ! '>='
       - !ruby/object:Gem::Version
         version: 1.9.4
 - !ruby/object:Gem::Dependency
@@ -43,6 +49,9 @@ dependencies:
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.6'
+    - - ! '>='
       - !ruby/object:Gem::Version
         version: 1.6.21
   type: :runtime
@@ -50,6 +59,9 @@ dependencies:
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.6'
+    - - ! '>='
       - !ruby/object:Gem::Version
         version: 1.6.21
 - !ruby/object:Gem::Dependency
@@ -57,6 +69,9 @@ dependencies:
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.3'
+    - - ! '>='
       - !ruby/object:Gem::Version
         version: 1.3.5
   type: :runtime
@@ -64,6 +79,9 @@ dependencies:
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.3'
+    - - ! '>='
       - !ruby/object:Gem::Version
         version: 1.3.5
 - !ruby/object:Gem::Dependency
@@ -111,6 +129,7 @@ files:
 - lib/kafkat/command/partitions.rb
 - lib/kafkat/command/reassign.rb
 - lib/kafkat/command/resign-rewrite.rb
+- lib/kafkat/command/set-replication-factor.rb
 - lib/kafkat/command/shutdown.rb
 - lib/kafkat/command/topics.rb
 - lib/kafkat/config.rb