kafkat 0.2.0 → 0.3.0

checksums.yaml

@@ -1,15 +1,15 @@
 ---
 !binary "U0hBMQ==":
   metadata.gz: !binary |-
-    NzIyMjdlYzlmM2JhZDI1YzAzZWM1ZTU2ZmU4NWVhNjI3NjY1NTkwMQ==
+    NGFmNTRiNDc3NzkxZmNmYzU1Y2YzZDA5ZmE3ODY5MWQyNGFmNzZmMA==
   data.tar.gz: !binary |-
-    YjNkZDUzZGZlYmIzNGRiYTNkNTBhNjAwYWE2NmI5NzEwMWRmOTVkMw==
+    NDA5MGExMTQwNWQ0NTU3MGY4YTdmODkxYTY2OTYxNzg1MzdlYWExMQ==
 SHA512:
   metadata.gz: !binary |-
-    NGQzMjRmOThmYmE3MDQ5ZDUyYTg4Zjg0ZTI3Zjc0MDRlOGY5NmYwMGNlZmI0
-    ZDlmZWE4OWM3MTI5MGE5NmU0MjU2NGQyYjk5NjNkZDFlZmRlMTllOWRkMDYx
-    NzhhNDcyYjIyZDU2M2E3NzZhYjE1ZGMyODI3ZjM4ZjJhODM5Mjk=
+    NmFiZTUxZjc2NzhlNmJmNThlMGZiNjE0ZjEzYjA5NThjMjEyYzEyMGFiZDg1
+    NWZkZTYxYjgwMDIxNjMwMGRlNzM0MzZmYTVkNzNiYmUyMmI4NWQxNDEzNzY2
+    ZGZiODVjZGI2ZTEwZmU3YzQ4YjBmMmM0ZDZmODI3MmY1NmRmNjc=
   data.tar.gz: !binary |-
-    MGIwYzM3NWMzMTZhNWJlMzdjY2JiNTRkYzNlMzBiZGQ4YmM2MWQ1MjI4ZWNm
-    Y2ExNGMxNTVjMTQ2Y2QzMWQzODlhZWE2YzljNTkyYTkyMWRiNzBiODkzZDdi
-    NjQ0OGE1MDlmNDYwY2M5NDZjNzJiYWUxOGE0NDdmMjY0ZjhhMjE=
+    Zjc1NmU5OWY0NDBjZTVjODlmNDY4MThiMWUyY2RmNWQ4OWRmOWZhMzEyMmVl
+    MzUyOTBjOGEzY2QxMWQ5NWZjMGJmMjRiNWM3ZmRiYzdmOWU5YTQwZDJhMGVh
+    NmIzNjg5ZGEyOTEwYzVlZWM5MTQ1MDI4ZWU4OTE3MDk2YjJjMjk=

data/CHANGELOG.md

@@ -1,6 +1,8 @@
 # CHANGE LOG
 
-
+## 0.3.0
+- [feature] add `verify-replicas` command
+- [bug-fix] use a pool of Zookeeper connections and handle Zookeeper errors correctly
 
 ## 0.2.0
-- add `cluster-restart` command
+- [feature] add `cluster-restart` command

data/lib/kafkat/command.rb

@@ -15,6 +15,7 @@ module Kafkat
     class Base
       include Formatting
       include CommandIO
+      include Kafkat::Logging
 
       attr_reader :config
 

data/lib/kafkat/command/drain.rb

@@ -46,6 +46,9 @@ module Kafkat
 
         assignments =
           generate_assignments(source_broker, topics, destination_brokers)
+
+        print "Num of topics got from zookeeper: #{topics.length}\n"
+        print "Num of partitions in the assignment: #{assignments.size}\n"
         prompt_and_execute_assignments(assignments)
       end
 
@@ -57,6 +60,7 @@ module Kafkat
 
           t.partitions.each do |p|
             if p.replicas.include? source_broker
+              replicas_size = p.replicas.length
               replicas = p.replicas - [source_broker]
               source_broker_is_leader = p.replicas.first == source_broker
               potential_broker_ids = destination_brokers - replicas
@@ -75,6 +79,11 @@ module Kafkat
               end
               partitions_by_broker[assigned_broker_id] += 1
 
+              if replicas.length != replicas_size
+                STDERR.print "ERROR: Number of replicas changes after reassignment topic: #{t.name}, partition: #{p.id} \n"
+                exit 1
+              end
+
               assignments << Assignment.new(t.name, p.id, replicas)
             end
           end

data/lib/kafkat/command/topics.rb

@@ -7,8 +7,9 @@ module Kafkat
             'Print all topics.'
 
       def run
-        ts = zookeeper.get_topics
-        ts.each { |name, t| print_topic(t) }
+        topic_names = zookeeper.get_topic_names
+
+        topic_names.each { |name| print_topic_name(name) }
       end
     end
   end

data/lib/kafkat/command/verify-replicas.rb

@@ -0,0 +1,92 @@
+module Kafkat
+  module Command
+    class VerifyReplicas < Base
+      register_as 'verify-replicas'
+
+      usage 'verify-replicas  [--topics] [--broker <id>] [--print-details] [--print-summary]',
+            'Check if all partitions in a topic have same number of replicas.'
+
+      def run
+        opts = Trollop.options do
+          opt :topics, "topic names", type: :string
+          opt :broker, "broker ID", type: :string
+          opt :print_details, "show replica size of mismatched partitions", :default => false
+          opt :print_summary, "show summary of mismatched partitions", :default => false
+        end
+
+        topic_names = opts[:topics]
+        print_details = opts[:print_details]
+        print_summary = opts[:print_summary]
+
+        if topic_names
+          topics_list = topic_names.split(',')
+          topics = zookeeper.get_topics(topics_list)
+        end
+        topics ||= zookeeper.get_topics
+        broker = opts[:broker] && opts[:broker].to_i
+
+        partition_replica_size, partition_replica_size_stat = verify_replicas(broker, topics)
+
+        print_summary = !print_details || print_summary
+        print_mismatched_partitions(partition_replica_size, partition_replica_size_stat, print_details, print_summary)
+      end
+
+      def verify_replicas(broker, topics)
+        partition_replica_size = {}
+        partition_replica_size_stat = {}
+
+        topics.each do |_, t|
+          partition_replica_size[t.name] = {}
+          partition_replica_size_stat[t.name] = {}
+
+          t.partitions.each do |p|
+            replica_size = p.replicas.length
+
+            next if broker && !p.replicas.include?(broker)
+
+            partition_replica_size_stat[t.name][replica_size] ||= 0
+            partition_replica_size_stat[t.name][replica_size] += 1
+
+            partition_replica_size[t.name][p.id] = replica_size
+          end
+
+        end
+
+        return partition_replica_size, partition_replica_size_stat
+      end
+
+      def print_mismatched_partitions(partition_replica_size, partition_replica_size_stat, print_details, print_summary)
+        topic_column_width = partition_replica_size.keys.max_by(&:length).length
+        if print_details
+          printf "%-#{topic_column_width}s %-10s %-15s %-20s\n", "topic", "partition", "replica_size", "replication_factor"
+
+          partition_replica_size.each do |topic_name, partition|
+            replication_factor = partition_replica_size_stat[topic_name].key(partition_replica_size_stat[topic_name].values.max)
+
+            partition.each do |id, replica_size|
+              if replica_size != replication_factor
+                printf "%-#{topic_column_width}s %-10d %-15d %-20d\n", topic_name, id, replica_size, replication_factor
+              end
+            end
+          end
+        end
+
+        if print_summary
+          printf "%-#{topic_column_width}s %-15s %-10s %-15s %-20s\n", "topic", "replica_size", "count", "percentage", "replication_factor"
+          partition_replica_size_stat.each do |topic_name, partition|
+            if partition.values.size > 1
+              replication_factor = partition_replica_size_stat[topic_name].key(partition_replica_size_stat[topic_name].values.max)
+              num_partitions = 0.0
+              partition.each { |key, value| num_partitions += value }
+
+              partition.each do |replica_size, cnt|
+                printf "%-#{topic_column_width}s %-15d %-10d %-15d %-20d\n", topic_name, replica_size, cnt, (cnt * 100 /num_partitions)
+                                                                                           .to_i, replication_factor
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/kafkat/interface/zookeeper.rb

@@ -1,4 +1,5 @@
 require 'thread'
+require 'zk'
 
 module Kafkat
   module Interface
@@ -44,47 +45,69 @@ module Kafkat
         raise NotFoundError
       end
 
+      def get_topic_names()
+        return zk.children(topics_path)
+      end
+
       def get_topics(names=nil)
+        error_msgs = {}
         topics = {}
-        names ||= zk.children(topics_path)
+
+        if names == nil
+          pool.with_connection do |cnx|
+            names = cnx.children(topics_path)
+          end
+        end
 
         threads = names.map do |name|
           Thread.new do
             begin
               topics[name] = get_topic(name)
             rescue => e
+              error_msgs[name] = e
             end
           end
         end
         threads.map(&:join)
 
+        unless error_msgs.empty?
+          STDERR.print "ERROR: zk cmds failed on get_topics: \n#{error_msgs.values.join("\n")}\n"
+          exit 1
+        end
         topics
       end
 
       def get_topic(name)
+        partition_queue = Queue.new
         path1 = topic_path(name)
-        topic_string = zk.get(path1).first
-        topic_json = JSON.parse(topic_string)
+        path2 = topic_partitions_path(name)
 
         partitions = []
-        path2 = topic_partitions_path(name)
+        topic_string = pool.with_connection { |cnx| cnx.get(path1).first }
+        partition_ids = pool.with_connection { |cnx| cnx.children(path2) }
 
-        threads = zk.children(path2).map do |id|
+        topic_json = JSON.parse(topic_string)
+
+        threads = partition_ids.map do |id|
           id = id.to_i
+
           Thread.new do
             path3 = topic_partition_state_path(name, id)
-            partition_string = zk.get(path3).first
+            partition_string = pool.with_connection { |cnx| cnx.get(path3).first }
             partition_json = JSON.parse(partition_string)
-
             replicas = topic_json['partitions'][id.to_s]
             leader = partition_json['leader']
             isr = partition_json['isr']
 
-            partitions << Partition.new(name, id, replicas, leader, isr)
+            partition_queue << Partition.new(name, id, replicas, leader, isr)
           end
         end
         threads.map(&:join)
 
+        until partition_queue.empty? do
+          partitions << partition_queue.pop
+        end
+
         partitions.sort_by!(&:id)
         Topic.new(name, partitions)
       rescue ZK::Exceptions::NoNode
@@ -115,6 +138,10 @@ module Kafkat
 
       private
 
+      def pool
+        @pool ||= ZK.new_pool(zk_path, :min_clients => 10, :max_clients => 300, :timeout => 1)
+      end
+
       def zk
         @zk ||= ZK.new(zk_path)
       end

data/lib/kafkat/utility.rb

@@ -1,2 +1,4 @@
 require 'kafkat/utility/formatting'
 require 'kafkat/utility/command_io'
+require 'kafkat/utility/logging'
+

data/lib/kafkat/utility/command_io.rb

@@ -18,4 +18,4 @@ module Kafkat
       end
     end
   end
-end
+end

data/lib/kafkat/utility/formatting.rb

@@ -23,6 +23,11 @@ module Kafkat
       print "\n"
     end
 
+    def print_topic_name(topic_name)
+      print justify(topic_name)
+      print "\n"
+    end
+
     def print_topic_header
       print justify('Topic')
       print "\n"

data/lib/kafkat/utility/logging.rb

@@ -0,0 +1,7 @@
+module Kafkat
+  module Logging
+   def print_err(message)
+    STDERR.print message
+   end
+  end
+end

data/lib/kafkat/version.rb

@@ -1,3 +1,3 @@
 module Kafkat
-  VERSION = '0.2.0'
+  VERSION = '0.3.0'
 end

data/spec/factories/topic.rb

@@ -34,6 +34,20 @@ module Kafkat
                      Partition.new(name, 3, [0, 1, 2], 0, [0]),
                      Partition.new(name, 4, [0, 1, 2], 1, [1])]}
       end
+
+      factory :topic_rep_factor_three_with_four_replicas_in_partition1 do
+        name "topic_name1"
+        partitions  [Partition.new("topic_name1", 0, [0, 1, 2], 0, [0]),
+                     Partition.new("topic_name1", 1, [0, 1, 2, 6], 1, [1]),
+                     Partition.new("topic_name1", 2, [0, 1, 2], 2, [2])]
+      end
+
+      factory :topic2_rep_factor_three do
+        name "topic_name2"
+        partitions [Partition.new("topic_name2", 0, [3, 4, 5], 0, [0]),
+                     Partition.new("topic_name2", 1, [3, 4, 5], 0, [0]),
+                     Partition.new("topic_name2", 2, [3, 4, 5], 1, [1])]
+      end
     end
   end
-end
+end

data/spec/lib/kafkat/command/verify-replicas_spec.rb

@@ -0,0 +1,50 @@
+require 'spec_helper'
+
+module Kafkat
+  RSpec.describe Command::VerifyReplicas do
+    let(:verify_replicas) { Command::VerifyReplicas.new({}) }
+
+    context 'two topics with replication factor 3' do
+      let(:topic_rep_factor_three_with_four_replicas_in_partition1) {
+        FactoryGirl.build(:topic_rep_factor_three_with_four_replicas_in_partition1) }
+      let(:topic2_rep_factor_three) { FactoryGirl.build(:topic2_rep_factor_three) }
+
+      it 'should return empty mismatched partitions for all brokers' do
+        partition_replica_size, partition_replica_size_stat = verify_replicas.verify_replicas(nil,
+                                                                                              {"topic_name2" => topic2_rep_factor_three})
+
+        expect(partition_replica_size).to eq({"topic_name2" => {0 => 3, 1 => 3, 2 => 3}})
+        expect(partition_replica_size_stat).to eq({"topic_name2" => {3 => 3}})
+      end
+
+      it 'should return topic 1 partition 1 for all brokers' do
+        partition_replica_size, partition_replica_size_stat = verify_replicas.verify_replicas(nil,
+                                                                                              {"topic_name1" => topic_rep_factor_three_with_four_replicas_in_partition1,
+                                                                                               "topic_name2" => topic2_rep_factor_three})
+
+        expect(partition_replica_size).to eq({"topic_name1" => {0 => 3, 1 => 4, 2 => 3}, "topic_name2" => {0 => 3, 1 => 3, 2 => 3}})
+        expect(partition_replica_size_stat).to eq({"topic_name1" => {3 => 2, 4 => 1}, "topic_name2" => {3 => 3}})
+      end
+
+      it 'should return topic 1 partition 1 for broker 6' do
+        partition_replica_size, partition_replica_size_stat = verify_replicas.verify_replicas(6,
+                                                                                              {"topic_name1" => topic_rep_factor_three_with_four_replicas_in_partition1,
+                                                                                               "topic_name2" => topic2_rep_factor_three})
+
+        expect(partition_replica_size).to eq({"topic_name1" => {1 => 4}, "topic_name2" => {}})
+        expect(partition_replica_size_stat).to eq({"topic_name1" => {4 => 1}, "topic_name2" => {}})
+      end
+
+      it 'should return empty mismatched partition for broker 3' do
+        partition_replica_size, partition_replica_size_stat = verify_replicas.verify_replicas(3,
+                                                                                              {"topic_name1" => topic_rep_factor_three_with_four_replicas_in_partition1,
+                                                                                               "topic_name2" => topic2_rep_factor_three})
+
+        expect(partition_replica_size).to eq({"topic_name1" => {}, "topic_name2" => {0 => 3, 1 => 3, 2 => 3}})
+        expect(partition_replica_size_stat).to eq({"topic_name1" => {}, "topic_name2" => {3 => 3}})
+      end
+    end
+
+
+  end
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: kafkat
 version: !ruby/object:Gem::Version
-  version: 0.2.0
+  version: 0.3.0
 platform: ruby
 authors:
 - Nelson Gauthier
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-10-19 00:00:00.000000000 Z
+date: 2016-11-28 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: zk
@@ -242,6 +242,7 @@ files:
 - lib/kafkat/command/set-replication-factor.rb
 - lib/kafkat/command/shutdown.rb
 - lib/kafkat/command/topics.rb
+- lib/kafkat/command/verify-replicas.rb
 - lib/kafkat/config.rb
 - lib/kafkat/interface.rb
 - lib/kafkat/interface/admin.rb
@@ -251,10 +252,12 @@ files:
 - lib/kafkat/utility.rb
 - lib/kafkat/utility/command_io.rb
 - lib/kafkat/utility/formatting.rb
+- lib/kafkat/utility/logging.rb
 - lib/kafkat/version.rb
 - spec/factories/topic.rb
 - spec/lib/kafkat/command/cluster_restart_spec.rb
 - spec/lib/kafkat/command/drain_spec.rb
+- spec/lib/kafkat/command/verify-replicas_spec.rb
 - spec/spec_helper.rb
 homepage: https://github.com/airbnb/kafkat
 licenses:
@@ -276,7 +279,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.4.1
+rubygems_version: 2.6.6
 signing_key: 
 specification_version: 4
 summary: Simplified command-line administration for Kafka brokers
@@ -284,4 +287,5 @@ test_files:
 - spec/factories/topic.rb
 - spec/lib/kafkat/command/cluster_restart_spec.rb
 - spec/lib/kafkat/command/drain_spec.rb
+- spec/lib/kafkat/command/verify-replicas_spec.rb
 - spec/spec_helper.rb