kafkat 0.2.0 → 0.3.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +8 -8
- data/CHANGELOG.md +4 -2
- data/lib/kafkat/command.rb +1 -0
- data/lib/kafkat/command/drain.rb +9 -0
- data/lib/kafkat/command/topics.rb +3 -2
- data/lib/kafkat/command/verify-replicas.rb +92 -0
- data/lib/kafkat/interface/zookeeper.rb +35 -8
- data/lib/kafkat/utility.rb +2 -0
- data/lib/kafkat/utility/command_io.rb +1 -1
- data/lib/kafkat/utility/formatting.rb +5 -0
- data/lib/kafkat/utility/logging.rb +7 -0
- data/lib/kafkat/version.rb +1 -1
- data/spec/factories/topic.rb +15 -1
- data/spec/lib/kafkat/command/verify-replicas_spec.rb +50 -0
- metadata +7 -3
checksums.yaml
CHANGED
@@ -1,15 +1,15 @@
|
|
1
1
|
---
|
2
2
|
!binary "U0hBMQ==":
|
3
3
|
metadata.gz: !binary |-
|
4
|
-
|
4
|
+
NGFmNTRiNDc3NzkxZmNmYzU1Y2YzZDA5ZmE3ODY5MWQyNGFmNzZmMA==
|
5
5
|
data.tar.gz: !binary |-
|
6
|
-
|
6
|
+
NDA5MGExMTQwNWQ0NTU3MGY4YTdmODkxYTY2OTYxNzg1MzdlYWExMQ==
|
7
7
|
SHA512:
|
8
8
|
metadata.gz: !binary |-
|
9
|
-
|
10
|
-
|
11
|
-
|
9
|
+
NmFiZTUxZjc2NzhlNmJmNThlMGZiNjE0ZjEzYjA5NThjMjEyYzEyMGFiZDg1
|
10
|
+
NWZkZTYxYjgwMDIxNjMwMGRlNzM0MzZmYTVkNzNiYmUyMmI4NWQxNDEzNzY2
|
11
|
+
ZGZiODVjZGI2ZTEwZmU3YzQ4YjBmMmM0ZDZmODI3MmY1NmRmNjc=
|
12
12
|
data.tar.gz: !binary |-
|
13
|
-
|
14
|
-
|
15
|
-
|
13
|
+
Zjc1NmU5OWY0NDBjZTVjODlmNDY4MThiMWUyY2RmNWQ4OWRmOWZhMzEyMmVl
|
14
|
+
MzUyOTBjOGEzY2QxMWQ5NWZjMGJmMjRiNWM3ZmRiYzdmOWU5YTQwZDJhMGVh
|
15
|
+
NmIzNjg5ZGEyOTEwYzVlZWM5MTQ1MDI4ZWU4OTE3MDk2YjJjMjk=
|
data/CHANGELOG.md
CHANGED
data/lib/kafkat/command.rb
CHANGED
data/lib/kafkat/command/drain.rb
CHANGED
@@ -46,6 +46,9 @@ module Kafkat
|
|
46
46
|
|
47
47
|
assignments =
|
48
48
|
generate_assignments(source_broker, topics, destination_brokers)
|
49
|
+
|
50
|
+
print "Num of topics got from zookeeper: #{topics.length}\n"
|
51
|
+
print "Num of partitions in the assignment: #{assignments.size}\n"
|
49
52
|
prompt_and_execute_assignments(assignments)
|
50
53
|
end
|
51
54
|
|
@@ -57,6 +60,7 @@ module Kafkat
|
|
57
60
|
|
58
61
|
t.partitions.each do |p|
|
59
62
|
if p.replicas.include? source_broker
|
63
|
+
replicas_size = p.replicas.length
|
60
64
|
replicas = p.replicas - [source_broker]
|
61
65
|
source_broker_is_leader = p.replicas.first == source_broker
|
62
66
|
potential_broker_ids = destination_brokers - replicas
|
@@ -75,6 +79,11 @@ module Kafkat
|
|
75
79
|
end
|
76
80
|
partitions_by_broker[assigned_broker_id] += 1
|
77
81
|
|
82
|
+
if replicas.length != replicas_size
|
83
|
+
STDERR.print "ERROR: Number of replicas changes after reassignment topic: #{t.name}, partition: #{p.id} \n"
|
84
|
+
exit 1
|
85
|
+
end
|
86
|
+
|
78
87
|
assignments << Assignment.new(t.name, p.id, replicas)
|
79
88
|
end
|
80
89
|
end
|
@@ -0,0 +1,92 @@
|
|
1
|
+
module Kafkat
|
2
|
+
module Command
|
3
|
+
class VerifyReplicas < Base
|
4
|
+
register_as 'verify-replicas'
|
5
|
+
|
6
|
+
usage 'verify-replicas [--topics] [--broker <id>] [--print-details] [--print-summary]',
|
7
|
+
'Check if all partitions in a topic have same number of replicas.'
|
8
|
+
|
9
|
+
def run
|
10
|
+
opts = Trollop.options do
|
11
|
+
opt :topics, "topic names", type: :string
|
12
|
+
opt :broker, "broker ID", type: :string
|
13
|
+
opt :print_details, "show replica size of mismatched partitions", :default => false
|
14
|
+
opt :print_summary, "show summary of mismatched partitions", :default => false
|
15
|
+
end
|
16
|
+
|
17
|
+
topic_names = opts[:topics]
|
18
|
+
print_details = opts[:print_details]
|
19
|
+
print_summary = opts[:print_summary]
|
20
|
+
|
21
|
+
if topic_names
|
22
|
+
topics_list = topic_names.split(',')
|
23
|
+
topics = zookeeper.get_topics(topics_list)
|
24
|
+
end
|
25
|
+
topics ||= zookeeper.get_topics
|
26
|
+
broker = opts[:broker] && opts[:broker].to_i
|
27
|
+
|
28
|
+
partition_replica_size, partition_replica_size_stat = verify_replicas(broker, topics)
|
29
|
+
|
30
|
+
print_summary = !print_details || print_summary
|
31
|
+
print_mismatched_partitions(partition_replica_size, partition_replica_size_stat, print_details, print_summary)
|
32
|
+
end
|
33
|
+
|
34
|
+
def verify_replicas(broker, topics)
|
35
|
+
partition_replica_size = {}
|
36
|
+
partition_replica_size_stat = {}
|
37
|
+
|
38
|
+
topics.each do |_, t|
|
39
|
+
partition_replica_size[t.name] = {}
|
40
|
+
partition_replica_size_stat[t.name] = {}
|
41
|
+
|
42
|
+
t.partitions.each do |p|
|
43
|
+
replica_size = p.replicas.length
|
44
|
+
|
45
|
+
next if broker && !p.replicas.include?(broker)
|
46
|
+
|
47
|
+
partition_replica_size_stat[t.name][replica_size] ||= 0
|
48
|
+
partition_replica_size_stat[t.name][replica_size] += 1
|
49
|
+
|
50
|
+
partition_replica_size[t.name][p.id] = replica_size
|
51
|
+
end
|
52
|
+
|
53
|
+
end
|
54
|
+
|
55
|
+
return partition_replica_size, partition_replica_size_stat
|
56
|
+
end
|
57
|
+
|
58
|
+
def print_mismatched_partitions(partition_replica_size, partition_replica_size_stat, print_details, print_summary)
|
59
|
+
topic_column_width = partition_replica_size.keys.max_by(&:length).length
|
60
|
+
if print_details
|
61
|
+
printf "%-#{topic_column_width}s %-10s %-15s %-20s\n", "topic", "partition", "replica_size", "replication_factor"
|
62
|
+
|
63
|
+
partition_replica_size.each do |topic_name, partition|
|
64
|
+
replication_factor = partition_replica_size_stat[topic_name].key(partition_replica_size_stat[topic_name].values.max)
|
65
|
+
|
66
|
+
partition.each do |id, replica_size|
|
67
|
+
if replica_size != replication_factor
|
68
|
+
printf "%-#{topic_column_width}s %-10d %-15d %-20d\n", topic_name, id, replica_size, replication_factor
|
69
|
+
end
|
70
|
+
end
|
71
|
+
end
|
72
|
+
end
|
73
|
+
|
74
|
+
if print_summary
|
75
|
+
printf "%-#{topic_column_width}s %-15s %-10s %-15s %-20s\n", "topic", "replica_size", "count", "percentage", "replication_factor"
|
76
|
+
partition_replica_size_stat.each do |topic_name, partition|
|
77
|
+
if partition.values.size > 1
|
78
|
+
replication_factor = partition_replica_size_stat[topic_name].key(partition_replica_size_stat[topic_name].values.max)
|
79
|
+
num_partitions = 0.0
|
80
|
+
partition.each { |key, value| num_partitions += value }
|
81
|
+
|
82
|
+
partition.each do |replica_size, cnt|
|
83
|
+
printf "%-#{topic_column_width}s %-15d %-10d %-15d %-20d\n", topic_name, replica_size, cnt, (cnt * 100 /num_partitions)
|
84
|
+
.to_i, replication_factor
|
85
|
+
end
|
86
|
+
end
|
87
|
+
end
|
88
|
+
end
|
89
|
+
end
|
90
|
+
end
|
91
|
+
end
|
92
|
+
end
|
@@ -1,4 +1,5 @@
|
|
1
1
|
require 'thread'
|
2
|
+
require 'zk'
|
2
3
|
|
3
4
|
module Kafkat
|
4
5
|
module Interface
|
@@ -44,47 +45,69 @@ module Kafkat
|
|
44
45
|
raise NotFoundError
|
45
46
|
end
|
46
47
|
|
48
|
+
def get_topic_names()
|
49
|
+
return zk.children(topics_path)
|
50
|
+
end
|
51
|
+
|
47
52
|
def get_topics(names=nil)
|
53
|
+
error_msgs = {}
|
48
54
|
topics = {}
|
49
|
-
|
55
|
+
|
56
|
+
if names == nil
|
57
|
+
pool.with_connection do |cnx|
|
58
|
+
names = cnx.children(topics_path)
|
59
|
+
end
|
60
|
+
end
|
50
61
|
|
51
62
|
threads = names.map do |name|
|
52
63
|
Thread.new do
|
53
64
|
begin
|
54
65
|
topics[name] = get_topic(name)
|
55
66
|
rescue => e
|
67
|
+
error_msgs[name] = e
|
56
68
|
end
|
57
69
|
end
|
58
70
|
end
|
59
71
|
threads.map(&:join)
|
60
72
|
|
73
|
+
unless error_msgs.empty?
|
74
|
+
STDERR.print "ERROR: zk cmds failed on get_topics: \n#{error_msgs.values.join("\n")}\n"
|
75
|
+
exit 1
|
76
|
+
end
|
61
77
|
topics
|
62
78
|
end
|
63
79
|
|
64
80
|
def get_topic(name)
|
81
|
+
partition_queue = Queue.new
|
65
82
|
path1 = topic_path(name)
|
66
|
-
|
67
|
-
topic_json = JSON.parse(topic_string)
|
83
|
+
path2 = topic_partitions_path(name)
|
68
84
|
|
69
85
|
partitions = []
|
70
|
-
|
86
|
+
topic_string = pool.with_connection { |cnx| cnx.get(path1).first }
|
87
|
+
partition_ids = pool.with_connection { |cnx| cnx.children(path2) }
|
71
88
|
|
72
|
-
|
89
|
+
topic_json = JSON.parse(topic_string)
|
90
|
+
|
91
|
+
threads = partition_ids.map do |id|
|
73
92
|
id = id.to_i
|
93
|
+
|
74
94
|
Thread.new do
|
75
95
|
path3 = topic_partition_state_path(name, id)
|
76
|
-
partition_string =
|
96
|
+
partition_string = pool.with_connection { |cnx| cnx.get(path3).first }
|
77
97
|
partition_json = JSON.parse(partition_string)
|
78
|
-
|
79
98
|
replicas = topic_json['partitions'][id.to_s]
|
80
99
|
leader = partition_json['leader']
|
81
100
|
isr = partition_json['isr']
|
82
101
|
|
83
|
-
|
102
|
+
partition_queue << Partition.new(name, id, replicas, leader, isr)
|
84
103
|
end
|
85
104
|
end
|
86
105
|
threads.map(&:join)
|
87
106
|
|
107
|
+
until partition_queue.empty? do
|
108
|
+
partitions << partition_queue.pop
|
109
|
+
end
|
110
|
+
|
88
111
|
partitions.sort_by!(&:id)
|
89
112
|
Topic.new(name, partitions)
|
90
113
|
rescue ZK::Exceptions::NoNode
|
@@ -115,6 +138,10 @@ module Kafkat
|
|
115
138
|
|
116
139
|
private
|
117
140
|
|
141
|
+
def pool
|
142
|
+
@pool ||= ZK.new_pool(zk_path, :min_clients => 10, :max_clients => 300, :timeout => 1)
|
143
|
+
end
|
144
|
+
|
118
145
|
def zk
|
119
146
|
@zk ||= ZK.new(zk_path)
|
120
147
|
end
|
data/lib/kafkat/utility.rb
CHANGED
data/lib/kafkat/version.rb
CHANGED
data/spec/factories/topic.rb
CHANGED
@@ -34,6 +34,20 @@ module Kafkat
|
|
34
34
|
Partition.new(name, 3, [0, 1, 2], 0, [0]),
|
35
35
|
Partition.new(name, 4, [0, 1, 2], 1, [1])]}
|
36
36
|
end
|
37
|
+
|
38
|
+
factory :topic_rep_factor_three_with_four_replicas_in_partition1 do
|
39
|
+
name "topic_name1"
|
40
|
+
partitions [Partition.new("topic_name1", 0, [0, 1, 2], 0, [0]),
|
41
|
+
Partition.new("topic_name1", 1, [0, 1, 2, 6], 1, [1]),
|
42
|
+
Partition.new("topic_name1", 2, [0, 1, 2], 2, [2])]
|
43
|
+
end
|
44
|
+
|
45
|
+
factory :topic2_rep_factor_three do
|
46
|
+
name "topic_name2"
|
47
|
+
partitions [Partition.new("topic_name2", 0, [3, 4, 5], 0, [0]),
|
48
|
+
Partition.new("topic_name2", 1, [3, 4, 5], 0, [0]),
|
49
|
+
Partition.new("topic_name2", 2, [3, 4, 5], 1, [1])]
|
50
|
+
end
|
37
51
|
end
|
38
52
|
end
|
39
|
-
end
|
53
|
+
end
|
@@ -0,0 +1,50 @@
|
|
1
|
+
require 'spec_helper'
|
2
|
+
|
3
|
+
module Kafkat
|
4
|
+
RSpec.describe Command::VerifyReplicas do
|
5
|
+
let(:verify_replicas) { Command::VerifyReplicas.new({}) }
|
6
|
+
|
7
|
+
context 'two topics with replication factor 3' do
|
8
|
+
let(:topic_rep_factor_three_with_four_replicas_in_partition1) {
|
9
|
+
FactoryGirl.build(:topic_rep_factor_three_with_four_replicas_in_partition1) }
|
10
|
+
let(:topic2_rep_factor_three) { FactoryGirl.build(:topic2_rep_factor_three) }
|
11
|
+
|
12
|
+
it 'should return empty mismatched partitions for all brokers' do
|
13
|
+
partition_replica_size, partition_replica_size_stat = verify_replicas.verify_replicas(nil,
|
14
|
+
{"topic_name2" => topic2_rep_factor_three})
|
15
|
+
|
16
|
+
expect(partition_replica_size).to eq({"topic_name2" => {0 => 3, 1 => 3, 2 => 3}})
|
17
|
+
expect(partition_replica_size_stat).to eq({"topic_name2" => {3 => 3}})
|
18
|
+
end
|
19
|
+
|
20
|
+
it 'should return topic 1 partition 1 for all brokers' do
|
21
|
+
partition_replica_size, partition_replica_size_stat = verify_replicas.verify_replicas(nil,
|
22
|
+
{"topic_name1" => topic_rep_factor_three_with_four_replicas_in_partition1,
|
23
|
+
"topic_name2" => topic2_rep_factor_three})
|
24
|
+
|
25
|
+
expect(partition_replica_size).to eq({"topic_name1" => {0 => 3, 1 => 4, 2 => 3}, "topic_name2" => {0 => 3, 1 => 3, 2 => 3}})
|
26
|
+
expect(partition_replica_size_stat).to eq({"topic_name1" => {3 => 2, 4 => 1}, "topic_name2" => {3 => 3}})
|
27
|
+
end
|
28
|
+
|
29
|
+
it 'should return topic 1 partition 1 for broker 6' do
|
30
|
+
partition_replica_size, partition_replica_size_stat = verify_replicas.verify_replicas(6,
|
31
|
+
{"topic_name1" => topic_rep_factor_three_with_four_replicas_in_partition1,
|
32
|
+
"topic_name2" => topic2_rep_factor_three})
|
33
|
+
|
34
|
+
expect(partition_replica_size).to eq({"topic_name1" => {1 => 4}, "topic_name2" => {}})
|
35
|
+
expect(partition_replica_size_stat).to eq({"topic_name1" => {4 => 1}, "topic_name2" => {}})
|
36
|
+
end
|
37
|
+
|
38
|
+
it 'should return empty mismatched partition for broker 3' do
|
39
|
+
partition_replica_size, partition_replica_size_stat = verify_replicas.verify_replicas(3,
|
40
|
+
{"topic_name1" => topic_rep_factor_three_with_four_replicas_in_partition1,
|
41
|
+
"topic_name2" => topic2_rep_factor_three})
|
42
|
+
|
43
|
+
expect(partition_replica_size).to eq({"topic_name1" => {}, "topic_name2" => {0 => 3, 1 => 3, 2 => 3}})
|
44
|
+
expect(partition_replica_size_stat).to eq({"topic_name1" => {}, "topic_name2" => {3 => 3}})
|
45
|
+
end
|
46
|
+
end
|
47
|
+
|
48
|
+
|
49
|
+
end
|
50
|
+
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: kafkat
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.3.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Nelson Gauthier
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2016-
|
11
|
+
date: 2016-11-28 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: zk
|
@@ -242,6 +242,7 @@ files:
|
|
242
242
|
- lib/kafkat/command/set-replication-factor.rb
|
243
243
|
- lib/kafkat/command/shutdown.rb
|
244
244
|
- lib/kafkat/command/topics.rb
|
245
|
+
- lib/kafkat/command/verify-replicas.rb
|
245
246
|
- lib/kafkat/config.rb
|
246
247
|
- lib/kafkat/interface.rb
|
247
248
|
- lib/kafkat/interface/admin.rb
|
@@ -251,10 +252,12 @@ files:
|
|
251
252
|
- lib/kafkat/utility.rb
|
252
253
|
- lib/kafkat/utility/command_io.rb
|
253
254
|
- lib/kafkat/utility/formatting.rb
|
255
|
+
- lib/kafkat/utility/logging.rb
|
254
256
|
- lib/kafkat/version.rb
|
255
257
|
- spec/factories/topic.rb
|
256
258
|
- spec/lib/kafkat/command/cluster_restart_spec.rb
|
257
259
|
- spec/lib/kafkat/command/drain_spec.rb
|
260
|
+
- spec/lib/kafkat/command/verify-replicas_spec.rb
|
258
261
|
- spec/spec_helper.rb
|
259
262
|
homepage: https://github.com/airbnb/kafkat
|
260
263
|
licenses:
|
@@ -276,7 +279,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
276
279
|
version: '0'
|
277
280
|
requirements: []
|
278
281
|
rubyforge_project:
|
279
|
-
rubygems_version: 2.
|
282
|
+
rubygems_version: 2.6.6
|
280
283
|
signing_key:
|
281
284
|
specification_version: 4
|
282
285
|
summary: Simplified command-line administration for Kafka brokers
|
@@ -284,4 +287,5 @@ test_files:
|
|
284
287
|
- spec/factories/topic.rb
|
285
288
|
- spec/lib/kafkat/command/cluster_restart_spec.rb
|
286
289
|
- spec/lib/kafkat/command/drain_spec.rb
|
290
|
+
- spec/lib/kafkat/command/verify-replicas_spec.rb
|
287
291
|
- spec/spec_helper.rb
|