kafkat 0.0.9 → 0.0.10
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +8 -8
- data/Gemfile.lock +5 -5
- data/README.md +14 -13
- data/kafkat.gemspec +3 -3
- data/lib/kafkat/cli.rb +2 -2
- data/lib/kafkat/command/elect-leaders.rb +1 -1
- data/lib/kafkat/command/reassign.rb +2 -2
- data/lib/kafkat/command/set-replication-factor.rb +173 -0
- data/lib/kafkat/command/shutdown.rb +1 -1
- data/lib/kafkat/version.rb +1 -1
- metadata +21 -2
checksums.yaml
CHANGED
@@ -1,15 +1,15 @@
|
|
1
1
|
---
|
2
2
|
!binary "U0hBMQ==":
|
3
3
|
metadata.gz: !binary |-
|
4
|
-
|
4
|
+
MTY0MTdhYjQ3YjAzNWExNWJjMzMyYThmOTY5ZWQ5NmQyOTcyMGM4Yw==
|
5
5
|
data.tar.gz: !binary |-
|
6
|
-
|
6
|
+
MTkzM2UzODgxNTIzMmJiYTAwNDQ2NmI5ZmVjYWZkYTdiYjNkZDk4Ng==
|
7
7
|
SHA512:
|
8
8
|
metadata.gz: !binary |-
|
9
|
-
|
10
|
-
|
11
|
-
|
9
|
+
N2U0ZTMyMjQ4MmI3M2QzMDkyNTM4ZWFjOWJmMTNhNmIxNjlkM2NjMzBkYTli
|
10
|
+
YmY3N2Y1ZTk5OWIwZjZlZjZhMzNjOGE2MmE4NDg2ZTc4NzFjYTYzZmZlZWNi
|
11
|
+
YzFmNzFjZGU5N2Y5Y2UyMGY5NDhiNmEyZWU5NjdiNzViMjVlNjU=
|
12
12
|
data.tar.gz: !binary |-
|
13
|
-
|
14
|
-
|
15
|
-
|
13
|
+
YjRiMzllYmZjNDBiMmYxNjRiNzExMzlkZGM2ZWQwOTA0NmFiMGRkZTQzZjMz
|
14
|
+
ZmRlMGM5OTczMTcwYzI4MmJiYmE1ZGYwMDZlNzhhMTI3ZmQ2YThkN2E4OTdj
|
15
|
+
ZmUzN2QyZDMxNDk5ODE1ZWIwNTdjZmMyNWFkMTYzYWRjOWJmZWY=
|
data/Gemfile.lock
CHANGED
@@ -1,12 +1,12 @@
|
|
1
1
|
PATH
|
2
2
|
remote: .
|
3
3
|
specs:
|
4
|
-
kafkat (0.0.
|
4
|
+
kafkat (0.0.10)
|
5
5
|
colored (~> 1.2)
|
6
|
-
highline (~> 1.6.21)
|
7
|
-
retryable (~> 1.3.5)
|
6
|
+
highline (~> 1.6, >= 1.6.21)
|
7
|
+
retryable (~> 1.3, >= 1.3.5)
|
8
8
|
trollop (~> 2.0)
|
9
|
-
zk (~> 1.9.4)
|
9
|
+
zk (~> 1.9, >= 1.9.4)
|
10
10
|
|
11
11
|
GEM
|
12
12
|
remote: https://rubygems.org/
|
@@ -23,7 +23,7 @@ GEM
|
|
23
23
|
zk (1.9.4)
|
24
24
|
logging (~> 1.8.2)
|
25
25
|
zookeeper (~> 1.4.0)
|
26
|
-
zookeeper (1.4.
|
26
|
+
zookeeper (1.4.9)
|
27
27
|
|
28
28
|
PLATFORMS
|
29
29
|
ruby
|
data/README.md
CHANGED
@@ -34,23 +34,24 @@ Kafkat searches for this file in two places, `~/.kafkatcfg` and `/etc/kafkatcfg`
|
|
34
34
|
|
35
35
|
```
|
36
36
|
$ kafkat
|
37
|
-
kafkat 0.0.
|
37
|
+
kafkat 0.0.10: Simplified command-line administration for Kafka brokers
|
38
38
|
usage: kafkat [command] [options]
|
39
39
|
|
40
40
|
Here's a list of supported commands:
|
41
41
|
|
42
|
-
brokers
|
43
|
-
clean-indexes
|
44
|
-
controller
|
45
|
-
elect-leaders [topic]
|
46
|
-
partitions [topic]
|
47
|
-
partitions [topic] --under-replicated
|
48
|
-
partitions [topic] --unavailable
|
49
|
-
reassign [topic] [--brokers <ids>] [--replicas <n>]
|
50
|
-
resign-rewrite <broker id>
|
51
|
-
resign-rewrite <broker id> --force
|
52
|
-
|
53
|
-
|
42
|
+
brokers Print available brokers from Zookeeper.
|
43
|
+
clean-indexes Delete untruncated Kafka log indexes from the filesystem.
|
44
|
+
controller Print the current controller.
|
45
|
+
elect-leaders [topic] Begin election of the preferred leaders.
|
46
|
+
partitions [topic] Print partitions by topic.
|
47
|
+
partitions [topic] --under-replicated Print partitions by topic (only under-replicated).
|
48
|
+
partitions [topic] --unavailable Print partitions by topic (only unavailable).
|
49
|
+
reassign [topic] [--brokers <ids>] [--replicas <n>] Begin reassignment of partitions.
|
50
|
+
resign-rewrite <broker id> Forcibly rewrite leaderships to exclude a broker.
|
51
|
+
resign-rewrite <broker id> --force Same as above but proceed if there are no available ISRs.
|
52
|
+
set-replication-factor [topic] [--newrf <n>] [--brokers id[,id]] Set the replication factor of
|
53
|
+
shutdown <broker id> Gracefully remove leaderships from a broker (requires JMX).
|
54
|
+
topics Print all topics.
|
54
55
|
|
55
56
|
```
|
56
57
|
|
data/kafkat.gemspec
CHANGED
@@ -18,9 +18,9 @@ Gem::Specification.new do |s|
|
|
18
18
|
s.test_files = s.files.grep(%r{^(test|spec|features)/})
|
19
19
|
s.require_path = 'lib'
|
20
20
|
|
21
|
-
s.add_runtime_dependency 'zk', '~> 1.9.4'
|
21
|
+
s.add_runtime_dependency 'zk', '~> 1.9', '>= 1.9.4'
|
22
22
|
s.add_runtime_dependency 'trollop', '~> 2.0'
|
23
|
-
s.add_runtime_dependency 'highline', '~> 1.6.21'
|
24
|
-
s.add_runtime_dependency 'retryable', '~> 1.3.5'
|
23
|
+
s.add_runtime_dependency 'highline', '~> 1.6', '>= 1.6.21'
|
24
|
+
s.add_runtime_dependency 'retryable', '~> 1.3', '>= 1.3.5'
|
25
25
|
s.add_runtime_dependency 'colored', '~> 1.2'
|
26
26
|
end
|
data/lib/kafkat/cli.rb
CHANGED
@@ -44,8 +44,8 @@ module Kafkat
|
|
44
44
|
Command.all.values.sort_by(&:command_name).each do |klass|
|
45
45
|
klass.usages.each do |usage|
|
46
46
|
format, description = usage[0], usage[1]
|
47
|
-
padding_length =
|
48
|
-
padding = " " * padding_length
|
47
|
+
padding_length = 68 - format.length
|
48
|
+
padding = " " * padding_length unless padding_length < 0
|
49
49
|
print " #{format}#{padding}#{description}\n"
|
50
50
|
end
|
51
51
|
end
|
@@ -42,7 +42,7 @@ module Kafkat
|
|
42
42
|
topic_replica_count = replica_count || t.partitions[0].replicas.size
|
43
43
|
|
44
44
|
if topic_replica_count > broker_count
|
45
|
-
print "ERROR: Replication factor is larger than brokers.\n"
|
45
|
+
print "ERROR: Replication factor (#{topic_replica_count}) is larger than brokers (#{broker_count}).\n"
|
46
46
|
exit 1
|
47
47
|
end
|
48
48
|
|
@@ -80,7 +80,7 @@ module Kafkat
|
|
80
80
|
print "\nBeginning.\n"
|
81
81
|
result = admin.reassign!(assignments)
|
82
82
|
print "Started.\n"
|
83
|
-
rescue Admin::ExecutionFailedError
|
83
|
+
rescue Interface::Admin::ExecutionFailedError
|
84
84
|
print result
|
85
85
|
end
|
86
86
|
end
|
@@ -0,0 +1,173 @@
|
|
1
|
+
module Kafkat
|
2
|
+
module Command
|
3
|
+
|
4
|
+
#
|
5
|
+
# Command to set the replication factor (RF) of a topic.
|
6
|
+
# The command accepts the topic name, the new desired replication factor,
|
7
|
+
# and, in case of an increase of the replication factor, a list of broker ids.
|
8
|
+
#
|
9
|
+
# When reducing the RF, a new partition assigment will be generated by
|
10
|
+
# removing the last partition replica of every replica set. The leader partition
|
11
|
+
# will not be removed from the replica set to prevent a leader election.
|
12
|
+
#
|
13
|
+
# When increasing the RF, a new partition assignment will be generated by allocating
|
14
|
+
# a new replica to every replica set. The new replica will be assigned to the provided
|
15
|
+
# broker list in a round robin fashion.
|
16
|
+
# If no broker id is specified on the command line, all brokers will be used.
|
17
|
+
#
|
18
|
+
#
|
19
|
+
class SetReplicationFactor < Base
|
20
|
+
register_as 'set-replication-factor'
|
21
|
+
|
22
|
+
usage 'set-replication-factor [topic] [--newrf <n>] [--brokers id[,id]]',
|
23
|
+
'Set the replication factor of'
|
24
|
+
|
25
|
+
def run
|
26
|
+
topic_name = ARGV.shift unless ARGV[0] && ARGV[0].start_with?('--')
|
27
|
+
|
28
|
+
all_brokers = zookeeper.get_brokers
|
29
|
+
topics = topic_name && zookeeper.get_topics([topic_name])
|
30
|
+
topics ||= zookeeper.get_topics
|
31
|
+
|
32
|
+
opts = Trollop.options do
|
33
|
+
opt :brokers, "the comma-separated list of broker the new partitions must be assigned to", type: :string
|
34
|
+
opt :newrf, "the new replication factor", type: :integer, required: true
|
35
|
+
end
|
36
|
+
|
37
|
+
broker_ids = opts[:brokers] && opts[:brokers].split(',').map(&:to_i)
|
38
|
+
new_rf = opts[:newrf]
|
39
|
+
|
40
|
+
if new_rf < 1
|
41
|
+
puts "ERROR: replication factor is smaller than 1"
|
42
|
+
exit 1
|
43
|
+
end
|
44
|
+
|
45
|
+
broker_ids ||= zookeeper.get_brokers.values.map(&:id)
|
46
|
+
|
47
|
+
all_brokers_id = all_brokers.values.map(&:id)
|
48
|
+
broker_ids.each do |id|
|
49
|
+
if !all_brokers_id.include?(id)
|
50
|
+
puts "ERROR: Broker #{id} is not currently active.\n"
|
51
|
+
exit 1
|
52
|
+
end
|
53
|
+
end
|
54
|
+
|
55
|
+
broker_count = broker_ids.size
|
56
|
+
if new_rf > broker_count
|
57
|
+
puts "ERROR: Replication factor is larger than number of brokers.\n"
|
58
|
+
exit 1
|
59
|
+
end
|
60
|
+
|
61
|
+
assignments = []
|
62
|
+
topics.each do |_, t|
|
63
|
+
current_rf = t.partitions[0].replicas.size
|
64
|
+
if new_rf < current_rf
|
65
|
+
warn_reduce_brokers if opts[:brokers]
|
66
|
+
assignments += reduce_rf(t, current_rf, new_rf)
|
67
|
+
elsif new_rf > current_rf
|
68
|
+
assignments += increase_rf(t, current_rf, new_rf, broker_ids)
|
69
|
+
end
|
70
|
+
end
|
71
|
+
|
72
|
+
# ****************
|
73
|
+
if assignments.empty?
|
74
|
+
puts "No partition reassignment required"
|
75
|
+
else
|
76
|
+
print "This operation executes the following assignments:\n\n"
|
77
|
+
print_assignment_header
|
78
|
+
assignments.each { |a| print_assignment(a) }
|
79
|
+
print "\n"
|
80
|
+
|
81
|
+
return unless agree("Proceed (y/n)?")
|
82
|
+
|
83
|
+
result = nil
|
84
|
+
begin
|
85
|
+
print "\nBeginning.\n"
|
86
|
+
result = admin.reassign!(assignments)
|
87
|
+
print "Started.\n"
|
88
|
+
rescue Admin::ExecutionFailedError
|
89
|
+
print result
|
90
|
+
end
|
91
|
+
end
|
92
|
+
end
|
93
|
+
|
94
|
+
|
95
|
+
#
|
96
|
+
# For every partition, remove the last replica from the replica list.
|
97
|
+
# If the last replica is the leader, then the previous replica is removed instead.
|
98
|
+
#
|
99
|
+
def reduce_rf(topic, current_rf, new_rf)
|
100
|
+
delta_rf = current_rf - new_rf
|
101
|
+
if current_rf == 1
|
102
|
+
raise 'Current replication factor if 1. Cannot reduce further.'
|
103
|
+
end
|
104
|
+
unless delta_rf > 0
|
105
|
+
raise "New replication factor (#{new_rf}) must be smaller than current replication factor (#{current_rf})"
|
106
|
+
end
|
107
|
+
assignments = []
|
108
|
+
topic.partitions.map do |p|
|
109
|
+
new_replicas = p.replicas
|
110
|
+
|
111
|
+
(0...delta_rf).each do |_|
|
112
|
+
(0...new_replicas.size).each do |i|
|
113
|
+
if new_replicas[new_replicas.size-1-i] != p.leader
|
114
|
+
new_replicas.delete_at(new_replicas.size-1-i)
|
115
|
+
break
|
116
|
+
end
|
117
|
+
end
|
118
|
+
end
|
119
|
+
|
120
|
+
if new_replicas.size != new_rf
|
121
|
+
raise 'Unexpected state'
|
122
|
+
end
|
123
|
+
assignments << Assignment.new(topic.name, p.id, new_replicas)
|
124
|
+
end
|
125
|
+
assignments
|
126
|
+
end
|
127
|
+
|
128
|
+
|
129
|
+
#
|
130
|
+
# For every partition, filter out the brokers that already have a replica for this partition,
|
131
|
+
# then pick (new_rf - current_rf) brokers and assign them new replicas.
|
132
|
+
#
|
133
|
+
# The count of new replicas assigned to the brokers is maintained in order to uniformly assign new replicas.
|
134
|
+
#
|
135
|
+
def increase_rf(topic, current_rf, new_rf, brokers)
|
136
|
+
unless new_rf > current_rf
|
137
|
+
raise 'New replication factor must be greater than the current replication factor'
|
138
|
+
end
|
139
|
+
|
140
|
+
delta_rf = new_rf - current_rf
|
141
|
+
if delta_rf > brokers.size
|
142
|
+
raise "#{delta_rf} new replicas requested for topic #{p.topic_name} but only #{brokers.size} brokers available"
|
143
|
+
end
|
144
|
+
|
145
|
+
broker_counts = brokers.map { |b| {:id => b, :count => 0} }
|
146
|
+
|
147
|
+
assignments = []
|
148
|
+
topic.partitions.map do |p|
|
149
|
+
existing_replicas = p.replicas
|
150
|
+
pick_from = broker_counts.reject { |b| existing_replicas.include?(b[:id]) }
|
151
|
+
if delta_rf > pick_from.size
|
152
|
+
raise "Cannot create #{delta_rf} new replicas for partition #{p.topic_name}.#{p.id}, not enough brokers"
|
153
|
+
end
|
154
|
+
new_replicas = pick_from.sort { |a, b| a[:count] <=> b[:count] }[0...delta_rf]
|
155
|
+
new_replicas.each { |b| b[:count] += 1 }
|
156
|
+
|
157
|
+
final_replicas = existing_replicas + new_replicas.map { |b| b[:id] }
|
158
|
+
|
159
|
+
assignments << Assignment.new(topic.name, p.id, final_replicas)
|
160
|
+
end
|
161
|
+
assignments
|
162
|
+
end
|
163
|
+
|
164
|
+
def warn_reduce_brokers
|
165
|
+
return if @did_warn_reduce_brokers
|
166
|
+
puts "When reducing the replication factor the list of specified brokers is ignored."
|
167
|
+
puts "Once the replication factor is set, you can use the reassign command."
|
168
|
+
@did_warn_reduce_brokers = true
|
169
|
+
end
|
170
|
+
|
171
|
+
end
|
172
|
+
end
|
173
|
+
end
|
data/lib/kafkat/version.rb
CHANGED
metadata
CHANGED
@@ -1,20 +1,23 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: kafkat
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.0.
|
4
|
+
version: 0.0.10
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Nelson Gauthier
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2015-02-01 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: zk
|
15
15
|
requirement: !ruby/object:Gem::Requirement
|
16
16
|
requirements:
|
17
17
|
- - ~>
|
18
|
+
- !ruby/object:Gem::Version
|
19
|
+
version: '1.9'
|
20
|
+
- - ! '>='
|
18
21
|
- !ruby/object:Gem::Version
|
19
22
|
version: 1.9.4
|
20
23
|
type: :runtime
|
@@ -22,6 +25,9 @@ dependencies:
|
|
22
25
|
version_requirements: !ruby/object:Gem::Requirement
|
23
26
|
requirements:
|
24
27
|
- - ~>
|
28
|
+
- !ruby/object:Gem::Version
|
29
|
+
version: '1.9'
|
30
|
+
- - ! '>='
|
25
31
|
- !ruby/object:Gem::Version
|
26
32
|
version: 1.9.4
|
27
33
|
- !ruby/object:Gem::Dependency
|
@@ -43,6 +49,9 @@ dependencies:
|
|
43
49
|
requirement: !ruby/object:Gem::Requirement
|
44
50
|
requirements:
|
45
51
|
- - ~>
|
52
|
+
- !ruby/object:Gem::Version
|
53
|
+
version: '1.6'
|
54
|
+
- - ! '>='
|
46
55
|
- !ruby/object:Gem::Version
|
47
56
|
version: 1.6.21
|
48
57
|
type: :runtime
|
@@ -50,6 +59,9 @@ dependencies:
|
|
50
59
|
version_requirements: !ruby/object:Gem::Requirement
|
51
60
|
requirements:
|
52
61
|
- - ~>
|
62
|
+
- !ruby/object:Gem::Version
|
63
|
+
version: '1.6'
|
64
|
+
- - ! '>='
|
53
65
|
- !ruby/object:Gem::Version
|
54
66
|
version: 1.6.21
|
55
67
|
- !ruby/object:Gem::Dependency
|
@@ -57,6 +69,9 @@ dependencies:
|
|
57
69
|
requirement: !ruby/object:Gem::Requirement
|
58
70
|
requirements:
|
59
71
|
- - ~>
|
72
|
+
- !ruby/object:Gem::Version
|
73
|
+
version: '1.3'
|
74
|
+
- - ! '>='
|
60
75
|
- !ruby/object:Gem::Version
|
61
76
|
version: 1.3.5
|
62
77
|
type: :runtime
|
@@ -64,6 +79,9 @@ dependencies:
|
|
64
79
|
version_requirements: !ruby/object:Gem::Requirement
|
65
80
|
requirements:
|
66
81
|
- - ~>
|
82
|
+
- !ruby/object:Gem::Version
|
83
|
+
version: '1.3'
|
84
|
+
- - ! '>='
|
67
85
|
- !ruby/object:Gem::Version
|
68
86
|
version: 1.3.5
|
69
87
|
- !ruby/object:Gem::Dependency
|
@@ -111,6 +129,7 @@ files:
|
|
111
129
|
- lib/kafkat/command/partitions.rb
|
112
130
|
- lib/kafkat/command/reassign.rb
|
113
131
|
- lib/kafkat/command/resign-rewrite.rb
|
132
|
+
- lib/kafkat/command/set-replication-factor.rb
|
114
133
|
- lib/kafkat/command/shutdown.rb
|
115
134
|
- lib/kafkat/command/topics.rb
|
116
135
|
- lib/kafkat/config.rb
|