kafkat 0.0.9 → 0.0.10
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +8 -8
- data/Gemfile.lock +5 -5
- data/README.md +14 -13
- data/kafkat.gemspec +3 -3
- data/lib/kafkat/cli.rb +2 -2
- data/lib/kafkat/command/elect-leaders.rb +1 -1
- data/lib/kafkat/command/reassign.rb +2 -2
- data/lib/kafkat/command/set-replication-factor.rb +173 -0
- data/lib/kafkat/command/shutdown.rb +1 -1
- data/lib/kafkat/version.rb +1 -1
- metadata +21 -2
checksums.yaml
CHANGED
@@ -1,15 +1,15 @@
|
|
1
1
|
---
|
2
2
|
!binary "U0hBMQ==":
|
3
3
|
metadata.gz: !binary |-
|
4
|
-
|
4
|
+
MTY0MTdhYjQ3YjAzNWExNWJjMzMyYThmOTY5ZWQ5NmQyOTcyMGM4Yw==
|
5
5
|
data.tar.gz: !binary |-
|
6
|
-
|
6
|
+
MTkzM2UzODgxNTIzMmJiYTAwNDQ2NmI5ZmVjYWZkYTdiYjNkZDk4Ng==
|
7
7
|
SHA512:
|
8
8
|
metadata.gz: !binary |-
|
9
|
-
|
10
|
-
|
11
|
-
|
9
|
+
N2U0ZTMyMjQ4MmI3M2QzMDkyNTM4ZWFjOWJmMTNhNmIxNjlkM2NjMzBkYTli
|
10
|
+
YmY3N2Y1ZTk5OWIwZjZlZjZhMzNjOGE2MmE4NDg2ZTc4NzFjYTYzZmZlZWNi
|
11
|
+
YzFmNzFjZGU5N2Y5Y2UyMGY5NDhiNmEyZWU5NjdiNzViMjVlNjU=
|
12
12
|
data.tar.gz: !binary |-
|
13
|
-
|
14
|
-
|
15
|
-
|
13
|
+
YjRiMzllYmZjNDBiMmYxNjRiNzExMzlkZGM2ZWQwOTA0NmFiMGRkZTQzZjMz
|
14
|
+
ZmRlMGM5OTczMTcwYzI4MmJiYmE1ZGYwMDZlNzhhMTI3ZmQ2YThkN2E4OTdj
|
15
|
+
ZmUzN2QyZDMxNDk5ODE1ZWIwNTdjZmMyNWFkMTYzYWRjOWJmZWY=
|
data/Gemfile.lock
CHANGED
@@ -1,12 +1,12 @@
|
|
1
1
|
PATH
|
2
2
|
remote: .
|
3
3
|
specs:
|
4
|
-
kafkat (0.0.
|
4
|
+
kafkat (0.0.10)
|
5
5
|
colored (~> 1.2)
|
6
|
-
highline (~> 1.6.21)
|
7
|
-
retryable (~> 1.3.5)
|
6
|
+
highline (~> 1.6, >= 1.6.21)
|
7
|
+
retryable (~> 1.3, >= 1.3.5)
|
8
8
|
trollop (~> 2.0)
|
9
|
-
zk (~> 1.9.4)
|
9
|
+
zk (~> 1.9, >= 1.9.4)
|
10
10
|
|
11
11
|
GEM
|
12
12
|
remote: https://rubygems.org/
|
@@ -23,7 +23,7 @@ GEM
|
|
23
23
|
zk (1.9.4)
|
24
24
|
logging (~> 1.8.2)
|
25
25
|
zookeeper (~> 1.4.0)
|
26
|
-
zookeeper (1.4.
|
26
|
+
zookeeper (1.4.9)
|
27
27
|
|
28
28
|
PLATFORMS
|
29
29
|
ruby
|
data/README.md
CHANGED
@@ -34,23 +34,24 @@ Kafkat searches for this file in two places, `~/.kafkatcfg` and `/etc/kafkatcfg`
|
|
34
34
|
|
35
35
|
```
|
36
36
|
$ kafkat
|
37
|
-
kafkat 0.0.
|
37
|
+
kafkat 0.0.10: Simplified command-line administration for Kafka brokers
|
38
38
|
usage: kafkat [command] [options]
|
39
39
|
|
40
40
|
Here's a list of supported commands:
|
41
41
|
|
42
|
-
brokers
|
43
|
-
clean-indexes
|
44
|
-
controller
|
45
|
-
elect-leaders [topic]
|
46
|
-
partitions [topic]
|
47
|
-
partitions [topic] --under-replicated
|
48
|
-
partitions [topic] --unavailable
|
49
|
-
reassign [topic] [--brokers <ids>] [--replicas <n>]
|
50
|
-
resign-rewrite <broker id>
|
51
|
-
resign-rewrite <broker id> --force
|
52
|
-
|
53
|
-
|
42
|
+
brokers Print available brokers from Zookeeper.
|
43
|
+
clean-indexes Delete untruncated Kafka log indexes from the filesystem.
|
44
|
+
controller Print the current controller.
|
45
|
+
elect-leaders [topic] Begin election of the preferred leaders.
|
46
|
+
partitions [topic] Print partitions by topic.
|
47
|
+
partitions [topic] --under-replicated Print partitions by topic (only under-replicated).
|
48
|
+
partitions [topic] --unavailable Print partitions by topic (only unavailable).
|
49
|
+
reassign [topic] [--brokers <ids>] [--replicas <n>] Begin reassignment of partitions.
|
50
|
+
resign-rewrite <broker id> Forcibly rewrite leaderships to exclude a broker.
|
51
|
+
resign-rewrite <broker id> --force Same as above but proceed if there are no available ISRs.
|
52
|
+
set-replication-factor [topic] [--newrf <n>] [--brokers id[,id]] Set the replication factor of
|
53
|
+
shutdown <broker id> Gracefully remove leaderships from a broker (requires JMX).
|
54
|
+
topics Print all topics.
|
54
55
|
|
55
56
|
```
|
56
57
|
|
data/kafkat.gemspec
CHANGED
@@ -18,9 +18,9 @@ Gem::Specification.new do |s|
|
|
18
18
|
s.test_files = s.files.grep(%r{^(test|spec|features)/})
|
19
19
|
s.require_path = 'lib'
|
20
20
|
|
21
|
-
s.add_runtime_dependency 'zk', '~> 1.9.4'
|
21
|
+
s.add_runtime_dependency 'zk', '~> 1.9', '>= 1.9.4'
|
22
22
|
s.add_runtime_dependency 'trollop', '~> 2.0'
|
23
|
-
s.add_runtime_dependency 'highline', '~> 1.6.21'
|
24
|
-
s.add_runtime_dependency 'retryable', '~> 1.3.5'
|
23
|
+
s.add_runtime_dependency 'highline', '~> 1.6', '>= 1.6.21'
|
24
|
+
s.add_runtime_dependency 'retryable', '~> 1.3', '>= 1.3.5'
|
25
25
|
s.add_runtime_dependency 'colored', '~> 1.2'
|
26
26
|
end
|
data/lib/kafkat/cli.rb
CHANGED
@@ -44,8 +44,8 @@ module Kafkat
|
|
44
44
|
Command.all.values.sort_by(&:command_name).each do |klass|
|
45
45
|
klass.usages.each do |usage|
|
46
46
|
format, description = usage[0], usage[1]
|
47
|
-
padding_length =
|
48
|
-
padding = " " * padding_length
|
47
|
+
padding_length = 68 - format.length
|
48
|
+
padding = " " * padding_length unless padding_length < 0
|
49
49
|
print " #{format}#{padding}#{description}\n"
|
50
50
|
end
|
51
51
|
end
|
@@ -42,7 +42,7 @@ module Kafkat
|
|
42
42
|
topic_replica_count = replica_count || t.partitions[0].replicas.size
|
43
43
|
|
44
44
|
if topic_replica_count > broker_count
|
45
|
-
print "ERROR: Replication factor is larger than brokers.\n"
|
45
|
+
print "ERROR: Replication factor (#{topic_replica_count}) is larger than brokers (#{broker_count}).\n"
|
46
46
|
exit 1
|
47
47
|
end
|
48
48
|
|
@@ -80,7 +80,7 @@ module Kafkat
|
|
80
80
|
print "\nBeginning.\n"
|
81
81
|
result = admin.reassign!(assignments)
|
82
82
|
print "Started.\n"
|
83
|
-
rescue Admin::ExecutionFailedError
|
83
|
+
rescue Interface::Admin::ExecutionFailedError
|
84
84
|
print result
|
85
85
|
end
|
86
86
|
end
|
@@ -0,0 +1,173 @@
|
|
1
|
+
module Kafkat
|
2
|
+
module Command
|
3
|
+
|
4
|
+
#
|
5
|
+
# Command to set the replication factor (RF) of a topic.
|
6
|
+
# The command accepts the topic name, the new desired replication factor,
|
7
|
+
# and, in case of an increase of the replication factor, a list of broker ids.
|
8
|
+
#
|
9
|
+
# When reducing the RF, a new partition assigment will be generated by
|
10
|
+
# removing the last partition replica of every replica set. The leader partition
|
11
|
+
# will not be removed from the replica set to prevent a leader election.
|
12
|
+
#
|
13
|
+
# When increasing the RF, a new partition assignment will be generated by allocating
|
14
|
+
# a new replica to every replica set. The new replica will be assigned to the provided
|
15
|
+
# broker list in a round robin fashion.
|
16
|
+
# If no broker id is specified on the command line, all brokers will be used.
|
17
|
+
#
|
18
|
+
#
|
19
|
+
class SetReplicationFactor < Base
|
20
|
+
register_as 'set-replication-factor'
|
21
|
+
|
22
|
+
usage 'set-replication-factor [topic] [--newrf <n>] [--brokers id[,id]]',
|
23
|
+
'Set the replication factor of'
|
24
|
+
|
25
|
+
def run
|
26
|
+
topic_name = ARGV.shift unless ARGV[0] && ARGV[0].start_with?('--')
|
27
|
+
|
28
|
+
all_brokers = zookeeper.get_brokers
|
29
|
+
topics = topic_name && zookeeper.get_topics([topic_name])
|
30
|
+
topics ||= zookeeper.get_topics
|
31
|
+
|
32
|
+
opts = Trollop.options do
|
33
|
+
opt :brokers, "the comma-separated list of broker the new partitions must be assigned to", type: :string
|
34
|
+
opt :newrf, "the new replication factor", type: :integer, required: true
|
35
|
+
end
|
36
|
+
|
37
|
+
broker_ids = opts[:brokers] && opts[:brokers].split(',').map(&:to_i)
|
38
|
+
new_rf = opts[:newrf]
|
39
|
+
|
40
|
+
if new_rf < 1
|
41
|
+
puts "ERROR: replication factor is smaller than 1"
|
42
|
+
exit 1
|
43
|
+
end
|
44
|
+
|
45
|
+
broker_ids ||= zookeeper.get_brokers.values.map(&:id)
|
46
|
+
|
47
|
+
all_brokers_id = all_brokers.values.map(&:id)
|
48
|
+
broker_ids.each do |id|
|
49
|
+
if !all_brokers_id.include?(id)
|
50
|
+
puts "ERROR: Broker #{id} is not currently active.\n"
|
51
|
+
exit 1
|
52
|
+
end
|
53
|
+
end
|
54
|
+
|
55
|
+
broker_count = broker_ids.size
|
56
|
+
if new_rf > broker_count
|
57
|
+
puts "ERROR: Replication factor is larger than number of brokers.\n"
|
58
|
+
exit 1
|
59
|
+
end
|
60
|
+
|
61
|
+
assignments = []
|
62
|
+
topics.each do |_, t|
|
63
|
+
current_rf = t.partitions[0].replicas.size
|
64
|
+
if new_rf < current_rf
|
65
|
+
warn_reduce_brokers if opts[:brokers]
|
66
|
+
assignments += reduce_rf(t, current_rf, new_rf)
|
67
|
+
elsif new_rf > current_rf
|
68
|
+
assignments += increase_rf(t, current_rf, new_rf, broker_ids)
|
69
|
+
end
|
70
|
+
end
|
71
|
+
|
72
|
+
# ****************
|
73
|
+
if assignments.empty?
|
74
|
+
puts "No partition reassignment required"
|
75
|
+
else
|
76
|
+
print "This operation executes the following assignments:\n\n"
|
77
|
+
print_assignment_header
|
78
|
+
assignments.each { |a| print_assignment(a) }
|
79
|
+
print "\n"
|
80
|
+
|
81
|
+
return unless agree("Proceed (y/n)?")
|
82
|
+
|
83
|
+
result = nil
|
84
|
+
begin
|
85
|
+
print "\nBeginning.\n"
|
86
|
+
result = admin.reassign!(assignments)
|
87
|
+
print "Started.\n"
|
88
|
+
rescue Admin::ExecutionFailedError
|
89
|
+
print result
|
90
|
+
end
|
91
|
+
end
|
92
|
+
end
|
93
|
+
|
94
|
+
|
95
|
+
#
|
96
|
+
# For every partition, remove the last replica from the replica list.
|
97
|
+
# If the last replica is the leader, then the previous replica is removed instead.
|
98
|
+
#
|
99
|
+
def reduce_rf(topic, current_rf, new_rf)
|
100
|
+
delta_rf = current_rf - new_rf
|
101
|
+
if current_rf == 1
|
102
|
+
raise 'Current replication factor if 1. Cannot reduce further.'
|
103
|
+
end
|
104
|
+
unless delta_rf > 0
|
105
|
+
raise "New replication factor (#{new_rf}) must be smaller than current replication factor (#{current_rf})"
|
106
|
+
end
|
107
|
+
assignments = []
|
108
|
+
topic.partitions.map do |p|
|
109
|
+
new_replicas = p.replicas
|
110
|
+
|
111
|
+
(0...delta_rf).each do |_|
|
112
|
+
(0...new_replicas.size).each do |i|
|
113
|
+
if new_replicas[new_replicas.size-1-i] != p.leader
|
114
|
+
new_replicas.delete_at(new_replicas.size-1-i)
|
115
|
+
break
|
116
|
+
end
|
117
|
+
end
|
118
|
+
end
|
119
|
+
|
120
|
+
if new_replicas.size != new_rf
|
121
|
+
raise 'Unexpected state'
|
122
|
+
end
|
123
|
+
assignments << Assignment.new(topic.name, p.id, new_replicas)
|
124
|
+
end
|
125
|
+
assignments
|
126
|
+
end
|
127
|
+
|
128
|
+
|
129
|
+
#
|
130
|
+
# For every partition, filter out the brokers that already have a replica for this partition,
|
131
|
+
# then pick (new_rf - current_rf) brokers and assign them new replicas.
|
132
|
+
#
|
133
|
+
# The count of new replicas assigned to the brokers is maintained in order to uniformly assign new replicas.
|
134
|
+
#
|
135
|
+
def increase_rf(topic, current_rf, new_rf, brokers)
|
136
|
+
unless new_rf > current_rf
|
137
|
+
raise 'New replication factor must be greater than the current replication factor'
|
138
|
+
end
|
139
|
+
|
140
|
+
delta_rf = new_rf - current_rf
|
141
|
+
if delta_rf > brokers.size
|
142
|
+
raise "#{delta_rf} new replicas requested for topic #{p.topic_name} but only #{brokers.size} brokers available"
|
143
|
+
end
|
144
|
+
|
145
|
+
broker_counts = brokers.map { |b| {:id => b, :count => 0} }
|
146
|
+
|
147
|
+
assignments = []
|
148
|
+
topic.partitions.map do |p|
|
149
|
+
existing_replicas = p.replicas
|
150
|
+
pick_from = broker_counts.reject { |b| existing_replicas.include?(b[:id]) }
|
151
|
+
if delta_rf > pick_from.size
|
152
|
+
raise "Cannot create #{delta_rf} new replicas for partition #{p.topic_name}.#{p.id}, not enough brokers"
|
153
|
+
end
|
154
|
+
new_replicas = pick_from.sort { |a, b| a[:count] <=> b[:count] }[0...delta_rf]
|
155
|
+
new_replicas.each { |b| b[:count] += 1 }
|
156
|
+
|
157
|
+
final_replicas = existing_replicas + new_replicas.map { |b| b[:id] }
|
158
|
+
|
159
|
+
assignments << Assignment.new(topic.name, p.id, final_replicas)
|
160
|
+
end
|
161
|
+
assignments
|
162
|
+
end
|
163
|
+
|
164
|
+
def warn_reduce_brokers
|
165
|
+
return if @did_warn_reduce_brokers
|
166
|
+
puts "When reducing the replication factor the list of specified brokers is ignored."
|
167
|
+
puts "Once the replication factor is set, you can use the reassign command."
|
168
|
+
@did_warn_reduce_brokers = true
|
169
|
+
end
|
170
|
+
|
171
|
+
end
|
172
|
+
end
|
173
|
+
end
|
data/lib/kafkat/version.rb
CHANGED
metadata
CHANGED
@@ -1,20 +1,23 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: kafkat
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.0.
|
4
|
+
version: 0.0.10
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Nelson Gauthier
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2015-02-01 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: zk
|
15
15
|
requirement: !ruby/object:Gem::Requirement
|
16
16
|
requirements:
|
17
17
|
- - ~>
|
18
|
+
- !ruby/object:Gem::Version
|
19
|
+
version: '1.9'
|
20
|
+
- - ! '>='
|
18
21
|
- !ruby/object:Gem::Version
|
19
22
|
version: 1.9.4
|
20
23
|
type: :runtime
|
@@ -22,6 +25,9 @@ dependencies:
|
|
22
25
|
version_requirements: !ruby/object:Gem::Requirement
|
23
26
|
requirements:
|
24
27
|
- - ~>
|
28
|
+
- !ruby/object:Gem::Version
|
29
|
+
version: '1.9'
|
30
|
+
- - ! '>='
|
25
31
|
- !ruby/object:Gem::Version
|
26
32
|
version: 1.9.4
|
27
33
|
- !ruby/object:Gem::Dependency
|
@@ -43,6 +49,9 @@ dependencies:
|
|
43
49
|
requirement: !ruby/object:Gem::Requirement
|
44
50
|
requirements:
|
45
51
|
- - ~>
|
52
|
+
- !ruby/object:Gem::Version
|
53
|
+
version: '1.6'
|
54
|
+
- - ! '>='
|
46
55
|
- !ruby/object:Gem::Version
|
47
56
|
version: 1.6.21
|
48
57
|
type: :runtime
|
@@ -50,6 +59,9 @@ dependencies:
|
|
50
59
|
version_requirements: !ruby/object:Gem::Requirement
|
51
60
|
requirements:
|
52
61
|
- - ~>
|
62
|
+
- !ruby/object:Gem::Version
|
63
|
+
version: '1.6'
|
64
|
+
- - ! '>='
|
53
65
|
- !ruby/object:Gem::Version
|
54
66
|
version: 1.6.21
|
55
67
|
- !ruby/object:Gem::Dependency
|
@@ -57,6 +69,9 @@ dependencies:
|
|
57
69
|
requirement: !ruby/object:Gem::Requirement
|
58
70
|
requirements:
|
59
71
|
- - ~>
|
72
|
+
- !ruby/object:Gem::Version
|
73
|
+
version: '1.3'
|
74
|
+
- - ! '>='
|
60
75
|
- !ruby/object:Gem::Version
|
61
76
|
version: 1.3.5
|
62
77
|
type: :runtime
|
@@ -64,6 +79,9 @@ dependencies:
|
|
64
79
|
version_requirements: !ruby/object:Gem::Requirement
|
65
80
|
requirements:
|
66
81
|
- - ~>
|
82
|
+
- !ruby/object:Gem::Version
|
83
|
+
version: '1.3'
|
84
|
+
- - ! '>='
|
67
85
|
- !ruby/object:Gem::Version
|
68
86
|
version: 1.3.5
|
69
87
|
- !ruby/object:Gem::Dependency
|
@@ -111,6 +129,7 @@ files:
|
|
111
129
|
- lib/kafkat/command/partitions.rb
|
112
130
|
- lib/kafkat/command/reassign.rb
|
113
131
|
- lib/kafkat/command/resign-rewrite.rb
|
132
|
+
- lib/kafkat/command/set-replication-factor.rb
|
114
133
|
- lib/kafkat/command/shutdown.rb
|
115
134
|
- lib/kafkat/command/topics.rb
|
116
135
|
- lib/kafkat/config.rb
|