kafkat 0.1.0 → 0.2.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +8 -8
- data/CHANGELOG.md +6 -0
- data/README.md +7 -0
- data/lib/kafkat/command/cluster_restart.rb +336 -0
- data/lib/kafkat/command/drain.rb +1 -0
- data/lib/kafkat/interface/zookeeper.rb +4 -0
- data/lib/kafkat/reboot.rb +0 -0
- data/lib/kafkat/version.rb +1 -1
- data/spec/lib/kafkat/command/cluster_restart_spec.rb +197 -0
- metadata +7 -2
checksums.yaml
CHANGED
@@ -1,15 +1,15 @@
|
|
1
1
|
---
|
2
2
|
!binary "U0hBMQ==":
|
3
3
|
metadata.gz: !binary |-
|
4
|
-
|
4
|
+
NzIyMjdlYzlmM2JhZDI1YzAzZWM1ZTU2ZmU4NWVhNjI3NjY1NTkwMQ==
|
5
5
|
data.tar.gz: !binary |-
|
6
|
-
|
6
|
+
YjNkZDUzZGZlYmIzNGRiYTNkNTBhNjAwYWE2NmI5NzEwMWRmOTVkMw==
|
7
7
|
SHA512:
|
8
8
|
metadata.gz: !binary |-
|
9
|
-
|
10
|
-
|
11
|
-
|
9
|
+
NGQzMjRmOThmYmE3MDQ5ZDUyYTg4Zjg0ZTI3Zjc0MDRlOGY5NmYwMGNlZmI0
|
10
|
+
ZDlmZWE4OWM3MTI5MGE5NmU0MjU2NGQyYjk5NjNkZDFlZmRlMTllOWRkMDYx
|
11
|
+
NzhhNDcyYjIyZDU2M2E3NzZhYjE1ZGMyODI3ZjM4ZjJhODM5Mjk=
|
12
12
|
data.tar.gz: !binary |-
|
13
|
-
|
14
|
-
|
15
|
-
|
13
|
+
MGIwYzM3NWMzMTZhNWJlMzdjY2JiNTRkYzNlMzBiZGQ4YmM2MWQ1MjI4ZWNm
|
14
|
+
Y2ExNGMxNTVjMTQ2Y2QzMWQzODlhZWE2YzljNTkyYTkyMWRiNzBiODkzZDdi
|
15
|
+
NjQ0OGE1MDlmNDYwY2M5NDZjNzJiYWUxOGE0NDdmMjY0ZjhhMjE=
|
data/CHANGELOG.md
ADDED
data/README.md
CHANGED
@@ -11,6 +11,11 @@ Simplified command-line administration for Kafka brokers.
|
|
11
11
|
## License & Attributions
|
12
12
|
This project is released under the Apache License Version 2.0 (APLv2).
|
13
13
|
|
14
|
+
## How to release
|
15
|
+
|
16
|
+
- update the version number in `lib/kafkat/version.rb`
|
17
|
+
- execute `bundle exec rake release`
|
18
|
+
|
14
19
|
|
15
20
|
## Usage
|
16
21
|
|
@@ -61,3 +66,5 @@ Here's a list of supported commands:
|
|
61
66
|
## Important Note
|
62
67
|
|
63
68
|
The gem needs read/write access to the Kafka log directory for some operations (clean indexes).
|
69
|
+
|
70
|
+
|
@@ -0,0 +1,336 @@
|
|
1
|
+
module Kafkat
|
2
|
+
module Command
|
3
|
+
class ClusterRestart < Base
|
4
|
+
|
5
|
+
register_as 'cluster-restart'
|
6
|
+
|
7
|
+
usage 'cluster-restart help', 'Determine the server restart sequence for kafka'
|
8
|
+
|
9
|
+
def run
|
10
|
+
subcommand_name = ARGV.shift || 'help'
|
11
|
+
begin
|
12
|
+
subcommand_class = ['Kafkat', 'ClusterRestart', 'Subcommands', subcommand_name.capitalize].inject(Object) do |mod, class_name|
|
13
|
+
mod.const_get(class_name)
|
14
|
+
end
|
15
|
+
subcommand_class.new(config).run
|
16
|
+
rescue NameError
|
17
|
+
print "ERROR: Unknown command #{subcommand_name}"
|
18
|
+
end
|
19
|
+
end
|
20
|
+
end
|
21
|
+
end
|
22
|
+
end
|
23
|
+
|
24
|
+
module Kafkat
|
25
|
+
module ClusterRestart
|
26
|
+
module Subcommands
|
27
|
+
|
28
|
+
class Help < ::Kafkat::Command::Base
|
29
|
+
def run
|
30
|
+
puts 'cluster-restart help Print Help and exit'
|
31
|
+
puts 'cluster-restart reset Clean up the restart state'
|
32
|
+
puts 'cluster-restart start Initialize the cluster-restart session for the cluster'
|
33
|
+
puts 'cluster-restart next Calculate the next broker to restart based on the current state'
|
34
|
+
puts 'cluster-restart good <broker> Mark this broker as successfully restarted'
|
35
|
+
puts 'cluster-restart log Print the state of the brokers'
|
36
|
+
puts 'cluster-restart restore <file> Start a new session and restore the state defined in that file'
|
37
|
+
end
|
38
|
+
end
|
39
|
+
|
40
|
+
class Start < ::Kafkat::Command::Base
|
41
|
+
|
42
|
+
attr_reader :session
|
43
|
+
|
44
|
+
def run
|
45
|
+
if Session.exists?
|
46
|
+
puts "ERROR: A session is already started"
|
47
|
+
puts "\n[Action] Please run 'next' or 'reset' commands"
|
48
|
+
exit 1
|
49
|
+
end
|
50
|
+
|
51
|
+
print "Starting a new Cluster-Restart session.\n"
|
52
|
+
|
53
|
+
@session = Session.from_zookeepers(zookeeper)
|
54
|
+
@session.save!
|
55
|
+
|
56
|
+
puts "\n[Action] Please run 'next' to select the broker with lowest restarting cost"
|
57
|
+
end
|
58
|
+
end
|
59
|
+
|
60
|
+
class Reset < ::Kafkat::Command::Base
|
61
|
+
|
62
|
+
def run
|
63
|
+
if Session.exists?
|
64
|
+
Session.reset!
|
65
|
+
end
|
66
|
+
puts "Session reset"
|
67
|
+
puts "\n[Action] Please run 'start' to start the session"
|
68
|
+
end
|
69
|
+
end
|
70
|
+
|
71
|
+
class Restore < ::Kafkat::Command::Base
|
72
|
+
|
73
|
+
attr_reader :session
|
74
|
+
|
75
|
+
def run
|
76
|
+
if Session.exists?
|
77
|
+
puts "ERROR: A session is already started"
|
78
|
+
puts "\n[Action] Please run 'next' or 'reset' commands"
|
79
|
+
exit 1
|
80
|
+
end
|
81
|
+
|
82
|
+
file_name = ARGV[0]
|
83
|
+
@session = Session.load!(file_name)
|
84
|
+
@session.save!
|
85
|
+
puts "Session restored"
|
86
|
+
puts "\m[Action] Please run 'next' to select the broker with lowest restarting cost"
|
87
|
+
end
|
88
|
+
end
|
89
|
+
|
90
|
+
class Next < ::Kafkat::Command::Base
|
91
|
+
|
92
|
+
attr_reader :session, :topics
|
93
|
+
|
94
|
+
def run
|
95
|
+
unless Session.exists?
|
96
|
+
puts "ERROR: no session in progress"
|
97
|
+
puts "\n[Action] Please run 'start' command"
|
98
|
+
exit 1
|
99
|
+
end
|
100
|
+
|
101
|
+
@session = Session.load!
|
102
|
+
if @session.all_restarted?
|
103
|
+
puts "All the brokers have been restarted"
|
104
|
+
else
|
105
|
+
pendings = @session.pending_brokers
|
106
|
+
if pendings.size > 1
|
107
|
+
puts "ERROR Illegal state: multiple brokers are in Pending state"
|
108
|
+
exit 1
|
109
|
+
elsif pendings.size == 1
|
110
|
+
next_broker = pendings[0]
|
111
|
+
puts "Broker #{next_broker} is in Pending state"
|
112
|
+
else
|
113
|
+
@topics = zookeeper.get_topics
|
114
|
+
next_broker, cost = ClusterRestartHelper.select_broker_with_min_cost(session, topics)
|
115
|
+
@session.update_states!(Session::STATE_PENDING, [next_broker])
|
116
|
+
@session.save!
|
117
|
+
puts "The next broker is: #{next_broker}"
|
118
|
+
end
|
119
|
+
puts "\n[Action-1] Restart broker #{next_broker} aka #{zookeeper.get_broker(next_broker).host}"
|
120
|
+
puts "\n[Action-2] Run 'good #{next_broker}' to mark it as restarted."
|
121
|
+
end
|
122
|
+
end
|
123
|
+
end
|
124
|
+
|
125
|
+
class Log < ::Kafkat::Command::Base
|
126
|
+
|
127
|
+
attr_reader :session
|
128
|
+
|
129
|
+
def run
|
130
|
+
unless Session.exists?
|
131
|
+
puts "ERROR: no session in progress"
|
132
|
+
puts "\n[Action] Please run 'start' command"
|
133
|
+
exit 1
|
134
|
+
end
|
135
|
+
|
136
|
+
@session = Session.load!
|
137
|
+
puts JSON.pretty_generate(@session.to_h)
|
138
|
+
end
|
139
|
+
end
|
140
|
+
|
141
|
+
class Good < ::Kafkat::Command::Base
|
142
|
+
|
143
|
+
attr_reader :session
|
144
|
+
|
145
|
+
def run
|
146
|
+
unless Session.exists?
|
147
|
+
puts "ERROR: no session in progress"
|
148
|
+
puts "\n[Action] Please run 'start' command"
|
149
|
+
exit 1
|
150
|
+
end
|
151
|
+
|
152
|
+
broker_id = ARGV[0]
|
153
|
+
if broker_id.nil?
|
154
|
+
puts "ERROR You must specify a broker id"
|
155
|
+
exit 1
|
156
|
+
end
|
157
|
+
restart(broker_id)
|
158
|
+
puts "Broker #{broker_id} has been marked as restarted"
|
159
|
+
puts "\n[Action] Please run 'next' to select the broker with lowest restarting cost"
|
160
|
+
end
|
161
|
+
|
162
|
+
def restart(broker_id)
|
163
|
+
@session = Session.load!
|
164
|
+
begin
|
165
|
+
if session.pending?(broker_id)
|
166
|
+
session.update_states!(Session::STATE_RESTARTED, [broker_id])
|
167
|
+
session.save!
|
168
|
+
else
|
169
|
+
puts "ERROR Broker state is #{session.state(broker_id)}"
|
170
|
+
exit 1
|
171
|
+
end
|
172
|
+
rescue UnknownBrokerError => e
|
173
|
+
puts "ERROR #{e.to_s}"
|
174
|
+
exit 1
|
175
|
+
end
|
176
|
+
end
|
177
|
+
end
|
178
|
+
end
|
179
|
+
|
180
|
+
class UnknownBrokerError < StandardError;
|
181
|
+
end
|
182
|
+
class UnknownStateError < StandardError;
|
183
|
+
end
|
184
|
+
|
185
|
+
class ClusterRestartHelper
|
186
|
+
|
187
|
+
def self.select_broker_with_min_cost(session, topics)
|
188
|
+
broker_to_partition = get_broker_to_leader_partition_mapping(topics)
|
189
|
+
broker_restart_cost = Hash.new(0)
|
190
|
+
session.broker_states.each do |broker_id, state|
|
191
|
+
if state == Session::STATE_NOT_RESTARTED
|
192
|
+
current_cost = calculate_cost(broker_id, broker_to_partition[broker_id], session)
|
193
|
+
broker_restart_cost[broker_id] = current_cost if current_cost != nil
|
194
|
+
end
|
195
|
+
end
|
196
|
+
|
197
|
+
# Sort by cost first, and then broker_id
|
198
|
+
broker_restart_cost.min_by { |broker_id, cost| [cost, broker_id] }
|
199
|
+
end
|
200
|
+
|
201
|
+
def self.get_broker_to_leader_partition_mapping(topics)
|
202
|
+
broker_to_partitions = Hash.new { |h, key| h[key] = [] }
|
203
|
+
|
204
|
+
topics.values.flat_map { |topic| topic.partitions }
|
205
|
+
.each do |partition|
|
206
|
+
broker_to_partitions[partition.leader] << partition
|
207
|
+
end
|
208
|
+
broker_to_partitions
|
209
|
+
end
|
210
|
+
|
211
|
+
def self.calculate_cost(broker_id, partitions, session)
|
212
|
+
raise UnknownBrokerError, "Unknown broker #{broker_id}" unless session.broker_states.key?(broker_id)
|
213
|
+
partitions.find_all { |partition| partition.leader == broker_id }
|
214
|
+
.reduce(0) do |cost, partition|
|
215
|
+
cost += partition.replicas.length
|
216
|
+
cost -= partition.replicas.find_all { |replica| session.restarted?(replica) }.size
|
217
|
+
cost
|
218
|
+
end
|
219
|
+
end
|
220
|
+
end
|
221
|
+
|
222
|
+
|
223
|
+
class Session
|
224
|
+
|
225
|
+
SESSION_PATH = '~/kafkat_cluster_restart_session.json'
|
226
|
+
STATE_RESTARTED = 'restarted' # use String instead of Symbol to facilitate JSON ser/deser
|
227
|
+
STATE_NOT_RESTARTED = 'not_restarted'
|
228
|
+
STATE_PENDING = 'pending'
|
229
|
+
STATES= [STATE_NOT_RESTARTED, STATE_RESTARTED, STATE_PENDING]
|
230
|
+
|
231
|
+
class NotFoundError < StandardError;
|
232
|
+
end
|
233
|
+
class ParseError < StandardError;
|
234
|
+
end
|
235
|
+
|
236
|
+
attr_reader :broker_states
|
237
|
+
|
238
|
+
def self.exists?
|
239
|
+
File.file?(File.expand_path(SESSION_PATH))
|
240
|
+
end
|
241
|
+
|
242
|
+
def self.load!(session_file = SESSION_PATH)
|
243
|
+
path = File.expand_path(session_file)
|
244
|
+
string = File.read(path)
|
245
|
+
|
246
|
+
json = JSON.parse(string)
|
247
|
+
self.new(json)
|
248
|
+
|
249
|
+
rescue Errno::ENOENT
|
250
|
+
raise NotFoundError
|
251
|
+
rescue JSON::JSONError
|
252
|
+
raise ParseError
|
253
|
+
end
|
254
|
+
|
255
|
+
def self.reset!(session_file = SESSION_PATH)
|
256
|
+
path = File.expand_path(session_file)
|
257
|
+
File.delete(path)
|
258
|
+
end
|
259
|
+
|
260
|
+
def self.from_zookeepers(zookeeper)
|
261
|
+
broker_ids = zookeeper.get_broker_ids
|
262
|
+
Session.from_brokers(broker_ids)
|
263
|
+
end
|
264
|
+
|
265
|
+
def self.from_brokers(brokers)
|
266
|
+
states = brokers.each_with_object({}) { |id, h| h[id] = STATE_NOT_RESTARTED }
|
267
|
+
Session.new('broker_states' => states)
|
268
|
+
end
|
269
|
+
|
270
|
+
def initialize(data = {})
|
271
|
+
@broker_states = data['broker_states'] || {}
|
272
|
+
end
|
273
|
+
|
274
|
+
def save!(session_file = SESSION_PATH)
|
275
|
+
File.open(File.expand_path(session_file), 'w') do |f|
|
276
|
+
f.puts JSON.pretty_generate(self.to_h)
|
277
|
+
end
|
278
|
+
end
|
279
|
+
|
280
|
+
def update_states!(state, ids)
|
281
|
+
state = state.to_s if state.is_a?(Symbol)
|
282
|
+
unless STATES.include?(state)
|
283
|
+
raise UnknownStateError, "Unknown State #{state}"
|
284
|
+
end
|
285
|
+
|
286
|
+
intersection = ids & broker_states.keys
|
287
|
+
unless intersection == ids
|
288
|
+
raise UnknownBrokerError, "Unknown brokers: #{(ids - intersection).join(', ')}"
|
289
|
+
end
|
290
|
+
|
291
|
+
ids.each { |id| broker_states[id] = state }
|
292
|
+
self
|
293
|
+
end
|
294
|
+
|
295
|
+
|
296
|
+
def state(broker_id)
|
297
|
+
raise UnknownBrokerError, "Unknown broker: #{broker_id}" unless @broker_states.key?(broker_id)
|
298
|
+
broker_states[broker_id]
|
299
|
+
end
|
300
|
+
|
301
|
+
def state?(broker_id, state)
|
302
|
+
raise UnknownBrokerError, "Unknown broker: #{broker_id}" unless @broker_states.key?(broker_id)
|
303
|
+
raise UnknownStateError, "Unknown state: #{state}" unless STATES.include?(state)
|
304
|
+
@broker_states[broker_id] == state
|
305
|
+
end
|
306
|
+
|
307
|
+
def pending?(broker_id)
|
308
|
+
state?(broker_id, STATE_PENDING)
|
309
|
+
end
|
310
|
+
|
311
|
+
def not_restarted?(broker_id)
|
312
|
+
state?(broker_id, STATE_NOT_RESTARTED)
|
313
|
+
end
|
314
|
+
|
315
|
+
def restarted?(broker_id)
|
316
|
+
state?(broker_id, STATE_RESTARTED)
|
317
|
+
end
|
318
|
+
|
319
|
+
def all_restarted?
|
320
|
+
@broker_states.values.all? { |state| state == STATE_RESTARTED }
|
321
|
+
end
|
322
|
+
|
323
|
+
def pending_brokers
|
324
|
+
broker_states.keys.find_all do |broker_id|
|
325
|
+
broker_states[broker_id] == STATE_PENDING
|
326
|
+
end
|
327
|
+
end
|
328
|
+
|
329
|
+
def to_h
|
330
|
+
{
|
331
|
+
:broker_states => broker_states,
|
332
|
+
}
|
333
|
+
end
|
334
|
+
end
|
335
|
+
end
|
336
|
+
end
|
data/lib/kafkat/command/drain.rb
CHANGED
File without changes
|
data/lib/kafkat/version.rb
CHANGED
@@ -0,0 +1,197 @@
|
|
1
|
+
require 'spec_helper'
|
2
|
+
|
3
|
+
module Kafkat
|
4
|
+
module ClusterRestart
|
5
|
+
|
6
|
+
describe Kafkat::ClusterRestart do
|
7
|
+
around(:all) do |example|
|
8
|
+
prev_home = ENV['HOME']
|
9
|
+
tmp = Dir.mktmpdir
|
10
|
+
ENV['HOME'] = tmp
|
11
|
+
begin
|
12
|
+
example.run
|
13
|
+
ensure
|
14
|
+
FileUtils.rm_rf tmp
|
15
|
+
ENV['HOME'] = prev_home
|
16
|
+
end
|
17
|
+
end
|
18
|
+
|
19
|
+
describe Session do
|
20
|
+
|
21
|
+
describe '#allBrokersRestarted?' do
|
22
|
+
context 'when some brokers have not been restarted' do
|
23
|
+
let (:session) {
|
24
|
+
Session.new('broker_states' => {'1' => Session::STATE_NOT_RESTARTED, '2' => Session::STATE_RESTARTED})
|
25
|
+
}
|
26
|
+
|
27
|
+
it do
|
28
|
+
expect(session.all_restarted?).to be false
|
29
|
+
end
|
30
|
+
end
|
31
|
+
|
32
|
+
context 'when all brokers have been restarted' do
|
33
|
+
let (:session) {
|
34
|
+
Session.new('broker_states' => {'1' => Session::STATE_RESTARTED, '2' => Session::STATE_RESTARTED})
|
35
|
+
}
|
36
|
+
|
37
|
+
it do
|
38
|
+
expect(session.all_restarted?).to be true
|
39
|
+
end
|
40
|
+
end
|
41
|
+
end
|
42
|
+
|
43
|
+
describe '#update_states!' do
|
44
|
+
let (:session) {
|
45
|
+
Session.new('broker_states' => {'1' => Session::STATE_NOT_RESTARTED, '2' => Session::STATE_RESTARTED})
|
46
|
+
}
|
47
|
+
|
48
|
+
it 'validates state' do
|
49
|
+
expect { session.update_states!('my_state', []) }.to raise_error UnknownStateError
|
50
|
+
end
|
51
|
+
|
52
|
+
it 'validates broker ids' do
|
53
|
+
expect { session.update_states!(Session::STATE_RESTARTED, ['1', '4']) }.to raise_error UnknownBrokerError
|
54
|
+
end
|
55
|
+
|
56
|
+
it 'changes the states' do
|
57
|
+
session.update_states!(Session::STATE_PENDING, ['1'])
|
58
|
+
expect(session.broker_states['1']).to eql(Session::STATE_PENDING)
|
59
|
+
end
|
60
|
+
end
|
61
|
+
|
62
|
+
end
|
63
|
+
|
64
|
+
|
65
|
+
describe Subcommands do
|
66
|
+
let(:p1) { Partition.new('topic1', 'p1', ['1', '2', '3'], '1', 1) }
|
67
|
+
let(:p2) { Partition.new('topic1', 'p2', ['1', '2', '3'], '2', 1) }
|
68
|
+
let(:p3) { Partition.new('topic1', 'p3', ['2', '3', '4'], '3', 1) }
|
69
|
+
let (:topics) {
|
70
|
+
{
|
71
|
+
'topic1' => Topic.new('topic1', [p1, p2, p3])
|
72
|
+
}
|
73
|
+
}
|
74
|
+
let(:zookeeper) { double('zookeeper') }
|
75
|
+
let(:broker_ids) { ['1', '2', '3', '4'] }
|
76
|
+
let(:broker_4) { Broker.new('4', 'i-xxxxxx.inst.aws.airbnb.com', 9092) }
|
77
|
+
let(:session) { Session.from_brokers(broker_ids) }
|
78
|
+
|
79
|
+
describe Subcommands::Next do
|
80
|
+
|
81
|
+
let(:next_command) { Subcommands::Next.new({}) }
|
82
|
+
|
83
|
+
it 'execute next with 4 brokers and 3 partitions' do
|
84
|
+
allow(zookeeper).to receive(:get_broker_ids).and_return(broker_ids)
|
85
|
+
allow(zookeeper).to receive(:get_broker).and_return(broker_4)
|
86
|
+
allow(zookeeper).to receive(:get_topics).and_return(topics)
|
87
|
+
allow(Session).to receive(:exists?).and_return(true)
|
88
|
+
allow(Session).to receive(:load!).and_return(session)
|
89
|
+
allow(session).to receive(:save!)
|
90
|
+
allow(next_command).to receive(:zookeeper).and_return(zookeeper)
|
91
|
+
|
92
|
+
expect(Session).to receive(:load!)
|
93
|
+
expect(session).to receive(:save!)
|
94
|
+
|
95
|
+
next_command.run
|
96
|
+
expect(next_command.session.broker_states['4']).to eq(Session::STATE_PENDING)
|
97
|
+
end
|
98
|
+
end
|
99
|
+
|
100
|
+
describe Subcommands::Good do
|
101
|
+
let(:good_command) { Subcommands::Good.new({}) }
|
102
|
+
|
103
|
+
let(:session){
|
104
|
+
Session.new('broker_states' => {'1' => Session::STATE_PENDING})
|
105
|
+
}
|
106
|
+
|
107
|
+
it 'set one broker to be restarted' do
|
108
|
+
allow(Session).to receive(:exists?).and_return(true)
|
109
|
+
allow(Session).to receive(:load!).and_return(session)
|
110
|
+
allow(session).to receive(:save!)
|
111
|
+
|
112
|
+
# expect(Session).to receive(:load!)
|
113
|
+
expect(session).to receive(:save!)
|
114
|
+
good_command.restart('1')
|
115
|
+
expect(good_command.session.broker_states['1']).to eq(Session::STATE_RESTARTED)
|
116
|
+
end
|
117
|
+
end
|
118
|
+
end
|
119
|
+
|
120
|
+
describe ClusterRestartHelper do
|
121
|
+
let(:p1) { Partition.new('topic1', 'p1', ['1', '2', '3'], '1', 1) }
|
122
|
+
let(:p2) { Partition.new('topic1', 'p2', ['1', '2', '3'], '2', 1) }
|
123
|
+
let(:p3) { Partition.new('topic1', 'p3', ['2', '3', '4'], '3', 1) }
|
124
|
+
let (:topics) {
|
125
|
+
{
|
126
|
+
'topic1' => Topic.new('topic1', [p1, p2, p3])
|
127
|
+
}
|
128
|
+
}
|
129
|
+
let(:zookeeper) { double('zookeeper') }
|
130
|
+
let(:broker_ids) { ['1', '2', '3', '4'] }
|
131
|
+
|
132
|
+
describe '#get_broker_to_leader_partition_mapping' do
|
133
|
+
it 'initialize a new mapping with 4 nodes' do
|
134
|
+
broker_to_partition = ClusterRestartHelper.get_broker_to_leader_partition_mapping(topics)
|
135
|
+
|
136
|
+
expect(broker_to_partition['1']).to eq([p1])
|
137
|
+
expect(broker_to_partition['2']).to eq([p2])
|
138
|
+
expect(broker_to_partition['3']).to eq([p3])
|
139
|
+
expect(broker_to_partition['4']).to eq([])
|
140
|
+
end
|
141
|
+
end
|
142
|
+
|
143
|
+
|
144
|
+
describe '#calculate_costs' do
|
145
|
+
context 'when no restarted brokers' do
|
146
|
+
it do
|
147
|
+
broker_to_partition = ClusterRestartHelper.get_broker_to_leader_partition_mapping(topics)
|
148
|
+
session = Session.from_brokers(broker_ids)
|
149
|
+
|
150
|
+
expect(ClusterRestartHelper.calculate_cost('1', broker_to_partition['1'], session)).to eq(3)
|
151
|
+
expect(ClusterRestartHelper.calculate_cost('2', broker_to_partition['2'], session)).to eq(3)
|
152
|
+
expect(ClusterRestartHelper.calculate_cost('3', broker_to_partition['3'], session)).to eq(3)
|
153
|
+
expect(ClusterRestartHelper.calculate_cost('4', broker_to_partition['4'], session)).to eq(0)
|
154
|
+
end
|
155
|
+
end
|
156
|
+
|
157
|
+
context 'when one broker has been restarted' do
|
158
|
+
it do
|
159
|
+
broker_to_partition = ClusterRestartHelper.get_broker_to_leader_partition_mapping(topics)
|
160
|
+
session = Session.from_brokers(broker_ids)
|
161
|
+
session.update_states!(Session::STATE_RESTARTED, ['4'])
|
162
|
+
|
163
|
+
expect(ClusterRestartHelper.calculate_cost('1', broker_to_partition['1'], session)).to eq(3)
|
164
|
+
expect(ClusterRestartHelper.calculate_cost('2', broker_to_partition['2'], session)).to eq(3)
|
165
|
+
expect(ClusterRestartHelper.calculate_cost('3', broker_to_partition['3'], session)).to eq(2)
|
166
|
+
expect(ClusterRestartHelper.calculate_cost('4', broker_to_partition['4'], session)).to eq(0)
|
167
|
+
end
|
168
|
+
end
|
169
|
+
end
|
170
|
+
|
171
|
+
describe '#select_broker_with_min_cost' do
|
172
|
+
context 'when no restarted brokers' do
|
173
|
+
it do
|
174
|
+
session = Session.from_brokers(broker_ids)
|
175
|
+
|
176
|
+
broker_id, cost = ClusterRestartHelper.select_broker_with_min_cost(session, topics)
|
177
|
+
expect(broker_id).to eq('4')
|
178
|
+
expect(cost).to eq(0)
|
179
|
+
end
|
180
|
+
end
|
181
|
+
|
182
|
+
context 'when next selection after one broker is restarted' do
|
183
|
+
it do
|
184
|
+
session = Session.from_brokers(broker_ids)
|
185
|
+
session.update_states!(Session::STATE_RESTARTED, ['4'])
|
186
|
+
|
187
|
+
broker_id, cost = ClusterRestartHelper.select_broker_with_min_cost(session, topics)
|
188
|
+
expect(broker_id).to eq('3')
|
189
|
+
expect(cost).to eq(2)
|
190
|
+
end
|
191
|
+
end
|
192
|
+
end
|
193
|
+
end
|
194
|
+
end
|
195
|
+
end
|
196
|
+
end
|
197
|
+
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: kafkat
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.2.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Nelson Gauthier
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2016-
|
11
|
+
date: 2016-10-19 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: zk
|
@@ -214,6 +214,7 @@ files:
|
|
214
214
|
- .rspec
|
215
215
|
- .simplecov
|
216
216
|
- .travis.yml
|
217
|
+
- CHANGELOG.md
|
217
218
|
- Gemfile
|
218
219
|
- LICENSE.txt
|
219
220
|
- README.md
|
@@ -231,6 +232,7 @@ files:
|
|
231
232
|
- lib/kafkat/command.rb
|
232
233
|
- lib/kafkat/command/brokers.rb
|
233
234
|
- lib/kafkat/command/clean-indexes.rb
|
235
|
+
- lib/kafkat/command/cluster_restart.rb
|
234
236
|
- lib/kafkat/command/controller.rb
|
235
237
|
- lib/kafkat/command/drain.rb
|
236
238
|
- lib/kafkat/command/elect-leaders.rb
|
@@ -245,11 +247,13 @@ files:
|
|
245
247
|
- lib/kafkat/interface/admin.rb
|
246
248
|
- lib/kafkat/interface/kafka_logs.rb
|
247
249
|
- lib/kafkat/interface/zookeeper.rb
|
250
|
+
- lib/kafkat/reboot.rb
|
248
251
|
- lib/kafkat/utility.rb
|
249
252
|
- lib/kafkat/utility/command_io.rb
|
250
253
|
- lib/kafkat/utility/formatting.rb
|
251
254
|
- lib/kafkat/version.rb
|
252
255
|
- spec/factories/topic.rb
|
256
|
+
- spec/lib/kafkat/command/cluster_restart_spec.rb
|
253
257
|
- spec/lib/kafkat/command/drain_spec.rb
|
254
258
|
- spec/spec_helper.rb
|
255
259
|
homepage: https://github.com/airbnb/kafkat
|
@@ -278,5 +282,6 @@ specification_version: 4
|
|
278
282
|
summary: Simplified command-line administration for Kafka brokers
|
279
283
|
test_files:
|
280
284
|
- spec/factories/topic.rb
|
285
|
+
- spec/lib/kafkat/command/cluster_restart_spec.rb
|
281
286
|
- spec/lib/kafkat/command/drain_spec.rb
|
282
287
|
- spec/spec_helper.rb
|