kafkat-onfocusio 0.3.1 → 0.3.2
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/CHANGELOG.md +4 -0
- data/lib/kafkat/command/reassign.rb +251 -11
- data/lib/kafkat/version.rb +1 -1
- metadata +3 -3
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: ad6b0512c4ca70f6fb22929bf9495dd1f15805a8
|
4
|
+
data.tar.gz: 3e85c394734b9603ed4137d4622a04e2fc25f69c
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: e78be4c2e45692cd6b4229c331a58ddf05a1d003b96141af7a61b3b1497aa726e4195dad4c437d0f9cbef22abb8c4a4c6e87fd83ae90a9dfc99f7c3e1e10100b
|
7
|
+
data.tar.gz: ac28a94e85505ff5ddf8439bad4ef2a5d86acc926590462381237ebcb5db99999ec8953242a4d8f321905cf5d9420b72d4e85f4e7cad1e7fcc8c7beec53f4435
|
data/CHANGELOG.md
CHANGED
@@ -1,3 +1,26 @@
|
|
1
|
+
class Hash
|
2
|
+
def hmap!(&block)
|
3
|
+
self.keys.each do |key|
|
4
|
+
hash = block.call(key, self[key])
|
5
|
+
|
6
|
+
self[hash.keys.first] = hash[hash.keys.first]
|
7
|
+
self.delete(key)
|
8
|
+
end
|
9
|
+
self
|
10
|
+
end
|
11
|
+
|
12
|
+
def hmap(&block)
|
13
|
+
h = Hash.new
|
14
|
+
self.keys.each do |key|
|
15
|
+
hash = block.call(key, self[key])
|
16
|
+
|
17
|
+
h[hash.keys.first] = hash[hash.keys.first]
|
18
|
+
end
|
19
|
+
h
|
20
|
+
end
|
21
|
+
end
|
22
|
+
|
23
|
+
|
1
24
|
module Kafkat
|
2
25
|
module Command
|
3
26
|
class Reassign < Base
|
@@ -41,6 +64,13 @@ module Kafkat
|
|
41
64
|
assignments = []
|
42
65
|
broker_count = broker_ids.size
|
43
66
|
|
67
|
+
# Helper structures
|
68
|
+
@replica_count = replica_count
|
69
|
+
@nodes_lists_replicas = {}
|
70
|
+
@nodes_lists_isr = {}
|
71
|
+
@partitions_lists={}
|
72
|
+
@partitions_lists_old={}
|
73
|
+
|
44
74
|
topics.each do |_, t|
|
45
75
|
# This is how Kafka's AdminUtils determines these values.
|
46
76
|
partition_count = t.partitions.size
|
@@ -51,26 +81,236 @@ module Kafkat
|
|
51
81
|
exit 1
|
52
82
|
end
|
53
83
|
|
54
|
-
start_index = Random.rand(broker_count)
|
55
|
-
replica_shift = Random.rand(broker_count)
|
84
|
+
# start_index = Random.rand(broker_count)
|
85
|
+
# replica_shift = Random.rand(broker_count)
|
86
|
+
|
87
|
+
# t.partitions.each do |p|
|
88
|
+
# replica_shift += 1 if p.id > 0 && p.id % broker_count == 0
|
89
|
+
# first_replica_index = (p.id + start_index) % broker_count
|
90
|
+
|
91
|
+
# replicas = [broker_ids[first_replica_index]]
|
92
|
+
|
93
|
+
# (0...topic_replica_count-1).each do |i|
|
94
|
+
# shift = 1 + (replica_shift + i) % (broker_count - 1)
|
95
|
+
# index = (first_replica_index + shift) % broker_count
|
96
|
+
# replicas << broker_ids[index]
|
97
|
+
# end
|
98
|
+
|
99
|
+
# replicas.reverse!
|
100
|
+
# assignments << Assignment.new(t.name, p.id, replicas)
|
101
|
+
# end
|
56
102
|
|
57
103
|
t.partitions.each do |p|
|
58
|
-
|
59
|
-
|
104
|
+
@partitions_lists[p.topic_name + ' ' + p.id.to_s] ||= {'name'=>p.topic_name, 'id'=>p.id, 'replicas'=>{},'isr'=>{}}
|
105
|
+
@partitions_lists_old[p.topic_name + ' ' + p.id.to_s] ||= {'name'=>p.topic_name, 'id'=>p.id, 'replicas'=>{},'isr'=>{}}
|
60
106
|
|
61
|
-
replicas
|
107
|
+
p.replicas.each do |node|
|
108
|
+
@partitions_lists[p.topic_name + ' ' + p.id.to_s]['replicas'][node] = ''
|
109
|
+
@partitions_lists_old[p.topic_name + ' ' + p.id.to_s]['replicas'][node] = ''
|
110
|
+
@nodes_lists_replicas[node] ||= {}
|
111
|
+
@nodes_lists_replicas[node][p.topic_name + ' ' + p.id.to_s] = ''
|
112
|
+
end
|
113
|
+
|
114
|
+
p.isr.each do |node|
|
115
|
+
@partitions_lists[p.topic_name + ' ' + p.id.to_s]['isr'][node] = ''
|
116
|
+
@partitions_lists_old[p.topic_name + ' ' + p.id.to_s]['isr'][node] = ''
|
62
117
|
|
63
|
-
|
64
|
-
|
65
|
-
|
66
|
-
|
118
|
+
# nodes_sizes_isr[node] ||= 0
|
119
|
+
# nodes_sizes_isr[node] += 1
|
120
|
+
@nodes_lists_isr[node] ||= {}
|
121
|
+
@nodes_lists_isr[node][p.topic_name + ' ' + p.id.to_s] = ''
|
67
122
|
end
|
123
|
+
end
|
124
|
+
end
|
125
|
+
|
126
|
+
|
127
|
+
# Add a topic partition to a specific node
|
128
|
+
def add_partition_to_node( tp, node )
|
129
|
+
|
130
|
+
@nodes_lists_replicas[node][tp] = ''
|
131
|
+
@partitions_lists[tp]['replicas'] ||= {}
|
132
|
+
@partitions_lists[tp]['replicas'][node] = ''
|
133
|
+
end
|
68
134
|
|
69
|
-
|
70
|
-
|
135
|
+
|
136
|
+
# Add a topic partition to lowest unbalanced nodes
|
137
|
+
# you can specify the number of times the partition must be added to fit its minimal replica count
|
138
|
+
def add_partition_to_all_lowest( tp, nb_partitions_to_add )
|
139
|
+
|
140
|
+
nodes_sizes_replicas = @nodes_lists_replicas.hmap { |k,v| { k => v.size } }
|
141
|
+
nodes_lowest = nodes_sizes_replicas.sort_by{|k,v| v}.map { |a| a[0] }
|
142
|
+
|
143
|
+
nodes_lowest.each do |node|
|
144
|
+
break if nb_partitions_to_add <= 0
|
145
|
+
|
146
|
+
unless @nodes_lists_replicas[node].has_key?(tp)
|
147
|
+
add_partition_to_node( tp, node )
|
148
|
+
nb_partitions_to_add -= 1
|
149
|
+
end
|
71
150
|
end
|
72
151
|
end
|
73
152
|
|
153
|
+
|
154
|
+
def remove_partition_from_node( tp, node )
|
155
|
+
|
156
|
+
@nodes_lists_replicas[node].delete tp
|
157
|
+
@nodes_lists_isr[node].delete tp
|
158
|
+
@partitions_lists[tp]['replicas'].delete(node)
|
159
|
+
@partitions_lists[tp]['isr'].delete(node)
|
160
|
+
end
|
161
|
+
|
162
|
+
|
163
|
+
# Remove excess replicas from node
|
164
|
+
# If nb_partitions_to_remove is nil, then remove all excess replicas
|
165
|
+
# First remove partitions which are not in ISR and already replicated enough times on other nodes of the cluster
|
166
|
+
# Then regardless of ISR
|
167
|
+
def remove_excess_replicas_from_node( node, nb_partitions_to_remove )
|
168
|
+
|
169
|
+
# First remove partitions which are not in ISR and already replicated enough times on other nodes of the cluster
|
170
|
+
|
171
|
+
replicas_not_isr = @nodes_lists_replicas[node].keys - @nodes_lists_isr[node].keys
|
172
|
+
|
173
|
+
replicas_not_isr_over_replicated = replicas_not_isr.keep_if { |tp| @partitions_lists[tp]['replicas'].keys.size > @replica_count }
|
174
|
+
|
175
|
+
replicas_not_isr_over_replicated.each do |tp|
|
176
|
+
break if (!nb_partitions_to_remove.nil? && nb_partitions_to_remove <= 0)
|
177
|
+
|
178
|
+
remove_partition_from_node( tp, node )
|
179
|
+
|
180
|
+
nb_partitions_to_remove -= 1 unless nb_partitions_to_remove.nil?
|
181
|
+
end
|
182
|
+
|
183
|
+
|
184
|
+
# Then remove partitions which regardless of ISR and already replicated enough times on other nodes of the cluster
|
185
|
+
|
186
|
+
replicas_over_replicated = @nodes_lists_replicas[node].keys.keep_if { |tp| @partitions_lists[tp]['replicas'].keys.size > @replica_count }
|
187
|
+
|
188
|
+
replicas_over_replicated.each do |tp|
|
189
|
+
break if (!nb_partitions_to_remove.nil? && nb_partitions_to_remove <= 0)
|
190
|
+
|
191
|
+
remove_partition_from_node( tp, node )
|
192
|
+
|
193
|
+
nb_partitions_to_remove -= 1 unless nb_partitions_to_remove.nil?
|
194
|
+
end
|
195
|
+
|
196
|
+
end
|
197
|
+
|
198
|
+
|
199
|
+
# Migrate excess replicas from node to lowest unbalanced nodes
|
200
|
+
# It chooses absent replicas from lowest unbalanced nodes, to make sure we fill the lowest
|
201
|
+
# BUG: There is very probably an issue here in case the lowest node has less than `nb_partitions_to_remove` absent partitions from source node
|
202
|
+
def migrate_excess_replicas_from_node( node, nb_partitions_to_remove )
|
203
|
+
|
204
|
+
# This method is not efficient, but that is not something you run often
|
205
|
+
nb_partitions_to_remove.times do |i|
|
206
|
+
|
207
|
+
# Find out the lowest balanced nodes
|
208
|
+
nodes_sizes_replicas = @nodes_lists_replicas.hmap { |k,v| { k => v.size } }
|
209
|
+
node_lowest = nodes_sizes_replicas.sort_by{|k,v| v}[0][0]
|
210
|
+
|
211
|
+
# Find out which partitions the lowest balanced node does not have and which can be migrated from this node
|
212
|
+
partition_to_migrate = (@nodes_lists_replicas[node].keys - @nodes_lists_replicas[node_lowest].keys)[0]
|
213
|
+
|
214
|
+
remove_partition_from_node( partition_to_migrate, node )
|
215
|
+
add_partition_to_node( partition_to_migrate, node_lowest )
|
216
|
+
|
217
|
+
end
|
218
|
+
|
219
|
+
# # Find out the lowest balanced nodes
|
220
|
+
# nodes_sizes_replicas = @nodes_lists_replicas.hmap { |k,v| { k => v.size } }
|
221
|
+
# nodes_lowest = nodes_sizes_replicas.sort_by{|k,v| v}.map { |a| a[0] }
|
222
|
+
|
223
|
+
# nodes_lowest.each do |node_lowest|
|
224
|
+
# break if nb_partitions_to_remove <= 0
|
225
|
+
|
226
|
+
# # Find out which partitions the lowest balanced node does not have and which can be migrated from this node
|
227
|
+
# partitions_to_migrate = @nodes_lists_replicas[node].keys - @nodes_lists_replicas[node_lowest].keys
|
228
|
+
|
229
|
+
# partitions_to_migrate.each do |tp|
|
230
|
+
# break if nb_partitions_to_remove <= 0
|
231
|
+
|
232
|
+
# remove_partition_from_node( tp, node )
|
233
|
+
# add_partition_to_node( tp, node_lowest )
|
234
|
+
|
235
|
+
# nb_partitions_to_remove -= 1
|
236
|
+
# end
|
237
|
+
# end
|
238
|
+
end
|
239
|
+
|
240
|
+
|
241
|
+
###
|
242
|
+
# At this point, the helper structures represent the current repartition of partitions
|
243
|
+
# We will modify the helper structures to balance the partitions accross brokers whith the minimum number of partitions moves
|
244
|
+
###
|
245
|
+
|
246
|
+
# Add under replicated partitions to lowest unbalanced nodes
|
247
|
+
partitions_under_replicated = @partitions_lists.keys.keep_if { |tp| @partitions_lists[tp]['replicas'].size < @replica_count }
|
248
|
+
partitions_under_replicated.each do |tp|
|
249
|
+
how_many = @replica_count - @partitions_lists[tp]['replicas'].size
|
250
|
+
add_x_to_all_lowest( tp, how_many )
|
251
|
+
end
|
252
|
+
|
253
|
+
|
254
|
+
# Begin leaning and draining nodes with the most partitions towards the lowest unbalanced nodes
|
255
|
+
2.times do |i|
|
256
|
+
|
257
|
+
# List all nodes from biggest to lowest balanced
|
258
|
+
nodes_sizes_replicas = @nodes_lists_replicas.hmap { |k,v| { k => v.size } }
|
259
|
+
nodes_biggest = nodes_sizes_replicas.sort_by{|k,v| v}.reverse.map { |a| a[0] }
|
260
|
+
|
261
|
+
# helper count of partition replicas to balance
|
262
|
+
nodes_left_to_balance = nodes_biggest.size
|
263
|
+
nb_partitions_left_to_balance = @replica_count * @partitions_lists.keys.size
|
264
|
+
|
265
|
+
# Remove excess replicas on all nodes from biggest to lowest unbalanced nodes, until balance is achieved
|
266
|
+
nodes_biggest.each do |node|
|
267
|
+
|
268
|
+
# Figure out the numbers of partitions each node should have to be balanced
|
269
|
+
# I am wary about float operations, for example to do a 5.00000001.ceil()
|
270
|
+
# nb_partitions_should_have = ( nb_partitions_left_to_balance / nodes_left_to_balance.to_f ).ceil
|
271
|
+
nb_partitions_should_have = nb_partitions_left_to_balance / nodes_left_to_balance
|
272
|
+
nb_partitions_should_have += 1 if (nb_partitions_left_to_balance % nodes_left_to_balance) > 0
|
273
|
+
|
274
|
+
|
275
|
+
# nb_partitions_to_remove = @nodes_lists_replicas[node].keys.size - nb_partitions_should_have.ceil
|
276
|
+
nb_partitions_to_remove = @nodes_lists_replicas[node].keys.size - nb_partitions_should_have
|
277
|
+
remove_excess_replicas_from_node( node, nb_partitions_to_remove) if nb_partitions_to_remove > 0
|
278
|
+
|
279
|
+
# nb_partitions_to_remove = @nodes_lists_replicas[node].keys.size - nb_partitions_should_have.ceil
|
280
|
+
nb_partitions_to_remove = @nodes_lists_replicas[node].keys.size - nb_partitions_should_have
|
281
|
+
migrate_excess_replicas_from_node( node, nb_partitions_to_remove) if nb_partitions_to_remove > 0
|
282
|
+
|
283
|
+
|
284
|
+
nodes_left_to_balance -= 1
|
285
|
+
nb_partitions_left_to_balance = nb_partitions_left_to_balance - nb_partitions_should_have
|
286
|
+
|
287
|
+
end
|
288
|
+
end
|
289
|
+
|
290
|
+
# Remove excess replicas on all nodes from lowest unbalanced nodes to biggest nodes. This is to enforce replica count, and if the whole algorithm is fine, it should do nothing
|
291
|
+
nodes_sizes_replicas = @nodes_lists_replicas.hmap { |k,v| { k => v.size } }
|
292
|
+
nodes_biggest = nodes_sizes_replicas.sort_by{|k,v| v}.reverse.map { |a| a[0] }
|
293
|
+
nodes_biggest.each do |node|
|
294
|
+
remove_excess_replicas_from_node( node, nil )
|
295
|
+
end
|
296
|
+
|
297
|
+
|
298
|
+
require 'pp'
|
299
|
+
puts "\nCurrent partition assignment:"
|
300
|
+
pp @partitions_lists_old.hmap { |k,v| { k => { 'replicas'=>v['replicas'], 'isr'=>v['isr']} } }
|
301
|
+
puts "\nOptimized partition assignment with minimal partition migrations:"
|
302
|
+
pp @partitions_lists.hmap { |k,v| { k => { 'replicas'=>v['replicas'], 'isr'=>v['isr']} } }
|
303
|
+
puts ''
|
304
|
+
|
305
|
+
|
306
|
+
@partitions_lists.each_key do |tp|
|
307
|
+
assignments << Assignment.new( @partitions_lists[tp]['name'],
|
308
|
+
@partitions_lists[tp]['id'],
|
309
|
+
@partitions_lists[tp]['replicas'].keys,
|
310
|
+
)
|
311
|
+
end
|
312
|
+
|
313
|
+
|
74
314
|
# ****************
|
75
315
|
|
76
316
|
prompt_and_execute_assignments(assignments)
|
data/lib/kafkat/version.rb
CHANGED
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: kafkat-onfocusio
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.3.
|
4
|
+
version: 0.3.2
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Nelson Gauthier
|
@@ -9,7 +9,7 @@ authors:
|
|
9
9
|
autorequire:
|
10
10
|
bindir: bin
|
11
11
|
cert_chain: []
|
12
|
-
date: 2017-
|
12
|
+
date: 2017-05-05 00:00:00.000000000 Z
|
13
13
|
dependencies:
|
14
14
|
- !ruby/object:Gem::Dependency
|
15
15
|
name: zk
|
@@ -282,7 +282,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
282
282
|
version: '0'
|
283
283
|
requirements: []
|
284
284
|
rubyforge_project:
|
285
|
-
rubygems_version: 2.
|
285
|
+
rubygems_version: 2.6.8
|
286
286
|
signing_key:
|
287
287
|
specification_version: 4
|
288
288
|
summary: Simplified command-line administration for Kafka brokers
|