kafkat-onfocusio 0.3.1 → 0.3.2

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: 88dab12e61dc76dfa68e5859ba87e49ad8877bf0
4
- data.tar.gz: 90a4a92de87795bae39af863fde0cff39e471d3f
3
+ metadata.gz: ad6b0512c4ca70f6fb22929bf9495dd1f15805a8
4
+ data.tar.gz: 3e85c394734b9603ed4137d4622a04e2fc25f69c
5
5
  SHA512:
6
- metadata.gz: 12b6343d27c4669335a0fb4ddb02b283757b1293e02e091e9a3b77f0c3516feae2ccfb088c788899cd5d6cde2db2f08e531a1f705591bce8fe70450f1935ba96
7
- data.tar.gz: f4d95781a0d137c93543b2a49e074331b9334781055cecd80288cf70d31be333e1e315db5b0e21f71da753b51ecec5a720d5a265499f9be7ae19a88cf3543637
6
+ metadata.gz: e78be4c2e45692cd6b4229c331a58ddf05a1d003b96141af7a61b3b1497aa726e4195dad4c437d0f9cbef22abb8c4a4c6e87fd83ae90a9dfc99f7c3e1e10100b
7
+ data.tar.gz: ac28a94e85505ff5ddf8439bad4ef2a5d86acc926590462381237ebcb5db99999ec8953242a4d8f321905cf5d9420b72d4e85f4e7cad1e7fcc8c7beec53f4435
data/CHANGELOG.md CHANGED
@@ -1,5 +1,9 @@
1
1
  # CHANGE LOG
2
2
 
3
+ ## 0.3.2
4
+
5
+ - [enhancement] updated the `reassign` command to rebalance partitions accross brokers whith the minimum number of partition moves
6
+
3
7
  ## 0.3.1
4
8
 
5
9
  - [feature] add `verify-reassign` command
@@ -1,3 +1,26 @@
1
+ class Hash
2
+ def hmap!(&block)
3
+ self.keys.each do |key|
4
+ hash = block.call(key, self[key])
5
+
6
+ self[hash.keys.first] = hash[hash.keys.first]
7
+ self.delete(key)
8
+ end
9
+ self
10
+ end
11
+
12
+ def hmap(&block)
13
+ h = Hash.new
14
+ self.keys.each do |key|
15
+ hash = block.call(key, self[key])
16
+
17
+ h[hash.keys.first] = hash[hash.keys.first]
18
+ end
19
+ h
20
+ end
21
+ end
22
+
23
+
1
24
  module Kafkat
2
25
  module Command
3
26
  class Reassign < Base
@@ -41,6 +64,13 @@ module Kafkat
41
64
  assignments = []
42
65
  broker_count = broker_ids.size
43
66
 
67
+ # Helper structures
68
+ @replica_count = replica_count
69
+ @nodes_lists_replicas = {}
70
+ @nodes_lists_isr = {}
71
+ @partitions_lists={}
72
+ @partitions_lists_old={}
73
+
44
74
  topics.each do |_, t|
45
75
  # This is how Kafka's AdminUtils determines these values.
46
76
  partition_count = t.partitions.size
@@ -51,26 +81,236 @@ module Kafkat
51
81
  exit 1
52
82
  end
53
83
 
54
- start_index = Random.rand(broker_count)
55
- replica_shift = Random.rand(broker_count)
84
+ # start_index = Random.rand(broker_count)
85
+ # replica_shift = Random.rand(broker_count)
86
+
87
+ # t.partitions.each do |p|
88
+ # replica_shift += 1 if p.id > 0 && p.id % broker_count == 0
89
+ # first_replica_index = (p.id + start_index) % broker_count
90
+
91
+ # replicas = [broker_ids[first_replica_index]]
92
+
93
+ # (0...topic_replica_count-1).each do |i|
94
+ # shift = 1 + (replica_shift + i) % (broker_count - 1)
95
+ # index = (first_replica_index + shift) % broker_count
96
+ # replicas << broker_ids[index]
97
+ # end
98
+
99
+ # replicas.reverse!
100
+ # assignments << Assignment.new(t.name, p.id, replicas)
101
+ # end
56
102
 
57
103
  t.partitions.each do |p|
58
- replica_shift += 1 if p.id > 0 && p.id % broker_count == 0
59
- first_replica_index = (p.id + start_index) % broker_count
104
+ @partitions_lists[p.topic_name + ' ' + p.id.to_s] ||= {'name'=>p.topic_name, 'id'=>p.id, 'replicas'=>{},'isr'=>{}}
105
+ @partitions_lists_old[p.topic_name + ' ' + p.id.to_s] ||= {'name'=>p.topic_name, 'id'=>p.id, 'replicas'=>{},'isr'=>{}}
60
106
 
61
- replicas = [broker_ids[first_replica_index]]
107
+ p.replicas.each do |node|
108
+ @partitions_lists[p.topic_name + ' ' + p.id.to_s]['replicas'][node] = ''
109
+ @partitions_lists_old[p.topic_name + ' ' + p.id.to_s]['replicas'][node] = ''
110
+ @nodes_lists_replicas[node] ||= {}
111
+ @nodes_lists_replicas[node][p.topic_name + ' ' + p.id.to_s] = ''
112
+ end
113
+
114
+ p.isr.each do |node|
115
+ @partitions_lists[p.topic_name + ' ' + p.id.to_s]['isr'][node] = ''
116
+ @partitions_lists_old[p.topic_name + ' ' + p.id.to_s]['isr'][node] = ''
62
117
 
63
- (0...topic_replica_count-1).each do |i|
64
- shift = 1 + (replica_shift + i) % (broker_count - 1)
65
- index = (first_replica_index + shift) % broker_count
66
- replicas << broker_ids[index]
118
+ # nodes_sizes_isr[node] ||= 0
119
+ # nodes_sizes_isr[node] += 1
120
+ @nodes_lists_isr[node] ||= {}
121
+ @nodes_lists_isr[node][p.topic_name + ' ' + p.id.to_s] = ''
67
122
  end
123
+ end
124
+ end
125
+
126
+
127
+ # Add a topic partition to a specific node
128
+ def add_partition_to_node( tp, node )
129
+
130
+ @nodes_lists_replicas[node][tp] = ''
131
+ @partitions_lists[tp]['replicas'] ||= {}
132
+ @partitions_lists[tp]['replicas'][node] = ''
133
+ end
68
134
 
69
- replicas.reverse!
70
- assignments << Assignment.new(t.name, p.id, replicas)
135
+
136
+ # Add a topic partition to lowest unbalanced nodes
137
+ # you can specify the number of times the partition must be added to fit its minimal replica count
138
+ def add_partition_to_all_lowest( tp, nb_partitions_to_add )
139
+
140
+ nodes_sizes_replicas = @nodes_lists_replicas.hmap { |k,v| { k => v.size } }
141
+ nodes_lowest = nodes_sizes_replicas.sort_by{|k,v| v}.map { |a| a[0] }
142
+
143
+ nodes_lowest.each do |node|
144
+ break if nb_partitions_to_add <= 0
145
+
146
+ unless @nodes_lists_replicas[node].has_key?(tp)
147
+ add_partition_to_node( tp, node )
148
+ nb_partitions_to_add -= 1
149
+ end
71
150
  end
72
151
  end
73
152
 
153
+
154
+ def remove_partition_from_node( tp, node )
155
+
156
+ @nodes_lists_replicas[node].delete tp
157
+ @nodes_lists_isr[node].delete tp
158
+ @partitions_lists[tp]['replicas'].delete(node)
159
+ @partitions_lists[tp]['isr'].delete(node)
160
+ end
161
+
162
+
163
+ # Remove excess replicas from node
164
+ # If nb_partitions_to_remove is nil, then remove all excess replicas
165
+ # First remove partitions which are not in ISR and already replicated enough times on other nodes of the cluster
166
+ # Then regardless of ISR
167
+ def remove_excess_replicas_from_node( node, nb_partitions_to_remove )
168
+
169
+ # First remove partitions which are not in ISR and already replicated enough times on other nodes of the cluster
170
+
171
+ replicas_not_isr = @nodes_lists_replicas[node].keys - @nodes_lists_isr[node].keys
172
+
173
+ replicas_not_isr_over_replicated = replicas_not_isr.keep_if { |tp| @partitions_lists[tp]['replicas'].keys.size > @replica_count }
174
+
175
+ replicas_not_isr_over_replicated.each do |tp|
176
+ break if (!nb_partitions_to_remove.nil? && nb_partitions_to_remove <= 0)
177
+
178
+ remove_partition_from_node( tp, node )
179
+
180
+ nb_partitions_to_remove -= 1 unless nb_partitions_to_remove.nil?
181
+ end
182
+
183
+
184
+ # Then remove partitions which regardless of ISR and already replicated enough times on other nodes of the cluster
185
+
186
+ replicas_over_replicated = @nodes_lists_replicas[node].keys.keep_if { |tp| @partitions_lists[tp]['replicas'].keys.size > @replica_count }
187
+
188
+ replicas_over_replicated.each do |tp|
189
+ break if (!nb_partitions_to_remove.nil? && nb_partitions_to_remove <= 0)
190
+
191
+ remove_partition_from_node( tp, node )
192
+
193
+ nb_partitions_to_remove -= 1 unless nb_partitions_to_remove.nil?
194
+ end
195
+
196
+ end
197
+
198
+
199
+ # Migrate excess replicas from node to lowest unbalanced nodes
200
+ # It chooses absent replicas from lowest unbalanced nodes, to make sure we fill the lowest
201
+ # BUG: There is very probably an issue here in case the lowest node has less than `nb_partitions_to_remove` absent partitions from source node
202
+ def migrate_excess_replicas_from_node( node, nb_partitions_to_remove )
203
+
204
+ # This method is not efficient, but that is not something you run often
205
+ nb_partitions_to_remove.times do |i|
206
+
207
+ # Find out the lowest balanced nodes
208
+ nodes_sizes_replicas = @nodes_lists_replicas.hmap { |k,v| { k => v.size } }
209
+ node_lowest = nodes_sizes_replicas.sort_by{|k,v| v}[0][0]
210
+
211
+ # Find out which partitions the lowest balanced node does not have and which can be migrated from this node
212
+ partition_to_migrate = (@nodes_lists_replicas[node].keys - @nodes_lists_replicas[node_lowest].keys)[0]
213
+
214
+ remove_partition_from_node( partition_to_migrate, node )
215
+ add_partition_to_node( partition_to_migrate, node_lowest )
216
+
217
+ end
218
+
219
+ # # Find out the lowest balanced nodes
220
+ # nodes_sizes_replicas = @nodes_lists_replicas.hmap { |k,v| { k => v.size } }
221
+ # nodes_lowest = nodes_sizes_replicas.sort_by{|k,v| v}.map { |a| a[0] }
222
+
223
+ # nodes_lowest.each do |node_lowest|
224
+ # break if nb_partitions_to_remove <= 0
225
+
226
+ # # Find out which partitions the lowest balanced node does not have and which can be migrated from this node
227
+ # partitions_to_migrate = @nodes_lists_replicas[node].keys - @nodes_lists_replicas[node_lowest].keys
228
+
229
+ # partitions_to_migrate.each do |tp|
230
+ # break if nb_partitions_to_remove <= 0
231
+
232
+ # remove_partition_from_node( tp, node )
233
+ # add_partition_to_node( tp, node_lowest )
234
+
235
+ # nb_partitions_to_remove -= 1
236
+ # end
237
+ # end
238
+ end
239
+
240
+
241
+ ###
242
+ # At this point, the helper structures represent the current repartition of partitions
243
+ # We will modify the helper structures to balance the partitions accross brokers whith the minimum number of partitions moves
244
+ ###
245
+
246
+ # Add under replicated partitions to lowest unbalanced nodes
247
+ partitions_under_replicated = @partitions_lists.keys.keep_if { |tp| @partitions_lists[tp]['replicas'].size < @replica_count }
248
+ partitions_under_replicated.each do |tp|
249
+ how_many = @replica_count - @partitions_lists[tp]['replicas'].size
250
+ add_x_to_all_lowest( tp, how_many )
251
+ end
252
+
253
+
254
+ # Begin leaning and draining nodes with the most partitions towards the lowest unbalanced nodes
255
+ 2.times do |i|
256
+
257
+ # List all nodes from biggest to lowest balanced
258
+ nodes_sizes_replicas = @nodes_lists_replicas.hmap { |k,v| { k => v.size } }
259
+ nodes_biggest = nodes_sizes_replicas.sort_by{|k,v| v}.reverse.map { |a| a[0] }
260
+
261
+ # helper count of partition replicas to balance
262
+ nodes_left_to_balance = nodes_biggest.size
263
+ nb_partitions_left_to_balance = @replica_count * @partitions_lists.keys.size
264
+
265
+ # Remove excess replicas on all nodes from biggest to lowest unbalanced nodes, until balance is achieved
266
+ nodes_biggest.each do |node|
267
+
268
+ # Figure out the numbers of partitions each node should have to be balanced
269
+ # I am wary about float operations, for example to do a 5.00000001.ceil()
270
+ # nb_partitions_should_have = ( nb_partitions_left_to_balance / nodes_left_to_balance.to_f ).ceil
271
+ nb_partitions_should_have = nb_partitions_left_to_balance / nodes_left_to_balance
272
+ nb_partitions_should_have += 1 if (nb_partitions_left_to_balance % nodes_left_to_balance) > 0
273
+
274
+
275
+ # nb_partitions_to_remove = @nodes_lists_replicas[node].keys.size - nb_partitions_should_have.ceil
276
+ nb_partitions_to_remove = @nodes_lists_replicas[node].keys.size - nb_partitions_should_have
277
+ remove_excess_replicas_from_node( node, nb_partitions_to_remove) if nb_partitions_to_remove > 0
278
+
279
+ # nb_partitions_to_remove = @nodes_lists_replicas[node].keys.size - nb_partitions_should_have.ceil
280
+ nb_partitions_to_remove = @nodes_lists_replicas[node].keys.size - nb_partitions_should_have
281
+ migrate_excess_replicas_from_node( node, nb_partitions_to_remove) if nb_partitions_to_remove > 0
282
+
283
+
284
+ nodes_left_to_balance -= 1
285
+ nb_partitions_left_to_balance = nb_partitions_left_to_balance - nb_partitions_should_have
286
+
287
+ end
288
+ end
289
+
290
+ # Remove excess replicas on all nodes from lowest unbalanced nodes to biggest nodes. This is to enforce replica count, and if the whole algorithm is fine, it should do nothing
291
+ nodes_sizes_replicas = @nodes_lists_replicas.hmap { |k,v| { k => v.size } }
292
+ nodes_biggest = nodes_sizes_replicas.sort_by{|k,v| v}.reverse.map { |a| a[0] }
293
+ nodes_biggest.each do |node|
294
+ remove_excess_replicas_from_node( node, nil )
295
+ end
296
+
297
+
298
+ require 'pp'
299
+ puts "\nCurrent partition assignment:"
300
+ pp @partitions_lists_old.hmap { |k,v| { k => { 'replicas'=>v['replicas'], 'isr'=>v['isr']} } }
301
+ puts "\nOptimized partition assignment with minimal partition migrations:"
302
+ pp @partitions_lists.hmap { |k,v| { k => { 'replicas'=>v['replicas'], 'isr'=>v['isr']} } }
303
+ puts ''
304
+
305
+
306
+ @partitions_lists.each_key do |tp|
307
+ assignments << Assignment.new( @partitions_lists[tp]['name'],
308
+ @partitions_lists[tp]['id'],
309
+ @partitions_lists[tp]['replicas'].keys,
310
+ )
311
+ end
312
+
313
+
74
314
  # ****************
75
315
 
76
316
  prompt_and_execute_assignments(assignments)
@@ -1,3 +1,3 @@
1
1
  module Kafkat
2
- VERSION = '0.3.1'
2
+ VERSION = '0.3.2'
3
3
  end
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: kafkat-onfocusio
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.3.1
4
+ version: 0.3.2
5
5
  platform: ruby
6
6
  authors:
7
7
  - Nelson Gauthier
@@ -9,7 +9,7 @@ authors:
9
9
  autorequire:
10
10
  bindir: bin
11
11
  cert_chain: []
12
- date: 2017-03-14 00:00:00.000000000 Z
12
+ date: 2017-05-05 00:00:00.000000000 Z
13
13
  dependencies:
14
14
  - !ruby/object:Gem::Dependency
15
15
  name: zk
@@ -282,7 +282,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
282
282
  version: '0'
283
283
  requirements: []
284
284
  rubyforge_project:
285
- rubygems_version: 2.2.2
285
+ rubygems_version: 2.6.8
286
286
  signing_key:
287
287
  specification_version: 4
288
288
  summary: Simplified command-line administration for Kafka brokers