kafkat 0.0.9

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,89 @@
1
+ module Kafkat
2
+ module Command
3
+ class Reassign < Base
4
+ register_as 'reassign'
5
+
6
+ usage 'reassign [topic] [--brokers <ids>] [--replicas <n>]',
7
+ 'Begin reassignment of partitions.'
8
+
9
+ def run
10
+ topic_name = ARGV.shift unless ARGV[0] && ARGV[0].start_with?('--')
11
+
12
+ all_brokers = zookeeper.get_brokers
13
+ topics = topic_name && zookeeper.get_topics([topic_name])
14
+ topics ||= zookeeper.get_topics
15
+
16
+ opts = Trollop.options do
17
+ opt :brokers, "replica set (broker IDs)", type: :string
18
+ opt :replicas, "number of replicas (count)", type: :integer
19
+ end
20
+
21
+ broker_ids = opts[:brokers] && opts[:brokers].split(',').map(&:to_i)
22
+ replica_count = opts[:replicas]
23
+
24
+ broker_ids ||= zookeeper.get_brokers.values.map(&:id)
25
+
26
+ all_brokers_id = all_brokers.values.map(&:id)
27
+ broker_ids.each do |id|
28
+ if !all_brokers_id.include?(id)
29
+ print "ERROR: Broker #{id} is not currently active.\n"
30
+ exit 1
31
+ end
32
+ end
33
+
34
+ # *** This logic is duplicated from Kakfa 0.8.1.1 ***
35
+
36
+ assignments = []
37
+ broker_count = broker_ids.size
38
+
39
+ topics.each do |_, t|
40
+ # This is how Kafka's AdminUtils determines these values.
41
+ partition_count = t.partitions.size
42
+ topic_replica_count = replica_count || t.partitions[0].replicas.size
43
+
44
+ if topic_replica_count > broker_count
45
+ print "ERROR: Replication factor is larger than brokers.\n"
46
+ exit 1
47
+ end
48
+
49
+ start_index = Random.rand(broker_count)
50
+ replica_shift = Random.rand(broker_count)
51
+
52
+ t.partitions.each do |p|
53
+ replica_shift += 1 if p.id > 0 && p.id % broker_count == 0
54
+ first_replica_index = (p.id + start_index) % broker_count
55
+
56
+ replicas = [broker_ids[first_replica_index]]
57
+
58
+ (0...topic_replica_count-1).each do |i|
59
+ shift = 1 + (replica_shift + i) % (broker_count - 1)
60
+ index = (first_replica_index + shift) % broker_count
61
+ replicas << broker_ids[index]
62
+ end
63
+
64
+ replicas.reverse!
65
+ assignments << Assignment.new(t.name, p.id, replicas)
66
+ end
67
+ end
68
+
69
+ # ****************
70
+
71
+ print "This operation executes the following assignments:\n\n"
72
+ print_assignment_header
73
+ assignments.each { |a| print_assignment(a) }
74
+ print "\n"
75
+
76
+ return unless agree("Proceed (y/n)?")
77
+
78
+ result = nil
79
+ begin
80
+ print "\nBeginning.\n"
81
+ result = admin.reassign!(assignments)
82
+ print "Started.\n"
83
+ rescue Admin::ExecutionFailedError
84
+ print result
85
+ end
86
+ end
87
+ end
88
+ end
89
+ end
@@ -0,0 +1,76 @@
1
+ module Kafkat
2
+ module Command
3
+ class ResignForce < Base
4
+ register_as 'resign-rewrite'
5
+
6
+ usage 'resign-rewrite <broker id>',
7
+ 'Forcibly rewrite leaderships to exclude a broker.'
8
+
9
+ usage 'resign-rewrite <broker id> --force',
10
+ 'Same as above but proceed if there are no available ISRs.'
11
+
12
+ def run
13
+ broker_id = ARGV[0] && ARGV.shift.to_i
14
+ if broker_id.nil?
15
+ puts "You must specify a broker ID."
16
+ exit 1
17
+ end
18
+
19
+ opts = Trollop.options do
20
+ opt :force, "force"
21
+ end
22
+
23
+ print "This operation rewrites leaderships in ZK to exclude broker '#{broker_id}'.\n"
24
+ print "WARNING: This is a last resort. Try the 'shutdown' command first!\n\n".red
25
+
26
+ return unless agree("Proceed (y/n)?")
27
+
28
+ brokers = zookeeper.get_brokers
29
+ topics = zookeeper.get_topics
30
+ force = opts[:force]
31
+
32
+ ops = {}
33
+ topics.each do |_, t|
34
+ t.partitions.each do |p|
35
+ next if p.leader != broker_id
36
+
37
+ alternates = p.isr.reject { |i| i == broker_id }
38
+ new_leader_id = alternates.sample
39
+
40
+ if !new_leader_id && !force
41
+ print "Partition #{t.name}-#{p.id} has no other ISRs!\n"
42
+ exit 1
43
+ end
44
+
45
+ new_leader_id ||= -1
46
+ ops[p] = new_leader_id
47
+ end
48
+ end
49
+
50
+ print "\n"
51
+ print "Summary of the new assignments:\n\n"
52
+
53
+ print "Partition\tLeader\n"
54
+ ops.each do |p, lid|
55
+ print justify("#{p.topic_name}-#{p.id}")
56
+ print justify(lid.to_s)
57
+ print "\n"
58
+ end
59
+
60
+ begin
61
+ print "\nStarting.\n"
62
+ ops.each do |p, lid|
63
+ retryable(tries: 3, on: Interface::Zookeeper::WriteConflictError) do
64
+ zookeeper.write_leader(p, lid)
65
+ end
66
+ end
67
+ rescue Interface::Zookeeper::WriteConflictError => e
68
+ print "Failed to update leaderships in ZK. Try re-running.\n\n"
69
+ exit 1
70
+ end
71
+
72
+ print "Done.\n"
73
+ end
74
+ end
75
+ end
76
+ end
@@ -0,0 +1,30 @@
1
+ module Kafkat
2
+ module Command
3
+ class Resign < Base
4
+ register_as 'shutdown'
5
+
6
+ usage 'shutdown <broker id>',
7
+ 'Gracefully remove leaderships from a broker (requires JMX).'
8
+
9
+ def run
10
+ broker_id = ARGV[0] && ARGV.shift.to_i
11
+ if broker_id.nil?
12
+ puts "You must specify a broker ID."
13
+ exit 1
14
+ end
15
+
16
+ print "This operation gracefully removes leaderships from broker '#{broker_id}'.\n"
17
+ return unless agree("Proceed (y/n)?")
18
+
19
+ result = nil
20
+ begin
21
+ print "\nBeginning shutdown.\n"
22
+ result = admin.shutdown!(broker_id)
23
+ print "Started.\n"
24
+ rescue Admin::ExecutionFailedError
25
+ print result
26
+ end
27
+ end
28
+ end
29
+ end
30
+ end
@@ -0,0 +1,15 @@
1
+ module Kafkat
2
+ module Command
3
+ class Topics < Base
4
+ register_as 'topics'
5
+
6
+ usage 'topics',
7
+ 'Print all topics.'
8
+
9
+ def run
10
+ ts = zookeeper.get_topics
11
+ ts.each { |name, t| print_topic(t) }
12
+ end
13
+ end
14
+ end
15
+ end
@@ -0,0 +1,68 @@
1
+ module Kafkat
2
+ module Command
3
+ class NotFoundError < StandardError; end
4
+
5
+ def self.all
6
+ @all ||= {}
7
+ end
8
+
9
+ def self.get(name)
10
+ klass = all[name.downcase]
11
+ raise NotFoundError if !klass
12
+ klass
13
+ end
14
+
15
+ class Base
16
+ include Formatting
17
+
18
+ attr_reader :config
19
+
20
+ class << self
21
+ attr_reader :command_name
22
+ end
23
+
24
+ def self.register_as(name)
25
+ @command_name = name
26
+ Command.all[name] = self
27
+ end
28
+
29
+ def self.usages
30
+ @usages ||= []
31
+ end
32
+
33
+ def self.usage(format, description)
34
+ usages << [format, description]
35
+ end
36
+
37
+ def initialize(config)
38
+ @config = config
39
+ end
40
+
41
+ def run
42
+ raise NotImplementedError
43
+ end
44
+
45
+ def admin
46
+ @admin ||= begin
47
+ Interface::Admin.new(config)
48
+ end
49
+ end
50
+
51
+ def zookeeper
52
+ @zookeeper ||= begin
53
+ Interface::Zookeeper.new(config)
54
+ end
55
+ end
56
+
57
+ def kafka_logs
58
+ @kafka_logs ||= begin
59
+ Interface::KafkaLogs.new(config)
60
+ end
61
+ end
62
+ end
63
+ end
64
+ end
65
+
66
+ # Require all of the commands.
67
+ command_glob = File.expand_path("../command/*.rb", __FILE__)
68
+ Dir[command_glob].each { |f| require f }
@@ -0,0 +1,45 @@
1
+ module Kafkat
2
+ class Config
3
+ CONFIG_PATHS = [
4
+ '~/.kafkatcfg',
5
+ '/etc/kafkatcfg'
6
+ ]
7
+
8
+ class NotFoundError < StandardError; end
9
+ class ParseError < StandardError; end
10
+
11
+ attr_reader :kafka_path
12
+ attr_reader :log_path
13
+ attr_reader :zk_path
14
+
15
+ def self.load!
16
+ string = nil
17
+ e = nil
18
+
19
+ CONFIG_PATHS.each do |rel_path|
20
+ begin
21
+ path = File.expand_path(rel_path)
22
+ string = File.read(path)
23
+ break
24
+ rescue => e
25
+ end
26
+ end
27
+
28
+ raise e if e && string.nil?
29
+
30
+ json = JSON.parse(string)
31
+ self.new(json)
32
+
33
+ rescue Errno::ENOENT
34
+ raise NotFoundError
35
+ rescue JSON::JSONError
36
+ raise ParseError
37
+ end
38
+
39
+ def initialize(json)
40
+ @kafka_path = json['kafka_path']
41
+ @log_path = json['log_path']
42
+ @zk_path = json['zk_path']
43
+ end
44
+ end
45
+ end
@@ -0,0 +1,90 @@
1
+ require 'tempfile'
2
+
3
+ module Kafkat
4
+ module Interface
5
+ class Admin
6
+ class ExecutionFailedError < StandardError; end
7
+
8
+ attr_reader :kafka_path
9
+ attr_reader :zk_path
10
+
11
+ def initialize(config)
12
+ @kafka_path = config.kafka_path
13
+ @zk_path = config.zk_path
14
+ end
15
+
16
+ def elect_leaders!(partitions)
17
+ file = Tempfile.new('kafkat-partitions.json')
18
+
19
+ json_partitions = []
20
+ partitions.each do |p|
21
+ json_partitions << {
22
+ 'topic' => p.topic_name,
23
+ 'partition' => p.id
24
+ }
25
+ end
26
+
27
+ json = {'partitions' => json_partitions}
28
+ file.write(JSON.dump(json))
29
+ file.close
30
+
31
+ run_tool(
32
+ 'kafka-preferred-replica-election',
33
+ '--path-to-json-file', file.path
34
+ )
35
+ ensure
36
+ file.unlink
37
+ end
38
+
39
+ def reassign!(assignments)
40
+ file = Tempfile.new('kafkat-partitions.json')
41
+
42
+ json_partitions = []
43
+ assignments.each do |a|
44
+ json_partitions << {
45
+ 'topic' => a.topic_name,
46
+ 'partition' => a.partition_id,
47
+ 'replicas' => a.replicas
48
+ }
49
+ end
50
+
51
+ json = {
52
+ 'partitions' => json_partitions,
53
+ 'version' => 1
54
+ }
55
+
56
+ file.write(JSON.dump(json))
57
+ file.close
58
+
59
+ run_tool(
60
+ 'kafka-reassign-partitions',
61
+ '--execute',
62
+ '--reassignment-json-file', file.path
63
+ )
64
+ ensure
65
+ file.unlink
66
+ end
67
+
68
+ def shutdown!(broker_id, options={})
69
+ args = ['--broker', broker_id]
70
+ args += ['--num.retries', options[:retries]] if options[:retries]
71
+ args += ['--retry.interval.ms', option[:interval]] if options[:interval]
72
+
73
+ run_tool(
74
+ 'kafka-run-class',
75
+ 'kafka.admin.ShutdownBroker',
76
+ *args
77
+ )
78
+ end
79
+
80
+ def run_tool(name, *args)
81
+ path = File.join(kafka_path, "bin/#{name}.sh")
82
+ args += ['--zookeeper', "\"#{zk_path}\""]
83
+ args_string = args.join(' ')
84
+ result = `#{path} #{args_string}`
85
+ raise ExecutionFailedError if $?.to_i > 0
86
+ result
87
+ end
88
+ end
89
+ end
90
+ end
@@ -0,0 +1,54 @@
1
+ module Kafkat
2
+ module Interface
3
+ class KafkaLogs
4
+ UNTRUNCATED_SIZE = 10 * 1024 * 1024 # 1MB
5
+
6
+ class NoLogsError < StandardError; end
7
+ class KafkaRunningError < StandardError; end
8
+
9
+ attr_reader :log_path
10
+
11
+ def initialize(config)
12
+ @log_path = config.log_path
13
+ end
14
+
15
+ def clean_indexes!
16
+ check_exists
17
+
18
+ to_remove = []
19
+ lock_for_write do
20
+ index_glob = File.join(log_path, '**/*.index')
21
+ Dir[index_glob].each do |index_path|
22
+ size = File.size(index_path)
23
+ to_remove << index_path if size == UNTRUNCATED_SIZE
24
+ end
25
+ end
26
+
27
+ to_remove.each do |path|
28
+ print "Removing #{path}.\n"
29
+ File.unlink(path)
30
+ end
31
+
32
+ to_remove.size
33
+ end
34
+
35
+ private
36
+
37
+ def check_exists
38
+ raise NoLogsError unless File.exists?(log_path)
39
+ end
40
+
41
+ def lock_for_write
42
+ File.open(lockfile_path, File::CREAT) do |lockfile|
43
+ locked = lockfile.flock(File::LOCK_EX | File::LOCK_NB)
44
+ raise KafkaRunningError unless locked
45
+ yield
46
+ end
47
+ end
48
+
49
+ def lockfile_path
50
+ File.join(log_path, '.lock')
51
+ end
52
+ end
53
+ end
54
+ end
@@ -0,0 +1,147 @@
1
+ require 'thread'
2
+
3
+ module Kafkat
4
+ module Interface
5
+ class Zookeeper
6
+ class NotFoundError < StandardError; end
7
+ class WriteConflictError < StandardError; end
8
+
9
+ attr_reader :zk_path
10
+
11
+ def initialize(config)
12
+ @zk_path = config.zk_path
13
+ end
14
+
15
+ def get_brokers(ids=nil)
16
+ brokers = {}
17
+ ids ||= zk.children(brokers_path)
18
+
19
+ threads = ids.map do |id|
20
+ id = id.to_i
21
+ Thread.new do
22
+ begin
23
+ brokers[id] = get_broker(id)
24
+ rescue
25
+ end
26
+ end
27
+ end
28
+ threads.map(&:join)
29
+
30
+ brokers
31
+ end
32
+
33
+ def get_broker(id)
34
+ path = broker_path(id)
35
+ string = zk.get(path).first
36
+ json = JSON.parse(string)
37
+ host, port = json['host'], json['port']
38
+ Broker.new(id, host, port)
39
+ rescue ZK::Exceptions::NoNode
40
+ raise NotFoundError
41
+ end
42
+
43
+ def get_topics(names=nil)
44
+ topics = {}
45
+ names ||= zk.children(topics_path)
46
+
47
+ threads = names.map do |name|
48
+ Thread.new do
49
+ begin
50
+ topics[name] = get_topic(name)
51
+ rescue => e
52
+ end
53
+ end
54
+ end
55
+ threads.map(&:join)
56
+
57
+ topics
58
+ end
59
+
60
+ def get_topic(name)
61
+ path1 = topic_path(name)
62
+ topic_string = zk.get(path1).first
63
+ topic_json = JSON.parse(topic_string)
64
+
65
+ partitions = []
66
+ path2 = topic_partitions_path(name)
67
+
68
+ threads = zk.children(path2).map do |id|
69
+ id = id.to_i
70
+ Thread.new do
71
+ path3 = topic_partition_state_path(name, id)
72
+ partition_string = zk.get(path3).first
73
+ partition_json = JSON.parse(partition_string)
74
+
75
+ replicas = topic_json['partitions'][id.to_s]
76
+ leader = partition_json['leader']
77
+ isr = partition_json['isr']
78
+
79
+ partitions << Partition.new(name, id, replicas, leader, isr)
80
+ end
81
+ end
82
+ threads.map(&:join)
83
+
84
+ partitions.sort_by!(&:id)
85
+ Topic.new(name, partitions)
86
+ rescue ZK::Exceptions::NoNode
87
+ raise NotFoundError
88
+ end
89
+
90
+ def get_controller
91
+ string = zk.get(controller_path).first
92
+ controller_json = JSON.parse(string)
93
+ controller_id = controller_json['brokerid']
94
+ get_broker(controller_id)
95
+ rescue ZK::Exceptions::NoNode
96
+ raise NotFoundError
97
+ end
98
+
99
+ def write_leader(partition, broker_id)
100
+ path = topic_partition_state_path(partition.topic_name, partition.id)
101
+ string, stat = zk.get(path)
102
+
103
+ partition_json = JSON.parse(string)
104
+ partition_json['leader'] = broker_id
105
+ new_string = JSON.dump(partition_json)
106
+
107
+ unless zk.set(path, new_string, version: stat.version)
108
+ raise ChangedDuringUpdateError
109
+ end
110
+ end
111
+
112
+ private
113
+
114
+ def zk
115
+ @zk ||= ZK.new(zk_path)
116
+ end
117
+
118
+ def brokers_path
119
+ '/brokers/ids'
120
+ end
121
+
122
+ def broker_path(id)
123
+ "/brokers/ids/#{id}"
124
+ end
125
+
126
+ def topics_path
127
+ '/brokers/topics'
128
+ end
129
+
130
+ def topic_path(name)
131
+ "/brokers/topics/#{name}"
132
+ end
133
+
134
+ def topic_partitions_path(name)
135
+ "/brokers/topics/#{name}/partitions"
136
+ end
137
+
138
+ def topic_partition_state_path(name, id)
139
+ "/brokers/topics/#{name}/partitions/#{id}/state"
140
+ end
141
+
142
+ def controller_path
143
+ "/controller"
144
+ end
145
+ end
146
+ end
147
+ end
@@ -0,0 +1,3 @@
1
+ require 'kafkat/interface/admin'
2
+ require 'kafkat/interface/kafka_logs'
3
+ require 'kafkat/interface/zookeeper'
@@ -0,0 +1,63 @@
1
+ module Kafkat
2
+ module Formatting
3
+ def justify(field, width=2)
4
+ field = field.to_s
5
+ count = [width - (field.length / 8), 0].max
6
+ field + "\t" * count
7
+ end
8
+
9
+ def print_broker(broker)
10
+ print justify(broker.id)
11
+ print justify("#{broker.host}:#{broker.port}")
12
+ print "\n"
13
+ end
14
+
15
+ def print_broker_header
16
+ print justify('Broker')
17
+ print justify('Socket')
18
+ print "\n"
19
+ end
20
+
21
+ def print_topic(topic)
22
+ print justify(topic.name)
23
+ print "\n"
24
+ end
25
+
26
+ def print_topic_header
27
+ print justify('Topic')
28
+ print "\n"
29
+ end
30
+
31
+ def print_partition(partition)
32
+ print justify(partition.topic_name)
33
+ print justify(partition.id)
34
+ print justify(partition.leader || 'none')
35
+ print justify(partition.replicas.inspect, 7)
36
+ print justify(partition.isr.inspect, 7)
37
+ print "\n"
38
+ end
39
+
40
+ def print_partition_header
41
+ print justify('Topic')
42
+ print justify('Partition')
43
+ print justify('Leader')
44
+ print justify('Replicas', 7)
45
+ print justify('ISRs', 7)
46
+ print "\n"
47
+ end
48
+
49
+ def print_assignment(assignment)
50
+ print justify(assignment.topic_name)
51
+ print justify(assignment.partition_id)
52
+ print justify(assignment.replicas.inspect)
53
+ print "\n"
54
+ end
55
+
56
+ def print_assignment_header
57
+ print justify('Topic')
58
+ print justify('Partition')
59
+ print justify('Replicas')
60
+ print "\n"
61
+ end
62
+ end
63
+ end
@@ -0,0 +1 @@
1
+ require 'kafkat/utility/formatting'
@@ -0,0 +1,3 @@
1
+ module Kafkat
2
+ VERSION = '0.0.9'
3
+ end