kazoo-ruby 0.3.3 → 0.4.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/lib/kazoo.rb +7 -0
- data/lib/kazoo/broker.rb +31 -16
- data/lib/kazoo/cli/consumers.rb +36 -1
- data/lib/kazoo/cli/topics.rb +14 -1
- data/lib/kazoo/cluster.rb +61 -29
- data/lib/kazoo/consumergroup.rb +148 -34
- data/lib/kazoo/partition.rb +9 -4
- data/lib/kazoo/replica_assigner.rb +74 -0
- data/lib/kazoo/subscription.rb +215 -0
- data/lib/kazoo/topic.rb +128 -82
- data/lib/kazoo/version.rb +1 -1
- data/test/functional/functional_consumergroup_test.rb +105 -53
- data/test/functional/functional_subscription_test.rb +62 -0
- data/test/functional/functional_topic_management_test.rb +25 -2
- data/test/replica_assigner_test.rb +80 -0
- data/test/subscription_test.rb +148 -0
- data/test/topic_test.rb +6 -19
- metadata +10 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 89757bee4b135ad94b20ef03dd75627928ac6430
|
4
|
+
data.tar.gz: babf18793c68d48949a54fadaa9c26a846cc90df
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 0fe533d27ff5eceb5cde6b60cb45137382c2498cf1ed97d51e5e59ed8755ec033ccc421915cd4cb16bf9729652d2d2c39675af31eeb1a9a528d468f8c8489625
|
7
|
+
data.tar.gz: d7cf53f56a7c2765d3f85b890e3acab71d5969970fc26ada8247404101d350f51f64b89a16c1987ad3303a2390909f94b8bc38bbbd4a3c56fb0e7e857becc5f7
|
data/lib/kazoo.rb
CHANGED
@@ -3,6 +3,8 @@ require 'json'
|
|
3
3
|
require 'thread'
|
4
4
|
require 'socket'
|
5
5
|
require 'securerandom'
|
6
|
+
require 'bigdecimal'
|
7
|
+
require 'time'
|
6
8
|
|
7
9
|
module Kazoo
|
8
10
|
Error = Class.new(StandardError)
|
@@ -14,6 +16,9 @@ module Kazoo
|
|
14
16
|
ConsumerInstanceRegistrationFailed = Class.new(Kazoo::Error)
|
15
17
|
PartitionAlreadyClaimed = Class.new(Kazoo::Error)
|
16
18
|
ReleasePartitionFailure = Class.new(Kazoo::Error)
|
19
|
+
InvalidSubscription = Class.new(Kazoo::Error)
|
20
|
+
InconsistentSubscriptions = Class.new(Kazoo::Error)
|
21
|
+
NoRunningInstances = Class.new(Kazoo::Error)
|
17
22
|
|
18
23
|
def self.connect(zookeeper)
|
19
24
|
Kazoo::Cluster.new(zookeeper)
|
@@ -24,5 +29,7 @@ require 'kazoo/cluster'
|
|
24
29
|
require 'kazoo/broker'
|
25
30
|
require 'kazoo/topic'
|
26
31
|
require 'kazoo/partition'
|
32
|
+
require 'kazoo/replica_assigner'
|
33
|
+
require 'kazoo/subscription'
|
27
34
|
require 'kazoo/consumergroup'
|
28
35
|
require 'kazoo/version'
|
data/lib/kazoo/broker.rb
CHANGED
@@ -1,4 +1,6 @@
|
|
1
1
|
module Kazoo
|
2
|
+
|
3
|
+
# Kazoo::Broker represents a Kafka broker in a Kafka cluster.
|
2
4
|
class Broker
|
3
5
|
attr_reader :cluster, :id, :host, :port, :jmx_port
|
4
6
|
|
@@ -8,45 +10,55 @@ module Kazoo
|
|
8
10
|
@jmx_port = jmx_port
|
9
11
|
end
|
10
12
|
|
13
|
+
# Returns a list of all partitions that are currently led by this broker.
|
11
14
|
def led_partitions
|
12
|
-
result,
|
13
|
-
cluster.partitions.
|
14
|
-
|
15
|
+
result, mutex = [], Mutex.new
|
16
|
+
threads = cluster.partitions.map do |partition|
|
17
|
+
Thread.new do
|
18
|
+
Thread.abort_on_exception = true
|
15
19
|
select = partition.leader == self
|
16
20
|
mutex.synchronize { result << partition } if select
|
17
21
|
end
|
18
|
-
threads.add(t)
|
19
22
|
end
|
20
|
-
threads.
|
23
|
+
threads.each(&:join)
|
21
24
|
result
|
22
25
|
end
|
23
26
|
|
27
|
+
# Returns a list of all partitions that host a replica on this broker.
|
24
28
|
def replicated_partitions
|
25
|
-
result,
|
26
|
-
cluster.partitions.
|
27
|
-
|
29
|
+
result, mutex = [], Mutex.new
|
30
|
+
threads = cluster.partitions.map do |partition|
|
31
|
+
Thread.new do
|
32
|
+
Thread.abort_on_exception = true
|
28
33
|
select = partition.replicas.include?(self)
|
29
34
|
mutex.synchronize { result << partition } if select
|
30
35
|
end
|
31
|
-
threads.add(t)
|
32
36
|
end
|
33
|
-
threads.
|
37
|
+
threads.each(&:join)
|
34
38
|
result
|
35
39
|
end
|
36
40
|
|
41
|
+
# Returns whether this broker is currently considered critical.
|
42
|
+
#
|
43
|
+
# A broker is considered critical if it is the only in sync replica
|
44
|
+
# of any of the partitions it hosts. This means that if this broker
|
45
|
+
# were to go down, the partition woild become unavailable for writes,
|
46
|
+
# and may also lose data depending on the configuration and settings.
|
37
47
|
def critical?(replicas: 1)
|
38
|
-
result,
|
39
|
-
replicated_partitions.
|
40
|
-
|
48
|
+
result, mutex = false, Mutex.new
|
49
|
+
threads = replicated_partitions.map do |partition|
|
50
|
+
Thread.new do
|
51
|
+
Thread.abort_on_exception = true
|
41
52
|
isr = partition.isr.reject { |r| r == self }
|
42
|
-
mutex.synchronize { result = true if isr.length < replicas }
|
53
|
+
mutex.synchronize { result = true if isr.length < Integer(replicas) }
|
43
54
|
end
|
44
|
-
threads.add(t)
|
45
55
|
end
|
46
|
-
threads.
|
56
|
+
threads.each(&:join)
|
47
57
|
result
|
48
58
|
end
|
49
59
|
|
60
|
+
# Returns the address of this broker, i.e. the hostname plus the port
|
61
|
+
# to connect to.
|
50
62
|
def addr
|
51
63
|
"#{host}:#{port}"
|
52
64
|
end
|
@@ -65,7 +77,10 @@ module Kazoo
|
|
65
77
|
"#<Kazoo::Broker id=#{id} addr=#{addr}>"
|
66
78
|
end
|
67
79
|
|
80
|
+
# Instantiates a Kazoo::Broker instance based on the Broker metadata that is stored
|
81
|
+
# in Zookeeper under `/brokers/<id>`.
|
68
82
|
def self.from_json(cluster, id, json)
|
83
|
+
raise Kazoo::VersionNotSupported unless json.fetch('version') == 1
|
69
84
|
new(cluster, id.to_i, json.fetch('host'), json.fetch('port'), jmx_port: json.fetch('jmx_port', nil))
|
70
85
|
end
|
71
86
|
end
|
data/lib/kazoo/cli/consumers.rb
CHANGED
@@ -24,7 +24,7 @@ module Kazoo
|
|
24
24
|
cg = kafka_cluster.consumergroup(name)
|
25
25
|
raise Kazoo::Error, "Consumergroup #{cg.name} is not registered in Zookeeper" unless cg.exists?
|
26
26
|
|
27
|
-
topics = cg.
|
27
|
+
topics = cg.subscribed_topics.sort_by(&:name)
|
28
28
|
|
29
29
|
puts "Consumer name: #{cg.name}"
|
30
30
|
puts "Created on: #{cg.created_at}"
|
@@ -89,6 +89,41 @@ module Kazoo
|
|
89
89
|
|
90
90
|
cg.reset_all_offsets
|
91
91
|
end
|
92
|
+
|
93
|
+
desc "clean-topic-claims [NAME]", "Removes all the topic claim Zookeeper nodes that are not needed for the consumer group's subscription"
|
94
|
+
def clean_topic_claims(name)
|
95
|
+
validate_class_options!
|
96
|
+
|
97
|
+
cg = kafka_cluster.consumergroup(name)
|
98
|
+
raise Kazoo::Error, "Consumergroup #{cg.name} is not registered in Zookeeper" unless cg.exists?
|
99
|
+
raise Kazoo::Error, "Cannot cleanup consumergroup #{cg.name} if it is not running" unless cg.active?
|
100
|
+
|
101
|
+
subscribed_topics = cg.subscribed_topics
|
102
|
+
claimed_topics = cg.claimed_topics
|
103
|
+
to_clean = claimed_topics - subscribed_topics
|
104
|
+
|
105
|
+
if to_clean.empty?
|
106
|
+
puts "The consumer group does not have any lingering topic claims."
|
107
|
+
else
|
108
|
+
puts "The following topics were once claimed, but are no longer part of #{cg.name}'s subscriptions:"
|
109
|
+
to_clean.each do |topic|
|
110
|
+
puts "- #{topic.name}"
|
111
|
+
end
|
112
|
+
|
113
|
+
cg.clean_topic_claims
|
114
|
+
end
|
115
|
+
end
|
116
|
+
|
117
|
+
desc "clean-stored-offsets [NAME]", "Removes all stored offsets for topics the consumer group is no longer subscribed to"
|
118
|
+
def clean_stored_offsets(name)
|
119
|
+
validate_class_options!
|
120
|
+
|
121
|
+
cg = kafka_cluster.consumergroup(name)
|
122
|
+
raise Kazoo::Error, "Consumergroup #{cg.name} is not registered in Zookeeper" unless cg.exists?
|
123
|
+
raise Kazoo::Error, "Cannot clean offsets for #{cg.name} if it is not running" unless cg.active?
|
124
|
+
|
125
|
+
cg.clean_stored_offsets
|
126
|
+
end
|
92
127
|
end
|
93
128
|
end
|
94
129
|
end
|
data/lib/kazoo/cli/topics.rb
CHANGED
@@ -28,7 +28,6 @@ module Kazoo
|
|
28
28
|
kafka_cluster.topics.fetch(name).destroy
|
29
29
|
end
|
30
30
|
|
31
|
-
option :topic, type: :string
|
32
31
|
desc "partitions TOPIC", "Lists partitions for a topic"
|
33
32
|
def partitions(topic)
|
34
33
|
validate_class_options!
|
@@ -38,6 +37,20 @@ module Kazoo
|
|
38
37
|
puts "#{partition.key}\tReplicas: #{partition.replicas.map(&:id).join(",")}\tISR: #{partition.isr.map(&:id).join(",")}"
|
39
38
|
end
|
40
39
|
end
|
40
|
+
|
41
|
+
option :partitions, type: :numeric, required: true
|
42
|
+
option :replication_factor, type: :numeric, required: false
|
43
|
+
desc "set_partitions TOPIC", "Lists partitions for a topic"
|
44
|
+
def set_partitions(topic)
|
45
|
+
validate_class_options!
|
46
|
+
|
47
|
+
topic = kafka_cluster.topics.fetch(topic)
|
48
|
+
new_partitions = options[:partitions] - topic.partitions.length
|
49
|
+
raise "You can only add partitions to a topic, not remove them" if new_partitions <= 0
|
50
|
+
|
51
|
+
replication_factor = options[:replication_factor] || topic.replication_factor
|
52
|
+
topic.add_partitions(partitions: new_partitions, replication_factor: replication_factor)
|
53
|
+
end
|
41
54
|
end
|
42
55
|
end
|
43
56
|
end
|
data/lib/kazoo/cluster.rb
CHANGED
@@ -1,6 +1,9 @@
|
|
1
1
|
module Kazoo
|
2
|
-
class Cluster
|
3
2
|
|
3
|
+
# Kazoo::Cluster represents a full Kafka cluster, based on how it is registered in Zookeeper.
|
4
|
+
# It allows you the inspect the brokers of the cluster, the topics and partition metadata,
|
5
|
+
# and the consumergroups that are registered against the cluster.
|
6
|
+
class Cluster
|
4
7
|
attr_reader :zookeeper
|
5
8
|
|
6
9
|
def initialize(zookeeper)
|
@@ -8,12 +11,14 @@ module Kazoo
|
|
8
11
|
@zk_mutex, @brokers_mutex, @topics_mutex = Mutex.new, Mutex.new, Mutex.new
|
9
12
|
end
|
10
13
|
|
14
|
+
# Returns a zookeeper connection
|
11
15
|
def zk
|
12
16
|
@zk_mutex.synchronize do
|
13
17
|
@zk ||= Zookeeper.new(zookeeper)
|
14
18
|
end
|
15
19
|
end
|
16
20
|
|
21
|
+
# Returns a hash of all the brokers in the
|
17
22
|
def brokers
|
18
23
|
@brokers_mutex.synchronize do
|
19
24
|
@brokers ||= begin
|
@@ -23,23 +28,24 @@ module Kazoo
|
|
23
28
|
raise NoClusterRegistered, "No Kafka cluster registered on this Zookeeper location."
|
24
29
|
end
|
25
30
|
|
26
|
-
result,
|
27
|
-
brokers.fetch(:children).map do |id|
|
28
|
-
|
31
|
+
result, mutex = {}, Mutex.new
|
32
|
+
threads = brokers.fetch(:children).map do |id|
|
33
|
+
Thread.new do
|
34
|
+
Thread.abort_on_exception = true
|
29
35
|
broker_info = zk.get(path: "/brokers/ids/#{id}")
|
30
36
|
raise Kazoo::Error, "Failed to retrieve broker info. Error code: #{broker_info.fetch(:rc)}" unless broker_info.fetch(:rc) == Zookeeper::Constants::ZOK
|
31
37
|
|
32
38
|
broker = Kazoo::Broker.from_json(self, id, JSON.parse(broker_info.fetch(:data)))
|
33
39
|
mutex.synchronize { result[id.to_i] = broker }
|
34
40
|
end
|
35
|
-
threads.add(t)
|
36
41
|
end
|
37
|
-
threads.
|
42
|
+
threads.each(&:join)
|
38
43
|
result
|
39
44
|
end
|
40
45
|
end
|
41
46
|
end
|
42
47
|
|
48
|
+
# Returns a list of consumer groups that are registered against the Kafka cluster.
|
43
49
|
def consumergroups
|
44
50
|
@consumergroups ||= begin
|
45
51
|
consumers = zk.get_children(path: "/consumers")
|
@@ -47,62 +53,66 @@ module Kazoo
|
|
47
53
|
end
|
48
54
|
end
|
49
55
|
|
56
|
+
# Returns a Kazoo::Consumergroup instance for a given consumer name.
|
57
|
+
#
|
58
|
+
# Note that this doesn't register a new consumer group in Zookeeper; you wil have to call
|
59
|
+
# Kazoo::Consumergroup.create to do that.
|
50
60
|
def consumergroup(name)
|
51
61
|
Kazoo::Consumergroup.new(self, name)
|
52
62
|
end
|
53
63
|
|
54
|
-
|
64
|
+
# Returns a hash of all the topics in the Kafka cluster, indexed by the topic name.
|
65
|
+
def topics(preload: Kazoo::Topic::DEFAULT_PRELOAD_METHODS)
|
55
66
|
@topics_mutex.synchronize do
|
56
67
|
@topics ||= begin
|
57
68
|
topics = zk.get_children(path: "/brokers/topics")
|
58
69
|
raise Kazoo::Error, "Failed to list topics. Error code: #{topics.fetch(:rc)}" unless topics.fetch(:rc) == Zookeeper::Constants::ZOK
|
59
|
-
|
60
|
-
result, threads, mutex = {}, ThreadGroup.new, Mutex.new
|
61
|
-
topics.fetch(:children).each do |name|
|
62
|
-
t = Thread.new do
|
63
|
-
topic_info = zk.get(path: "/brokers/topics/#{name}")
|
64
|
-
raise Kazoo::Error, "Failed to get topic info. Error code: #{topic_info.fetch(:rc)}" unless topic_info.fetch(:rc) == Zookeeper::Constants::ZOK
|
65
|
-
|
66
|
-
topic = Kazoo::Topic.from_json(self, name, JSON.parse(topic_info.fetch(:data)))
|
67
|
-
mutex.synchronize { result[name] = topic }
|
68
|
-
end
|
69
|
-
threads.add(t)
|
70
|
-
end
|
71
|
-
threads.list.each(&:join)
|
72
|
-
result
|
70
|
+
preload_topics_from_names(topics.fetch(:children), preload: preload)
|
73
71
|
end
|
74
72
|
end
|
75
73
|
end
|
76
74
|
|
75
|
+
# Returns a Kazoo::Topic for a given topic name.
|
77
76
|
def topic(name)
|
78
77
|
Kazoo::Topic.new(self, name)
|
79
78
|
end
|
80
79
|
|
81
|
-
|
80
|
+
# Creates a topic on the Kafka cluster, with the provided number of partitions and
|
81
|
+
# replication factor.
|
82
|
+
def create_topic(name, partitions: nil, replication_factor: nil, config: nil)
|
82
83
|
raise ArgumentError, "partitions must be a positive integer" if Integer(partitions) <= 0
|
83
84
|
raise ArgumentError, "replication_factor must be a positive integer" if Integer(replication_factor) <= 0
|
84
85
|
|
85
|
-
Kazoo::Topic.create(self, name, partitions: Integer(partitions), replication_factor: Integer(replication_factor))
|
86
|
+
Kazoo::Topic.create(self, name, partitions: Integer(partitions), replication_factor: Integer(replication_factor), config: config)
|
86
87
|
end
|
87
88
|
|
89
|
+
# Returns a list of all partitions hosted by the cluster
|
88
90
|
def partitions
|
89
91
|
topics.values.flat_map(&:partitions)
|
90
92
|
end
|
91
93
|
|
94
|
+
# Resets the locally cached list of brokers and topics, which will mean they will be fetched
|
95
|
+
# freshly from Zookeeper the next time they are requested.
|
92
96
|
def reset_metadata
|
93
|
-
@topics, @brokers = nil, nil
|
97
|
+
@topics, @brokers, @consumergroups = nil, nil, nil
|
94
98
|
end
|
95
99
|
|
100
|
+
# Returns true if any of the partitions hosted by the cluster
|
96
101
|
def under_replicated?
|
97
102
|
partitions.any?(&:under_replicated?)
|
98
103
|
end
|
99
104
|
|
105
|
+
# Closes the zookeeper connection and clears all the local caches.
|
100
106
|
def close
|
101
107
|
zk.close
|
108
|
+
@zk = nil
|
109
|
+
reset_metadata
|
102
110
|
end
|
103
111
|
|
104
112
|
protected
|
105
113
|
|
114
|
+
# Recursively creates a node in Zookeeper, by recusrively trying to create its
|
115
|
+
# parent if it doesn not yet exist.
|
106
116
|
def recursive_create(path: nil)
|
107
117
|
raise ArgumentError, "path is a required argument" if path.nil?
|
108
118
|
|
@@ -113,29 +123,51 @@ module Kazoo
|
|
113
123
|
when Zookeeper::Constants::ZNONODE
|
114
124
|
recursive_create(path: File.dirname(path))
|
115
125
|
result = zk.create(path: path)
|
116
|
-
|
126
|
+
|
127
|
+
case result.fetch(:rc)
|
128
|
+
when Zookeeper::Constants::ZOK, Zookeeper::Constants::ZNODEEXISTS
|
129
|
+
return
|
130
|
+
else
|
131
|
+
raise Kazoo::Error, "Failed to create node #{path}. Result code: #{result.fetch(:rc)}"
|
132
|
+
end
|
117
133
|
else
|
118
134
|
raise Kazoo::Error, "Failed to create node #{path}. Result code: #{result.fetch(:rc)}"
|
119
135
|
end
|
120
136
|
end
|
121
137
|
|
138
|
+
# Deletes a node and all of its children from Zookeeper.
|
122
139
|
def recursive_delete(path: nil)
|
123
140
|
raise ArgumentError, "path is a required argument" if path.nil?
|
124
141
|
|
125
142
|
result = zk.get_children(path: path)
|
126
143
|
raise Kazoo::Error, "Failed to list children of #{path} to delete them. Result code: #{result.fetch(:rc)}" if result.fetch(:rc) != Zookeeper::Constants::ZOK
|
127
144
|
|
128
|
-
threads =
|
129
|
-
|
130
|
-
threads << Thread.new do
|
145
|
+
threads = result.fetch(:children).map do |name|
|
146
|
+
Thread.new do
|
131
147
|
Thread.abort_on_exception = true
|
132
148
|
recursive_delete(path: File.join(path, name))
|
133
149
|
end
|
134
|
-
threads.each(&:join)
|
135
150
|
end
|
151
|
+
threads.each(&:join)
|
136
152
|
|
137
153
|
result = zk.delete(path: path)
|
138
154
|
raise Kazoo::Error, "Failed to delete node #{path}. Result code: #{result.fetch(:rc)}" if result.fetch(:rc) != Zookeeper::Constants::ZOK
|
139
155
|
end
|
156
|
+
|
157
|
+
private
|
158
|
+
|
159
|
+
def preload_topics_from_names(names, preload: Kazoo::Topic::DEFAULT_PRELOAD_METHODS)
|
160
|
+
result, mutex = {}, Mutex.new
|
161
|
+
threads = names.map do |name|
|
162
|
+
Thread.new do
|
163
|
+
Thread.abort_on_exception = true
|
164
|
+
topic = topic(name)
|
165
|
+
(preload & Kazoo::Topic::ALL_PRELOAD_METHODS).each { |method| topic.send(method) }
|
166
|
+
mutex.synchronize { result[name] = topic }
|
167
|
+
end
|
168
|
+
end
|
169
|
+
threads.each(&:join)
|
170
|
+
result
|
171
|
+
end
|
140
172
|
end
|
141
173
|
end
|
data/lib/kazoo/consumergroup.rb
CHANGED
@@ -9,10 +9,12 @@ module Kazoo
|
|
9
9
|
def create
|
10
10
|
cluster.send(:recursive_create, path: "/consumers/#{name}/ids")
|
11
11
|
cluster.send(:recursive_create, path: "/consumers/#{name}/owners")
|
12
|
+
cluster.reset_metadata
|
12
13
|
end
|
13
14
|
|
14
15
|
def destroy
|
15
16
|
cluster.send(:recursive_delete, path: "/consumers/#{name}")
|
17
|
+
cluster.reset_metadata
|
16
18
|
end
|
17
19
|
|
18
20
|
def exists?
|
@@ -27,9 +29,18 @@ module Kazoo
|
|
27
29
|
Time.at(result.fetch(:stat).mtime / 1000.0)
|
28
30
|
end
|
29
31
|
|
32
|
+
def instantiate(id: nil, subscription: nil)
|
33
|
+
Instance.new(self, id: id, subscription: subscription)
|
34
|
+
end
|
35
|
+
|
36
|
+
def subscription
|
37
|
+
subscriptions = instances.map(&:subscription).compact
|
38
|
+
raise NoRunningInstances, "Consumergroup #{name} has no running instances; cannot determine subscription" if subscriptions.length == 0
|
39
|
+
|
40
|
+
subscriptions.uniq!
|
41
|
+
raise InconsistentSubscriptions, "Subscriptions of running instances are different from each other" if subscriptions.length != 1
|
30
42
|
|
31
|
-
|
32
|
-
Instance.new(self, id: id)
|
43
|
+
subscriptions.first
|
33
44
|
end
|
34
45
|
|
35
46
|
def active?
|
@@ -40,7 +51,7 @@ module Kazoo
|
|
40
51
|
result = cluster.zk.get_children(path: "/consumers/#{name}/ids")
|
41
52
|
case result.fetch(:rc)
|
42
53
|
when Zookeeper::Constants::ZOK
|
43
|
-
result.fetch(:children)
|
54
|
+
instances_with_subscription(result.fetch(:children))
|
44
55
|
when Zookeeper::Constants::ZNONODE
|
45
56
|
[]
|
46
57
|
else
|
@@ -51,16 +62,18 @@ module Kazoo
|
|
51
62
|
def watch_instances(&block)
|
52
63
|
cb = Zookeeper::Callbacks::WatcherCallback.create(&block)
|
53
64
|
result = cluster.zk.get_children(path: "/consumers/#{name}/ids", watcher: cb)
|
54
|
-
|
55
|
-
|
56
|
-
|
65
|
+
instances = case result.fetch(:rc)
|
66
|
+
when Zookeeper::Constants::ZOK
|
67
|
+
instances_with_subscription(result.fetch(:children))
|
68
|
+
when Zookeeper::Constants::ZNONODE
|
69
|
+
[]
|
70
|
+
else
|
71
|
+
raise Kazoo::Error, "Failed getting a list of runniong instances for #{name}. Error code: #{result.fetch(:rc)}"
|
57
72
|
end
|
58
73
|
|
59
|
-
instances = result.fetch(:children).map { |id| Instance.new(self, id: id) }
|
60
74
|
[instances, cb]
|
61
75
|
end
|
62
76
|
|
63
|
-
|
64
77
|
def watch_partition_claim(partition, &block)
|
65
78
|
cb = Zookeeper::Callbacks::WatcherCallback.create(&block)
|
66
79
|
|
@@ -76,7 +89,7 @@ module Kazoo
|
|
76
89
|
end
|
77
90
|
end
|
78
91
|
|
79
|
-
def
|
92
|
+
def claimed_topics
|
80
93
|
topic_result = cluster.zk.get_children(path: "/consumers/#{name}/owners")
|
81
94
|
case topic_result.fetch(:rc)
|
82
95
|
when Zookeeper::Constants::ZOK
|
@@ -88,6 +101,12 @@ module Kazoo
|
|
88
101
|
end
|
89
102
|
end
|
90
103
|
|
104
|
+
def subscribed_topics
|
105
|
+
subscription.topics(cluster)
|
106
|
+
end
|
107
|
+
|
108
|
+
alias_method :topics, :subscribed_topics
|
109
|
+
|
91
110
|
def partitions
|
92
111
|
partitions, threads, mutex = [], [], Mutex.new
|
93
112
|
topics.each do |topic|
|
@@ -154,35 +173,72 @@ module Kazoo
|
|
154
173
|
case topic_result.fetch(:rc)
|
155
174
|
when Zookeeper::Constants::ZOK; # continue
|
156
175
|
when Zookeeper::Constants::ZNONODE; return {}
|
157
|
-
else raise Kazoo::Error, "Failed to
|
176
|
+
else raise Kazoo::Error, "Failed to get topic offsets. Result code: #{topic_result.fetch(:rc)}"
|
158
177
|
end
|
159
178
|
|
160
|
-
offsets,
|
161
|
-
topic_result.fetch(:children).
|
162
|
-
|
179
|
+
offsets, mutex = {}, Mutex.new
|
180
|
+
topic_threads = topic_result.fetch(:children).map do |topic_name|
|
181
|
+
Thread.new do
|
163
182
|
Thread.abort_on_exception = true
|
164
183
|
|
165
|
-
topic =
|
184
|
+
topic = cluster.topic(topic_name)
|
166
185
|
partition_result = cluster.zk.get_children(path: "/consumers/#{name}/offsets/#{topic.name}")
|
167
|
-
raise Kazoo::Error, "Failed to
|
186
|
+
raise Kazoo::Error, "Failed to get partition offsets. Result code: #{partition_result.fetch(:rc)}" if partition_result.fetch(:rc) != Zookeeper::Constants::ZOK
|
168
187
|
|
169
|
-
partition_threads =
|
170
|
-
|
171
|
-
partition_threads << Thread.new do
|
188
|
+
partition_threads = partition_result.fetch(:children).map do |partition_id|
|
189
|
+
Thread.new do
|
172
190
|
Thread.abort_on_exception = true
|
173
191
|
|
174
192
|
partition = topic.partition(partition_id.to_i)
|
175
193
|
offset_result = cluster.zk.get(path: "/consumers/#{name}/offsets/#{topic.name}/#{partition.id}")
|
176
|
-
|
194
|
+
offset = case offset_result.fetch(:rc)
|
195
|
+
when Zookeeper::Constants::ZOK
|
196
|
+
offset_result.fetch(:data).to_i
|
197
|
+
when Zookeeper::Constants::ZNONODE
|
198
|
+
nil
|
199
|
+
else
|
200
|
+
raise Kazoo::Error, "Failed to retrieve offset for #{partition.key}. Error code: #{offset_result.fetch(:rc)}"
|
201
|
+
end
|
202
|
+
mutex.synchronize { offsets[partition] = offset }
|
203
|
+
end
|
204
|
+
end
|
205
|
+
partition_threads.each(&:join)
|
206
|
+
end
|
207
|
+
end
|
177
208
|
|
178
|
-
|
209
|
+
topic_threads.each(&:join)
|
210
|
+
return offsets
|
211
|
+
end
|
212
|
+
|
213
|
+
def retrieve_offsets(subscription = self.subscription)
|
214
|
+
subscription = Kazoo::Subscription.build(subscription)
|
215
|
+
|
216
|
+
offsets, mutex = {}, Mutex.new
|
217
|
+
topic_threads = subscription.topics(cluster).map do |topic|
|
218
|
+
Thread.new do
|
219
|
+
Thread.abort_on_exception = true
|
220
|
+
|
221
|
+
partition_threads = topic.partitions.map do |partition|
|
222
|
+
Thread.new do
|
223
|
+
Thread.abort_on_exception = true
|
224
|
+
|
225
|
+
offset_result = cluster.zk.get(path: "/consumers/#{name}/offsets/#{topic.name}/#{partition.id}")
|
226
|
+
offset = case offset_result.fetch(:rc)
|
227
|
+
when Zookeeper::Constants::ZOK
|
228
|
+
offset_result.fetch(:data).to_i
|
229
|
+
when Zookeeper::Constants::ZNONODE
|
230
|
+
nil
|
231
|
+
else
|
232
|
+
raise Kazoo::Error, "Failed to retrieve offset for #{partition.key}. Error code: #{offset_result.fetch(:rc)}"
|
233
|
+
end
|
234
|
+
mutex.synchronize { offsets[partition] = offset }
|
179
235
|
end
|
180
236
|
end
|
181
237
|
partition_threads.each(&:join)
|
182
238
|
end
|
183
239
|
end
|
184
240
|
|
185
|
-
|
241
|
+
topic_threads.each(&:join)
|
186
242
|
return offsets
|
187
243
|
end
|
188
244
|
|
@@ -192,8 +248,8 @@ module Kazoo
|
|
192
248
|
|
193
249
|
result = cluster.zk.set(path: partition_offset_path, data: next_offset_data)
|
194
250
|
if result.fetch(:rc) == Zookeeper::Constants::ZNONODE
|
195
|
-
cluster.send(:recursive_create, path:
|
196
|
-
result = cluster.zk.
|
251
|
+
cluster.send(:recursive_create, path: partition_offset_path)
|
252
|
+
result = cluster.zk.set(path: partition_offset_path, data: next_offset_data)
|
197
253
|
end
|
198
254
|
|
199
255
|
if result.fetch(:rc) != Zookeeper::Constants::ZOK
|
@@ -205,6 +261,40 @@ module Kazoo
|
|
205
261
|
cluster.send(:recursive_delete, path: "/consumers/#{name}/offsets")
|
206
262
|
end
|
207
263
|
|
264
|
+
def clean_topic_claims(subscription = nil)
|
265
|
+
subscription = subscription.nil? ? self.subscription : Kazoo::Subscription.build(subscription)
|
266
|
+
|
267
|
+
threads = claimed_topics.map do |topic|
|
268
|
+
Thread.new do
|
269
|
+
Thread.abort_on_exception = true
|
270
|
+
unless subscription.topics(cluster).include?(topic)
|
271
|
+
cluster.send(:recursive_delete, path: "/consumers/#{name}/owners/#{topic.name}")
|
272
|
+
end
|
273
|
+
end
|
274
|
+
end
|
275
|
+
|
276
|
+
threads.each(&:join)
|
277
|
+
end
|
278
|
+
|
279
|
+
def clean_stored_offsets(subscription = nil)
|
280
|
+
subscription = subscription.nil? ? self.subscription : Kazoo::Subscription.build(subscription)
|
281
|
+
|
282
|
+
topics_result = cluster.zk.get_children(path: "/consumers/#{name}/offsets")
|
283
|
+
raise Kazoo::Error, "Failed to retrieve list of topics. Error code: #{topics_result.fetch(:rc)}" if topics_result.fetch(:rc) != Zookeeper::Constants::ZOK
|
284
|
+
|
285
|
+
threads = topics_result.fetch(:children).map do |topic_name|
|
286
|
+
Thread.new do
|
287
|
+
Thread.abort_on_exception = true
|
288
|
+
topic = cluster.topic(topic_name)
|
289
|
+
unless subscription.topics(cluster).include?(topic)
|
290
|
+
cluster.send(:recursive_delete, path: "/consumers/#{name}/offsets/#{topic.name}")
|
291
|
+
end
|
292
|
+
end
|
293
|
+
end
|
294
|
+
|
295
|
+
threads.each(&:join)
|
296
|
+
end
|
297
|
+
|
208
298
|
def inspect
|
209
299
|
"#<Kazoo::Consumergroup name=#{name}>"
|
210
300
|
end
|
@@ -219,17 +309,36 @@ module Kazoo
|
|
219
309
|
[cluster, name].hash
|
220
310
|
end
|
221
311
|
|
312
|
+
protected
|
313
|
+
|
314
|
+
def instances_with_subscription(instance_ids)
|
315
|
+
instances, threads, mutex = [], [], Mutex.new
|
316
|
+
instance_ids.each do |id|
|
317
|
+
threads << Thread.new do
|
318
|
+
Thread.abort_on_exception = true
|
319
|
+
|
320
|
+
subscription_result = cluster.zk.get(path: "/consumers/#{name}/ids/#{id}")
|
321
|
+
raise Kazoo::Error, "Failed to retrieve subscription for instance. Error code: #{result.fetch(:rc)}" if subscription_result.fetch(:rc) != Zookeeper::Constants::ZOK
|
322
|
+
subscription = Kazoo::Subscription.from_json(subscription_result.fetch(:data))
|
323
|
+
mutex.synchronize { instances << Instance.new(self, id: id, subscription: subscription) }
|
324
|
+
end
|
325
|
+
end
|
326
|
+
threads.each(&:join)
|
327
|
+
instances
|
328
|
+
end
|
329
|
+
|
222
330
|
class Instance
|
223
331
|
|
224
332
|
def self.generate_id
|
225
333
|
"#{Socket.gethostname}:#{SecureRandom.uuid}"
|
226
334
|
end
|
227
335
|
|
228
|
-
attr_reader :group, :id
|
336
|
+
attr_reader :group, :id, :subscription
|
229
337
|
|
230
|
-
def initialize(group, id: nil)
|
338
|
+
def initialize(group, id: nil, subscription: nil)
|
231
339
|
@group = group
|
232
340
|
@id = id || self.class.generate_id
|
341
|
+
@subscription = Kazoo::Subscription.build(subscription) unless subscription.nil?
|
233
342
|
end
|
234
343
|
|
235
344
|
def registered?
|
@@ -237,23 +346,21 @@ module Kazoo
|
|
237
346
|
stat.fetch(:stat).exists?
|
238
347
|
end
|
239
348
|
|
240
|
-
def register(
|
349
|
+
def register(subscription_deprecated = nil)
|
350
|
+
# Don't provide the subscription here, but provide it when instantiating the consumer instance.
|
351
|
+
@subscription = Kazoo::Subscription.build(subscription_deprecated) unless subscription_deprecated.nil?
|
352
|
+
|
241
353
|
result = cluster.zk.create(
|
242
354
|
path: "/consumers/#{group.name}/ids/#{id}",
|
243
355
|
ephemeral: true,
|
244
|
-
data:
|
245
|
-
version: 1,
|
246
|
-
timestamp: Time.now.to_i,
|
247
|
-
pattern: "static",
|
248
|
-
subscription: Hash[*subscription.flat_map { |topic| [topic.name, 1] } ]
|
249
|
-
})
|
356
|
+
data: subscription.to_json,
|
250
357
|
)
|
251
358
|
|
252
359
|
if result.fetch(:rc) != Zookeeper::Constants::ZOK
|
253
360
|
raise Kazoo::ConsumerInstanceRegistrationFailed, "Failed to register instance #{id} for consumer group #{group.name}! Error code: #{result.fetch(:rc)}"
|
254
361
|
end
|
255
362
|
|
256
|
-
subscription.each do |topic|
|
363
|
+
subscription.topics(cluster).each do |topic|
|
257
364
|
stat = cluster.zk.stat(path: "/consumers/#{group.name}/owners/#{topic.name}")
|
258
365
|
unless stat.fetch(:stat).exists?
|
259
366
|
result = cluster.zk.create(path: "/consumers/#{group.name}/owners/#{topic.name}")
|
@@ -262,6 +369,8 @@ module Kazoo
|
|
262
369
|
end
|
263
370
|
end
|
264
371
|
end
|
372
|
+
|
373
|
+
return self
|
265
374
|
end
|
266
375
|
|
267
376
|
def created_at
|
@@ -273,7 +382,12 @@ module Kazoo
|
|
273
382
|
|
274
383
|
|
275
384
|
def deregister
|
276
|
-
cluster.zk.delete(path: "/consumers/#{group.name}/ids/#{id}")
|
385
|
+
result = cluster.zk.delete(path: "/consumers/#{group.name}/ids/#{id}")
|
386
|
+
if result.fetch(:rc) != Zookeeper::Constants::ZOK
|
387
|
+
raise Kazoo::Error, "Failed to deregister instance #{id} for consumer group #{group.name}! Error code: #{result.fetch(:rc)}"
|
388
|
+
end
|
389
|
+
|
390
|
+
return self
|
277
391
|
end
|
278
392
|
|
279
393
|
def claim_partition(partition)
|