ruby-kafka 0.3.0 → 0.3.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +5 -0
- data/README.md +5 -5
- data/examples/firehose-consumer.rb +63 -0
- data/examples/firehose-producer.rb +3 -0
- data/lib/kafka/consumer.rb +8 -2
- data/lib/kafka/consumer_group.rb +9 -5
- data/lib/kafka/protocol/message_set.rb +11 -7
- data/lib/kafka/version.rb +1 -1
- metadata +2 -1
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 2842846dbe233b635e3dae0c3e14d9d660d19089
|
4
|
+
data.tar.gz: f7a9b7a9aea3df0e3175b0b239fe9d72f743fed5
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: a4c354e1677e76d0d0eb5483fd56b9d0fba7b8d16baf0f1fe450960977e8c18edba2c00d0a650168d280d03784170e078ae77e7914b6451bd20fc1784aea4867
|
7
|
+
data.tar.gz: 25e871eb218789b094a167371560db89edb1be38387969c2c74fa892701c527abb9d1cd9be0b14b41b65e4805cbbab19dd729c56c963b7555f6d7e15136ea20d
|
data/CHANGELOG.md
CHANGED
@@ -4,6 +4,11 @@ Changes and additions to the library will be listed here.
|
|
4
4
|
|
5
5
|
## Unreleased
|
6
6
|
|
7
|
+
## v0.3.1
|
8
|
+
|
9
|
+
- Simplify the heartbeat algorithm.
|
10
|
+
- Handle partial messages at the end of message sets received from the brokers.
|
11
|
+
|
7
12
|
## v0.3.0
|
8
13
|
|
9
14
|
- Add support for encryption and authentication with SSL (Tom Crayford).
|
data/README.md
CHANGED
@@ -18,11 +18,11 @@ Although parts of this library work with Kafka 0.8 – specifically, the Produce
|
|
18
18
|
5. [Message Delivery Guarantees](#message-delivery-guarantees)
|
19
19
|
6. [Compression](#compression)
|
20
20
|
2. [Consuming Messages from Kafka](#consuming-messages-from-kafka)
|
21
|
-
3. [Logging](#logging)
|
22
|
-
4. [Understanding Timeouts](#understanding-timeouts)
|
23
|
-
5. [Encryption and Authentication using SSL](#encryption-and-authentication-using-ssl)
|
24
|
-
|
25
|
-
|
21
|
+
3. [Logging](#logging)
|
22
|
+
4. [Understanding Timeouts](#understanding-timeouts)
|
23
|
+
5. [Encryption and Authentication using SSL](#encryption-and-authentication-using-ssl)
|
24
|
+
3. [Development](#development)
|
25
|
+
4. [Roadmap](#roadmap)
|
26
26
|
|
27
27
|
## Installation
|
28
28
|
|
@@ -0,0 +1,63 @@
|
|
1
|
+
$LOAD_PATH.unshift(File.expand_path("../../lib", __FILE__))
|
2
|
+
|
3
|
+
require "kafka"
|
4
|
+
|
5
|
+
KAFKA_CLIENT_CERT = ENV.fetch("KAFKA_CLIENT_CERT")
|
6
|
+
KAFKA_CLIENT_CERT_KEY = ENV.fetch("KAFKA_CLIENT_CERT_KEY")
|
7
|
+
KAFKA_SERVER_CERT = ENV.fetch("KAFKA_SERVER_CERT")
|
8
|
+
KAFKA_URL = ENV.fetch("KAFKA_URL")
|
9
|
+
KAFKA_BROKERS = KAFKA_URL.gsub("kafka+ssl://", "").split(",")
|
10
|
+
KAFKA_TOPIC = "test-messages"
|
11
|
+
|
12
|
+
NUM_THREADS = 4
|
13
|
+
|
14
|
+
queue = Queue.new
|
15
|
+
|
16
|
+
threads = NUM_THREADS.times.map do |worker_id|
|
17
|
+
Thread.new do
|
18
|
+
logger = Logger.new($stderr)
|
19
|
+
logger.level = Logger::INFO
|
20
|
+
|
21
|
+
logger.formatter = proc {|severity, datetime, progname, msg|
|
22
|
+
"[#{worker_id}] #{severity.ljust(5)} -- #{msg}\n"
|
23
|
+
}
|
24
|
+
|
25
|
+
kafka = Kafka.new(
|
26
|
+
seed_brokers: KAFKA_BROKERS,
|
27
|
+
logger: logger,
|
28
|
+
connect_timeout: 30,
|
29
|
+
socket_timeout: 30,
|
30
|
+
ssl_client_cert: KAFKA_CLIENT_CERT,
|
31
|
+
ssl_client_cert_key: KAFKA_CLIENT_CERT_KEY,
|
32
|
+
ssl_ca_cert: KAFKA_SERVER_CERT,
|
33
|
+
)
|
34
|
+
|
35
|
+
consumer = kafka.consumer(group_id: "firehose")
|
36
|
+
consumer.subscribe(KAFKA_TOPIC)
|
37
|
+
|
38
|
+
begin
|
39
|
+
i = 0
|
40
|
+
consumer.each_message do |message|
|
41
|
+
i += 1
|
42
|
+
|
43
|
+
if i % 1000 == 0
|
44
|
+
queue << i
|
45
|
+
i = 0
|
46
|
+
end
|
47
|
+
|
48
|
+
sleep 0.01
|
49
|
+
end
|
50
|
+
ensure
|
51
|
+
consumer.shutdown
|
52
|
+
end
|
53
|
+
end
|
54
|
+
end
|
55
|
+
|
56
|
+
threads.each {|t| t.abort_on_exception = true }
|
57
|
+
|
58
|
+
received_messages = 0
|
59
|
+
|
60
|
+
loop do
|
61
|
+
received_messages += queue.pop
|
62
|
+
puts "===> Received #{received_messages} messages"
|
63
|
+
end
|
data/lib/kafka/consumer.rb
CHANGED
@@ -57,6 +57,9 @@ module Kafka
|
|
57
57
|
@group = group
|
58
58
|
@offset_manager = offset_manager
|
59
59
|
@session_timeout = session_timeout
|
60
|
+
|
61
|
+
# Send two heartbeats in each session window, just to be sure.
|
62
|
+
@heartbeat_interval = @session_timeout / 2
|
60
63
|
end
|
61
64
|
|
62
65
|
# Subscribes the consumer to a topic.
|
@@ -122,6 +125,9 @@ module Kafka
|
|
122
125
|
rescue RebalanceInProgress
|
123
126
|
@logger.error "Group is rebalancing; rejoining"
|
124
127
|
join_group
|
128
|
+
rescue IllegalGeneration
|
129
|
+
@logger.error "Group has transitioned to a new generation; rejoining"
|
130
|
+
join_group
|
125
131
|
end
|
126
132
|
end
|
127
133
|
end
|
@@ -197,9 +203,9 @@ module Kafka
|
|
197
203
|
# `session_timeout`.
|
198
204
|
#
|
199
205
|
def send_heartbeat_if_necessary
|
200
|
-
@last_heartbeat ||= Time.
|
206
|
+
@last_heartbeat ||= Time.now
|
201
207
|
|
202
|
-
if
|
208
|
+
if Time.now > @last_heartbeat + @heartbeat_interval
|
203
209
|
@group.heartbeat
|
204
210
|
@last_heartbeat = Time.now
|
205
211
|
end
|
data/lib/kafka/consumer_group.rb
CHANGED
@@ -28,6 +28,10 @@ module Kafka
|
|
28
28
|
end
|
29
29
|
|
30
30
|
def join
|
31
|
+
if @topics.empty?
|
32
|
+
raise Kafka::Error, "Cannot join group without at least one topic subscription"
|
33
|
+
end
|
34
|
+
|
31
35
|
join_group
|
32
36
|
synchronize
|
33
37
|
rescue NotCoordinatorForGroup
|
@@ -42,7 +46,7 @@ module Kafka
|
|
42
46
|
end
|
43
47
|
|
44
48
|
def leave
|
45
|
-
@logger.info "
|
49
|
+
@logger.info "Leaving group `#{@group_id}`"
|
46
50
|
coordinator.leave_group(group_id: @group_id, member_id: @member_id)
|
47
51
|
rescue ConnectionError
|
48
52
|
end
|
@@ -70,7 +74,7 @@ module Kafka
|
|
70
74
|
end
|
71
75
|
|
72
76
|
def heartbeat
|
73
|
-
@logger.info "
|
77
|
+
@logger.info "Sending heartbeat..."
|
74
78
|
|
75
79
|
response = coordinator.heartbeat(
|
76
80
|
group_id: @group_id,
|
@@ -99,7 +103,7 @@ module Kafka
|
|
99
103
|
@leader_id = response.leader_id
|
100
104
|
@members = response.members
|
101
105
|
|
102
|
-
@logger.info "
|
106
|
+
@logger.info "Joined group `#{@group_id}` with member id `#{@member_id}`"
|
103
107
|
rescue UnknownMemberId
|
104
108
|
@logger.error "Failed to join group; resetting member id and retrying in 1s..."
|
105
109
|
|
@@ -117,7 +121,7 @@ module Kafka
|
|
117
121
|
group_assignment = {}
|
118
122
|
|
119
123
|
if group_leader?
|
120
|
-
@logger.info "
|
124
|
+
@logger.info "Chosen as leader of group `#{@group_id}`"
|
121
125
|
|
122
126
|
group_assignment = @assignment_strategy.assign(
|
123
127
|
members: @members.keys,
|
@@ -135,7 +139,7 @@ module Kafka
|
|
135
139
|
Protocol.handle_error(response.error_code)
|
136
140
|
|
137
141
|
response.member_assignment.topics.each do |topic, assigned_partitions|
|
138
|
-
@logger.info "
|
142
|
+
@logger.info "Partitions assigned for `#{topic}`: #{assigned_partitions.join(', ')}"
|
139
143
|
end
|
140
144
|
|
141
145
|
@assigned_partitions.replace(response.member_assignment.topics)
|
@@ -27,13 +27,17 @@ module Kafka
|
|
27
27
|
fetched_messages = []
|
28
28
|
|
29
29
|
until decoder.eof?
|
30
|
-
|
31
|
-
|
32
|
-
|
33
|
-
|
34
|
-
|
35
|
-
|
36
|
-
|
30
|
+
begin
|
31
|
+
message = Message.decode(decoder)
|
32
|
+
|
33
|
+
if message.compressed?
|
34
|
+
wrapped_message_set = message.decompress
|
35
|
+
fetched_messages.concat(wrapped_message_set.messages)
|
36
|
+
else
|
37
|
+
fetched_messages << message
|
38
|
+
end
|
39
|
+
rescue EOFError
|
40
|
+
# We tried to decode a partial message; just skip it.
|
37
41
|
end
|
38
42
|
end
|
39
43
|
|
data/lib/kafka/version.rb
CHANGED
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: ruby-kafka
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.3.
|
4
|
+
version: 0.3.1
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Daniel Schierbeck
|
@@ -163,6 +163,7 @@ files:
|
|
163
163
|
- bin/console
|
164
164
|
- bin/setup
|
165
165
|
- circle.yml
|
166
|
+
- examples/firehose-consumer.rb
|
166
167
|
- examples/firehose-producer.rb
|
167
168
|
- examples/simple-consumer.rb
|
168
169
|
- examples/simple-producer.rb
|