ruby-kafka 0.5.0.beta2 → 0.5.0.beta3
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/lib/kafka/consumer.rb +8 -0
- data/lib/kafka/protocol/message.rb +13 -5
- data/lib/kafka/protocol/produce_response.rb +1 -1
- data/lib/kafka/statsd.rb +3 -0
- data/lib/kafka/version.rb +1 -1
- metadata +2 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 473deb1157c274182f15656009eb4f181f764888
|
4
|
+
data.tar.gz: 634892fc04c6e91bbe703547d7a70e037e85fbcb
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 464247f865b176e319700a6b484146f025d7e931ea9af58b01dc2ac31660bc448741bdeb61ac0d9031d6d13aaed0c7dcc6a7e3d2b8b786c96c3a83bf50b77f17
|
7
|
+
data.tar.gz: '09b1db2e9955325ad9c07453ef51b34cb2e5f0455289668bc3c04572024db35b135715360e139b320efee5a7d32ed71c880d7f1fb8e827135509bce6a071e256'
|
data/lib/kafka/consumer.rb
CHANGED
@@ -216,6 +216,8 @@ module Kafka
|
|
216
216
|
|
217
217
|
# We may not have received any messages, but it's still a good idea to
|
218
218
|
# commit offsets if we've processed messages in the last set of batches.
|
219
|
+
# This also ensures the offsets are retained if we haven't read any messages
|
220
|
+
# since the offset retention period has elapsed.
|
219
221
|
@offset_manager.commit_offsets_if_necessary
|
220
222
|
end
|
221
223
|
end
|
@@ -279,6 +281,12 @@ module Kafka
|
|
279
281
|
|
280
282
|
return if !@running
|
281
283
|
end
|
284
|
+
|
285
|
+
# We may not have received any messages, but it's still a good idea to
|
286
|
+
# commit offsets if we've processed messages in the last set of batches.
|
287
|
+
# This also ensures the offsets are retained if we haven't read any messages
|
288
|
+
# since the offset retention period has elapsed.
|
289
|
+
@offset_manager.commit_offsets_if_necessary
|
282
290
|
end
|
283
291
|
end
|
284
292
|
|
@@ -66,13 +66,21 @@ module Kafka
|
|
66
66
|
|
67
67
|
crc = message_decoder.int32
|
68
68
|
magic_byte = message_decoder.int8
|
69
|
+
attributes = message_decoder.int8
|
69
70
|
|
70
|
-
|
71
|
+
# The magic byte indicates the message format version. There are situations
|
72
|
+
# where an old message format can be returned from a newer version of Kafka,
|
73
|
+
# because old messages are not necessarily rewritten on upgrades.
|
74
|
+
case magic_byte
|
75
|
+
when 0
|
76
|
+
# No timestamp in the pre-0.10 message format.
|
77
|
+
timestamp = nil
|
78
|
+
when 1
|
79
|
+
timestamp = message_decoder.int64
|
80
|
+
else
|
71
81
|
raise Kafka::Error, "Invalid magic byte: #{magic_byte}"
|
72
82
|
end
|
73
83
|
|
74
|
-
attributes = message_decoder.int8
|
75
|
-
timestamp = message_decoder.int64
|
76
84
|
key = message_decoder.bytes
|
77
85
|
value = message_decoder.bytes
|
78
86
|
|
@@ -80,7 +88,7 @@ module Kafka
|
|
80
88
|
# attributes.
|
81
89
|
codec_id = attributes & 0b111
|
82
90
|
|
83
|
-
new(key: key, value: value, codec_id: codec_id, offset: offset, create_time: Time.at(timestamp/1000.0))
|
91
|
+
new(key: key, value: value, codec_id: codec_id, offset: offset, create_time: Time.at(timestamp / 1000.0))
|
84
92
|
end
|
85
93
|
|
86
94
|
private
|
@@ -104,7 +112,7 @@ module Kafka
|
|
104
112
|
|
105
113
|
encoder.write_int8(MAGIC_BYTE)
|
106
114
|
encoder.write_int8(@codec_id)
|
107
|
-
encoder.write_int64((@create_time.to_f*1000).to_i)
|
115
|
+
encoder.write_int64((@create_time.to_f * 1000).to_i)
|
108
116
|
encoder.write_bytes(@key)
|
109
117
|
encoder.write_bytes(@value)
|
110
118
|
|
data/lib/kafka/statsd.rb
CHANGED
@@ -97,6 +97,7 @@ module Kafka
|
|
97
97
|
end
|
98
98
|
|
99
99
|
def process_batch(event)
|
100
|
+
lag = event.payload.fetch(:offset_lag)
|
100
101
|
messages = event.payload.fetch(:message_count)
|
101
102
|
client = event.payload.fetch(:client_id)
|
102
103
|
group_id = event.payload.fetch(:group_id)
|
@@ -109,6 +110,8 @@ module Kafka
|
|
109
110
|
timing("consumer.#{client}.#{group_id}.#{topic}.#{partition}.process_batch.latency", event.duration)
|
110
111
|
count("consumer.#{client}.#{group_id}.#{topic}.#{partition}.messages", messages)
|
111
112
|
end
|
113
|
+
|
114
|
+
gauge("consumer.#{client}.#{group_id}.#{topic}.#{partition}.lag", lag)
|
112
115
|
end
|
113
116
|
|
114
117
|
def join_group(event)
|
data/lib/kafka/version.rb
CHANGED
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: ruby-kafka
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.5.0.
|
4
|
+
version: 0.5.0.beta3
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Daniel Schierbeck
|
8
8
|
autorequire:
|
9
9
|
bindir: exe
|
10
10
|
cert_chain: []
|
11
|
-
date: 2017-10-
|
11
|
+
date: 2017-10-24 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: bundler
|