ruby-kafka 0.7.1.beta1 → 0.7.1.beta2
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/CHANGELOG.md +4 -0
- data/lib/kafka/fetch_operation.rb +1 -0
- data/lib/kafka/fetched_batch_generator.rb +10 -7
- data/lib/kafka/fetcher.rb +10 -0
- data/lib/kafka/version.rb +1 -1
- metadata +2 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: def137ba9f120b344452139b52dd8826740af0343fd17da25f96ee819e1f7c87
|
4
|
+
data.tar.gz: 8ffb491a63f5a7fbea9771962e521169d11ea84c810effecbf0140ebf1a8f1fe
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 21e457a2830b7e3524654b4e04fd939a4532fc4f531eab0c0bb2572e71a5426fd8ec484721c8ae40b740c77f3be1762642ec698bb1e6177f502d20952509320a
|
7
|
+
data.tar.gz: cda0dcf607d826c027eb4c91f923038353048b22296a79a4cfa2e9edc16adeb83e50ed66d992aba4335c9ee94dcba0bba06c5b0c7f7ece3b6595e90ebbbba6d5
|
data/CHANGELOG.md
CHANGED
@@ -4,9 +4,13 @@ Changes and additions to the library will be listed here.
|
|
4
4
|
|
5
5
|
## Unreleased
|
6
6
|
|
7
|
+
- Fix consuming duplication due to redundant messages returned from Kafka (#636).
|
8
|
+
- Fresh cluster info on fetch error (#641).
|
9
|
+
|
7
10
|
## 0.7.1
|
8
11
|
|
9
12
|
- Exactly Once Delivery and Transactional Messaging Support (#608).
|
13
|
+
- Support extra client certificates in the SSL Context when authenticating with Kafka (#633).
|
10
14
|
|
11
15
|
## 0.7.0
|
12
16
|
|
@@ -7,10 +7,11 @@ module Kafka
|
|
7
7
|
COMMITTED_TRANSACTION_SIGNAL = "\x00\x00\x00\x01".freeze
|
8
8
|
ABORTED_TRANSACTION_SIGNAL = "\x00\x00\x00\x00".freeze
|
9
9
|
|
10
|
-
def initialize(topic, fetched_partition, logger:)
|
10
|
+
def initialize(topic, fetched_partition, offset, logger:)
|
11
11
|
@topic = topic
|
12
12
|
@fetched_partition = fetched_partition
|
13
13
|
@logger = logger
|
14
|
+
@offset = offset
|
14
15
|
end
|
15
16
|
|
16
17
|
def generate
|
@@ -40,11 +41,13 @@ module Kafka
|
|
40
41
|
messages = @fetched_partition.messages.flat_map do |message_set|
|
41
42
|
message_set.messages.map do |message|
|
42
43
|
last_offset = message.offset if last_offset.nil? || last_offset < message.offset
|
43
|
-
|
44
|
-
|
45
|
-
|
46
|
-
|
47
|
-
|
44
|
+
if message.offset >= @offset
|
45
|
+
FetchedMessage.new(
|
46
|
+
message: message,
|
47
|
+
topic: @topic,
|
48
|
+
partition: @fetched_partition.partition
|
49
|
+
)
|
50
|
+
end
|
48
51
|
end
|
49
52
|
end
|
50
53
|
FetchedBatch.new(
|
@@ -82,7 +85,7 @@ module Kafka
|
|
82
85
|
end
|
83
86
|
|
84
87
|
record_batch.records.each do |record|
|
85
|
-
|
88
|
+
if !record.is_control_record && record.offset >= @offset
|
86
89
|
records << FetchedMessage.new(
|
87
90
|
message: record,
|
88
91
|
topic: @topic,
|
data/lib/kafka/fetcher.rb
CHANGED
@@ -178,6 +178,16 @@ module Kafka
|
|
178
178
|
end
|
179
179
|
|
180
180
|
operation.execute
|
181
|
+
rescue UnknownTopicOrPartition
|
182
|
+
@logger.error "Failed to fetch from some partitions. Maybe a rebalance has happened? Refreshing cluster info."
|
183
|
+
|
184
|
+
# Our cluster information has become stale, we need to refresh it.
|
185
|
+
@cluster.refresh_metadata!
|
186
|
+
|
187
|
+
# Don't overwhelm the brokers in case this keeps happening.
|
188
|
+
sleep 10
|
189
|
+
|
190
|
+
retry
|
181
191
|
rescue NoPartitionsToFetchFrom
|
182
192
|
backoff = @max_wait_time > 0 ? @max_wait_time : 1
|
183
193
|
|
data/lib/kafka/version.rb
CHANGED
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: ruby-kafka
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.7.1.
|
4
|
+
version: 0.7.1.beta2
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Daniel Schierbeck
|
8
8
|
autorequire:
|
9
9
|
bindir: exe
|
10
10
|
cert_chain: []
|
11
|
-
date: 2018-
|
11
|
+
date: 2018-09-11 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: digest-crc
|