ruby-kafka 0.7.1.beta1 → 0.7.1.beta2

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: c6a5d1dd1969107f9107675cad6848630f8cf13ce4ce223b0fa7c84bc4facf03
4
- data.tar.gz: 2797042cbef4be8b219d2103dc5f63cbe36bb5c68aae738178ed02b197579254
3
+ metadata.gz: def137ba9f120b344452139b52dd8826740af0343fd17da25f96ee819e1f7c87
4
+ data.tar.gz: 8ffb491a63f5a7fbea9771962e521169d11ea84c810effecbf0140ebf1a8f1fe
5
5
  SHA512:
6
- metadata.gz: d3dbfbb842fbc7b370d197a6f8020d4c88814c64383fe9cacd8e4338e4cafbdf222573476ba6bcf4d4ab05263b30ab0b63de2b7bc861d06730a303aec0a8d7a9
7
- data.tar.gz: 12aba0aeacb8ceb731591f037ef075b3218d6c44710c5852fdfa1e5c769649bea95b4d609a9920ba48ce80b207a82628c488759d4c7cb48317ce71bbee36bfd0
6
+ metadata.gz: 21e457a2830b7e3524654b4e04fd939a4532fc4f531eab0c0bb2572e71a5426fd8ec484721c8ae40b740c77f3be1762642ec698bb1e6177f502d20952509320a
7
+ data.tar.gz: cda0dcf607d826c027eb4c91f923038353048b22296a79a4cfa2e9edc16adeb83e50ed66d992aba4335c9ee94dcba0bba06c5b0c7f7ece3b6595e90ebbbba6d5
data/CHANGELOG.md CHANGED
@@ -4,9 +4,13 @@ Changes and additions to the library will be listed here.
4
4
 
5
5
  ## Unreleased
6
6
 
7
+ - Fix consuming duplication due to redundant messages returned from Kafka (#636).
8
+ - Fresh cluster info on fetch error (#641).
9
+
7
10
  ## 0.7.1
8
11
 
9
12
  - Exactly Once Delivery and Transactional Messaging Support (#608).
13
+ - Support extra client certificates in the SSL Context when authenticating with Kafka (#633).
10
14
 
11
15
  ## 0.7.0
12
16
 
@@ -100,6 +100,7 @@ module Kafka
100
100
  Kafka::FetchedBatchGenerator.new(
101
101
  fetched_topic.name,
102
102
  fetched_partition,
103
+ topics.fetch(fetched_topic.name).fetch(fetched_partition.partition).fetch(:fetch_offset),
103
104
  logger: @logger
104
105
  ).generate
105
106
  end
@@ -7,10 +7,11 @@ module Kafka
7
7
  COMMITTED_TRANSACTION_SIGNAL = "\x00\x00\x00\x01".freeze
8
8
  ABORTED_TRANSACTION_SIGNAL = "\x00\x00\x00\x00".freeze
9
9
 
10
- def initialize(topic, fetched_partition, logger:)
10
+ def initialize(topic, fetched_partition, offset, logger:)
11
11
  @topic = topic
12
12
  @fetched_partition = fetched_partition
13
13
  @logger = logger
14
+ @offset = offset
14
15
  end
15
16
 
16
17
  def generate
@@ -40,11 +41,13 @@ module Kafka
40
41
  messages = @fetched_partition.messages.flat_map do |message_set|
41
42
  message_set.messages.map do |message|
42
43
  last_offset = message.offset if last_offset.nil? || last_offset < message.offset
43
- FetchedMessage.new(
44
- message: message,
45
- topic: @topic,
46
- partition: @fetched_partition.partition
47
- )
44
+ if message.offset >= @offset
45
+ FetchedMessage.new(
46
+ message: message,
47
+ topic: @topic,
48
+ partition: @fetched_partition.partition
49
+ )
50
+ end
48
51
  end
49
52
  end
50
53
  FetchedBatch.new(
@@ -82,7 +85,7 @@ module Kafka
82
85
  end
83
86
 
84
87
  record_batch.records.each do |record|
85
- unless record.is_control_record
88
+ if !record.is_control_record && record.offset >= @offset
86
89
  records << FetchedMessage.new(
87
90
  message: record,
88
91
  topic: @topic,
data/lib/kafka/fetcher.rb CHANGED
@@ -178,6 +178,16 @@ module Kafka
178
178
  end
179
179
 
180
180
  operation.execute
181
+ rescue UnknownTopicOrPartition
182
+ @logger.error "Failed to fetch from some partitions. Maybe a rebalance has happened? Refreshing cluster info."
183
+
184
+ # Our cluster information has become stale, we need to refresh it.
185
+ @cluster.refresh_metadata!
186
+
187
+ # Don't overwhelm the brokers in case this keeps happening.
188
+ sleep 10
189
+
190
+ retry
181
191
  rescue NoPartitionsToFetchFrom
182
192
  backoff = @max_wait_time > 0 ? @max_wait_time : 1
183
193
 
data/lib/kafka/version.rb CHANGED
@@ -1,5 +1,5 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  module Kafka
4
- VERSION = "0.7.1.beta1"
4
+ VERSION = "0.7.1.beta2"
5
5
  end
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: ruby-kafka
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.7.1.beta1
4
+ version: 0.7.1.beta2
5
5
  platform: ruby
6
6
  authors:
7
7
  - Daniel Schierbeck
8
8
  autorequire:
9
9
  bindir: exe
10
10
  cert_chain: []
11
- date: 2018-08-21 00:00:00.000000000 Z
11
+ date: 2018-09-11 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: digest-crc