ruby-kafka 0.7.1.beta1 → 0.7.1.beta2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: c6a5d1dd1969107f9107675cad6848630f8cf13ce4ce223b0fa7c84bc4facf03
-  data.tar.gz: 2797042cbef4be8b219d2103dc5f63cbe36bb5c68aae738178ed02b197579254
+  metadata.gz: def137ba9f120b344452139b52dd8826740af0343fd17da25f96ee819e1f7c87
+  data.tar.gz: 8ffb491a63f5a7fbea9771962e521169d11ea84c810effecbf0140ebf1a8f1fe
 SHA512:
-  metadata.gz: d3dbfbb842fbc7b370d197a6f8020d4c88814c64383fe9cacd8e4338e4cafbdf222573476ba6bcf4d4ab05263b30ab0b63de2b7bc861d06730a303aec0a8d7a9
-  data.tar.gz: 12aba0aeacb8ceb731591f037ef075b3218d6c44710c5852fdfa1e5c769649bea95b4d609a9920ba48ce80b207a82628c488759d4c7cb48317ce71bbee36bfd0
+  metadata.gz: 21e457a2830b7e3524654b4e04fd939a4532fc4f531eab0c0bb2572e71a5426fd8ec484721c8ae40b740c77f3be1762642ec698bb1e6177f502d20952509320a
+  data.tar.gz: cda0dcf607d826c027eb4c91f923038353048b22296a79a4cfa2e9edc16adeb83e50ed66d992aba4335c9ee94dcba0bba06c5b0c7f7ece3b6595e90ebbbba6d5

data/CHANGELOG.md

@@ -4,9 +4,13 @@ Changes and additions to the library will be listed here.
 
 ## Unreleased
 
+- Fix consuming duplication due to redundant messages returned from Kafka (#636).
+- Fresh cluster info on fetch error (#641).
+
 ## 0.7.1
 
 - Exactly Once Delivery and Transactional Messaging Support (#608).
+- Support extra client certificates in the SSL Context when authenticating with Kafka (#633).
 
 ## 0.7.0
 

data/lib/kafka/fetch_operation.rb

@@ -100,6 +100,7 @@ module Kafka
             Kafka::FetchedBatchGenerator.new(
               fetched_topic.name,
               fetched_partition,
+              topics.fetch(fetched_topic.name).fetch(fetched_partition.partition).fetch(:fetch_offset),
               logger: @logger
             ).generate
           end

data/lib/kafka/fetched_batch_generator.rb

@@ -7,10 +7,11 @@ module Kafka
     COMMITTED_TRANSACTION_SIGNAL = "\x00\x00\x00\x01".freeze
     ABORTED_TRANSACTION_SIGNAL = "\x00\x00\x00\x00".freeze
 
-    def initialize(topic, fetched_partition, logger:)
+    def initialize(topic, fetched_partition, offset, logger:)
       @topic = topic
       @fetched_partition = fetched_partition
       @logger = logger
+      @offset = offset
     end
 
     def generate
@@ -40,11 +41,13 @@ module Kafka
       messages = @fetched_partition.messages.flat_map do |message_set|
         message_set.messages.map do |message|
           last_offset = message.offset if last_offset.nil? || last_offset < message.offset
-          FetchedMessage.new(
-            message: message,
-            topic: @topic,
-            partition: @fetched_partition.partition
-          )
+          if message.offset >= @offset
+            FetchedMessage.new(
+              message: message,
+              topic: @topic,
+              partition: @fetched_partition.partition
+            )
+          end
         end
       end
       FetchedBatch.new(
@@ -82,7 +85,7 @@ module Kafka
         end
 
         record_batch.records.each do |record|
-          unless record.is_control_record
+          if !record.is_control_record && record.offset >= @offset
             records << FetchedMessage.new(
               message: record,
               topic: @topic,

data/lib/kafka/fetcher.rb

@@ -178,6 +178,16 @@ module Kafka
       end
 
       operation.execute
+    rescue UnknownTopicOrPartition
+      @logger.error "Failed to fetch from some partitions. Maybe a rebalance has happened? Refreshing cluster info."
+
+      # Our cluster information has become stale, we need to refresh it.
+      @cluster.refresh_metadata!
+
+      # Don't overwhelm the brokers in case this keeps happening.
+      sleep 10
+
+      retry
     rescue NoPartitionsToFetchFrom
       backoff = @max_wait_time > 0 ? @max_wait_time : 1
 

data/lib/kafka/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Kafka
-  VERSION = "0.7.1.beta1"
+  VERSION = "0.7.1.beta2"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: ruby-kafka
 version: !ruby/object:Gem::Version
-  version: 0.7.1.beta1
+  version: 0.7.1.beta2
 platform: ruby
 authors:
 - Daniel Schierbeck
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2018-08-21 00:00:00.000000000 Z
+date: 2018-09-11 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: digest-crc