ruby-kafka 0.3.13.beta3 → 0.3.13.beta4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 0185da2c30e9d614d899d9c64257ce6e4f3c3623
-  data.tar.gz: 88d010e2aa98c3177e1591db1b89a87603c7872d
+  metadata.gz: 74dfff5457403137f170938a453a12e1288fd3f7
+  data.tar.gz: 03074ef585e3a46b45b703ed1efa13ef89e32062
 SHA512:
-  metadata.gz: a693bc3483e3e5d621b3f29ac59978fcdaa107dba84c601edbe46a2991158ec32180f88447e5f94ada737463f3fafbd572e4fbd5f0dd5e4e7afcb6d4e17612de
-  data.tar.gz: aeda8eb706dd0a96004f847c0ff6e5ae97a2973849f95b70b80c99f58fa049439fe2bbb51f0a0efcb80bf6526a0a1ca6b61761486304ca458a9101a47a65b4f4
+  metadata.gz: 88a1fc91d4d4b6b1a5aa1250e6f9a5b2a178d1bdb3233531d588ba8440072b64282ae17e866e0812ecbc502ff7ad418d3a0925d04ba710767d28f65af63890e0
+  data.tar.gz: 5a3a5b05c8d30679c528bf532584946d762a6d8c038f93e62958119db7f79699e503cb7a14164de43d008e9681cf8f6aca14be2f1ed2da0c7d36fdf1b7791573

data/CHANGELOG.md

@@ -4,6 +4,10 @@ Changes and additions to the library will be listed here.
 
 ## Unreleased
 
+## v0.3.13.beta4
+
+- Automatically recover from invalid consumer checkpoints.
+
 ## v0.3.13.beta1
 
 - Minimize the number of times messages are reprocessed after a consumer group resync.

data/lib/kafka.rb

@@ -19,6 +19,7 @@ module Kafka
   end
 
   class OffsetOutOfRange < ProtocolError
+    attr_accessor :topic, :partition
   end
 
   # The request is for a topic or partition that does not exist on the broker.

data/lib/kafka/consumer.rb

@@ -263,6 +263,12 @@ module Kafka
       end
 
       operation.execute
+    rescue OffsetOutOfRange => e
+      @logger.error "Invalid offset for #{e.topic}/#{e.partition}, resetting to default offset"
+
+      @offset_manager.seek_to_default(e.topic, e.partition)
+
+      retry
     rescue ConnectionError => e
       @logger.error "Connection error while fetching messages: #{e}"
 

data/lib/kafka/consumer_group.rb

@@ -113,7 +113,7 @@ module Kafka
     rescue UnknownMemberId
       @logger.error "Failed to join group; resetting member id and retrying in 1s..."
 
-      @member_id = nil
+      @member_id = ""
       sleep 1
 
       retry

data/lib/kafka/fetch_operation.rb

@@ -71,6 +71,11 @@ module Kafka
           fetched_topic.partitions.map {|fetched_partition|
             begin
               Protocol.handle_error(fetched_partition.error_code)
+            rescue Kafka::OffsetOutOfRange => e
+              e.topic = fetched_topic.name
+              e.partition = fetched_partition.partition
+
+              raise e
             rescue Kafka::Error => e
               topic = fetched_topic.name
               partition = fetched_partition.partition

data/lib/kafka/offset_manager.rb

@@ -26,6 +26,11 @@ module Kafka
       @logger.debug "Marking #{topic}/#{partition}:#{offset} as committed"
     end
 
+    def seek_to_default(topic, partition)
+      @processed_offsets[topic] ||= {}
+      @processed_offsets[topic][partition] = -1
+    end
+
     def next_offset_for(topic, partition)
       offset = @processed_offsets.fetch(topic, {}).fetch(partition) {
         committed_offset_for(topic, partition)

data/lib/kafka/producer.rb

@@ -360,7 +360,10 @@ module Kafka
       end
 
       if failed_messages.any?
-        @logger.error "Failed to assign partitions to #{failed_messages.count} messages"
+        failed_messages.group_by(&:topic).each do |topic, messages|
+          @logger.error "Failed to assign partitions to #{messages.count} messages in #{topic}"
+        end
+
         @cluster.mark_as_stale!
       end
 

data/lib/kafka/snappy_codec.rb

@@ -2,6 +2,9 @@ module Kafka
   class SnappyCodec
     def initialize
       require "snappy"
+    rescue LoadError
+      raise LoadError,
+        "Using snappy compression requires adding a dependency on the `snappy` gem to your Gemfile."
     end
 
     def codec_id

data/lib/kafka/version.rb

@@ -1,3 +1,3 @@
 module Kafka
-  VERSION = "0.3.13.beta3"
+  VERSION = "0.3.13.beta4"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: ruby-kafka
 version: !ruby/object:Gem::Version
-  version: 0.3.13.beta3
+  version: 0.3.13.beta4
 platform: ruby
 authors:
 - Daniel Schierbeck
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2016-08-03 00:00:00.000000000 Z
+date: 2016-08-05 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler