ruby-kafka 0.3.11 → 0.3.12

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 7b1f414825e2d64312d9b4ad80747fe56b591408
-  data.tar.gz: ed5ec3a67e16f375061a5040829c2830e0b57395
+  metadata.gz: 29c5259a5c2650e5a9eec0c513aa997a90834f9d
+  data.tar.gz: b4eb6a8de42d27f2a3b215cb690d7c803f706bad
 SHA512:
-  metadata.gz: a2cadfb3b2be1d37d79c6139f9fc5081c5d1315d60306c9c2ce9dd325a94d5d0e720cd0b826624e14003bbd89cab2b916f337bc671b273ec947f50f2149cc9bf
-  data.tar.gz: 1a64728b38159b9a26685563826a970882262c46449c1cebc2470a90e68dc8ec52fdd3a3626d620d6047cabf2fe8627ddc48dea5f72f75d4e81c4ae259eae4c9
+  metadata.gz: 43c219f2e1956264a864a9ce41e75f2a3b46ac6cc5c248cdc99a28a15b2871ed8fdb1554422797c9ca8e913996874981aadf3f35f23fea30110e616bfc3013e0
+  data.tar.gz: ad849644e4f75761164f41ffad9d69e67e0ac71eb2796d222afc224f1745bc9199491165d38fd346ebf0b8f7da6ea25e60fb9fe7a853b12bc1699f069331d8d5

data/CHANGELOG.md

@@ -4,6 +4,10 @@ Changes and additions to the library will be listed here.
 
 ## Unreleased
 
+## v0.3.12
+
+- Fix a bug in the consumer.
+
 ## v0.3.11
 
 - Fix bug in the simple consumer loop.

data/lib/kafka/client.rb

@@ -60,6 +60,21 @@ module Kafka
       @cluster = initialize_cluster
     end
 
+    # Delivers a single message to the Kafka cluster.
+    #
+    # **Note:** Only use this API for low-throughput scenarios. If you want to deliver
+    # many messages at a high rate, or if you want to configure the way messages are
+    # sent, use the {#producer} or {#async_producer} APIs instead.
+    #
+    # @param value [String, nil] the message value.
+    # @param key [String, nil] the message key.
+    # @param topic [String] the topic that the message should be written to.
+    # @param partition [Integer, nil] the partition that the message should be written
+    #   to, or `nil` if either `partition_key` is passed or the partition should be
+    #   chosen at random.
+    # @param partition_key [String] a value used to deterministically choose a
+    #   partition to write to.
+    # @return [nil]
     def deliver_message(value, key: nil, topic:, partition: nil, partition_key: nil)
       create_time = Time.now
 

data/lib/kafka/connection.rb

@@ -123,6 +123,11 @@ module Kafka
       raise ConnectionError, e
     end
 
+    def reopen
+      close
+      open
+    end
+
     # Writes a request over the connection.
     #
     # @param request [#encode] the request that should be encoded and written.
@@ -140,14 +145,29 @@ module Kafka
       )
 
       data = Kafka::Protocol::Encoder.encode_with(message)
+      retried = false
       notification[:request_size] = data.bytesize
 
-      @encoder.write_bytes(data)
+      begin
+        @encoder.write_bytes(data)
+      rescue Errno::ETIMEDOUT
+        @logger.error "Timed out while writing request #{@correlation_id}"
+        raise
+      rescue Errno::EPIPE, Errno::ECONNRESET, EOFError
+        # Kafka brokers automatically close client connections after a period of
+        # inactivity. If this has happened, it's safe to re-open the connection
+        # and retry the request.
+        if retried
+          raise
+        else
+          @logger.warn "Connection has been closed by the server, retrying..."
+          retried = true
+          reopen
+          retry
+        end
+      end
 
       nil
-    rescue Errno::ETIMEDOUT
-      @logger.error "Timed out while writing request #{@correlation_id}"
-      raise
     end
 
     # Reads a response from the connection.

data/lib/kafka/consumer.rb

@@ -130,6 +130,10 @@ module Kafka
             return if !@running
           end
         end
+
+        # We may not have received any messages, but it's still a good idea to
+        # commit offsets if we've processed messages in the last set of batches.
+        @offset_manager.commit_offsets_if_necessary
       end
     end
 

data/lib/kafka/offset_manager.rb

@@ -11,7 +11,7 @@ module Kafka
       @processed_offsets = {}
       @default_offsets = {}
       @committed_offsets = nil
-      @last_commit = Time.at(0)
+      @last_commit = Time.now
     end
 
     def set_default_offset(topic, default_offset)
@@ -21,7 +21,8 @@ module Kafka
     def mark_as_processed(topic, partition, offset)
       @uncommitted_offsets += 1
       @processed_offsets[topic] ||= {}
-      @processed_offsets[topic][partition] = offset + 1
+      @processed_offsets[topic][partition] = offset
+      @logger.debug "Marking #{topic}/#{partition}:#{offset} as committed"
     end
 
     def next_offset_for(topic, partition)
@@ -32,20 +33,21 @@ module Kafka
       # A negative offset means that no offset has been committed, so we need to
       # resolve the default offset for the topic.
       if offset < 0
-        offset = @default_offsets.fetch(topic)
-        offset = @cluster.resolve_offset(topic, partition, offset)
-
-        # Make sure we commit this offset so that we don't repeat have to
-        # resolve the default offset every time.
-        mark_as_processed(topic, partition, offset - 1)
+        default_offset = @default_offsets.fetch(topic)
+        @cluster.resolve_offset(topic, partition, default_offset)
+      else
+        # The next offset is the last offset plus one.
+        offset + 1
       end
-
-      offset
     end
 
     def commit_offsets
       unless @processed_offsets.empty?
-        @logger.info "Committing offsets for #{@uncommitted_offsets} messages"
+        pretty_offsets = @processed_offsets.flat_map {|topic, partitions|
+          partitions.map {|partition, offset| "#{topic}/#{partition}:#{offset}" }
+        }.join(", ")
+
+        @logger.info "Committing offsets: #{pretty_offsets}"
 
         @group.commit_offsets(@processed_offsets)
 

data/lib/kafka/version.rb

@@ -1,3 +1,3 @@
 module Kafka
-  VERSION = "0.3.11"
+  VERSION = "0.3.12"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: ruby-kafka
 version: !ruby/object:Gem::Version
-  version: 0.3.11
+  version: 0.3.12
 platform: ruby
 authors:
 - Daniel Schierbeck
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2016-07-14 00:00:00.000000000 Z
+date: 2016-07-28 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler