ruby-kafka 0.5.0.beta5 → 0.5.0.beta6

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: '049f2fa2e6a04ff652cffe3ed630dbfe69e87009'
-  data.tar.gz: ed051c1dbb21aaac9360e58c74fca58d7706a76a
+  metadata.gz: 9893c7609209a4d58c3ac399cbbdce89c8df0124
+  data.tar.gz: 820e9d1bcd07f6d7c8de1d0ba1d286f184b64d8c
 SHA512:
-  metadata.gz: '08b01eb42a5082deac74cd5ff3136aa50ec16ac34c5ec4818d5a3722b9b46273b3601bd355073d3165c8917a9daf51356a43ff5ec97eceedbfcc6e0450018b79'
-  data.tar.gz: 16114b7e4abe72d82f3ae37cc0cfba4cbfdf798073adc04f1eaea892ac64eed2d96b66f093df1e2b3f1c6ac40f7a5b87146efadbed888d46a32abc4113e1f289
+  metadata.gz: baf63463ad738c02a1ad07d60db31ff6546a3cc2e97ad411ea02b31dbad7654b003ae61836bc67f10db30650096e278a3e621f3456a44aac4a0c7f574f80b728
+  data.tar.gz: 54be82a578abcf241b387f70cd2a07b2ce069f2e9d80e0c24104780af4b1ae8b55dccf8f5962ac31456d19bd2770f2a2af6911fcd8ef7200f501633c36fe81be

data/examples/consumer-group.rb

@@ -20,6 +20,7 @@ consumer = kafka.consumer(group_id: "test")
 consumer.subscribe(topic)
 
 trap("TERM") { consumer.stop }
+trap("INT") { consumer.stop }
 
 consumer.each_message do |message|
   puts message.value

data/lib/kafka/connection.rb

@@ -109,10 +109,10 @@ module Kafka
 
         response
       end
-    rescue Errno::EPIPE, Errno::ECONNRESET, Errno::ETIMEDOUT, Errno::ESHUTDOWN, EOFError => e
+    rescue SystemCallError, EOFError => e
       close
 
-      raise ConnectionError, "Connection error: #{e}"
+      raise ConnectionError, "Connection error #{e.class}: #{e}"
     end
 
     private

data/lib/kafka/consumer.rb

@@ -57,6 +57,17 @@ module Kafka
 
       # The maximum number of bytes to fetch from a single partition, by topic.
       @max_bytes = {}
+
+      # Hash containing offsets for each topic and partition that has the
+      # automatically_mark_as_processed feature disabled. Offset manager is only active
+      # when everything is suppose to happen automatically. Otherwise we need to keep track of the
+      # offset manually in memory for all the time
+      # The key structure for this equals an array with topic and partition [topic, partition]
+      # The value is equal to the offset of the last message we've received
+      # @note It won't be updated in case user marks message as processed, because for the case
+      #   when user commits message other than last in a batch, this would make ruby-kafka refetch
+      #   some already consumed messages
+      @current_offsets = Hash.new { |h, k| h[k] = {} }
     end
 
     # Subscribes the consumer to a topic.
@@ -181,7 +192,11 @@ module Kafka
     # @return [nil]
     def each_message(min_bytes: 1, max_wait_time: 1, automatically_mark_as_processed: true)
       consumer_loop do
-        batches = fetch_batches(min_bytes: min_bytes, max_wait_time: max_wait_time)
+        batches = fetch_batches(
+          min_bytes: min_bytes,
+          max_wait_time: max_wait_time,
+          automatically_mark_as_processed: automatically_mark_as_processed
+        )
 
         batches.each do |batch|
           batch.messages.each do |message|
@@ -197,6 +212,7 @@ module Kafka
 
               begin
                 yield message
+                @current_offsets[message.topic][message.partition] = message.offset
               rescue => e
                 location = "#{message.topic}/#{message.partition} at offset #{message.offset}"
                 backtrace = e.backtrace.join("\n")
@@ -247,7 +263,11 @@ module Kafka
     # @return [nil]
     def each_batch(min_bytes: 1, max_wait_time: 1, automatically_mark_as_processed: true)
       consumer_loop do
-        batches = fetch_batches(min_bytes: min_bytes, max_wait_time: max_wait_time)
+        batches = fetch_batches(
+          min_bytes: min_bytes,
+          max_wait_time: max_wait_time,
+          automatically_mark_as_processed: automatically_mark_as_processed
+        )
 
         batches.each do |batch|
           unless batch.empty?
@@ -262,6 +282,7 @@ module Kafka
 
               begin
                 yield batch
+                @current_offsets[batch.topic][batch.partition] = batch.last_offset
               rescue => e
                 offset_range = (batch.first_offset..batch.last_offset)
                 location = "#{batch.topic}/#{batch.partition} in offset range #{offset_range}"
@@ -379,7 +400,7 @@ module Kafka
       end
     end
 
-    def fetch_batches(min_bytes:, max_wait_time:)
+    def fetch_batches(min_bytes:, max_wait_time:, automatically_mark_as_processed:)
       join_group unless @group.member?
 
       subscribed_partitions = @group.subscribed_partitions
@@ -395,7 +416,18 @@ module Kafka
 
       subscribed_partitions.each do |topic, partitions|
         partitions.each do |partition|
-          offset = @offset_manager.next_offset_for(topic, partition)
+          if automatically_mark_as_processed
+            offset = @offset_manager.next_offset_for(topic, partition)
+          else
+            # When automatic marking is off, the first poll needs to be based on the last committed
+            # offset from Kafka, that's why we fallback in case of nil (it may not be 0)
+            if @current_offsets[topic].key?(partition)
+              offset = @current_offsets[topic][partition] + 1
+            else
+              offset = @offset_manager.next_offset_for(topic, partition)
+            end
+          end
+
           max_bytes = @max_bytes.fetch(topic)
 
           if paused?(topic, partition)

data/lib/kafka/consumer_group.rb

@@ -48,6 +48,7 @@ module Kafka
     rescue ConnectionError
       @logger.error "Connection error while trying to join group `#{@group_id}`; retrying..."
       sleep 1
+      @cluster.mark_as_stale!
       @coordinator = nil
       retry
     end

data/lib/kafka/socket_with_timeout.rb

@@ -24,9 +24,6 @@ module Kafka
 
       @timeout = timeout
 
-      # This pipe is used to cancel IO.select calls when sockets are closed.
-      @cancel_reader, @cancel_writer = IO.pipe
-
       @socket = Socket.new(Socket.const_get(addr[0][0]), Socket::SOCK_STREAM, 0)
       @socket.setsockopt(Socket::IPPROTO_TCP, Socket::TCP_NODELAY, 1)
 
@@ -60,20 +57,11 @@ module Kafka
     # @raise [Errno::ETIMEDOUT] if the timeout is exceeded.
     # @return [String] the data that was read from the socket.
     def read(num_bytes)
-      rs, _, _ = IO.select([@socket, @cancel_reader], nil, nil, @timeout)
-
-      # The read timed out.
-      raise Errno::ETIMEDOUT if rs.nil?
-
-      # The socket has been closed.
-      raise Errno::ECONNABORTED if rs.include?(@cancel_reader)
+      unless IO.select([@socket], nil, nil, @timeout)
+        raise Errno::ETIMEDOUT
+      end
 
       @socket.read(num_bytes)
-    rescue Errno::EBADF
-      # We'll get EBADF if `select` is called with a closed socket, or
-      # if it's closed in the middle of things.
-      raise Errno::ESHUTDOWN if @socket.closed?
-      raise
     rescue IO::EAGAINWaitReadable
       retry
     end
@@ -92,7 +80,6 @@ module Kafka
     end
 
     def close
-      @cancel_writer.puts
       @socket.close
     end
 

data/lib/kafka/version.rb

@@ -1,3 +1,3 @@
 module Kafka
-  VERSION = "0.5.0.beta5"
+  VERSION = "0.5.0.beta6"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: ruby-kafka
 version: !ruby/object:Gem::Version
-  version: 0.5.0.beta5
+  version: 0.5.0.beta6
 platform: ruby
 authors:
 - Daniel Schierbeck
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2017-10-26 00:00:00.000000000 Z
+date: 2017-10-27 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler