ruby-kafka 0.5.0.beta2 → 0.5.0.beta3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: d91d5937445f7d3e85fd36801ac52d91f19fb192
-  data.tar.gz: b7baea6a96bd70ba6ec9ff4ad42e206c22e53c7e
+  metadata.gz: 473deb1157c274182f15656009eb4f181f764888
+  data.tar.gz: 634892fc04c6e91bbe703547d7a70e037e85fbcb
 SHA512:
-  metadata.gz: 9e0dac8db89b3002a49538f0daf34c7df3c821a22eae2d1615d4bbc97c25f2b4312a0f7cab1c0682a96079213ee53e85e64784a01096d2ca252c9726635d9236
-  data.tar.gz: ef97106a320e4c933189a935f42f8d383dc0619176cee8df5f8b30c060adee157de9b3293e522795f2f276fd063140f7e2284db4e1ee3343c6104eb0773b0468
+  metadata.gz: 464247f865b176e319700a6b484146f025d7e931ea9af58b01dc2ac31660bc448741bdeb61ac0d9031d6d13aaed0c7dcc6a7e3d2b8b786c96c3a83bf50b77f17
+  data.tar.gz: '09b1db2e9955325ad9c07453ef51b34cb2e5f0455289668bc3c04572024db35b135715360e139b320efee5a7d32ed71c880d7f1fb8e827135509bce6a071e256'

data/lib/kafka/consumer.rb

@@ -216,6 +216,8 @@ module Kafka
 
         # We may not have received any messages, but it's still a good idea to
         # commit offsets if we've processed messages in the last set of batches.
+        # This also ensures the offsets are retained if we haven't read any messages
+        # since the offset retention period has elapsed.
         @offset_manager.commit_offsets_if_necessary
       end
     end
@@ -279,6 +281,12 @@ module Kafka
 
           return if !@running
         end
+
+        # We may not have received any messages, but it's still a good idea to
+        # commit offsets if we've processed messages in the last set of batches.
+        # This also ensures the offsets are retained if we haven't read any messages
+        # since the offset retention period has elapsed.
+        @offset_manager.commit_offsets_if_necessary
       end
     end
 

data/lib/kafka/protocol/message.rb

@@ -66,13 +66,21 @@ module Kafka
 
         crc = message_decoder.int32
         magic_byte = message_decoder.int8
+        attributes = message_decoder.int8
 
-        unless magic_byte == MAGIC_BYTE
+        # The magic byte indicates the message format version. There are situations
+        # where an old message format can be returned from a newer version of Kafka,
+        # because old messages are not necessarily rewritten on upgrades.
+        case magic_byte
+        when 0
+          # No timestamp in the pre-0.10 message format.
+          timestamp = nil
+        when 1
+          timestamp = message_decoder.int64
+        else
           raise Kafka::Error, "Invalid magic byte: #{magic_byte}"
         end
 
-        attributes = message_decoder.int8
-        timestamp = message_decoder.int64
         key = message_decoder.bytes
         value = message_decoder.bytes
 
@@ -80,7 +88,7 @@ module Kafka
         # attributes.
         codec_id = attributes & 0b111
 
-        new(key: key, value: value, codec_id: codec_id, offset: offset, create_time: Time.at(timestamp/1000.0))
+        new(key: key, value: value, codec_id: codec_id, offset: offset, create_time: Time.at(timestamp / 1000.0))
       end
 
       private
@@ -104,7 +112,7 @@ module Kafka
 
         encoder.write_int8(MAGIC_BYTE)
         encoder.write_int8(@codec_id)
-        encoder.write_int64((@create_time.to_f*1000).to_i)
+        encoder.write_int64((@create_time.to_f * 1000).to_i)
         encoder.write_bytes(@key)
         encoder.write_bytes(@value)
 

data/lib/kafka/protocol/produce_response.rb

@@ -45,7 +45,7 @@ module Kafka
               partition: decoder.int32,
               error_code: decoder.int16,
               offset: decoder.int64,
-              timestamp: Time.at(decoder.int64/1000.0),
+              timestamp: Time.at(decoder.int64 / 1000.0),
             )
           end
 

data/lib/kafka/statsd.rb

@@ -97,6 +97,7 @@ module Kafka
       end
 
       def process_batch(event)
+        lag = event.payload.fetch(:offset_lag)
         messages = event.payload.fetch(:message_count)
         client = event.payload.fetch(:client_id)
         group_id = event.payload.fetch(:group_id)
@@ -109,6 +110,8 @@ module Kafka
           timing("consumer.#{client}.#{group_id}.#{topic}.#{partition}.process_batch.latency", event.duration)
           count("consumer.#{client}.#{group_id}.#{topic}.#{partition}.messages", messages)
         end
+
+        gauge("consumer.#{client}.#{group_id}.#{topic}.#{partition}.lag", lag)
       end
 
       def join_group(event)

data/lib/kafka/version.rb

@@ -1,3 +1,3 @@
 module Kafka
-  VERSION = "0.5.0.beta2"
+  VERSION = "0.5.0.beta3"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: ruby-kafka
 version: !ruby/object:Gem::Version
-  version: 0.5.0.beta2
+  version: 0.5.0.beta3
 platform: ruby
 authors:
 - Daniel Schierbeck
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2017-10-23 00:00:00.000000000 Z
+date: 2017-10-24 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler