ruby-kafka 0.5.1.beta1 → 0.5.1.beta2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 872944b4c0fb6b670fac704b827cb2ffa713e8ae
-  data.tar.gz: 17507f698aa6eb93007164d72cb24d2f79d9f2d8
+  metadata.gz: aee148ccfa605884f691d00ae8c82432a6d6bca1
+  data.tar.gz: 86f831463bc276e54a72d67e64aff5847ea41ec7
 SHA512:
-  metadata.gz: a5ac176d6dfa6db9ac431cf6a3c7bb1eda4183ef2a0f19864d0904ebb3024861b92ef98083fdf06512ab85c5545c2b93db10e83d8539858817953e9fc2689a4b
-  data.tar.gz: d72d3db50e98dd12840494c0691943d2f332dca639caf0cda65e750ce3109bd03ad194f720129a887100242f19ef52d2a710dd290fa02ec43eafa3f8fa025c2e
+  metadata.gz: d28036dc879a07b3580db2cece96241b5002dca3bff5e7de109cbe699e15e893a4e1ddd13283ead236a5eca9e9b25db1e0bd7de02d5e4f8c5ace24e148c89c10
+  data.tar.gz: f3c969dfcd6cb2dca36a98a1cd80cfd20f63780e9b762fe96d6f8b39ee98ad376d8b7e641f97d5d15a647b2d63688168ec84636386d302ccbbcfbb00d1f4b093

data/CHANGELOG.md

@@ -11,6 +11,7 @@ Requires Kafka 0.10.1+ due to usage of a few new APIs.
 - Make `#deliver_message` more resilient using retries and backoff.
 - Add support for SASL SCRAM authentication (#465).
 - Refactor and simplify SASL code.
+- Fix issue when a consumer resets a partition to its default offset.
 
 ## v0.5.0
 

data/lib/kafka/connection.rb

@@ -89,11 +89,11 @@ module Kafka
         response_size: 0,
       }
 
+      raise IdleConnection if idle?
+
       @instrumenter.instrument("request.connection", notification) do
         open unless open?
 
-        raise IdleConnection if idle?
-
         @correlation_id += 1
 
         @logger.debug "Sending #{api_name} API request #{@correlation_id} to #{to_s}"

data/lib/kafka/consumer.rb

@@ -209,6 +209,7 @@ module Kafka
                 partition: message.partition,
                 offset: message.offset,
                 offset_lag: batch.highwater_mark_offset - message.offset - 1,
+                create_time: message.create_time,
                 key: message.key,
                 value: message.value,
               )

data/lib/kafka/datadog.rb

@@ -122,7 +122,9 @@ module Kafka
 
     class ConsumerSubscriber < StatsdSubscriber
       def process_message(event)
-        lag = event.payload.fetch(:offset_lag)
+        offset_lag = event.payload.fetch(:offset_lag)
+        create_time = event.payload.fetch(:create_time)
+        time_lag = create_time && ((Time.now - create_time) * 1000).to_i
 
         tags = {
           client: event.payload.fetch(:client_id),
@@ -138,7 +140,12 @@ module Kafka
           increment("consumer.messages", tags: tags)
         end
 
-        gauge("consumer.lag", lag, tags: tags)
+        gauge("consumer.lag", offset_lag, tags: tags)
+
+        # Not all messages have timestamps.
+        if time_lag
+          gauge("consumer.time_lag", time_lag, tags: tags)
+        end
       end
 
       def process_batch(event)

data/lib/kafka/offset_manager.rb

@@ -66,7 +66,9 @@ module Kafka
       # Remove any cached offset, in case things have changed broker-side.
       clear_resolved_offset(topic)
 
-      seek_to(topic, partition, -1)
+      offset = resolve_offset(topic, partition)
+
+      seek_to(topic, partition, offset)
     end
 
     # Move the consumer's position in the partition to the specified offset.

data/lib/kafka/statsd.rb

@@ -80,12 +80,15 @@ module Kafka
 
     class ConsumerSubscriber < StatsdSubscriber
       def process_message(event)
-        lag = event.payload.fetch(:offset_lag)
+        offset_lag = event.payload.fetch(:offset_lag)
+        create_time = event.payload.fetch(:create_time)
         client = event.payload.fetch(:client_id)
         group_id = event.payload.fetch(:group_id)
         topic = event.payload.fetch(:topic)
         partition = event.payload.fetch(:partition)
 
+        time_lag = create_time && ((Time.now - create_time) * 1000).to_i
+
         if event.payload.key?(:exception)
           increment("consumer.#{client}.#{group_id}.#{topic}.#{partition}.process_message.errors")
         else
@@ -93,7 +96,12 @@ module Kafka
           increment("consumer.#{client}.#{group_id}.#{topic}.#{partition}.messages")
         end
 
-        gauge("consumer.#{client}.#{group_id}.#{topic}.#{partition}.lag", lag)
+        gauge("consumer.#{client}.#{group_id}.#{topic}.#{partition}.lag", offset_lag)
+
+        # Not all messages have timestamps.
+        if time_lag
+          gauge("consumer.#{client}.#{group_id}.#{topic}.#{partition}.time_lag", time_lag)
+        end
       end
 
       def process_batch(event)

data/lib/kafka/version.rb

@@ -1,3 +1,3 @@
 module Kafka
-  VERSION = "0.5.1.beta1"
+  VERSION = "0.5.1.beta2"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: ruby-kafka
 version: !ruby/object:Gem::Version
-  version: 0.5.1.beta1
+  version: 0.5.1.beta2
 platform: ruby
 authors:
 - Daniel Schierbeck
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2017-11-09 00:00:00.000000000 Z
+date: 2017-11-10 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler