ruby-kafka 0.7.4 → 0.7.5.beta1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 5f4441fd6947e65aaf63f57a7248000af3b1140eca6b33ed128cc51f24fa415b
-  data.tar.gz: ea4de0c4d19991767378cc4672bd7da47a3847b2931edd900d3d2f8e463bca0a
+  metadata.gz: '0619c0e62d82ad058d2b8890e011725bbfd50f472f473c5ea460bcbdde689031'
+  data.tar.gz: 5e9a31e162be2adcb7898fe444916097faa24ea0721d794b402f110fdf37022d
 SHA512:
-  metadata.gz: ffd480a08e25377139e65aab9aeb0d8d85056f033142104ae6593b7b39bafbd9db0de3b4487224a608550b567b072f32b30e6d61c2f2bbcfdc47423d3aece3f8
-  data.tar.gz: b0acbf1f1024d50d1169a68316a67041a837965a76d79c0143228b5e142cdd412b4b9813257e6a40a400bf5a89a1fb70f246932ffb9f280e243848a73a66653c
+  metadata.gz: 41fa3e9d58f03213c0970fd4b4f94ba2479db67776f8b543f1bf65ea4a33078649afac066ff2c87d0269001ed3184d0cad78621cb12c0c2871c45cc7b581b160
+  data.tar.gz: 73c1347bdc0c4ffa354cea563b4e421993222de0908366a32daeea0bca280d5fc01e37b19dd3c5b7e3c22ce329bad643497c295be8b3536aa5162307346a32eb

data/CHANGELOG.md

@@ -3,6 +3,11 @@
 Changes and additions to the library will be listed here.
 
 ## Unreleased
+- Distribute partitions across consumer groups when there are few partitions per topic (#681)
+- Fix an issue where a consumer would fail to fetch any messages (#689)
+- Instrumentation for heartbeat event
+- Synchronously stop the fetcher to prevent race condition when processing commands
+- Instrument batch fetching (#694)
 
 ## 0.7.4
 - Fix wrong encoding calculation that leads to message corruption (#682, #680).

data/lib/kafka/client.rb

@@ -374,6 +374,7 @@ module Kafka
       heartbeat = Heartbeat.new(
         group: group,
         interval: heartbeat_interval,
+        instrumenter: instrumenter
       )
 
       Consumer.new(

data/lib/kafka/datadog.rb

@@ -167,7 +167,6 @@ module Kafka
 
       def process_batch(event)
         offset = event.payload.fetch(:last_offset)
-        lag = event.payload.fetch(:offset_lag)
         messages = event.payload.fetch(:message_count)
 
         tags = {
@@ -185,6 +184,20 @@ module Kafka
         end
 
         gauge("consumer.offset", offset, tags: tags)
+      end
+
+      def fetch_batch(event)
+        lag = event.payload.fetch(:offset_lag)
+        batch_size = event.payload.fetch(:message_count)
+
+        tags = {
+          client: event.payload.fetch(:client_id),
+          group_id: event.payload.fetch(:group_id),
+          topic: event.payload.fetch(:topic),
+          partition: event.payload.fetch(:partition),
+        }
+
+        histogram("consumer.batch_size", batch_size, tags: tags)
         gauge("consumer.lag", lag, tags: tags)
       end
 

data/lib/kafka/fetcher.rb

@@ -49,6 +49,8 @@ module Kafka
     def start
       return if @running
 
+      @running = true
+
       @thread = Thread.new do
         while @running
           loop
@@ -56,13 +58,12 @@ module Kafka
         @logger.info "Fetcher thread exited."
       end
       @thread.abort_on_exception = true
-
-      @running = true
     end
 
     def stop
       return unless @running
       @commands << [:stop, []]
+      @thread.join
     end
 
     def reset

data/lib/kafka/heartbeat.rb

@@ -2,15 +2,20 @@
 
 module Kafka
   class Heartbeat
-    def initialize(group:, interval:)
+    def initialize(group:, interval:, instrumenter:)
       @group = group
       @interval = interval
       @last_heartbeat = Time.now
+      @instrumenter = instrumenter
     end
 
     def trigger!
-      @group.heartbeat
-      @last_heartbeat = Time.now
+      @instrumenter.instrument('heartbeat.consumer',
+                               group_id: @group.group_id,
+                               topic_partitions: @group.assigned_partitions) do
+        @group.heartbeat
+        @last_heartbeat = Time.now
+      end
     end
 
     def trigger

data/lib/kafka/round_robin_assignment_strategy.rb

@@ -24,20 +24,23 @@ module Kafka
         group_assignment[member_id] = Protocol::MemberAssignment.new
       end
 
-      topics.each do |topic|
+      topic_partitions = topics.flat_map do |topic|
         begin
           partitions = @cluster.partitions_for(topic).map(&:partition_id)
         rescue UnknownTopicOrPartition
           raise UnknownTopicOrPartition, "unknown topic #{topic}"
         end
+        Array.new(partitions.count) { topic }.zip(partitions)
+      end
 
-        partitions_per_member = partitions.group_by {|partition_id|
-          partition_id % members.count
-        }.values
+      partitions_per_member = topic_partitions.group_by.with_index do |_, index|
+        index % members.count
+      end.values
 
-        members.zip(partitions_per_member).each do |member_id, member_partitions|
-          unless member_partitions.nil?
-            group_assignment[member_id].assign(topic, member_partitions)
+      members.zip(partitions_per_member).each do |member_id, member_partitions|
+        unless member_partitions.nil?
+          member_partitions.each do |topic, partition|
+            group_assignment[member_id].assign(topic, [partition])
           end
         end
       end

data/lib/kafka/statsd.rb

@@ -107,7 +107,6 @@ module Kafka
       end
 
       def process_batch(event)
-        lag = event.payload.fetch(:offset_lag)
         messages = event.payload.fetch(:message_count)
         client = event.payload.fetch(:client_id)
         group_id = event.payload.fetch(:group_id)
@@ -120,7 +119,17 @@ module Kafka
           timing("consumer.#{client}.#{group_id}.#{topic}.#{partition}.process_batch.latency", event.duration)
           count("consumer.#{client}.#{group_id}.#{topic}.#{partition}.messages", messages)
         end
+      end
+
+      def fetch_batch(event)
+        lag = event.payload.fetch(:offset_lag)
+        batch_size = event.payload.fetch(:message_count)
+        client = event.payload.fetch(:client_id)
+        group_id = event.payload.fetch(:group_id)
+        topic = event.payload.fetch(:topic)
+        partition = event.payload.fetch(:partition)
 
+        count("consumer.#{client}.#{group_id}.#{topic}.#{partition}.batch_size", batch_size)
         gauge("consumer.#{client}.#{group_id}.#{topic}.#{partition}.lag", lag)
       end
 

data/lib/kafka/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Kafka
-  VERSION = "0.7.4"
+  VERSION = "0.7.5.beta1"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: ruby-kafka
 version: !ruby/object:Gem::Version
-  version: 0.7.4
+  version: 0.7.5.beta1
 platform: ruby
 authors:
 - Daniel Schierbeck
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2018-11-13 00:00:00.000000000 Z
+date: 2018-12-04 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: digest-crc
@@ -446,9 +446,9 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: 2.1.0
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">="
+  - - ">"
     - !ruby/object:Gem::Version
-      version: '0'
+      version: 1.3.1
 requirements: []
 rubyforge_project: 
 rubygems_version: 2.7.6