ruby-kafka 0.3.12 → 0.3.13.beta1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 29c5259a5c2650e5a9eec0c513aa997a90834f9d
-  data.tar.gz: b4eb6a8de42d27f2a3b215cb690d7c803f706bad
+  metadata.gz: e6c48d1b7996a28caddd510f0daf032fdfca84fe
+  data.tar.gz: 302d2211024a2fdb904947c97cb0c258dc8583eb
 SHA512:
-  metadata.gz: 43c219f2e1956264a864a9ce41e75f2a3b46ac6cc5c248cdc99a28a15b2871ed8fdb1554422797c9ca8e913996874981aadf3f35f23fea30110e616bfc3013e0
-  data.tar.gz: ad849644e4f75761164f41ffad9d69e67e0ac71eb2796d222afc224f1745bc9199491165d38fd346ebf0b8f7da6ea25e60fb9fe7a853b12bc1699f069331d8d5
+  metadata.gz: d5524a95270673acb7e7e0ba2d6907eb774c715fb467925a052a633c1ab0e76de0fe5a16806ef3a7b95bbc29d54bd832a2b0a43d496753f4b5f1a162e4f91dde
+  data.tar.gz: 34f6417d7d0fdb987f1c6fdf69595a9de413ad010ca391f1d5d5fa3632fe781fe9514ae3568d20c2ed120887e288bb42848ebb22ab42125215d50ca16c0523da

data/CHANGELOG.md

@@ -4,6 +4,11 @@ Changes and additions to the library will be listed here.
 
 ## Unreleased
 
+## v0.3.13.beta1
+
+- Minimize the number of times messages are reprocessed after a consumer group resync.
+- Improve instrumentation of the async producer.
+
 ## v0.3.12
 
 - Fix a bug in the consumer.

data/README.md

@@ -26,8 +26,9 @@ Although parts of this library work with Kafka 0.8 – specifically, the Produce
     1. [Consumer Groups](#consumer-groups)
     2. [Consumer Checkpointing](#consumer-checkpointing)
     3. [Topic Subscriptions](#topic-subscriptions)
-    4. [Consuming Messages in Batches](#consuming-messages-in-batches)
-    5. [Balancing Throughput and Latency](#balancing-throughput-and-latency)
+    4. [Shutting Down a Consumer](#shutting-down-a-consumer)
+    5. [Consuming Messages in Batches](#consuming-messages-in-batches)
+    6. [Balancing Throughput and Latency](#balancing-throughput-and-latency)
   4. [Thread Safety](#thread-safety)
   5. [Logging](#logging)
   6. [Instrumentation](#instrumentation)
@@ -526,6 +527,23 @@ consumer.subscribe("notifications", start_from_beginning: false)
 Once the consumer group has checkpointed its progress in the topic's partitions, the consumers will always start from the checkpointed offsets, regardless of `start_from_beginning`. As such, this setting only applies when the consumer initially starts consuming from a topic.
 
 
+#### Shutting Down a Consumer
+
+In order to shut down a running consumer process cleanly, call `#stop` on it. A common pattern is to trap a process signal and initiate the shutdown from there:
+
+```ruby
+consumer = kafka.consumer(...)
+
+# The consumer can be stopped from the command line by executing
+# `kill -s QUIT <process-id>`.
+trap("QUIT") { consumer.stop }
+
+consumer.each_message do |message|
+  ...
+end
+```
+
+
 #### Consuming Messages in Batches
 
 Sometimes it is easier to deal with messages in batches rather than individually. A _batch_ is a sequence of one or more Kafka messages that all belong to the same topic and partition. One common reason to want to use batches is when some external system has a batch or transactional API.

data/lib/kafka/async_producer.rb

@@ -69,7 +69,7 @@ module Kafka
     # @param delivery_interval [Integer] if greater than zero, the number of
     #   seconds between automatic message deliveries.
     #
-    def initialize(sync_producer:, max_queue_size: 1000, delivery_threshold: 0, delivery_interval: 0, instrumenter:)
+    def initialize(sync_producer:, max_queue_size: 1000, delivery_threshold: 0, delivery_interval: 0, instrumenter:, logger:)
       raise ArgumentError unless max_queue_size > 0
       raise ArgumentError unless delivery_threshold >= 0
       raise ArgumentError unless delivery_interval >= 0
@@ -77,6 +77,7 @@ module Kafka
       @queue = Queue.new
       @max_queue_size = max_queue_size
       @instrumenter = instrumenter
+      @logger = logger
 
       @worker = Worker.new(
         queue: @queue,
@@ -102,6 +103,12 @@ module Kafka
       args = [value, **options.merge(topic: topic)]
       @queue << [:produce, args]
 
+      @instrumenter.instrument("enqueue_message.async_producer", {
+        topic: topic,
+        queue_size: @queue.size,
+        max_queue_size: @max_queue_size,
+      })
+
       nil
     end
 
@@ -122,6 +129,7 @@ module Kafka
     # @see Kafka::Producer#shutdown
     # @return [nil]
     def shutdown
+      @timer_thread && @timer_thread.exit
       @queue << [:shutdown, nil]
       @worker_thread && @worker_thread.join
 
@@ -149,6 +157,8 @@ module Kafka
         topic: topic,
       })
 
+      @logger.error "Buffer overflow: failed to enqueue message for #{topic}"
+
       raise BufferOverflow
     end
 

data/lib/kafka/client.rb

@@ -200,6 +200,7 @@ module Kafka
         delivery_threshold: delivery_threshold,
         max_queue_size: max_queue_size,
         instrumenter: @instrumenter,
+        logger: @logger,
       )
     end
 

data/lib/kafka/consumer.rb

@@ -211,8 +211,22 @@ module Kafka
     end
 
     def join_group
-      @offset_manager.clear_offsets
+      old_generation_id = @group.generation_id
+
       @group.join
+
+      if old_generation_id && @group.generation_id != old_generation_id + 1
+        # We've been out of the group for at least an entire generation, no
+        # sense in trying to hold on to offset data
+        @offset_manager.clear_offsets
+      else
+        # After rejoining the group we may have been assigned a new set of
+        # partitions. Keeping the old offset commits around forever would risk
+        # having the consumer go back and reprocess messages if it's assigned
+        # a partition it used to be assigned to way back. For that reason, we
+        # only keep commits for the partitions that we're still assigned.
+        @offset_manager.clear_offsets_excluding(@group.assigned_partitions)
+      end
     end
 
     def fetch_batches(min_bytes:, max_wait_time:)

data/lib/kafka/consumer_group.rb

@@ -3,7 +3,7 @@ require "kafka/round_robin_assignment_strategy"
 
 module Kafka
   class ConsumerGroup
-    attr_reader :assigned_partitions
+    attr_reader :assigned_partitions, :generation_id
 
     def initialize(cluster:, logger:, group_id:, session_timeout:)
       @cluster = cluster

data/lib/kafka/datadog.rb

@@ -206,5 +206,27 @@ module Kafka
 
       attach_to "producer.kafka"
     end
+
+    class AsyncProducerSubscriber < StatsdSubscriber
+      def enqueue_message(event)
+        client = event.payload.fetch(:client_id)
+        topic = event.payload.fetch(:topic)
+        queue_size = event.payload.fetch(:queue_size)
+        max_queue_size = event.payload.fetch(:max_queue_size)
+        queue_fill_ratio = queue_size.to_f / max_queue_size.to_f
+
+        tags = {
+          client: client,
+        }
+
+        # This gets us the avg/max queue size per producer.
+        histogram("producer.queue.size", queue_size, tags: tags)
+
+        # This gets us the avg/max queue fill ratio per producer.
+        histogram("producer.queue.fill_ratio", queue_fill_ratio, tags: tags)
+      end
+
+      attach_to "async_producer.kafka"
+    end
   end
 end

data/lib/kafka/offset_manager.rb

@@ -59,14 +59,27 @@ module Kafka
     end
 
     def commit_offsets_if_necessary
-      if seconds_since_last_commit >= @commit_interval || commit_threshold_reached?
+      if commit_timeout_reached? || commit_threshold_reached?
         commit_offsets
       end
     end
 
     def clear_offsets
-      @uncommitted_offsets = 0
       @processed_offsets.clear
+
+      # Clear the cached commits from the brokers.
+      @committed_offsets = nil
+    end
+
+    def clear_offsets_excluding(excluded)
+      # Clear all offsets that aren't in `excluded`.
+      @processed_offsets.each do |topic, partitions|
+        partitions.keep_if do |partition, _|
+          excluded.fetch(topic, []).include?(partition)
+        end
+      end
+
+      # Clear the cached commits from the brokers.
       @committed_offsets = nil
     end
 
@@ -81,6 +94,10 @@ module Kafka
       @committed_offsets.offset_for(topic, partition)
     end
 
+    def commit_timeout_reached?
+      @commit_interval != 0 && seconds_since_last_commit >= @commit_interval
+    end
+
     def commit_threshold_reached?
       @commit_threshold != 0 && @uncommitted_offsets >= @commit_threshold
     end

data/lib/kafka/version.rb

@@ -1,3 +1,3 @@
 module Kafka
-  VERSION = "0.3.12"
+  VERSION = "0.3.13.beta1"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: ruby-kafka
 version: !ruby/object:Gem::Version
-  version: 0.3.12
+  version: 0.3.13.beta1
 platform: ruby
 authors:
 - Daniel Schierbeck
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2016-07-28 00:00:00.000000000 Z
+date: 2016-08-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -303,9 +303,9 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: 2.1.0
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">="
+  - - ">"
     - !ruby/object:Gem::Version
-      version: '0'
+      version: 1.3.1
 requirements: []
 rubyforge_project: 
 rubygems_version: 2.4.5.1