ruby-kafka 0.6.0.beta3 → 0.6.0.beta4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: b0de3ad482f6130c080cc00b0dd7f4d862861838cfd6c762f6235defada9652f
-  data.tar.gz: c97c1432402dc5d78816ffb0c251e1c808c4ebaaad7609e3f8b7265d0661db8a
+  metadata.gz: 58773abdf15d690fb38890feb0788e3a47cb26312aa8022f869691dc6b56c3c6
+  data.tar.gz: e63116cf01fd6a140976179d63b9368b0a78855a3cbbb580ad84da74750b31a0
 SHA512:
-  metadata.gz: 647f4e5908073b08c597e90bc229c9962cf53121590297857caa641ebed7007331aa434238a719584b5c4ef66668a3a9fbe9a732105577b51f3fbc2539b6b3c9
-  data.tar.gz: dce76f84ed3ad8a010935321adc225f5ac757c33cfb39fee8d84c67c4f36bc4fdb053123f84638324695a4ec834d5765a0f44abd66eeb0c153762907f1737e55
+  metadata.gz: 421cb2a9813a72cb8b6821323427e2c2dd86467b9c5658b0a095650c87578dccc69bd48aa3c094a8c9827ea5bb18e752c24fe846c92540522c3bf2710ac41f07
+  data.tar.gz: cbef389f1c4b6336fcc957d27f18d0e37e76149a814fbfee39aeb7b8ec21c3aa39d5f4f3945ad953a70c67aea06273e096937da7be9e9daa3978ec7c84994fde

data/CHANGELOG.md

@@ -5,6 +5,8 @@ Changes and additions to the library will be listed here.
 ## Unreleased
 
 - Fetch messages asynchronously (#526).
+- Add support for exponential backoff in pauses (#566).
+- Instrument pause durations (#574).
 
 ## v0.5.5
 

data/lib/kafka/consumer.rb

@@ -1,6 +1,7 @@
 require "kafka/consumer_group"
 require "kafka/offset_manager"
 require "kafka/fetcher"
+require "kafka/pause"
 
 module Kafka
 
@@ -50,8 +51,11 @@ module Kafka
       @fetcher = fetcher
       @heartbeat = heartbeat
 
-      # A list of partitions that have been paused, per topic.
-      @paused_partitions = {}
+      @pauses = Hash.new {|h, k|
+        h[k] = Hash.new {|h2, k2|
+          h2[k2] = Pause.new
+        }
+      }
 
       # Whether or not the consumer is currently consuming messages.
       @running = false
@@ -113,16 +117,28 @@ module Kafka
     # the rest of the partitions to continue being processed.
     #
     # If the `timeout` argument is passed, the partition will automatically be
-    # resumed when the timeout expires.
+    # resumed when the timeout expires. If `exponential_backoff` is enabled, each
+    # subsequent pause will cause the timeout to double until a message from the
+    # partition has been successfully processed.
     #
     # @param topic [String]
     # @param partition [Integer]
-    # @param timeout [Integer] the number of seconds to pause the partition for,
+    # @param timeout [nil, Integer] the number of seconds to pause the partition for,
     #   or `nil` if the partition should not be automatically resumed.
+    # @param max_timeout [nil, Integer] the maximum number of seconds to pause for,
+    #   or `nil` if no maximum should be enforced.
+    # @param exponential_backoff [Boolean] whether to enable exponential backoff.
     # @return [nil]
-    def pause(topic, partition, timeout: nil)
-      @paused_partitions[topic] ||= {}
-      @paused_partitions[topic][partition] = timeout && Time.now + timeout
+    def pause(topic, partition, timeout: nil, max_timeout: nil, exponential_backoff: false)
+      if max_timeout && !exponential_backoff
+        raise ArgumentError, "`max_timeout` only makes sense when `exponential_backoff` is enabled"
+      end
+
+      pause_for(topic, partition).pause!(
+        timeout: timeout,
+        max_timeout: max_timeout,
+        exponential_backoff: exponential_backoff,
+      )
     end
 
     # Resume processing of a topic partition.
@@ -132,8 +148,7 @@ module Kafka
     # @param partition [Integer]
     # @return [nil]
     def resume(topic, partition)
-      paused_partitions = @paused_partitions.fetch(topic, {})
-      paused_partitions.delete(partition)
+      pause_for(topic, partition).resume!
 
       seek_to_next(topic, partition)
     end
@@ -145,16 +160,7 @@ module Kafka
     # @param partition [Integer]
     # @return [Boolean] true if the partition is paused, false otherwise.
     def paused?(topic, partition)
-      partitions = @paused_partitions.fetch(topic, {})
-
-      if partitions.key?(partition)
-        # Users can set an optional timeout, after which the partition is
-        # automatically resumed. When pausing, the timeout is translated to an
-        # absolute point in time.
-        timeout = partitions.fetch(partition)
-
-        timeout.nil? || Time.now < timeout
-      end
+      pause_for(topic, partition).paused?
     end
 
     # Fetches and enumerates the messages in the topics that the consumer group
@@ -230,6 +236,10 @@ module Kafka
 
             return if !@running
           end
+
+          # We've successfully processed a batch from the partition, so we can clear
+          # the pause.
+          pause_for(batch.topic, batch.partition).reset!
         end
 
         # We may not have received any messages, but it's still a good idea to
@@ -305,6 +315,10 @@ module Kafka
             end
 
             mark_message_as_processed(batch.messages.last) if automatically_mark_as_processed
+
+            # We've successfully processed a batch from the partition, so we can clear
+            # the pause.
+            pause_for(batch.topic, batch.partition).reset!
           end
 
           @offset_manager.commit_offsets_if_necessary
@@ -449,9 +463,15 @@ module Kafka
     end
 
     def resume_paused_partitions!
-      @paused_partitions.each do |topic, partitions|
-        partitions.keys.each do |partition|
-          unless paused?(topic, partition)
+      @pauses.each do |topic, partitions|
+        partitions.each do |partition, pause|
+          @instrumenter.instrument("pause_status.consumer", {
+            topic: topic,
+            partition: partition,
+            duration: pause.pause_duration,
+          })
+
+          if pause.paused? && pause.expired?
             @logger.info "Automatically resuming partition #{topic}/#{partition}, pause timeout expired"
             resume(topic, partition)
           end
@@ -494,5 +514,9 @@ module Kafka
 
       raise FetchError, e
     end
+
+    def pause_for(topic, partition)
+      @pauses[topic][partition]
+    end
   end
 end

data/lib/kafka/datadog.rb

@@ -217,6 +217,19 @@ module Kafka
         end
       end
 
+      def pause_status(event)
+        tags = {
+          client: event.payload.fetch(:client_id),
+          group_id: event.payload.fetch(:group_id),
+          topic: event.payload.fetch(:topic),
+          partition: event.payload.fetch(:partition),
+        }
+
+        duration = event.payload.fetch(:duration)
+
+        gauge("consumer.pause.duration", duration, tags: tags)
+      end
+
       attach_to "consumer.kafka"
     end
 

data/lib/kafka/pause.rb

@@ -0,0 +1,90 @@
+module Kafka
+  # Manages the pause state of a partition.
+  #
+  # The processing of messages in a partition can be paused, e.g. if there was
+  # an exception during processing. This could be caused by a downstream service
+  # not being available. A typical way of solving such an issue is to back off
+  # for a little while and then try again. In order to do that, _pause_ the
+  # partition.
+  class Pause
+    def initialize(clock: Time)
+      @clock = clock
+      @started_at = nil
+      @pauses = 0
+      @timeout = nil
+      @max_timeout = nil
+      @exponential_backoff = false
+    end
+
+    # Mark the partition as paused.
+    #
+    # If exponential backoff is enabled, each subsequent pause of a partition will
+    # cause a doubling of the actual timeout, i.e. for pause number _n_, the actual
+    # timeout will be _2^n * timeout_.
+    #
+    # Only when {#reset!} is called is this state cleared.
+    #
+    # @param timeout [nil, Integer] if specified, the partition will automatically
+    #   resume after this many seconds.
+    # @param exponential_backoff [Boolean] whether to enable exponential timeouts.
+    def pause!(timeout: nil, max_timeout: nil, exponential_backoff: false)
+      @started_at = @clock.now
+      @timeout = timeout
+      @max_timeout = max_timeout
+      @exponential_backoff = exponential_backoff
+      @pauses += 1
+    end
+
+    # Resumes the partition.
+    #
+    # The number of pauses is still retained, and if the partition is paused again
+    # it may be with an exponential backoff.
+    def resume!
+      @started_at = nil
+      @timeout = nil
+      @max_timeout = nil
+    end
+
+    # Whether the partition is currently paused.
+    def paused?
+      # This is nil if we're not currently paused.
+      return false if @started_at.nil?
+
+      # If no timeout is set we pause forever.
+      return true if @timeout.nil?
+
+      !expired?
+    end
+
+    def pause_duration
+      if paused?
+        Time.now - @started_at
+      else
+        0
+      end
+    end
+
+    # Whether the pause has expired.
+    def expired?
+      !@timeout.nil? && @clock.now >= ends_at
+    end
+
+    # Resets the pause state, ensuring that the next pause is not exponential.
+    def reset!
+      @pauses = 0
+    end
+
+    private
+
+    def ends_at
+      # Apply an exponential backoff to the timeout.
+      backoff_factor = @exponential_backoff ? 2**(@pauses - 1) : 1
+      timeout = backoff_factor * @timeout
+
+      # If set, don't allow a timeout longer than max_timeout.
+      timeout = @max_timeout if @max_timeout && timeout > @max_timeout
+
+      @started_at + timeout
+    end
+  end
+end

data/lib/kafka/statsd.rb

@@ -155,6 +155,17 @@ module Kafka
         end
       end
 
+      def pause_status(event)
+        client = event.payload.fetch(:client_id)
+        group_id = event.payload.fetch(:group_id)
+        topic = event.payload.fetch(:topic)
+        partition = event.payload.fetch(:partition)
+
+        duration = event.payload.fetch(:duration)
+
+        gauge("consumer.#{client}.#{group_id}.#{topic}.#{partition}.pause.duration", duration)
+      end
+
       attach_to "consumer.kafka"
     end
 

data/lib/kafka/version.rb

@@ -1,3 +1,3 @@
 module Kafka
-  VERSION = "0.6.0.beta3"
+  VERSION = "0.6.0.beta4"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: ruby-kafka
 version: !ruby/object:Gem::Version
-  version: 0.6.0.beta3
+  version: 0.6.0.beta4
 platform: ruby
 authors:
 - Daniel Schierbeck
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2018-04-24 00:00:00.000000000 Z
+date: 2018-04-26 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -331,6 +331,7 @@ files:
 - lib/kafka/message_buffer.rb
 - lib/kafka/offset_manager.rb
 - lib/kafka/partitioner.rb
+- lib/kafka/pause.rb
 - lib/kafka/pending_message.rb
 - lib/kafka/pending_message_queue.rb
 - lib/kafka/produce_operation.rb