ruby-kafka 0.4.2 → 0.4.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 5c001c97f73b001e170f23d7fd970781aa6bb196
-  data.tar.gz: 39f0b03b56ad97eee788fac63ec467599ad8cd8e
+  metadata.gz: 6c934181fd72d8f78eb9b4438b7297a6a71a51eb
+  data.tar.gz: e3a970abf6839ab03bdd34caeb53dfe9fe91666d
 SHA512:
-  metadata.gz: 6f9312f25ab7493d5803b581b22af270399f0f5ff2fbf4012da1f4a30525c430fb1fa21085cb4042aa4b59074a0abe3af9038ee6e97f0b48770baa50eaab6393
-  data.tar.gz: 7523013dbc08d7a193faf6b2b2419a6704cb4b7918f9ed993545b534e356084a765a1c8cfba937a55d7236c26985e3cf252cbe50df6c3c98a45e1bc47e32caad
+  metadata.gz: 266b2624a721b56c991797a6848531e00675274a24241db59de6d8649ac70ba41799b40d39efbfd8c90a1b467aa142fe68526533e23933c606fbaac7b9ae780d
+  data.tar.gz: 1d41e52b87201e803a3edf62a56a0086c1f8aed86dc255a9c73eee5b295acc27113e7d82adba4558666142509664610f5531c654049e596c546c89e468f153b5

data/.gitignore

@@ -3,6 +3,7 @@
 /_yardoc/
 /coverage/
 /doc/
+/Gemfile.lock
 /pkg/
 /spec/reports/
 /tmp/

data/.rubocop.yml

@@ -0,0 +1,44 @@
+AllCops:
+  DisplayCopNames: true
+  TargetRubyVersion: 2.1
+
+Lint:
+  Enabled: false
+Metrics:
+  Enabled: false
+Performance:
+  Enabled: false
+Style:
+  Enabled: false
+
+# Configured cops
+
+Layout/CaseIndentation:
+  EnforcedStyle: end
+Layout/FirstParameterIndentation:
+  EnforcedStyle: consistent
+Layout/IndentArray:
+  EnforcedStyle: consistent
+Layout/IndentHash:
+  EnforcedStyle: consistent
+Layout/MultilineMethodCallIndentation:
+  EnforcedStyle: indented
+Layout/MultilineOperationIndentation:
+  EnforcedStyle: indented
+Lint/EndAlignment:
+  EnforcedStyleAlignWith: variable
+
+#
+# Disabled cops
+#
+
+Layout/AlignHash:
+  Enabled: false
+Layout/AlignParameters:
+  Enabled: false
+Layout/EmptyLinesAroundClassBody:
+  Enabled: false
+Layout/EmptyLinesAroundModuleBody:
+  Enabled: false
+Layout/SpaceInsideBlockBraces:
+  Enabled: false

data/CHANGELOG.md

@@ -4,6 +4,20 @@ Changes and additions to the library will be listed here.
 
 ## Unreleased
 
+## v0.4.3
+
+- Restart the async producer thread automatically after errors.
+- Include the offset lag in batch consumer metrics (Statsd).
+- Make the default `max_wait_time` more sane.
+- Fix issue with cached default offset lookups (#431).
+- Upgrade to Datadog client version 3.
+
+## v0.4.2
+
+- Fix connection issue on SASL connections (#401).
+- Add more instrumentation of consumer groups (#407).
+- Improve error logging (#385)
+
 ## v0.4.1
 
 - Allow seeking the consumer position (#386).

data/Gemfile

@@ -1,4 +1,3 @@
 source "https://rubygems.org"
-ruby "2.2.3"
 
 gemspec

data/README.md

@@ -31,8 +31,9 @@ Although parts of this library work with Kafka 0.8 – specifically, the Produce
     5. [Logging](#logging)
     6. [Instrumentation](#instrumentation)
     7. [Monitoring](#monitoring)
-        1. [Reporting Metrics to Statsd](#reporting-metrics-to-statsd)
-        2. [Reporting Metrics to Datadog](#reporting-metrics-to-datadog)
+        1. [What to Monitor](#what-to-monitor)
+        2. [Reporting Metrics to Statsd](#reporting-metrics-to-statsd)
+        3. [Reporting Metrics to Datadog](#reporting-metrics-to-datadog)
     8. [Understanding Timeouts](#understanding-timeouts)
     9. [Security](#security)
         1. [Encryption and Authentication using SSL](#encryption-and-authentication-using-ssl)
@@ -654,14 +655,15 @@ In order to optimize for low latency, you want to process a message as soon as p
 There are three values that can be tuned in order to balance these two concerns.
 
 * `min_bytes` is the minimum number of bytes to return from a single message fetch. By setting this to a high value you can increase the processing throughput. The default value is one byte.
-* `max_wait_time` is the maximum number of seconds to wait before returning data from a single message fetch. By setting this high you also increase the processing throughput – and by setting it low you set a bound on latency. This configuration overrides `min_bytes`, so you'll _always_ get data back within the time specified. The default value is five seconds. If you want to have at most one second of latency, set `max_wait_time` to 1.
+* `max_wait_time` is the maximum number of seconds to wait before returning data from a single message fetch. By setting this high you also increase the processing throughput – and by setting it low you set a bound on latency. This configuration overrides `min_bytes`, so you'll _always_ get data back within the time specified. The default value is one second. If you want to have at most five seconds of latency, set `max_wait_time` to 5. You should make sure `max_wait_time` * num brokers + `heartbeat_interval` is less than `session_timeout`.
 * `max_bytes_per_partition` is the maximum amount of data a broker will return for a single partition when fetching new messages. The default is 1MB, but increasing this number may lead to better throughtput since you'll need to fetch less frequently. Setting it to a lower value is not recommended unless you have so many partitions that it's causing network and latency issues to transfer a fetch response from a broker to a client. Setting the number too high may result in instability, so be careful.
 
 The first two settings can be passed to either `#each_message` or `#each_batch`, e.g.
 
 ```ruby
-# Waits for data for up to 30 seconds, preferring to fetch at least 5KB at a time.
-consumer.each_message(min_bytes: 1024 * 5, max_wait_time: 30) do |message|
+# Waits for data for up to 5 seconds on each broker, preferring to fetch at least 5KB at a time.
+# This can wait up to num brokers * 5 seconds.
+consumer.each_message(min_bytes: 1024 * 5, max_wait_time: 5) do |message|
   # ...
 end
 ```
@@ -729,6 +731,33 @@ end
   * `message_count` is the number of messages for which delivery was attempted.
   * `delivered_message_count` is the number of messages that were acknowledged by the brokers - if this number is smaller than `message_count` not all messages were successfully delivered.
 
+#### Consumer Notifications
+
+* `process_message.consumer.kafka` is sent whenever a message is processed by a consumer. It includes the following payload:
+  * `value` is the message value.
+  * `key` is the message key.
+  * `topic` is the topic that the message was consumed from.
+  * `partition` is the topic partition that the message was consumed from.
+  * `offset` is the message's offset within the topic partition. 
+  * `offset_lag` is the number of messages within the topic partition that have not yet been consumed.
+
+* `process_batch.consumer.kafka` is sent whenever a message batch is processed by a consumer. It includes the following payload:
+  * `message_count` is the number of messages in the batch.
+  * `topic` is the topic that the message batch was consumed from.
+  * `partition` is the topic partition that the message batch was consumed from.
+  * `highwater_mark_offset` is the message batch's highest offset within the topic partition. 
+  * `offset_lag` is the number of messages within the topic partition that have not yet been consumed. 
+
+* `join_group.consumer.kafka` is sent whenever a consumer joins a consumer group. It includes the following payload:
+  * `group_id` is the consumer group id.
+
+* `sync_group.consumer.kafka` is sent whenever a consumer is assigned topic partitions within a consumer group. It includes the following payload:
+  * `group_id` is the consumer group id.  
+  
+* `leave_group.consumer.kafka` is sent whenever a consumer leaves a consumer group. It includes the following payload:
+  * `group_id` is the consumer group id.
+ 
+  
 #### Connection Notifications
 
 * `request.connection.kafka` is sent whenever a network request is sent to a Kafka broker. It includes the following payload:
@@ -749,6 +778,29 @@ It is highly recommended that you monitor your Kafka client applications in prod
 You can quite easily build monitoring on top of the provided [instrumentation hooks](#instrumentation). In order to further help with monitoring, a prebuilt [Statsd](https://github.com/etsy/statsd) and [Datadog](https://www.datadoghq.com/) reporter is included with ruby-kafka.
 
 
+#### What to Monitor
+
+We recommend monitoring the following:
+
+* Low-level Kafka API calls:
+    * The rate of API call errors to the total number of calls by both API and broker.
+    * The API call throughput by both API and broker.
+    * The API call latency by both API and broker.
+* Producer-level metrics:
+    * Delivery throughput by topic.
+    * The latency of deliveries.
+    * The producer buffer fill ratios.
+    * The async producer queue sizes.
+    * Message delivery delays.
+    * Failed delivery attempts.
+* Consumer-level metrics:
+    * Message processing throughput by topic.
+    * Processing latency by topic.
+    * Processing errors by topic.
+    * Consumer lag (how many messages are yet to be processed) by topic/partition.
+    * Group join/sync/leave by client host.
+
+
 #### Reporting Metrics to Statsd
 
 The Statsd reporter is automatically enabled when the `kafka/statsd` library is required. You can optionally change the configuration.
@@ -938,6 +990,8 @@ Currently, there are three actively developed frameworks based on ruby-kafka, th
 
 * [Racecar](https://github.com/zendesk/racecar) - A simple framework that integrates with Ruby on Rails to provide a seamless way to write, test, configure, and run Kafka consumers. It comes with sensible defaults and conventions.
 
+* [DeliveryBoy](https://github.com/zendesk/delivery_boy) – A library that integrates with Ruby on Rails, making it easy to publish Kafka messages from any Rails application.
+
 * [Karafka](https://github.com/karafka/karafka) - Framework used to simplify Apache Kafka based Ruby and Rails applications development. Karafka provides higher abstraction layers, including Capistrano, Docker and Heroku support.
 
 * [Phobos](https://github.com/klarna/phobos) - Micro framework and library for applications dealing with Apache Kafka. It wraps common behaviors needed by consumers and producers in an easy and convenient API.

data/ci/init.rb

@@ -4,7 +4,7 @@ require "kafka"
 
 logger = Logger.new(STDOUT)
 logger.level = Logger::INFO
-logger.formatter = -> (_, _, _, msg) { msg }
+logger.formatter = ->(_, _, _, msg) { msg }
 
 STDOUT.sync = true
 

data/circle.yml

@@ -5,17 +5,19 @@ machine:
     - docker
   environment:
     LOG_LEVEL: DEBUG
+  ruby:
+    version: 2.4.1
 
 dependencies:
   pre:
     - docker -v
     - docker pull ches/kafka:0.9.0.1
     - docker pull jplock/zookeeper:3.4.6
-    - gem install bundler -v 1.9.5
 
 test:
   override:
     - bundle exec rspec -r rspec_junit_formatter --format RspecJunitFormatter -o $CIRCLE_TEST_REPORTS/rspec/unit.xml
     - bundle exec rspec -r rspec_junit_formatter --format RspecJunitFormatter -o $CIRCLE_TEST_REPORTS/rspec/functional.xml --tag functional
   post:
+    - bundle exec rubocop
     - cp *.log $CIRCLE_ARTIFACTS/ || true

data/lib/kafka.rb

@@ -203,6 +203,13 @@ module Kafka
 
   # Raised if not all messages could be sent by a producer.
   class DeliveryFailed < Error
+    attr_reader :failed_messages
+
+    def initialize(message, failed_messages)
+      @failed_messages = failed_messages
+
+      super(message)
+    end
   end
 
   class HeartbeatError < Error

data/lib/kafka/async_producer.rb

@@ -142,16 +142,10 @@ module Kafka
 
     def ensure_threads_running!
       @worker_thread = nil unless @worker_thread && @worker_thread.alive?
-      @worker_thread ||= start_thread { @worker.run }
+      @worker_thread ||= Thread.new { @worker.run }
 
       @timer_thread = nil unless @timer_thread && @timer_thread.alive?
-      @timer_thread ||= start_thread { @timer.run }
-    end
-
-    def start_thread(&block)
-      thread = Thread.new(&block)
-      thread.abort_on_exception = true
-      thread
+      @timer_thread ||= Thread.new { @timer.run }
     end
 
     def buffer_overflow(topic)
@@ -191,6 +185,8 @@ module Kafka
       end
 
       def run
+        @logger.info "Starting async producer in the background..."
+
         loop do
           operation, payload = @queue.pop
 
@@ -218,6 +214,15 @@ module Kafka
             raise "Unknown operation #{operation.inspect}"
           end
         end
+      rescue Kafka::Error => e
+        @logger.error "Unexpected Kafka error #{e.class}: #{e.message}\n#{e.backtrace.join("\n")}"
+        @logger.info "Restarting in 10 seconds..."
+
+        sleep 10
+        retry
+      rescue Exception => e
+        @logger.error "Unexpected Kafka error #{e.class}: #{e.message}\n#{e.backtrace.join("\n")}"
+        @logger.error "Async producer crashed!"
       ensure
         @producer.shutdown
       end

data/lib/kafka/client.rb

@@ -14,12 +14,13 @@ require "kafka/sasl_authenticator"
 
 module Kafka
   class Client
+    URI_SCHEMES = ["kafka", "kafka+ssl"]
 
     # Initializes a new Kafka client.
     #
     # @param seed_brokers [Array<String>, String] the list of brokers used to initialize
     #   the client. Either an Array of connections, or a comma separated string of connections.
-    #   Connections can either be a string of "port:protocol" or a full URI with a scheme.
+    #   A connection can either be a string of "host:port" or a full URI with a scheme.
     #   If there's a scheme it's ignored and only host/port are used.
     #
     # @param client_id [String] the identifier for this application.
@@ -142,7 +143,7 @@ module Kafka
       operation.execute
 
       unless buffer.empty?
-        raise DeliveryFailed
+        raise DeliveryFailed.new(nil, [message])
       end
     end
 
@@ -435,7 +436,6 @@ module Kafka
       @cluster.resolve_offset(topic, partition, :latest) - 1
     end
 
-
     # Retrieve the offset of the last message in each partition of the specified topics.
     #
     # @param topics [Array<String>] topic names.
@@ -516,6 +516,11 @@ module Kafka
         connection = "kafka://" + connection unless connection =~ /:\/\//
         uri = URI.parse(connection)
         uri.port ||= 9092 # Default Kafka port.
+
+        unless URI_SCHEMES.include?(uri.scheme)
+          raise Kafka::Error, "invalid protocol `#{uri.scheme}` in `#{connection}`"
+        end
+
         uri
       end
     end

data/lib/kafka/consumer.rb

@@ -165,10 +165,10 @@ module Kafka
     # at the last committed offsets.
     #
     # @param min_bytes [Integer] the minimum number of bytes to read before
-    #   returning messages from the server; if `max_wait_time` is reached, this
+    #   returning messages from each broker; if `max_wait_time` is reached, this
     #   is ignored.
     # @param max_wait_time [Integer, Float] the maximum duration of time to wait before
-    #   returning messages from the server, in seconds.
+    #   returning messages from each broker, in seconds.
     # @param automatically_mark_as_processed [Boolean] whether to automatically
     #   mark a message as successfully processed when the block returns
     #   without an exception. Once marked successful, the offsets of processed
@@ -178,7 +178,7 @@ module Kafka
     #   The original exception will be returned by calling `#cause` on the
     #   {Kafka::ProcessingError} instance.
     # @return [nil]
-    def each_message(min_bytes: 1, max_wait_time: 5, automatically_mark_as_processed: true)
+    def each_message(min_bytes: 1, max_wait_time: 1, automatically_mark_as_processed: true)
       consumer_loop do
         batches = fetch_batches(min_bytes: min_bytes, max_wait_time: max_wait_time)
 
@@ -232,17 +232,17 @@ module Kafka
     # at the last committed offsets.
     #
     # @param min_bytes [Integer] the minimum number of bytes to read before
-    #   returning messages from the server; if `max_wait_time` is reached, this
+    #   returning messages from each broker; if `max_wait_time` is reached, this
     #   is ignored.
     # @param max_wait_time [Integer, Float] the maximum duration of time to wait before
-    #   returning messages from the server, in seconds.
+    #   returning messages from each broker, in seconds.
     # @param automatically_mark_as_processed [Boolean] whether to automatically
     #   mark a batch's messages as successfully processed when the block returns
     #   without an exception. Once marked successful, the offsets of processed
     #   messages can be committed to Kafka.
     # @yieldparam batch [Kafka::FetchedBatch] a message batch fetched from Kafka.
     # @return [nil]
-    def each_batch(min_bytes: 1, max_wait_time: 5, automatically_mark_as_processed: true)
+    def each_batch(min_bytes: 1, max_wait_time: 1, automatically_mark_as_processed: true)
       consumer_loop do
         batches = fetch_batches(min_bytes: min_bytes, max_wait_time: max_wait_time)
 
@@ -260,7 +260,7 @@ module Kafka
               begin
                 yield batch
               rescue => e
-                offset_range = (batch.first_offset .. batch.last_offset)
+                offset_range = (batch.first_offset..batch.last_offset)
                 location = "#{batch.topic}/#{batch.partition} in offset range #{offset_range}"
                 backtrace = e.backtrace.join("\n")
 
@@ -289,9 +289,9 @@ module Kafka
     # you will want to do this in every consumer group member in order to make sure
     # that the member that's assigned the partition knows where to start.
     #
-    # @param topic [String] 
-    # @param partition [Integer] 
-    # @param offset [Integer] 
+    # @param topic [String]
+    # @param partition [Integer]
+    # @param offset [Integer]
     # @return [nil]
     def seek(topic, partition, offset)
       @offset_manager.seek_to(topic, partition, offset)

data/lib/kafka/datadog.rb

@@ -27,24 +27,53 @@ module Kafka
   module Datadog
     STATSD_NAMESPACE = "ruby_kafka"
 
-    def self.statsd
-      @statsd ||= ::Datadog::Statsd.new(::Datadog::Statsd::DEFAULT_HOST, ::Datadog::Statsd::DEFAULT_PORT, namespace: STATSD_NAMESPACE)
-    end
+    class << self
+      def statsd
+        @statsd ||= ::Datadog::Statsd.new(host, port, namespace: namespace, tags: tags)
+      end
 
-    def self.host=(host)
-      statsd.host = host
-    end
+      def host
+        @host ||= ::Datadog::Statsd::DEFAULT_HOST
+      end
 
-    def self.port=(port)
-      statsd.port = port
-    end
+      def host=(host)
+        @host = host
+        clear
+      end
 
-    def self.namespace=(namespace)
-      statsd.namespace = namespace
-    end
+      def port
+        @port ||= ::Datadog::Statsd::DEFAULT_PORT
+      end
+
+      def port=(port)
+        @port = port
+        clear
+      end
+
+      def namespace
+        @namespace ||= STATSD_NAMESPACE
+      end
 
-    def self.tags=(tags)
-      statsd.tags = tags
+      def namespace=(namespace)
+        @namespace = namespace
+        clear
+      end
+
+      def tags
+        @tags ||= []
+      end
+
+      def tags=(tags)
+        @tags = tags
+        clear
+      end
+
+      private
+
+      def clear
+        @statsd && @statsd.close
+        @statsd = nil
+      end
     end
 
     class StatsdSubscriber < ActiveSupport::Subscriber
@@ -113,6 +142,7 @@ module Kafka
       end
 
       def process_batch(event)
+        lag = event.payload.fetch(:offset_lag)
         messages = event.payload.fetch(:message_count)
 
         tags = {
@@ -128,6 +158,8 @@ module Kafka
           timing("consumer.process_batch.latency", event.duration, tags: tags)
           count("consumer.messages", messages, tags: tags)
         end
+
+        gauge("consumer.lag", lag, tags: tags)
       end
 
       def join_group(event)

data/lib/kafka/offset_manager.rb

@@ -1,4 +1,7 @@
 module Kafka
+
+  # Manages a consumer's position in partitions, figures out where to resume processing
+  # from, etc.
   class OffsetManager
 
     # The default broker setting for offsets.retention.minutes is 1440.
@@ -21,10 +24,28 @@ module Kafka
       @recommit_interval = (offset_retention_time || DEFAULT_RETENTION_TIME) / 2
     end
 
+    # Set the default offset for a topic.
+    #
+    # When the consumer is started for the first time, or in cases where it gets stuck and
+    # has to reset its position, it must start either with the earliest messages or with
+    # the latest, skipping to the very end of each partition.
+    #
+    # @param topic [String] the name of the topic.
+    # @param default_offset [Symbol] either `:earliest` or `:latest`.
+    # @return [nil]
     def set_default_offset(topic, default_offset)
       @default_offsets[topic] = default_offset
     end
 
+    # Mark a message as having been processed.
+    #
+    # When offsets are committed, the message's offset will be stored in Kafka so
+    # that we can resume from this point at a later time.
+    #
+    # @param topic [String] the name of the topic.
+    # @param partition [Integer] the partition number.
+    # @param offset [Integer] the offset of the message that should be marked as processed.
+    # @return [nil]
     def mark_as_processed(topic, partition, offset)
       @uncommitted_offsets += 1
       @processed_offsets[topic] ||= {}
@@ -35,15 +56,35 @@ module Kafka
       @logger.debug "Marking #{topic}/#{partition}:#{offset} as processed"
     end
 
+    # Move the consumer's position in the partition back to the configured default
+    # offset, either the first or latest in the partition.
+    #
+    # @param topic [String] the name of the topic.
+    # @param partition [Integer] the partition number.
+    # @return [nil]
     def seek_to_default(topic, partition)
+      # Remove any cached offset, in case things have changed broker-side.
+      clear_resolved_offset(topic)
+
       seek_to(topic, partition, -1)
     end
 
+    # Move the consumer's position in the partition to the specified offset.
+    #
+    # @param topic [String] the name of the topic.
+    # @param partition [Integer] the partition number.
+    # @param offset [Integer] the offset that the consumer position should be moved to.
+    # @return [nil]
     def seek_to(topic, partition, offset)
       @processed_offsets[topic] ||= {}
       @processed_offsets[topic][partition] = offset
     end
 
+    # Return the next offset that should be fetched for the specified partition.
+    #
+    # @param topic [String] the name of the topic.
+    # @param partition [Integer] the partition number.
+    # @return [Integer] the next offset that should be fetched.
     def next_offset_for(topic, partition)
       offset = @processed_offsets.fetch(topic, {}).fetch(partition) {
         committed_offset_for(topic, partition)
@@ -59,6 +100,16 @@ module Kafka
       end
     end
 
+    # Commit offsets of messages that have been marked as processed.
+    #
+    # If `recommit` is set to true, we will also commit the existing positions
+    # even if no messages have been processed on a partition. This is done
+    # in order to avoid the offset information expiring in cases where messages
+    # are very rare -- it's essentially a keep-alive.
+    #
+    # @param recommit [Boolean] whether to recommit offsets that have already been
+    #   committed.
+    # @return [nil]
     def commit_offsets(recommit = false)
       offsets = offsets_to_commit(recommit)
       unless offsets.empty?
@@ -74,6 +125,10 @@ module Kafka
       end
     end
 
+    # Commit offsets if necessary, according to the offset commit policy specified
+    # when initializing the class.
+    #
+    # @return [nil]
     def commit_offsets_if_necessary
       recommit = recommit_timeout_reached?
       if recommit || commit_timeout_reached? || commit_threshold_reached?
@@ -81,6 +136,9 @@ module Kafka
       end
     end
 
+    # Clear all stored offset information.
+    #
+    # @return [nil]
     def clear_offsets
       @processed_offsets.clear
       @resolved_offsets.clear
@@ -89,6 +147,12 @@ module Kafka
       @committed_offsets = nil
     end
 
+    # Clear stored offset information for all partitions except those specified
+    # in `excluded`.
+    #
+    #     offset_manager.clear_offsets_excluding("my-topic" => [1, 2, 3])
+    #
+    # @return [nil]
     def clear_offsets_excluding(excluded)
       # Clear all offsets that aren't in `excluded`.
       @processed_offsets.each do |topic, partitions|
@@ -104,6 +168,10 @@ module Kafka
 
     private
 
+    def clear_resolved_offset(topic)
+      @resolved_offsets.delete(topic)
+    end
+
     def resolve_offset(topic, partition)
       @resolved_offsets[topic] ||= fetch_resolved_offsets(topic)
       @resolved_offsets[topic].fetch(partition)

data/lib/kafka/pending_message.rb

@@ -11,5 +11,15 @@ module Kafka
       @create_time = create_time
       @bytesize = key.to_s.bytesize + value.to_s.bytesize
     end
+
+    def ==(other)
+      @value == other.value &&
+        @key == other.key &&
+        @topic == other.topic &&
+        @partition == other.partition &&
+        @partition_key == other.partition_key &&
+        @create_time == other.create_time &&
+        @bytesize == other.bytesize
+    end
   end
 end

data/lib/kafka/produce_operation.rb

@@ -98,7 +98,7 @@ module Kafka
             timeout: @ack_timeout * 1000, # Kafka expects the timeout in milliseconds.
           )
 
-          handle_response(response) if response
+          handle_response(broker, response) if response
         rescue ConnectionError => e
           @logger.error "Could not connect to broker #{broker}: #{e}"
 
@@ -108,7 +108,7 @@ module Kafka
       end
     end
 
-    def handle_response(response)
+    def handle_response(broker, response)
       response.each_partition do |topic_info, partition_info|
         topic = topic_info.topic
         partition = partition_info.partition
@@ -138,24 +138,24 @@ module Kafka
             })
           end
         rescue Kafka::CorruptMessage
-          @logger.error "Corrupt message when writing to #{topic}/#{partition}"
+          @logger.error "Corrupt message when writing to #{topic}/#{partition} on #{broker}"
         rescue Kafka::UnknownTopicOrPartition
-          @logger.error "Unknown topic or partition #{topic}/#{partition}"
+          @logger.error "Unknown topic or partition #{topic}/#{partition} on #{broker}"
           @cluster.mark_as_stale!
         rescue Kafka::LeaderNotAvailable
           @logger.error "Leader currently not available for #{topic}/#{partition}"
           @cluster.mark_as_stale!
         rescue Kafka::NotLeaderForPartition
-          @logger.error "Broker not currently leader for #{topic}/#{partition}"
+          @logger.error "Broker #{broker} not currently leader for #{topic}/#{partition}"
           @cluster.mark_as_stale!
         rescue Kafka::RequestTimedOut
-          @logger.error "Timed out while writing to #{topic}/#{partition}"
+          @logger.error "Timed out while writing to #{topic}/#{partition} on #{broker}"
         rescue Kafka::NotEnoughReplicas
           @logger.error "Not enough in-sync replicas for #{topic}/#{partition}"
         rescue Kafka::NotEnoughReplicasAfterAppend
           @logger.error "Messages written, but to fewer in-sync replicas than required for #{topic}/#{partition}"
         else
-          @logger.debug "Successfully appended #{messages.count} messages to #{topic}/#{partition}"
+          @logger.debug "Successfully appended #{messages.count} messages to #{topic}/#{partition} on #{broker}"
 
           # The messages were successfully written; clear them from the buffer.
           @buffer.clear_messages(topic: topic, partition: partition)

data/lib/kafka/producer.rb

@@ -294,7 +294,11 @@ module Kafka
 
         notification[:attempts] = attempt
 
-        @cluster.refresh_metadata_if_necessary!
+        begin
+          @cluster.refresh_metadata_if_necessary!
+        rescue ConnectionError => e
+          raise DeliveryFailed.new(e, buffer_messages)
+        end
 
         assign_partitions!
         operation.execute
@@ -321,13 +325,13 @@ module Kafka
       unless @pending_message_queue.empty?
         # Mark the cluster as stale in order to force a cluster metadata refresh.
         @cluster.mark_as_stale!
-        raise DeliveryFailed, "Failed to assign partitions to #{@pending_message_queue.size} messages"
+        raise DeliveryFailed.new("Failed to assign partitions to #{@pending_message_queue.size} messages", buffer_messages)
       end
 
       unless @buffer.empty?
         partitions = @buffer.map {|topic, partition, _| "#{topic}/#{partition}" }.join(", ")
 
-        raise DeliveryFailed, "Failed to send messages to #{partitions}"
+        raise DeliveryFailed.new("Failed to send messages to #{partitions}", buffer_messages)
       end
     end
 
@@ -380,6 +384,29 @@ module Kafka
       @pending_message_queue.replace(failed_messages)
     end
 
+    def buffer_messages
+      messages = []
+
+      @pending_message_queue.each do |message|
+        messages << message
+      end
+
+      @buffer.each do |topic, partition, messages_for_partition|
+        messages_for_partition.each do |message|
+          messages << PendingMessage.new(
+            message.value,
+            message.key,
+            topic,
+            partition,
+            nil,
+            message.create_time
+          )
+        end
+      end
+
+      messages
+    end
+
     def buffer_overflow(topic, message)
       @instrumenter.instrument("buffer_overflow.producer", {
         topic: topic,

data/lib/kafka/sasl_gssapi_authenticator.rb

@@ -38,7 +38,7 @@ module Kafka
       end
 
       # we can continue, so send OK
-      @encoder.write([0,2].pack('l>c'))
+      @encoder.write([0, 2].pack('l>c'))
 
       # read wrapped message and return it back with principal
       handshake_messages

data/lib/kafka/ssl_socket_with_timeout.rb

@@ -31,15 +31,15 @@ module Kafka
 
       # first initiate the TCP socket
       begin
-        # Initiate the socket connection in the background. If it doesn't fail 
-        # immediately it will raise an IO::WaitWritable (Errno::EINPROGRESS) 
+        # Initiate the socket connection in the background. If it doesn't fail
+        # immediately it will raise an IO::WaitWritable (Errno::EINPROGRESS)
         # indicating the connection is in progress.
         @tcp_socket.connect_nonblock(sockaddr)
       rescue IO::WaitWritable
         # select will block until the socket is writable or the timeout
         # is exceeded, whichever comes first.
         unless select_with_timeout(@tcp_socket, :connect_write)
-          # select returns nil when the socket is not ready before timeout 
+          # select returns nil when the socket is not ready before timeout
           # seconds have elapsed
           @tcp_socket.close
           raise Errno::ETIMEDOUT
@@ -57,8 +57,8 @@ module Kafka
       @ssl_socket = OpenSSL::SSL::SSLSocket.new(@tcp_socket, ssl_context)
 
       begin
-        # Initiate the socket connection in the background. If it doesn't fail 
-        # immediately it will raise an IO::WaitWritable (Errno::EINPROGRESS) 
+        # Initiate the socket connection in the background. If it doesn't fail
+        # immediately it will raise an IO::WaitWritable (Errno::EINPROGRESS)
         # indicating the connection is in progress.
         # Unlike waiting for a tcp socket to connect, you can't time out ssl socket
         # connections during the connect phase properly, because IO.select only partially works.
@@ -130,7 +130,7 @@ module Kafka
           # our write buffer.
           written += @ssl_socket.write_nonblock(bytes)
         rescue Errno::EFAULT => error
-            raise error
+          raise error
         rescue OpenSSL::SSL::SSLError, Errno::EAGAIN, Errno::EWOULDBLOCK, IO::WaitWritable => error
           if error.is_a?(OpenSSL::SSL::SSLError) && error.message == 'write would block'
             if select_with_timeout(@ssl_socket, :write)

data/lib/kafka/statsd.rb

@@ -29,7 +29,7 @@ module Kafka
     DEFAULT_PORT = 8125
 
     def self.statsd
-      @statsd ||= ::Statsd.new(DEFAULT_HOST, DEFAULT_PORT).tap{ |sd| sd.namespace = DEFAULT_NAMESPACE }
+      @statsd ||= ::Statsd.new(DEFAULT_HOST, DEFAULT_PORT).tap { |sd| sd.namespace = DEFAULT_NAMESPACE }
     end
 
     def self.host=(host)

data/lib/kafka/version.rb

@@ -1,3 +1,3 @@
 module Kafka
-  VERSION = "0.4.2"
+  VERSION = "0.4.3"
 end

data/ruby-kafka.gemspec

@@ -1,4 +1,5 @@
 # coding: utf-8
+
 lib = File.expand_path('../lib', __FILE__)
 $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 require 'kafka/version'
@@ -36,9 +37,10 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency "snappy"
   spec.add_development_dependency "colored"
   spec.add_development_dependency "rspec_junit_formatter", "0.2.2"
-  spec.add_development_dependency "dogstatsd-ruby", ">= 2.0.0"
+  spec.add_development_dependency "dogstatsd-ruby", ">= 3.0.0"
   spec.add_development_dependency "statsd-ruby"
   spec.add_development_dependency "ruby-prof"
   spec.add_development_dependency "timecop"
+  spec.add_development_dependency "rubocop", "~> 0.49.1"
   spec.add_development_dependency "gssapi", '>=1.2.0'
 end

data/vendor/bundle/bin/rubocop

@@ -0,0 +1,17 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+#
+# This file was generated by Bundler.
+#
+# The application 'rubocop' is installed as part of a gem, and
+# this file is here to facilitate running it.
+#
+
+require "pathname"
+ENV["BUNDLE_GEMFILE"] ||= File.expand_path("../../../../Gemfile",
+  Pathname.new(__FILE__).realpath)
+
+require "rubygems"
+require "bundler/setup"
+
+load Gem.bin_path("rubocop", "rubocop")

data/vendor/bundle/bin/ruby-parse

@@ -0,0 +1,17 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+#
+# This file was generated by Bundler.
+#
+# The application 'ruby-parse' is installed as part of a gem, and
+# this file is here to facilitate running it.
+#
+
+require "pathname"
+ENV["BUNDLE_GEMFILE"] ||= File.expand_path("../../../../Gemfile",
+  Pathname.new(__FILE__).realpath)
+
+require "rubygems"
+require "bundler/setup"
+
+load Gem.bin_path("parser", "ruby-parse")

data/vendor/bundle/bin/ruby-rewrite

@@ -0,0 +1,17 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+#
+# This file was generated by Bundler.
+#
+# The application 'ruby-rewrite' is installed as part of a gem, and
+# this file is here to facilitate running it.
+#
+
+require "pathname"
+ENV["BUNDLE_GEMFILE"] ||= File.expand_path("../../../../Gemfile",
+  Pathname.new(__FILE__).realpath)
+
+require "rubygems"
+require "bundler/setup"
+
+load Gem.bin_path("parser", "ruby-rewrite")

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: ruby-kafka
 version: !ruby/object:Gem::Version
-  version: 0.4.2
+  version: 0.4.3
 platform: ruby
 authors:
 - Daniel Schierbeck
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2017-09-08 00:00:00.000000000 Z
+date: 2017-10-20 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -170,14 +170,14 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 2.0.0
+        version: 3.0.0
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 2.0.0
+        version: 3.0.0
 - !ruby/object:Gem::Dependency
   name: statsd-ruby
   requirement: !ruby/object:Gem::Requirement
@@ -220,6 +220,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: rubocop
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.49.1
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.49.1
 - !ruby/object:Gem::Dependency
   name: gssapi
   requirement: !ruby/object:Gem::Requirement
@@ -243,10 +257,10 @@ extra_rdoc_files: []
 files:
 - ".gitignore"
 - ".rspec"
+- ".rubocop.yml"
 - ".yardopts"
 - CHANGELOG.md
 - Gemfile
-- Gemfile.lock
 - ISSUE_TEMPLATE.md
 - LICENSE.txt
 - Procfile
@@ -341,8 +355,11 @@ files:
 - vendor/bundle/bin/pry
 - vendor/bundle/bin/rake
 - vendor/bundle/bin/rspec
+- vendor/bundle/bin/rubocop
+- vendor/bundle/bin/ruby-parse
 - vendor/bundle/bin/ruby-prof
 - vendor/bundle/bin/ruby-prof-check-trace
+- vendor/bundle/bin/ruby-rewrite
 homepage: https://github.com/zendesk/ruby-kafka
 licenses:
 - Apache License Version 2.0
@@ -363,7 +380,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.4.5.1
+rubygems_version: 2.6.11
 signing_key: 
 specification_version: 4
 summary: A client library for the Kafka distributed commit log.

data/Gemfile.lock

@@ -1,92 +0,0 @@
-PATH
-  remote: .
-  specs:
-    ruby-kafka (0.4.1)
-
-GEM
-  remote: https://rubygems.org/
-  specs:
-    activesupport (4.2.5)
-      i18n (~> 0.7)
-      json (~> 1.7, >= 1.7.7)
-      minitest (~> 5.1)
-      thread_safe (~> 0.3, >= 0.3.4)
-      tzinfo (~> 1.1)
-    benchmark-perf (0.1.0)
-    builder (3.2.2)
-    coderay (1.1.0)
-    colored (1.2)
-    diff-lcs (1.2.5)
-    docker-api (1.32.1)
-      excon (>= 0.38.0)
-      json
-    dogstatsd-ruby (2.1.0)
-    dotenv (2.1.0)
-    excon (0.54.0)
-    ffi (1.9.18)
-    gssapi (1.2.0)
-      ffi (>= 1.0.1)
-    i18n (0.7.0)
-    json (1.8.3)
-    method_source (0.8.2)
-    minitest (5.8.3)
-    pry (0.9.12.6)
-      coderay (~> 1.0)
-      method_source (~> 0.8)
-      slop (~> 3.4)
-    rake (10.5.0)
-    rspec (3.4.0)
-      rspec-core (~> 3.4.0)
-      rspec-expectations (~> 3.4.0)
-      rspec-mocks (~> 3.4.0)
-    rspec-benchmark (0.1.0)
-      benchmark-perf (~> 0.1.0)
-      rspec (>= 3.0.0, < 4.0.0)
-    rspec-core (3.4.1)
-      rspec-support (~> 3.4.0)
-    rspec-expectations (3.4.0)
-      diff-lcs (>= 1.2.0, < 2.0)
-      rspec-support (~> 3.4.0)
-    rspec-mocks (3.4.1)
-      diff-lcs (>= 1.2.0, < 2.0)
-      rspec-support (~> 3.4.0)
-    rspec-support (3.4.1)
-    rspec_junit_formatter (0.2.2)
-      builder (< 4)
-      rspec-core (>= 2, < 4, != 2.12.0)
-    ruby-prof (0.15.9)
-    slop (3.6.0)
-    snappy (0.0.12)
-    statsd-ruby (1.4.0)
-    thread_safe (0.3.5)
-    timecop (0.8.0)
-    tzinfo (1.2.2)
-      thread_safe (~> 0.1)
-
-PLATFORMS
-  ruby
-
-DEPENDENCIES
-  activesupport
-  bundler (>= 1.9.5)
-  colored
-  docker-api
-  dogstatsd-ruby (>= 2.0.0)
-  dotenv
-  gssapi (>= 1.2.0)
-  pry
-  rake (~> 10.0)
-  rspec
-  rspec-benchmark
-  rspec_junit_formatter (= 0.2.2)
-  ruby-kafka!
-  ruby-prof
-  snappy
-  statsd-ruby
-  timecop
-
-RUBY VERSION
-   ruby 2.2.3p173
-
-BUNDLED WITH
-   1.15.3