ruby-kafka 0.3.2 → 0.3.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: c742496ae66067502319aecdfd06536baf455636
-  data.tar.gz: c5b8d1c26990d1d00460907fb4d5838d75d64635
+  metadata.gz: bf114586f8ec65d1db3ade1d082fc82de140f241
+  data.tar.gz: a202f5dd06b0339a8e3895e48824acba6d266ba5
 SHA512:
-  metadata.gz: 70392fc8aed22ceabdfaeef0f4d41bc076537edc4a188d927999ab4ecdca8b1c9169d8441e6a2d0b7f2cbea136e8e87fa13fb05ef5d1cb68699770c24ba28acf
-  data.tar.gz: cac553e7b994d4e4bce613d1fa08f6757d82f088e6add7c8f6cce51233901882d4f6297e54890fb7852afd5fd9a64a356faa8bfe17baa1ebc70132a74eca18c8
+  metadata.gz: 4788eabf90baf70adb425860c98073058dd601247f1dd447aeb3d154025a5466c19130c7a8d00984ff0e9a5771f980a39a33ffec8bc4384695388999ceda2a13
+  data.tar.gz: 8f5572d8e9dcf46737236ac6a6467fc5d1bd60e091e889ffe056131c084bd47699c59bd404e11b0ce8ec3c5b5c800d8d07c318c6d3abd2e75585d4168079c70c

data/CHANGELOG.md

@@ -4,6 +4,12 @@ Changes and additions to the library will be listed here.
 
 ## Unreleased
 
+## v0.3.3
+
+- Allow clearing a producer's buffer (Martin Nowak).
+- Improved Consumer API.
+- Instrument producer errors.
+
 ## v0.3.2
 
 - Experimental batch consumer API.

data/README.md

@@ -6,7 +6,7 @@ The Producer API is currently beta level and used in production. There's an alph
 
 Although parts of this library work with Kafka 0.8 – specifically, the Producer API – it's being tested and developed against Kafka 0.9. The Consumer API will be 0.9 only.
 
-#### Table of Contents
+## Table of Contents
 
 1. [Installation](#installation)
 2. [Usage](#usage)
@@ -244,7 +244,7 @@ producer = kafka.producer
 
 producer.produce("hello", topic: "greetings")
 
-# If this line fails with Kafka::DeliveryFailed we *may* have succeeded in deliverying
+# If this line fails with Kafka::DeliveryFailed we *may* have succeeded in delivering
 # the message to Kafka but won't know for sure.
 producer.deliver_messages
 

data/circle.yml

@@ -1,3 +1,3 @@
 dependencies:
   pre:
-    - gem install bundler
+    - gem install bundler -v 1.9.5

data/examples/firehose-consumer.rb

@@ -1,6 +1,9 @@
 $LOAD_PATH.unshift(File.expand_path("../../lib", __FILE__))
 
 require "kafka"
+require "dotenv"
+
+Dotenv.load
 
 KAFKA_CLIENT_CERT = ENV.fetch("KAFKA_CLIENT_CERT")
 KAFKA_CLIENT_CERT_KEY = ENV.fetch("KAFKA_CLIENT_CERT_KEY")

data/lib/kafka.rb

@@ -119,6 +119,15 @@ module Kafka
   class DeliveryFailed < Error
   end
 
+  class HeartbeatError < Error
+  end
+
+  class OffsetCommitError < Error
+  end
+
+  class FetchError < Error
+  end
+
   # Initializes a new Kafka client.
   #
   # @see Client#initialize

data/lib/kafka/broker.rb

@@ -4,11 +4,6 @@ require "kafka/protocol"
 
 module Kafka
   class Broker
-    def self.connect(node_id: nil, logger:, **options)
-      connection = Connection.new(logger: logger, **options)
-      new(connection: connection, node_id: node_id, logger: logger)
-    end
-
     def initialize(connection:, node_id: nil, logger:)
       @connection = connection
       @node_id = node_id

data/lib/kafka/broker_pool.rb

@@ -2,27 +2,19 @@ require "kafka/broker"
 
 module Kafka
   class BrokerPool
-    def initialize(client_id:, connect_timeout: nil, socket_timeout: nil, logger:, ssl_context: nil)
-      @client_id = client_id
-      @connect_timeout = connect_timeout
-      @socket_timeout = socket_timeout
+    def initialize(connection_builder:, logger:)
       @logger = logger
+      @connection_builder = connection_builder
       @brokers = {}
-      @ssl_context = ssl_context
     end
 
     def connect(host, port, node_id: nil)
       return @brokers.fetch(node_id) if @brokers.key?(node_id)
 
-      broker = Broker.connect(
-        host: host,
-        port: port,
+      broker = Broker.new(
+        connection: @connection_builder.build_connection(host, port),
         node_id: node_id,
-        client_id: @client_id,
-        connect_timeout: @connect_timeout,
-        socket_timeout: @socket_timeout,
         logger: @logger,
-        ssl_context: @ssl_context,
       )
 
       @brokers[node_id] = broker unless node_id.nil?

data/lib/kafka/client.rb

@@ -3,9 +3,12 @@ require "openssl"
 require "kafka/cluster"
 require "kafka/producer"
 require "kafka/consumer"
+require "kafka/heartbeat"
 require "kafka/async_producer"
 require "kafka/fetched_message"
 require "kafka/fetch_operation"
+require "kafka/connection_builder"
+require "kafka/instrumenter"
 
 module Kafka
   class Client
@@ -37,15 +40,22 @@ module Kafka
     # @return [Client]
     def initialize(seed_brokers:, client_id: "ruby-kafka", logger: nil, connect_timeout: nil, socket_timeout: nil, ssl_ca_cert: nil, ssl_client_cert: nil, ssl_client_cert_key: nil)
       @logger = logger || Logger.new(nil)
+      @instrumenter = Instrumenter.new(client_id: client_id)
 
       ssl_context = build_ssl_context(ssl_ca_cert, ssl_client_cert, ssl_client_cert_key)
 
-      broker_pool = BrokerPool.new(
+      connection_builder = ConnectionBuilder.new(
         client_id: client_id,
         connect_timeout: connect_timeout,
         socket_timeout: socket_timeout,
-        logger: @logger,
         ssl_context: ssl_context,
+        logger: @logger,
+        instrumenter: @instrumenter,
+      )
+
+      broker_pool = BrokerPool.new(
+        connection_builder: connection_builder,
+        logger: @logger,
       )
 
       @cluster = Cluster.new(
@@ -89,11 +99,13 @@ module Kafka
       compressor = Compressor.new(
         codec_name: compression_codec,
         threshold: compression_threshold,
+        instrumenter: @instrumenter,
       )
 
       Producer.new(
         cluster: @cluster,
         logger: @logger,
+        instrumenter: @instrumenter,
         compressor: compressor,
         ack_timeout: ack_timeout,
         required_acks: required_acks,
@@ -139,8 +151,10 @@ module Kafka
     # @param offset_commit_threshold [Integer] the number of messages that can be
     #   processed before their offsets are committed. If zero, offset commits are
     #   not triggered by message processing.
+    # @param heartbeat_interval [Integer] the interval between heartbeats; must be less
+    #   than the session window.
     # @return [Consumer]
-    def consumer(group_id:, session_timeout: 30, offset_commit_interval: 10, offset_commit_threshold: 0)
+    def consumer(group_id:, session_timeout: 30, offset_commit_interval: 10, offset_commit_threshold: 0, heartbeat_interval: 10)
       group = ConsumerGroup.new(
         cluster: @cluster,
         logger: @logger,
@@ -155,12 +169,19 @@ module Kafka
         commit_threshold: offset_commit_threshold,
       )
 
+      heartbeat = Heartbeat.new(
+        group: group,
+        interval: heartbeat_interval,
+      )
+
       Consumer.new(
         cluster: @cluster,
         logger: @logger,
+        instrumenter: @instrumenter,
         group: group,
         offset_manager: offset_manager,
         session_timeout: session_timeout,
+        heartbeat: heartbeat,
       )
     end
 

data/lib/kafka/compressor.rb

@@ -20,9 +20,10 @@ module Kafka
     # @param codec_name [Symbol, nil]
     # @param threshold [Integer] the minimum number of messages in a message set
     #   that will trigger compression.
-    def initialize(codec_name:, threshold:)
+    def initialize(codec_name:, threshold:, instrumenter:)
       @codec = Compression.find_codec(codec_name)
       @threshold = threshold
+      @instrumenter = instrumenter
     end
 
     # @param message_set [Protocol::MessageSet]
@@ -45,7 +46,7 @@ module Kafka
     def compress_data(message_set)
       data = Protocol::Encoder.encode_with(message_set)
 
-      Instrumentation.instrument("compress.compressor.kafka") do |notification|
+      @instrumenter.instrument("compress.compressor") do |notification|
         compressed_data = @codec.compress(data)
 
         notification[:message_count] = message_set.size

data/lib/kafka/connection.rb

@@ -1,7 +1,6 @@
 require "stringio"
 require "kafka/socket_with_timeout"
 require "kafka/ssl_socket_with_timeout"
-require "kafka/instrumentation"
 require "kafka/protocol/request_message"
 require "kafka/protocol/encoder"
 require "kafka/protocol/decoder"
@@ -43,9 +42,10 @@ module Kafka
     #   broker. Default is 10 seconds.
     #
     # @return [Connection] a new connection.
-    def initialize(host:, port:, client_id:, logger:, connect_timeout: nil, socket_timeout: nil, ssl_context: nil)
+    def initialize(host:, port:, client_id:, logger:, instrumenter:, connect_timeout: nil, socket_timeout: nil, ssl_context: nil)
       @host, @port, @client_id = host, port, client_id
       @logger = logger
+      @instrumenter = instrumenter
 
       @connect_timeout = connect_timeout || CONNECT_TIMEOUT
       @socket_timeout = socket_timeout || SOCKET_TIMEOUT
@@ -82,7 +82,7 @@ module Kafka
         response_size: 0,
       }
 
-      Instrumentation.instrument("request.connection.kafka", notification) do
+      @instrumenter.instrument("request.connection", notification) do
         open unless open?
 
         @correlation_id += 1

data/lib/kafka/connection_builder.rb

@@ -0,0 +1,25 @@
+module Kafka
+  class ConnectionBuilder
+    def initialize(client_id:, logger:, instrumenter:, connect_timeout:, socket_timeout:, ssl_context:)
+      @client_id = client_id
+      @logger = logger
+      @instrumenter = instrumenter
+      @connect_timeout = connect_timeout
+      @socket_timeout = socket_timeout
+      @ssl_context = ssl_context
+    end
+
+    def build_connection(host, port)
+      Connection.new(
+        host: host,
+        port: port,
+        client_id: @client_id,
+        connect_timeout: @connect_timeout,
+        socket_timeout: @socket_timeout,
+        logger: @logger,
+        instrumenter: @instrumenter,
+        ssl_context: @ssl_context,
+      )
+    end
+  end
+end

data/lib/kafka/consumer.rb

@@ -43,15 +43,14 @@ module Kafka
   #
   class Consumer
 
-    def initialize(cluster:, logger:, group:, offset_manager:, session_timeout:)
+    def initialize(cluster:, logger:, instrumenter:, group:, offset_manager:, session_timeout:, heartbeat:)
       @cluster = cluster
       @logger = logger
+      @instrumenter = instrumenter
       @group = group
       @offset_manager = offset_manager
       @session_timeout = session_timeout
-
-      # Send two heartbeats in each session window, just to be sure.
-      @heartbeat_interval = @session_timeout / 2
+      @heartbeat = heartbeat
 
       # Whether or not the consumer is currently consuming messages.
       @running = false
@@ -75,6 +74,10 @@ module Kafka
       nil
     end
 
+    def stop
+      @running = false
+    end
+
     # Fetches and enumerates the messages in the topics that the consumer group
     # subscribes to.
     #
@@ -86,96 +89,99 @@ module Kafka
     # that is tasked with taking over processing of these partitions will resume
     # at the last committed offsets.
     #
+    # @param min_bytes [Integer] the minimum number of bytes to read before
+    #   returning messages from the server; if `max_wait_time` is reached, this
+    #   is ignored.
+    # @param max_wait_time [Integer] the maximum duration of time to wait before
+    #   returning messages from the server, in seconds.
     # @yieldparam message [Kafka::FetchedMessage] a message fetched from Kafka.
     # @return [nil]
-    def each_message
-      @running = true
-
-      while @running
-        begin
-          fetch_batches.each do |batch|
-            batch.messages.each do |message|
-              Instrumentation.instrument("process_message.consumer.kafka") do |notification|
-                notification.update(
-                  topic: message.topic,
-                  partition: message.partition,
-                  offset: message.offset,
-                  offset_lag: batch.highwater_mark_offset - message.offset,
-                  key: message.key,
-                  value: message.value,
-                )
-
-                yield message
-              end
-
-              @offset_manager.commit_offsets_if_necessary
-
-              send_heartbeat_if_necessary
-              mark_message_as_processed(message)
-
-              break if !@running
+    def each_message(min_bytes: 1, max_wait_time: 5)
+      consumer_loop do
+        batches = fetch_batches(min_bytes: min_bytes, max_wait_time: max_wait_time)
+
+        batches.each do |batch|
+          batch.messages.each do |message|
+            @instrumenter.instrument("process_message.consumer") do |notification|
+              notification.update(
+                topic: message.topic,
+                partition: message.partition,
+                offset: message.offset,
+                offset_lag: batch.highwater_mark_offset - message.offset,
+                key: message.key,
+                value: message.value,
+              )
+
+              yield message
             end
+
+            @offset_manager.commit_offsets_if_necessary
+
+            @heartbeat.send_if_necessary
+            mark_message_as_processed(message)
+
+            return if !@running
           end
-        rescue ConnectionError => e
-          @logger.error "Connection error while sending heartbeat; rejoining"
-          join_group
-        rescue UnknownMemberId
-          @logger.error "Kicked out of group; rejoining"
-          join_group
-        rescue RebalanceInProgress
-          @logger.error "Group is rebalancing; rejoining"
-          join_group
-        rescue IllegalGeneration
-          @logger.error "Group has transitioned to a new generation; rejoining"
-          join_group
         end
       end
-    ensure
-      # In order to quickly have the consumer group re-balance itself, it's
-      # important that members explicitly tell Kafka when they're leaving.
-      @offset_manager.commit_offsets
-      @group.leave
-      @running = false
     end
 
-    def stop
-      @running = false
+    # Fetches and enumerates the messages in the topics that the consumer group
+    # subscribes to.
+    #
+    # Each batch of messages is yielded to the provided block. If the block returns
+    # without raising an exception, the batch will be considered successfully
+    # processed. At regular intervals the offset of the most recent successfully
+    # processed message batch in each partition will be committed to the Kafka
+    # offset store. If the consumer crashes or leaves the group, the group member
+    # that is tasked with taking over processing of these partitions will resume
+    # at the last committed offsets.
+    #
+    # @param min_bytes [Integer] the minimum number of bytes to read before
+    #   returning messages from the server; if `max_wait_time` is reached, this
+    #   is ignored.
+    # @param max_wait_time [Integer] the maximum duration of time to wait before
+    #   returning messages from the server, in seconds.
+    # @yieldparam batch [Kafka::FetchedBatch] a message batch fetched from Kafka.
+    # @return [nil]
+    def each_batch(min_bytes: 1, max_wait_time: 5)
+      consumer_loop do
+        batches = fetch_batches(min_bytes: min_bytes, max_wait_time: max_wait_time)
+
+        batches.each do |batch|
+          unless batch.empty?
+            @instrumenter.instrument("process_batch.consumer") do |notification|
+              notification.update(
+                topic: batch.topic,
+                partition: batch.partition,
+                highwater_mark_offset: batch.highwater_mark_offset,
+                message_count: batch.messages.count,
+              )
+
+              yield batch
+            end
+
+            mark_message_as_processed(batch.messages.last)
+          end
+
+          @offset_manager.commit_offsets_if_necessary
+
+          @heartbeat.send_if_necessary
+
+          return if !@running
+        end
+      end
     end
 
-    def each_batch
-      loop do
-        begin
-          fetch_batches.each do |batch|
-            unless batch.empty?
-              Instrumentation.instrument("process_batch.consumer.kafka") do |notification|
-                notification.update(
-                  topic: batch.topic,
-                  partition: batch.partition,
-                  highwater_mark_offset: batch.highwater_mark_offset,
-                  message_count: batch.messages.count,
-                )
-
-                yield batch
-              end
-
-              mark_message_as_processed(batch.messages.last)
-            end
+    private
 
-            @offset_manager.commit_offsets_if_necessary
+    def consumer_loop
+      @running = true
 
-            send_heartbeat_if_necessary
-          end
-        rescue ConnectionError => e
-          @logger.error "Connection error while sending heartbeat; rejoining"
-          join_group
-        rescue UnknownMemberId
-          @logger.error "Kicked out of group; rejoining"
-          join_group
-        rescue RebalanceInProgress
-          @logger.error "Group is rebalancing; rejoining"
-          join_group
-        rescue IllegalGeneration
-          @logger.error "Group has transitioned to a new generation; rejoining"
+      while @running
+        begin
+          yield
+        rescue HeartbeatError, OffsetCommitError, FetchError
           join_group
         end
       end
@@ -184,29 +190,28 @@ module Kafka
       # important that members explicitly tell Kafka when they're leaving.
       @offset_manager.commit_offsets
       @group.leave
+      @running = false
     end
 
-    private
-
     def join_group
       @offset_manager.clear_offsets
       @group.join
     end
 
-    def fetch_batches
+    def fetch_batches(min_bytes:, max_wait_time:)
       join_group unless @group.member?
 
       assigned_partitions = @group.assigned_partitions
 
-      send_heartbeat_if_necessary
+      @heartbeat.send_if_necessary
 
       raise "No partitions assigned!" if assigned_partitions.empty?
 
       operation = FetchOperation.new(
         cluster: @cluster,
         logger: @logger,
-        min_bytes: 1,
-        max_wait_time: 5,
+        min_bytes: min_bytes,
+        max_wait_time: max_wait_time,
       )
 
       assigned_partitions.each do |topic, partitions|
@@ -223,22 +228,7 @@ module Kafka
     rescue ConnectionError => e
       @logger.error "Connection error while fetching messages: #{e}"
 
-      return []
-    end
-
-    # Sends a heartbeat if it would be necessary in order to avoid getting
-    # kicked out of the consumer group.
-    #
-    # Each consumer needs to send a heartbeat with a frequency defined by
-    # `session_timeout`.
-    #
-    def send_heartbeat_if_necessary
-      @last_heartbeat ||= Time.now
-
-      if Time.now > @last_heartbeat + @heartbeat_interval
-        @group.heartbeat
-        @last_heartbeat = Time.now
-      end
+      raise FetchError, e
     end
 
     def mark_message_as_processed(message)

data/lib/kafka/consumer_group.rb

@@ -71,6 +71,9 @@ module Kafka
           Protocol.handle_error(error_code)
         end
       end
+    rescue ConnectionError, UnknownMemberId, RebalanceInProgress, IllegalGeneration => e
+      @logger.error "Error committing offsets: #{e}"
+      raise OffsetCommitError, e
     end
 
     def heartbeat
@@ -83,6 +86,9 @@ module Kafka
       )
 
       Protocol.handle_error(response.error_code)
+    rescue ConnectionError, UnknownMemberId, RebalanceInProgress, IllegalGeneration => e
+      @logger.error "Error sending heartbeat: #{e}"
+      raise HeartbeatError, e
     end
 
     private

data/lib/kafka/fetched_batch.rb

@@ -1,6 +1,18 @@
 module Kafka
+
+  # An ordered sequence of messages fetched from a Kafka partition.
   class FetchedBatch
-    attr_reader :topic, :partition, :highwater_mark_offset, :messages
+    # @return [String]
+    attr_reader :topic
+
+    # @return [Integer]
+    attr_reader :partition
+
+    # @return [Integer] the offset of the most recent message in the partition.
+    attr_reader :highwater_mark_offset
+
+    # @return [Array<Kafka::FetchedMessage>]
+    attr_reader :messages
 
     def initialize(topic:, partition:, highwater_mark_offset:, messages:)
       @topic = topic

data/lib/kafka/heartbeat.rb

@@ -0,0 +1,16 @@
+module Kafka
+  class Heartbeat
+    def initialize(group:, interval:)
+      @group = group
+      @interval = interval
+      @last_heartbeat = Time.now
+    end
+
+    def send_if_necessary
+      if Time.now > @last_heartbeat + @interval
+        @group.heartbeat
+        @last_heartbeat = Time.now
+      end
+    end
+  end
+end

data/lib/kafka/instrumenter.rb

@@ -0,0 +1,25 @@
+module Kafka
+  class Instrumenter
+    NAMESPACE = "kafka"
+
+    def initialize(default_payload = {})
+      @default_payload = default_payload
+
+      if defined?(ActiveSupport::Notifications)
+        @backend = ActiveSupport::Notifications
+      else
+        @backend = nil
+      end
+    end
+
+    def instrument(event_name, payload = {}, &block)
+      if @backend
+        payload.update(@default_payload)
+
+        @backend.instrument("#{event_name}.#{NAMESPACE}", payload, &block)
+      else
+        block.call(payload) if block
+      end
+    end
+  end
+end

data/lib/kafka/pending_message_queue.rb

@@ -7,9 +7,7 @@ module Kafka
     attr_reader :size, :bytesize
 
     def initialize
-      @messages = []
-      @size = 0
-      @bytesize = 0
+      clear
     end
 
     def write(message)
@@ -22,6 +20,12 @@ module Kafka
       @messages.empty?
     end
 
+    def clear
+      @messages = []
+      @size = 0
+      @bytesize = 0
+    end
+
     # Yields each message in the queue to the provided block, removing the
     # message after the block has processed it. If the block raises an
     # exception, the message will be retained in the queue.

data/lib/kafka/produce_operation.rb

@@ -25,17 +25,18 @@ module Kafka
   # * `sent_message_count` – the number of messages that were successfully sent.
   #
   class ProduceOperation
-    def initialize(cluster:, buffer:, compressor:, required_acks:, ack_timeout:, logger:)
+    def initialize(cluster:, buffer:, compressor:, required_acks:, ack_timeout:, logger:, instrumenter:)
       @cluster = cluster
       @buffer = buffer
       @required_acks = required_acks
       @ack_timeout = ack_timeout
       @compressor = compressor
       @logger = logger
+      @instrumenter = instrumenter
     end
 
     def execute
-      Instrumentation.instrument("send_messages.producer.kafka") do |notification|
+      @instrumenter.instrument("send_messages.producer") do |notification|
         message_count = @buffer.size
 
         notification[:message_count] = message_count
@@ -62,7 +63,13 @@ module Kafka
           messages_for_broker[broker] ||= MessageBuffer.new
           messages_for_broker[broker].concat(messages, topic: topic, partition: partition)
         rescue Kafka::Error => e
-          @logger.error "Could not connect to leader for partition #{topic}/#{partition}: #{e}"
+          @logger.error "Could not connect to leader for partition #{topic}/#{partition}: #{e.message}"
+
+          @instrumenter.instrument("partition_error.producer", {
+            topic: topic,
+            partition: partition,
+            exception: [e.class.to_s, e.message],
+          })
 
           # We can't send the messages right now, so we'll just keep them in the buffer.
           # We'll mark the cluster as stale in order to force a metadata refresh.
@@ -108,10 +115,20 @@ module Kafka
         ack_time = Time.now
 
         begin
-          Protocol.handle_error(partition_info.error_code)
+          begin
+            Protocol.handle_error(partition_info.error_code)
+          rescue ProtocolError => e
+            @instrumenter.instrument("partition_error.producer", {
+              topic: topic,
+              partition: partition,
+              exception: [e.class.to_s, e.message],
+            })
+
+            raise e
+          end
 
           messages.each do |message|
-            Instrumentation.instrument("ack_message.producer.kafka", {
+            @instrumenter.instrument("ack_message.producer", {
               key: message.key,
               value: message.value,
               topic: topic,

data/lib/kafka/producer.rb

@@ -130,9 +130,10 @@ module Kafka
   #
   class Producer
 
-    def initialize(cluster:, logger:, compressor:, ack_timeout:, required_acks:, max_retries:, retry_backoff:, max_buffer_size:, max_buffer_bytesize:)
+    def initialize(cluster:, logger:, instrumenter:, compressor:, ack_timeout:, required_acks:, max_retries:, retry_backoff:, max_buffer_size:, max_buffer_bytesize:)
       @cluster = cluster
       @logger = logger
+      @instrumenter = instrumenter
       @required_acks = required_acks
       @ack_timeout = ack_timeout
       @max_retries = max_retries
@@ -203,7 +204,7 @@ module Kafka
       @target_topics.add(topic)
       @pending_message_queue.write(message)
 
-      Instrumentation.instrument("produce_message.producer.kafka", {
+      @instrumenter.instrument("produce_message.producer", {
         value: value,
         key: key,
         topic: topic,
@@ -228,7 +229,7 @@ module Kafka
       # There's no need to do anything if the buffer is empty.
       return if buffer_size == 0
 
-      Instrumentation.instrument("deliver_messages.producer.kafka") do |notification|
+      @instrumenter.instrument("deliver_messages.producer") do |notification|
         message_count = buffer_size
 
         notification[:message_count] = message_count
@@ -253,6 +254,14 @@ module Kafka
       @pending_message_queue.bytesize + @buffer.bytesize
     end
 
+    # Deletes all buffered messages.
+    #
+    # @return [nil]
+    def clear_buffer
+      @buffer.clear
+      @pending_message_queue.clear
+    end
+
     # Closes all connections to the brokers.
     #
     # @return [nil]
@@ -274,6 +283,7 @@ module Kafka
         ack_timeout: @ack_timeout,
         compressor: @compressor,
         logger: @logger,
+        instrumenter: @instrumenter,
       )
 
       loop do
@@ -286,6 +296,13 @@ module Kafka
         assign_partitions!
         operation.execute
 
+        if @required_acks.zero?
+          # No response is returned by the brokers, so we can't know which messages
+          # have been successfully written. Our only option is to assume that they all
+          # have.
+          @buffer.clear
+        end
+
         if buffer_size.zero?
           break
         elsif attempt <= @max_retries
@@ -298,11 +315,8 @@ module Kafka
         end
       end
 
-      if @required_acks == 0
-        # No response is returned by the brokers, so we can't know which messages
-        # have been successfully written. Our only option is to assume that they all
-        # have.
-        @buffer.clear
+      unless @pending_message_queue.empty?
+        raise DeliveryFailed, "Failed to assign partitions to #{@pending_message_queue.size} messages"
       end
 
       unless @buffer.empty?

data/lib/kafka/round_robin_assignment_strategy.rb

@@ -30,7 +30,9 @@ module Kafka
         }.values
 
         members.zip(partitions_per_member).each do |member_id, member_partitions|
-          group_assignment[member_id].assign(topic, member_partitions)
+          unless member_partitions.nil?
+            group_assignment[member_id].assign(topic, member_partitions)
+          end
         end
       end
 

data/lib/kafka/version.rb

@@ -1,3 +1,3 @@
 module Kafka
-  VERSION = "0.3.2"
+  VERSION = "0.3.3"
 end

data/ruby-kafka.gemspec

@@ -27,7 +27,7 @@ Gem::Specification.new do |spec|
   spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
   spec.require_paths = ["lib"]
 
-  spec.add_development_dependency "bundler", "~> 1.10"
+  spec.add_development_dependency "bundler", ">= 1.9.5"
   spec.add_development_dependency "rake", "~> 10.0"
   spec.add_development_dependency "rspec"
   spec.add_development_dependency "pry"

metadata

@@ -1,29 +1,29 @@
 --- !ruby/object:Gem::Specification
 name: ruby-kafka
 version: !ruby/object:Gem::Version
-  version: 0.3.2
+  version: 0.3.3
 platform: ruby
 authors:
 - Daniel Schierbeck
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2016-03-15 00:00:00.000000000 Z
+date: 2016-04-12 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '1.10'
+        version: 1.9.5
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '1.10'
+        version: 1.9.5
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
@@ -177,13 +177,15 @@ files:
 - lib/kafka/compression.rb
 - lib/kafka/compressor.rb
 - lib/kafka/connection.rb
+- lib/kafka/connection_builder.rb
 - lib/kafka/consumer.rb
 - lib/kafka/consumer_group.rb
 - lib/kafka/fetch_operation.rb
 - lib/kafka/fetched_batch.rb
 - lib/kafka/fetched_message.rb
 - lib/kafka/gzip_codec.rb
-- lib/kafka/instrumentation.rb
+- lib/kafka/heartbeat.rb
+- lib/kafka/instrumenter.rb
 - lib/kafka/message_buffer.rb
 - lib/kafka/offset_manager.rb
 - lib/kafka/partitioner.rb

data/lib/kafka/instrumentation.rb

@@ -1,13 +0,0 @@
-module Kafka
-  class NullInstrumentation
-    def self.instrument(name, payload = {})
-      yield payload if block_given?
-    end
-  end
-
-  if defined?(ActiveSupport::Notifications)
-    Instrumentation = ActiveSupport::Notifications
-  else
-    Instrumentation = NullInstrumentation
-  end
-end