ruby-kafka 0.1.0.pre.beta1 → 0.1.0.pre.beta2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: eb0de1c04f8551ffe3750b3eac702b51777e525d
-  data.tar.gz: 1e6ed1b055a754d7eb958a03314f50f62138de59
+  metadata.gz: 6d8c3294bd19a1c201194ff051f6d98d7a6dc273
+  data.tar.gz: 1769607ad012c3913b1ad707922ec271cc7a4567
 SHA512:
-  metadata.gz: 5ac63915c1bead98581a2b4ddd577887ec5fa7f8d7c14ebcedac3d77e06d24a718a1df3b88961edd13927acdb16e4992acdaaecd2e05dc7fe1e20c511a8ad6c1
-  data.tar.gz: 2273d4da86a6ee82c5c3cc310e1549056638e1c07839abcbbb511f9dc8cb055ed8680e4ee9b5415bbb00b21155c4955f66d7be385423e639c5c27e0e2b38f8c8
+  metadata.gz: 02b14e2350078b1555cadf35a5de7eba1ca5f218a65ddfdef7e9e76f8a7f7089c85a8a1ae40623566f37ee1675c63354a240c9ce97c3e7797180b137155de2fe
+  data.tar.gz: 603e25244557ba9bd684c5f30543654b224ffabfb7bde6eff59ed88805e6618d89f846afcf40eb2113c9fbc44d02f57ab7c825ee03c9535c3f0be139739a8725

data/README.md

@@ -2,7 +2,7 @@
 
 A Ruby client library for the Kafka distributed log system. The focus of this library will be operational simplicity, with good logging and metrics that can make debugging issues easier.
 
-This library is still in pre-alpha stage, but development is ongoing. Current efforts are focused on implementing a solid Producer client. The next step will be implementing a client for the Kafka 0.9 Consumer API.
+This library is still in pre-beta stage, but development is ongoing. Current efforts are focused on implementing a solid Producer client. The next step will be implementing a client for the Kafka 0.9 Consumer API.
 
 ## Installation
 
@@ -22,13 +22,18 @@ Or install it yourself as:
 
 ## Usage
 
+Currently, only the Producer API is supported. A Kafka 0.9 compatible Consumer API is on the roadmap.
+
 ```ruby
+# The client must be initialized with at least one Kafka broker.
 kafka = Kafka.new(
   seed_brokers: ["kafka1:9092", "kafka2:9092"],
   client_id: "my-app",
   logger: Logger.new($stderr),
 )
 
+# Each producer keeps a separate pool of broker connections. Don't use the same
+# producer from more than one thread.
 producer = kafka.get_producer
 
 # `write` will buffer the message in the producer.
@@ -43,7 +48,25 @@ producer.flush
 
 After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake spec` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
 
-To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org).
+**Note:** the specs require a working [Docker](https://www.docker.com/) instance, but should work out of the box if you have Docker installed. Please create an issue if that's not the case.
+
+## Roadmap
+
+v0.1 is targeted for release in February. Other milestones do not have firm target dates, but v0.2 will be released as soon as we are confident that it is ready to run in critical production environments and that the API shouldn't be changed.
+
+### v0.1: Producer API for non-critical production data
+
+We need to actually run this in production for a while before we can say that it won't lose data, so initially the library should only be deployed for non-critical use cases.
+
+The API may also be changed.
+
+### v0.2: Stable Producer API
+
+The API should now have stabilized and the library should be battle tested enough to deploy for critical use cases.
+
+### v1.0: Consumer API
+
+The Consumer API defined by Kafka 0.9 will be implemented.
 
 ## Contributing
 

data/bin/console

@@ -3,12 +3,5 @@
 require "bundler/setup"
 require "kafka"
 
-# You can add fixtures and/or initialization code here to make experimenting
-# with your gem easier. You can also use a different console, if you like.
-
-# (If you use this, don't forget to add pry to your Gemfile!)
-# require "pry"
-# Pry.start
-
-require "irb"
-IRB.start
+require "pry"
+Pry.start

data/bin/setup

@@ -3,5 +3,3 @@ set -euo pipefail
 IFS=$'\n\t'
 
 bundle install
-
-# Do any other automated setup that you need to do here

data/circle.yml

@@ -0,0 +1,3 @@
+dependencies:
+  pre:
+    - gem install bundler

data/kafka.gemspec

@@ -22,4 +22,5 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency "bundler", "~> 1.10"
   spec.add_development_dependency "rake", "~> 10.0"
   spec.add_development_dependency "rspec"
+  spec.add_development_dependency "pry"
 end

data/lib/kafka.rb

@@ -1,22 +1,113 @@
 require "kafka/version"
-require "kafka/client"
 
 module Kafka
-  Error = Class.new(StandardError)
-  ConnectionError = Class.new(Error)
-  CorruptMessage = Class.new(Error)
-  UnknownError = Class.new(Error)
-  OffsetOutOfRange = Class.new(Error)
-  UnknownTopicOrPartition = Class.new(Error)
-  InvalidMessageSize = Class.new(Error)
-  LeaderNotAvailable = Class.new(Error)
-  NotLeaderForPartition = Class.new(Error)
-  RequestTimedOut = Class.new(Error)
+  class Error < StandardError
+  end
+
+  # Subclasses of this exception class map to an error code described in the
+  # Kafka protocol specification.
+  #
+  # See https://cwiki.apache.org/confluence/display/KAFKA/A+Guide+To+The+Kafka+Protocol
+  class ProtocolError < StandardError
+  end
+
+  # This indicates that a message contents does not match its CRC.
+  class CorruptMessage < ProtocolError
+  end
+
+  class UnknownError < ProtocolError
+  end
+
+  class OffsetOutOfRange < ProtocolError
+  end
+
+  # The request is for a topic or partition that does not exist on the broker.
+  class UnknownTopicOrPartition < ProtocolError
+  end
+
+  # The message has a negative size.
+  class InvalidMessageSize < ProtocolError
+  end
+
+  # This error is thrown if we are in the middle of a leadership election and
+  # there is currently no leader for this partition and hence it is unavailable
+  # for writes.
+  class LeaderNotAvailable < ProtocolError
+  end
+
+  # This error is thrown if the client attempts to send messages to a replica
+  # that is not the leader for some partition. It indicates that the client's
+  # metadata is out of date.
+  class NotLeaderForPartition < ProtocolError
+  end
+
+  # This error is thrown if the request exceeds the user-specified time limit
+  # in the request.
+  class RequestTimedOut < ProtocolError
+  end
+
+  class BrokerNotAvailable < ProtocolError
+  end
+
+  # The server has a configurable maximum message size to avoid unbounded memory
+  # allocation. This error is thrown if the client attempt to produce a message
+  # larger than this maximum.
+  class MessageSizeTooLarge < ProtocolError
+  end
+
+  # If you specify a string larger than configured maximum for offset metadata.
+  class OffsetMetadataTooLarge < ProtocolError
+  end
+
+  # For a request which attempts to access an invalid topic (e.g. one which has
+  # an illegal name), or if an attempt is made to write to an internal topic
+  # (such as the consumer offsets topic).
+  class InvalidTopic < ProtocolError
+  end
+
+  # If a message batch in a produce request exceeds the maximum configured
+  # segment size.
+  class RecordListTooLarge < ProtocolError
+  end
+
+  # Returned from a produce request when the number of in-sync replicas is
+  # lower than the configured minimum and requiredAcks is -1.
+  class NotEnoughReplicas < ProtocolError
+  end
+
+  # Returned from a produce request when the message was written to the log,
+  # but with fewer in-sync replicas than required.
+  class NotEnoughReplicasAfterAppend < ProtocolError
+  end
+
+  # Returned from a produce request if the requested requiredAcks is invalid
+  # (anything other than -1, 1, or 0).
+  class InvalidRequiredAcks < ProtocolError
+  end
 
   # Raised if a replica is expected on a broker, but is not. Can be safely ignored.
-  ReplicaNotAvailable = Class.new(Error)
+  class ReplicaNotAvailable < ProtocolError
+  end
 
+  # Raised when there's a network connection error.
+  class ConnectionError < Error
+  end
+
+  # Raised when a producer buffer has reached its maximum size.
+  class BufferOverflow < Error
+  end
+
+  # Raised if not all messages could be sent by a producer.
+  class FailedToSendMessages < Error
+  end
+
+  # Initializes a new Kafka client.
+  #
+  # @see Client#initialize
+  # @return [Client]
   def self.new(**options)
     Client.new(**options)
   end
 end
+
+require "kafka/client"

data/lib/kafka/broker_pool.rb

@@ -8,12 +8,6 @@ module Kafka
   # partitions to the current leader for those partitions.
   class BrokerPool
 
-    # The number of times to try to connect to a broker before giving up.
-    MAX_CONNECTION_ATTEMPTS = 3
-
-    # The backoff period between connection retries, in seconds.
-    RETRY_BACKOFF_TIMEOUT = 5
-
     # Initializes a broker pool with a set of seed brokers.
     #
     # The pool will try to fetch cluster metadata from one of the brokers.
@@ -27,19 +21,52 @@ module Kafka
       @socket_timeout = socket_timeout
       @brokers = {}
       @seed_brokers = seed_brokers
+      @cluster_info = nil
+    end
 
-      refresh
+    def mark_as_stale!
+      @cluster_info = nil
     end
 
-    # Refreshes the cluster metadata.
+    # Finds the broker acting as the leader of the given topic and partition.
+    #
+    # @param topic [String]
+    # @param partition [Integer]
+    # @return [Integer] the broker id.
+    def get_leader_id(topic, partition)
+      cluster_info.find_leader_id(topic, partition)
+    end
+
+    def get_broker(broker_id)
+      @brokers[broker_id] ||= connect_to_broker(broker_id)
+    end
+
+    def partitions_for(topic)
+      cluster_info.partitions_for(topic)
+    end
+
+    def shutdown
+      @brokers.each do |id, broker|
+        @logger.info "Disconnecting broker #{id}"
+        broker.disconnect
+      end
+    end
+
+    private
+
+    def cluster_info
+      @cluster_info ||= fetch_cluster_info
+    end
+
+    # Fetches the cluster metadata.
     #
     # This is used to update the partition leadership information, among other things.
-    # The methods will go through each node listed in `seed_brokers`, connecting to the
+    # The methods will go through each node listed in +seed_brokers+, connecting to the
     # first one that is available. This node will be queried for the cluster metadata.
     #
-    # @raise [ConnectionError] if none of the nodes in `seed_brokers` are available.
-    # @return [nil]
-    def refresh
+    # @raise [ConnectionError] if none of the nodes in +seed_brokers+ are available.
+    # @return [Protocol::MetadataResponse] the cluster metadata.
+    def fetch_cluster_info
       @seed_brokers.each do |node|
         @logger.info "Trying to initialize broker pool from node #{node}"
 
@@ -54,76 +81,21 @@ module Kafka
             logger: @logger,
           )
 
-          @cluster_info = broker.fetch_metadata
+          cluster_info = broker.fetch_metadata
 
-          @logger.info "Initialized broker pool with brokers: #{@cluster_info.brokers.inspect}"
+          @logger.info "Initialized broker pool with brokers: #{cluster_info.brokers.inspect}"
 
-          return
+          return cluster_info
         rescue Error => e
-          @logger.error "Failed to fetch metadata from broker #{broker}: #{e}"
+          @logger.error "Failed to fetch metadata from #{node}: #{e}"
         end
       end
 
       raise ConnectionError, "Could not connect to any of the seed brokers: #{@seed_brokers.inspect}"
     end
 
-    # Finds the broker acting as the leader of the given topic and partition and connects to it.
-    #
-    # Note that this call may take a considerable amount of time, since the cached cluster
-    # metadata may be out of date. In that case, the cluster needs to be re-discovered. This
-    # can happen when a broker becomes unavailable, which would trigger a leader election for
-    # the partitions previously owned by that broker. Since this can take some time, this method
-    # will retry up to `MAX_CONNECTION_ATTEMPTS` times, waiting `RETRY_BACKOFF_TIMEOUT` seconds
-    # between each attempt.
-    #
-    # @param topic [String]
-    # @param partition [Integer]
-    # @raise [ConnectionError] if it was not possible to connect to the leader.
-    # @return [Broker] the broker that's currently acting as leader of the partition.
-    def get_leader(topic, partition)
-      attempt = 0
-
-      begin
-        leader_id = @cluster_info.find_leader_id(topic, partition)
-        broker_for_id(leader_id)
-      rescue ConnectionError => e
-        @logger.error "Failed to connect to leader for topic `#{topic}`, partition #{partition}"
-
-        if attempt < MAX_CONNECTION_ATTEMPTS
-          attempt += 1
-
-          @logger.info "Rediscovering cluster and retrying"
-
-          sleep RETRY_BACKOFF_TIMEOUT
-          refresh
-          retry
-        else
-          @logger.error "Giving up trying to find leader for topic `#{topic}`, partition #{partition}"
-
-          raise e
-        end
-      end
-    end
-
-    def partitions_for(topic)
-      @cluster_info.partitions_for(topic)
-    end
-
-    def shutdown
-      @brokers.each do |id, broker|
-        @logger.info "Disconnecting broker #{id}"
-        broker.disconnect
-      end
-    end
-
-    private
-
-    def broker_for_id(broker_id)
-      @brokers[broker_id] ||= connect_to_broker(broker_id)
-    end
-
     def connect_to_broker(broker_id)
-      broker_info = @cluster_info.find_broker(broker_id)
+      broker_info = cluster_info.find_broker(broker_id)
 
       Broker.connect(
         host: broker_info.host,

data/lib/kafka/client.rb

@@ -3,6 +3,20 @@ require "kafka/producer"
 
 module Kafka
   class Client
+
+    # Initializes a new Kafka client.
+    #
+    # @param seed_brokers [Array<String>] the list of brokers used to initialize
+    #   the client.
+    #
+    # @param client_id [String] the identifier for this application.
+    #
+    # @param logger [Logger]
+    #
+    # @param socket_timeout [Integer, nil] the timeout setting for socket
+    #   connections. See {BrokerPool#initialize}.
+    #
+    # @return [Client]
     def initialize(seed_brokers:, client_id:, logger:, socket_timeout: nil)
       @seed_brokers = seed_brokers
       @client_id = client_id

data/lib/kafka/connection.rb

@@ -44,7 +44,7 @@ module Kafka
 
       # Correlation id is initialized to zero and bumped for each request.
       @correlation_id = 0
-    rescue Errno::ETIMEDOUT
+    rescue Errno::ETIMEDOUT => e
       @logger.error "Timed out while trying to connect to #{host}:#{port}: #{e}"
       raise ConnectionError, e
     rescue SocketError, Errno::ECONNREFUSED => e
@@ -67,12 +67,26 @@ module Kafka
     # @param request [#encode] the request that should be encoded and written.
     # @param response_class [#decode] an object that can decode the response.
     #
-    # @return [Object] the response that was decoded by `response_class`.
+    # @return [Object] the response that was decoded by +response_class+.
     def request(api_key, request, response_class)
       write_request(api_key, request)
 
       unless response_class.nil?
-        read_response(response_class)
+        loop do
+          correlation_id, response = read_response(response_class)
+
+          # There may have been a previous request that timed out before the client
+          # was able to read the response. In that case, the response will still be
+          # sitting in the socket waiting to be read. If the response we just read
+          # was to a previous request, we can safely skip it.
+          if correlation_id < @correlation_id
+            @logger.error "Received out-of-order response id #{correlation_id}, was expecting #{@correlation_id}"
+          elsif correlation_id > @correlation_id
+            raise Kafka::Error, "Correlation id mismatch: expected #{@correlation_id} but got #{correlation_id}"
+          else
+            break response
+          end
+        end
       end
     end
 
@@ -131,7 +145,7 @@ module Kafka
 
       @logger.debug "Received response #{correlation_id} from #{to_s}"
 
-      response
+      return correlation_id, response
     end
   end
 end

data/lib/kafka/message_buffer.rb

@@ -0,0 +1,64 @@
+module Kafka
+
+  # Buffers messages for specific topics/partitions.
+  class MessageBuffer
+    include Enumerable
+
+    def initialize
+      @buffer = {}
+    end
+
+    def write(message, topic:, partition:)
+      buffer_for(topic, partition) << message
+    end
+
+    def concat(messages, topic:, partition:)
+      buffer_for(topic, partition).concat(messages)
+    end
+
+    def to_h
+      @buffer
+    end
+
+    def size
+      @buffer.values.inject(0) {|sum, messages| messages.values.flatten.size + sum }
+    end
+
+    def empty?
+      @buffer.empty?
+    end
+
+    def each
+      @buffer.each do |topic, messages_for_topic|
+        messages_for_topic.each do |partition, messages_for_partition|
+          yield topic, partition, messages_for_partition
+        end
+      end
+    end
+
+    # Clears buffered messages for the given topic and partition.
+    #
+    # @param topic [String] the name of the topic.
+    # @param partition [Integer] the partition id.
+    #
+    # @return [nil]
+    def clear_messages(topic:, partition:)
+      @buffer[topic].delete(partition)
+      @buffer.delete(topic) if @buffer[topic].empty?
+    end
+
+    # Clears messages across all topics and partitions.
+    #
+    # @return [nil]
+    def clear
+      @buffer = {}
+    end
+
+    private
+
+    def buffer_for(topic, partition)
+      @buffer[topic] ||= {}
+      @buffer[topic][partition] ||= []
+    end
+  end
+end

data/lib/kafka/partitioner.rb

@@ -1,13 +1,28 @@
 require "zlib"
 
 module Kafka
+
+  # Assigns partitions to messages.
   class Partitioner
     def initialize(partitions)
       @partitions = partitions
     end
 
+    # Assigns a partition number based on a key.
+    #
+    # If the key is nil, then a random partition is selected. Otherwise, a digest
+    # of the key is used to deterministically find a partition. As long as the
+    # number of partitions doesn't change, the same key will always be assigned
+    # to the same partition.
+    #
+    # @param key [String, nil] the key to base the partition assignment on, or nil.
+    # @return [Integer] the partition number.
     def partition_for_key(key)
-      Zlib.crc32(key) % @partitions.count
+      if key.nil?
+        rand(@partitions.count)
+      else
+        Zlib.crc32(key) % @partitions.count
+      end
     end
   end
 end

data/lib/kafka/producer.rb

@@ -1,19 +1,68 @@
-require "kafka/message"
-require "kafka/message_set"
 require "kafka/partitioner"
+require "kafka/message_buffer"
+require "kafka/protocol/message"
 
 module Kafka
+
+  # Allows sending messages to a Kafka cluster.
+  #
+  # == Buffering
+  #
+  # The producer buffers pending messages until {#flush} is called. Note that there is
+  # a maximum buffer size (default is 1,000 messages) and writing messages after the
+  # buffer has reached this size will result in a BufferOverflow exception. Make sure
+  # to periodically call {#flush} or set +max_buffer_size+ to an appropriate value.
+  #
+  # Buffering messages and sending them in batches greatly improves performance, so
+  # try to avoid flushing after every write. The tradeoff between throughput and
+  # message delays depends on your use case.
+  #
+  # == Error Handling and Retries
+  #
+  # The design of the error handling is based on having a {MessageBuffer} hold messages
+  # for all topics/partitions. Whenever we want to flush messages to the cluster, we
+  # group the buffered messages by the broker they need to be sent to and fire off a
+  # request to each broker. A request can be a partial success, so we go through the
+  # response and inspect the error code for each partition that we wrote to. If the
+  # write to a given partition was successful, we clear the corresponding messages
+  # from the buffer -- otherwise, we log the error and keep the messages in the buffer.
+  #
+  # After this, we check if the buffer is empty. If it is, we're all done. If it's
+  # not, we do another round of requests, this time with just the remaining messages.
+  # We do this for as long as +max_retries+ permits.
+  #
   class Producer
-    # @param timeout [Integer] The number of seconds to wait for an
-    #   acknowledgement from the broker before timing out.
+
+    # Initializes a new Producer.
+    #
+    # @param broker_pool [BrokerPool] the broker pool representing the cluster.
+    #
+    # @param logger [Logger]
+    #
+    # @param timeout [Integer] The number of seconds a broker can wait for
+    #   replicas to acknowledge a write before responding with a timeout.
+    #
     # @param required_acks [Integer] The number of replicas that must acknowledge
     #   a write.
-    def initialize(broker_pool:, logger:, timeout: 10, required_acks: 1)
+    #
+    # @param max_retries [Integer] the number of retries that should be attempted
+    #   before giving up sending messages to the cluster. Does not include the
+    #   original attempt.
+    #
+    # @param retry_backoff [Integer] the number of seconds to wait between retries.
+    #
+    # @param max_buffer_size [Integer] the number of messages allowed in the buffer
+    #   before new writes will raise BufferOverflow exceptions.
+    #
+    def initialize(broker_pool:, logger:, timeout: 10, required_acks: 1, max_retries: 2, retry_backoff: 1, max_buffer_size: 1000)
       @broker_pool = broker_pool
       @logger = logger
       @required_acks = required_acks
       @timeout = timeout
-      @buffered_messages = []
+      @max_retries = max_retries
+      @retry_backoff = retry_backoff
+      @max_buffer_size = max_buffer_size
+      @buffer = MessageBuffer.new
     end
 
     # Writes a message to the specified topic. Note that messages are buffered in
@@ -22,13 +71,20 @@ module Kafka
     # == Partitioning
     #
     # There are several options for specifying the partition that the message should
-    # be written to. The simplest option is to not specify a partition or partition
-    # key, in which case the message key will be used to select one of the available
-    # partitions. You can also specify the `partition` parameter yourself. This
-    # requires you to know which partitions are available, however. Oftentimes the
-    # best option is to specify the `partition_key` parameter: messages with the
-    # same partition key will always be assigned to the same partition, as long as
-    # the number of partitions doesn't change.
+    # be written to.
+    #
+    # The simplest option is to not specify a message key, partition key, or
+    # partition number, in which case the message will be assigned a partition at
+    # random.
+    #
+    # You can also specify the +partition+ parameter yourself. This requires you to
+    # know which partitions are available, however. Oftentimes the best option is
+    # to specify the +partition_key+ parameter: messages with the same partition
+    # key will always be assigned to the same partition, as long as the number of
+    # partitions doesn't change. You can also omit the partition key and specify
+    # a message key instead. The message key is part of the message payload, and
+    # so can carry semantic value--whether you want to have the message key double
+    # as a partition key is up to you.
     #
     # @param value [String] the message data.
     # @param key [String] the message key.
@@ -36,8 +92,13 @@ module Kafka
     # @param partition [Integer] the partition that the message should be written to.
     # @param partition_key [String] the key that should be used to assign a partition.
     #
-    # @return [Message] the message that was written.
-    def write(value, key:, topic:, partition: nil, partition_key: nil)
+    # @raise [BufferOverflow] if the maximum buffer size has been reached.
+    # @return [nil]
+    def write(value, key: nil, topic:, partition: nil, partition_key: nil)
+      unless buffer_size < @max_buffer_size
+        raise BufferOverflow, "Max buffer size #{@max_buffer_size} exceeded"
+      end
+
       if partition.nil?
         # If no explicit partition key is specified we use the message key instead.
         partition_key ||= key
@@ -45,58 +106,135 @@ module Kafka
         partition = partitioner.partition_for_key(partition_key)
       end
 
-      message = Message.new(value, key: key, topic: topic, partition: partition)
+      message = Protocol::Message.new(key: key, value: value)
 
-      @buffered_messages << message
+      @buffer.write(message, topic: topic, partition: partition)
 
-      message
+      partition
     end
 
     # Flushes all messages to the Kafka brokers.
     #
-    # Depending on the value of `required_acks` used when initializing the producer,
+    # Depending on the value of +required_acks+ used when initializing the producer,
     # this call may block until the specified number of replicas have acknowledged
-    # the writes. The `timeout` setting places an upper bound on the amount of time
+    # the writes. The +timeout+ setting places an upper bound on the amount of time
     # the call will block before failing.
     #
+    # @raise [FailedToSendMessages] if not all messages could be successfully sent.
     # @return [nil]
     def flush
+      attempt = 0
+
+      loop do
+        @logger.info "Flushing #{@buffer.size} messages"
+
+        attempt += 1
+        transmit_messages
+
+        if @buffer.empty?
+          @logger.info "Successfully transmitted all messages"
+          break
+        elsif attempt <= @max_retries
+          @logger.warn "Failed to transmit all messages, retry #{attempt} of #{@max_retries}"
+          @logger.info "Waiting #{@retry_backoff}s before retrying"
+
+          sleep @retry_backoff
+        else
+          @logger.error "Failed to transmit all messages; keeping remaining messages in buffer"
+          break
+        end
+      end
+
+      if @required_acks == 0
+        # No response is returned by the brokers, so we can't know which messages
+        # have been successfully written. Our only option is to assume that they all
+        # have.
+        @buffer.clear
+      end
+
+      unless @buffer.empty?
+        partitions = @buffer.map {|topic, partition, _| "#{topic}/#{partition}" }.join(", ")
+
+        raise FailedToSendMessages, "Failed to send messages to #{partitions}"
+      end
+    end
+
+    # Returns the number of messages currently held in the buffer.
+    #
+    # @return [Integer] buffer size.
+    def buffer_size
+      @buffer.size
+    end
+
+    def shutdown
+      @broker_pool.shutdown
+    end
+
+    private
+
+    def transmit_messages
       messages_for_broker = {}
 
-      @buffered_messages.each do |message|
-        broker = @broker_pool.get_leader(message.topic, message.partition)
+      @buffer.each do |topic, partition, messages|
+        broker_id = @broker_pool.get_leader_id(topic, partition)
 
-        messages_for_broker[broker] ||= []
-        messages_for_broker[broker] << message
+        @logger.debug "Current leader for #{topic}/#{partition} is node #{broker_id}"
+
+        messages_for_broker[broker_id] ||= MessageBuffer.new
+        messages_for_broker[broker_id].concat(messages, topic: topic, partition: partition)
       end
 
-      messages_for_broker.each do |broker, messages|
-        @logger.info "Sending #{messages.count} messages to broker #{broker}"
+      messages_for_broker.each do |broker_id, message_set|
+        begin
+          broker = @broker_pool.get_broker(broker_id)
 
-        message_set = MessageSet.new(messages)
+          response = broker.produce(
+            messages_for_topics: message_set.to_h,
+            required_acks: @required_acks,
+            timeout: @timeout * 1000, # Kafka expects the timeout in milliseconds.
+          )
 
-        response = broker.produce(
-          messages_for_topics: message_set.to_h,
-          required_acks: @required_acks,
-          timeout: @timeout * 1000, # Kafka expects the timeout in milliseconds.
-        )
+          handle_response(response) if response
+        rescue ConnectionError => e
+          @logger.error "Could not connect to broker #{broker_id}: #{e}"
 
-        if response
-          response.topics.each do |topic_info|
-            topic_info.partitions.each do |partition_info|
-              Protocol.handle_error(partition_info.error_code)
-            end
-          end
+          # Mark the broker pool as stale in order to force a cluster metadata refresh.
+          @broker_pool.mark_as_stale!
         end
       end
+    end
 
-      @buffered_messages.clear
+    def handle_response(response)
+      response.each_partition do |topic_info, partition_info|
+        topic = topic_info.topic
+        partition = partition_info.partition
 
-      nil
-    end
+        begin
+          Protocol.handle_error(partition_info.error_code)
+        rescue Kafka::CorruptMessage
+          @logger.error "Corrupt message when writing to #{topic}/#{partition}"
+        rescue Kafka::UnknownTopicOrPartition
+          @logger.error "Unknown topic or partition #{topic}/#{partition}"
+        rescue Kafka::LeaderNotAvailable
+          @logger.error "Leader currently not available for #{topic}/#{partition}"
+          @broker_pool.mark_as_stale!
+        rescue Kafka::NotLeaderForPartition
+          @logger.error "Broker not currently leader for #{topic}/#{partition}"
+          @broker_pool.mark_as_stale!
+        rescue Kafka::RequestTimedOut
+          @logger.error "Timed out while writing to #{topic}/#{partition}"
+        rescue Kafka::NotEnoughReplicas
+          @logger.error "Not enough in-sync replicas for #{topic}/#{partition}"
+        rescue Kafka::NotEnoughReplicasAfterAppend
+          @logger.error "Messages written, but to fewer in-sync replicas than required for #{topic}/#{partition}"
+        else
+          offset = partition_info.offset
+          @logger.info "Successfully flushed messages for #{topic}/#{partition}; new offset is #{offset}"
 
-    def shutdown
-      @broker_pool.shutdown
+          # The messages were successfully written; clear them from the buffer.
+          @buffer.clear_messages(topic: topic, partition: partition)
+        end
+      end
     end
   end
 end

data/lib/kafka/protocol.rb

@@ -3,19 +3,34 @@ module Kafka
     PRODUCE_API_KEY = 0
     TOPIC_METADATA_API_KEY = 3
 
+    ERRORS = {
+      -1 => UnknownError,
+      1 => OffsetOutOfRange,
+      2 => CorruptMessage,
+      3 => UnknownTopicOrPartition,
+      4 => InvalidMessageSize,
+      5 => LeaderNotAvailable,
+      6 => NotLeaderForPartition,
+      7 => RequestTimedOut,
+      8 => BrokerNotAvailable,
+      9 => ReplicaNotAvailable,
+      10 => MessageSizeTooLarge,
+      12 => OffsetMetadataTooLarge,
+      17 => InvalidTopic,
+      18 => RecordListTooLarge,
+      19 => NotEnoughReplicas,
+      20 => NotEnoughReplicasAfterAppend,
+      21 => InvalidRequiredAcks,
+    }
+
+
     def self.handle_error(error_code)
-      case error_code
-      when -1 then raise UnknownError
-      when 0 then nil # no error, yay!
-      when 1 then raise OffsetOutOfRange
-      when 2 then raise CorruptMessage
-      when 3 then raise UnknownTopicOrPartition
-      when 4 then raise InvalidMessageSize
-      when 5 then raise LeaderNotAvailable
-      when 6 then raise NotLeaderForPartition
-      when 7 then raise RequestTimedOut
-      when 9 then raise ReplicaNotAvailable
-      else raise UnknownError, "Unknown error with code #{error_code}"
+      if error_code == 0
+        # No errors, yay!
+      elsif error = ERRORS[error_code]
+        raise error
+      else
+        raise UnknownError, "Unknown error with code #{error_code}"
       end
     end
   end

data/lib/kafka/protocol/metadata_response.rb

@@ -8,7 +8,7 @@ module Kafka
     #
     # * For each broker a node id, host, and port is provided.
     # * For each topic partition the node id of the broker acting as partition leader,
-    #   as well as a list of node ids for the set of replicas, are given. The `isr` list is
+    #   as well as a list of node ids for the set of replicas, are given. The +isr+ list is
     #   the subset of replicas that are "in sync", i.e. have fully caught up with the
     #   leader.
     #

data/lib/kafka/protocol/produce_response.rb

@@ -26,6 +26,14 @@ module Kafka
         @topics = topics
       end
 
+      def each_partition
+        @topics.each do |topic_info|
+          topic_info.partitions.each do |partition_info|
+            yield topic_info, partition_info
+          end
+        end
+      end
+
       def self.decode(decoder)
         topics = decoder.array do
           topic = decoder.string

data/lib/kafka/version.rb

@@ -1,3 +1,3 @@
 module Kafka
-  VERSION = "0.1.0-beta1"
+  VERSION = "0.1.0-beta2"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: ruby-kafka
 version: !ruby/object:Gem::Version
-  version: 0.1.0.pre.beta1
+  version: 0.1.0.pre.beta2
 platform: ruby
 authors:
 - Daniel Schierbeck
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2016-01-25 00:00:00.000000000 Z
+date: 2016-01-28 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -52,6 +52,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: pry
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 description: A client library for the Kafka distributed commit log. Still very much
   at the alpha stage.
 email:
@@ -62,22 +76,20 @@ extra_rdoc_files: []
 files:
 - ".gitignore"
 - ".rspec"
-- ".travis.yml"
 - Gemfile
 - LICENSE.txt
 - README.md
 - Rakefile
 - bin/console
 - bin/setup
-- docker-compose.yml
+- circle.yml
 - kafka.gemspec
 - lib/kafka.rb
 - lib/kafka/broker.rb
 - lib/kafka/broker_pool.rb
 - lib/kafka/client.rb
 - lib/kafka/connection.rb
-- lib/kafka/message.rb
-- lib/kafka/message_set.rb
+- lib/kafka/message_buffer.rb
 - lib/kafka/partitioner.rb
 - lib/kafka/producer.rb
 - lib/kafka/protocol.rb
@@ -91,7 +103,6 @@ files:
 - lib/kafka/protocol/topic_metadata_request.rb
 - lib/kafka/version.rb
 - lib/ruby-kafka.rb
-- test-setup.sh
 homepage: https://github.com/zendesk/ruby-kafka
 licenses:
 - Apache License Version 2.0

data/.travis.yml

@@ -1,4 +0,0 @@
-language: ruby
-rvm:
-  - 2.2.3
-before_install: gem install bundler -v 1.10.6

data/docker-compose.yml

@@ -1,35 +0,0 @@
-zookeeper:
-  image: wurstmeister/zookeeper
-  ports: 
-    - "2181"
-kafka1:
-  image: wurstmeister/kafka:0.8.2.0
-  ports:
-    - "9992:9092"
-  links:
-    - zookeeper:zk
-  environment:
-    KAFKA_ADVERTISED_HOST_NAME: "192.168.42.45"
-    KAFKA_CREATE_TOPICS: "test-messages:5:3"
-  volumes:
-    - /var/run/docker.sock:/var/run/docker.sock
-kafka2:
-  image: wurstmeister/kafka:0.8.2.0
-  ports:
-    - "9993:9092"
-  links:
-    - zookeeper:zk
-  environment:
-    KAFKA_ADVERTISED_HOST_NAME: "192.168.42.45"
-  volumes:
-    - /var/run/docker.sock:/var/run/docker.sock
-kafka3:
-  image: wurstmeister/kafka:0.8.2.0
-  ports:
-    - "9994:9092"
-  links:
-    - zookeeper:zk
-  environment:
-    KAFKA_ADVERTISED_HOST_NAME: "192.168.42.45"
-  volumes:
-    - /var/run/docker.sock:/var/run/docker.sock

data/lib/kafka/message.rb

@@ -1,12 +0,0 @@
-module Kafka
-  class Message
-    attr_reader :value, :key, :topic, :partition
-
-    def initialize(value, key:, topic:, partition:)
-      @value = value
-      @key = key
-      @topic = topic
-      @partition = partition
-    end
-  end
-end

data/lib/kafka/message_set.rb

@@ -1,24 +0,0 @@
-require "kafka/protocol/message"
-
-module Kafka
-  class MessageSet
-    def initialize(messages)
-      @messages = messages
-    end
-
-    def to_h
-      hsh = {}
-      
-      @messages.each do |message|
-        value, key = message.value, message.key
-        topic, partition = message.topic, message.partition
-
-        hsh[topic] ||= {}
-        hsh[topic][partition] ||= []
-        hsh[topic][partition] << Protocol::Message.new(value: value, key: key)
-      end
-
-      hsh
-    end
-  end
-end

data/test-setup.sh

@@ -1,3 +0,0 @@
-#!/bin/bash
-
-docker-compose run kafka1 /opt/kafka_2.10-0.8.2.0/bin/kafka-topics.sh --create --topic test-messages --replication-factor 3 --partitions 5 --zookeeper zk