ruby-kafka 0.1.0.pre.alpha → 0.1.0.pre.alpha2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 1a697874e8d60e20fefd56d530fae7af5d9a0122
-  data.tar.gz: c93b6564dde9e1630a1675a13dbc4227f4aa0053
+  metadata.gz: 945e87a87fcfebd2808de4203027613846a2f7ad
+  data.tar.gz: f43c576a56aea2f49ef1047aec56f376ec2be0b0
 SHA512:
-  metadata.gz: e64575cd8677e03e565f9ed64d7b9d6b548a80ab68c7f37d108a0f6d95161843acef2805a625db9d9502d5861fe166200b048119835209162235d9c4783d679b
-  data.tar.gz: 4caa3012931623f4f3372368c195736aee833995b76937b7e2dd8bea5dec2fa9ce31dc1bbb209d435144884a158e674b6fd55fb5087a0f971cc9245c678229cd
+  metadata.gz: c51ccf72b3822a773d013c68fe77665d7d5e9b4021131cb97d8bed65e47c681be18d89f718b63840ef5c463991b0c56c042b3ab747753c23b7f88c0b6131d8d3
+  data.tar.gz: 4c5c41e3858562fcdac0540652f5d3d8f21334a841b717f7f6281f1ff15a58c3ba165f518fbce9778669d5d35279e5242bb3ca3d1afce62a5c40c01b790e1720

data/README.md

@@ -1,15 +1,15 @@
 # Kafka
 
-Welcome to your new gem! In this directory, you'll find the files you need to be able to package up your Ruby library into a gem. Put your Ruby code in the file `lib/kafka`. To experiment with that code, run `bin/console` for an interactive prompt.
+A Ruby client library for the Kafka distributed log system. The focus of this library will be operational simplicity, with good logging and metrics that can make debugging issues easier.
 
-TODO: Delete this and the text above, and describe your gem
+This library is still in pre-alpha stage, but development is ongoing. Current efforts are focused on implementing a solid Producer client. The next step will be implementing a client for the Kafka 0.9 Consumer API.
 
 ## Installation
 
 Add this line to your application's Gemfile:
 
 ```ruby
-gem 'kafka'
+gem 'ruby-kafka'
 ```
 
 And then execute:
@@ -18,11 +18,26 @@ And then execute:
 
 Or install it yourself as:
 
-    $ gem install kafka
+    $ gem install ruby-kafka
 
 ## Usage
 
-TODO: Write usage instructions here
+```ruby
+kafka = Kafka.new(
+  seed_brokers: ["kafka1:9092", "kafka2:9092"],
+  client_id: "my-app",
+  logger: Logger.new($stderr),
+)
+
+producer = kafka.get_producer
+
+# `write` will buffer the message in the producer.
+producer.write("hello1", key: "x", topic: "test-messages", partition: 0)
+producer.write("hello2", key: "y", topic: "test-messages", partition: 1)
+
+# `flush` will send the buffered messages to the cluster.
+producer.flush
+```
 
 ## Development
 
@@ -32,7 +47,7 @@ To install this gem onto your local machine, run `bundle exec rake install`. To
 
 ## Contributing
 
-Bug reports and pull requests are welcome on GitHub at https://github.com/[USERNAME]/kafka.
+Bug reports and pull requests are welcome on GitHub at https://github.com/zendesk/ruby-kafka.
 
 
 ## Copyright and license

data/docker-compose.yml

@@ -0,0 +1,35 @@
+zookeeper:
+  image: wurstmeister/zookeeper
+  ports: 
+    - "2181"
+kafka1:
+  image: wurstmeister/kafka:0.8.2.0
+  ports:
+    - "9992:9092"
+  links:
+    - zookeeper:zk
+  environment:
+    KAFKA_ADVERTISED_HOST_NAME: "192.168.42.45"
+    KAFKA_CREATE_TOPICS: "test-messages:5:3"
+  volumes:
+    - /var/run/docker.sock:/var/run/docker.sock
+kafka2:
+  image: wurstmeister/kafka:0.8.2.0
+  ports:
+    - "9993:9092"
+  links:
+    - zookeeper:zk
+  environment:
+    KAFKA_ADVERTISED_HOST_NAME: "192.168.42.45"
+  volumes:
+    - /var/run/docker.sock:/var/run/docker.sock
+kafka3:
+  image: wurstmeister/kafka:0.8.2.0
+  ports:
+    - "9994:9092"
+  links:
+    - zookeeper:zk
+  environment:
+    KAFKA_ADVERTISED_HOST_NAME: "192.168.42.45"
+  volumes:
+    - /var/run/docker.sock:/var/run/docker.sock

data/lib/kafka.rb

@@ -1,9 +1,19 @@
 require "kafka/version"
-require "kafka/cluster"
-require "kafka/producer"
+require "kafka/client"
 
 module Kafka
+  Error = Class.new(StandardError)
+  ConnectionError = Class.new(Error)
+  CorruptMessage = Class.new(Error)
+  UnknownError = Class.new(Error)
+  OffsetOutOfRange = Class.new(Error)
+  UnknownTopicOrPartition = Class.new(Error)
+  InvalidMessageSize = Class.new(Error)
+  LeaderNotAvailable = Class.new(Error)
+  NotLeaderForPartition = Class.new(Error)
+  RequestTimedOut = Class.new(Error)
+
   def self.new(**options)
-    Cluster.new(**options)
+    Client.new(**options)
   end
 end

data/lib/kafka/broker.rb

@@ -0,0 +1,50 @@
+require "logger"
+require "kafka/connection"
+require "kafka/protocol"
+
+module Kafka
+  class Broker
+    def initialize(host:, port:, node_id: nil, client_id:, logger:)
+      @host, @port, @node_id = host, port, node_id
+
+      @connection = Connection.new(
+        host: host,
+        port: port,
+        client_id: client_id,
+        logger: logger
+      )
+
+      @logger = logger
+    end
+
+    def to_s
+      "#{@host}:#{@port} (node_id=#{@node_id.inspect})"
+    end
+
+    def fetch_metadata(**options)
+      api_key = Protocol::TOPIC_METADATA_API_KEY
+      request = Protocol::TopicMetadataRequest.new(**options)
+      response_class = Protocol::MetadataResponse
+
+      response = @connection.request(api_key, request, response_class)
+
+      response.topics.each do |topic|
+        Protocol.handle_error(topic.topic_error_code)
+
+        topic.partitions.each do |partition|
+          Protocol.handle_error(partition.partition_error_code)
+        end
+      end
+
+      response
+    end
+
+    def produce(**options)
+      api_key = Protocol::PRODUCE_API_KEY
+      request = Protocol::ProduceRequest.new(**options)
+      response_class = request.requires_acks? ? Protocol::ProduceResponse : nil
+
+      @connection.request(api_key, request, response_class)
+    end
+  end
+end

data/lib/kafka/broker_pool.rb

@@ -0,0 +1,77 @@
+require "kafka/broker"
+
+module Kafka
+
+  # A broker pool represents the set of brokers in a cluster. It needs to be initialized
+  # with a non-empty list of seed brokers. The first seed broker that the pool can connect
+  # to will be asked for the cluster metadata, which allows the pool to map topic
+  # partitions to the current leader for those partitions.
+  class BrokerPool
+
+    # Initializes a broker pool with a set of seed brokers.
+    #
+    # The pool will try to fetch cluster metadata from one of the brokers.
+    #
+    # @param seed_brokers [Array<String>]
+    # @param client_id [String]
+    # @param logger [Logger]
+    def initialize(seed_brokers:, client_id:, logger:)
+      @client_id = client_id
+      @logger = logger
+      @brokers = {}
+
+      initialize_from_seed_brokers(seed_brokers)
+    end
+
+    # Gets the leader of the given topic and partition.
+    #
+    # @param topic [String]
+    # @param partition [Integer]
+    # @return [Broker] the broker that's currently acting as leader of the partition.
+    def get_leader(topic, partition)
+      leader_id = @cluster_info.find_leader_id(topic, partition)
+
+      broker_for_id(leader_id)
+    end
+
+    private
+
+    def broker_for_id(broker_id)
+      @brokers[broker_id] ||= connect_to_broker(broker_id)
+    end
+
+    def connect_to_broker(broker_id)
+      broker_info = @cluster_info.find_broker(broker_id)
+
+      Broker.new(
+        host: broker_info.host,
+        port: broker_info.port,
+        node_id: broker_info.node_id,
+        client_id: @client_id,
+        logger: @logger,
+      )
+    end
+
+    def initialize_from_seed_brokers(seed_brokers)
+      seed_brokers.each do |node|
+        @logger.info "Trying to initialize broker pool from node #{node}"
+
+        begin
+          host, port = node.split(":", 2)
+
+          broker = Broker.new(host: host, port: port, client_id: @client_id, logger: @logger)
+
+          @cluster_info = broker.fetch_metadata
+
+          @logger.info "Initialized broker pool with brokers: #{@cluster_info.brokers.inspect}"
+
+          return
+        rescue Error => e
+          @logger.error "Failed to fetch metadata from broker #{broker}: #{e}"
+        end
+      end
+
+      raise ConnectionError, "Could not connect to any of the seed brokers: #{seed_brokers.inspect}"
+    end
+  end
+end

data/lib/kafka/client.rb

@@ -0,0 +1,22 @@
+require "kafka/broker_pool"
+require "kafka/producer"
+
+module Kafka
+  class Client
+    def initialize(seed_brokers:, client_id:, logger:)
+      @seed_brokers = seed_brokers
+      @client_id = client_id
+      @logger = logger
+    end
+
+    def get_producer(**options)
+      broker_pool = BrokerPool.new(
+        seed_brokers: @seed_brokers,
+        client_id: @client_id,
+        logger: @logger
+      )
+
+      Producer.new(broker_pool: broker_pool, logger: @logger, **options)
+    end
+  end
+end

data/lib/kafka/connection.rb

@@ -4,64 +4,109 @@ require "kafka/protocol/encoder"
 require "kafka/protocol/decoder"
 
 module Kafka
-  ConnectionError = Class.new(StandardError)
 
+  # A connection to a single Kafka broker.
+  #
+  # Usually you'll need a separate connection to each broker in a cluster, since most
+  # requests must be directed specifically to the broker that is currently leader for
+  # the set of topic partitions you want to produce to or consumer from.
   class Connection
+    API_VERSION = 0
+
+    # Opens a connection to a Kafka broker.
+    #
+    # @param host [String] the hostname of the broker.
+    # @param port [Integer] the port of the broker.
+    # @param client_id [String] the client id is a user-specified string sent in each
+    #   request to help trace calls and should logically identify the application
+    #   making the request.
+    # @param logger [Logger] the logger used to log trace messages.
+    #
+    # @return [Connection] a new connection.
     def initialize(host:, port:, client_id:, logger:)
-      @host = host
-      @port = port
-      @client_id = client_id
+      @host, @port, @client_id = host, port, client_id
       @logger = logger
-      @socket = nil
-      @correlation_id = 0
-      @socket_timeout = 1000
-    end
 
-    def open
       @logger.info "Opening connection to #{@host}:#{@port} with client id #{@client_id}..."
 
-      @socket = TCPSocket.new(@host, @port)
+      @socket = TCPSocket.new(host, port)
+
+      @encoder = Kafka::Protocol::Encoder.new(@socket)
+      @decoder = Kafka::Protocol::Decoder.new(@socket)
+
+      # Correlation id is initialized to zero and bumped for each request.
+      @correlation_id = 0
     rescue SocketError => e
-      @logger.error "Failed to connect to #{@host}:#{@port}: #{e}"
+      @logger.error "Failed to connect to #{host}:#{port}: #{e}"
 
       raise ConnectionError, e
     end
 
+    def to_s
+      "#{@host}:#{@port}"
+    end
+
+    # Sends a request over the connection.
+    #
+    # @param api_key [Integer] the integer code for the API that is invoked.
+    # @param request [#encode] the request that should be encoded and written.
+    # @param response_class [#decode] an object that can decode the response.
+    #
+    # @return [Object] the response that was decoded by `response_class`.
+    def request(api_key, request, response_class)
+      write_request(api_key, request)
+
+      unless response_class.nil?
+        read_response(response_class)
+      end
+    end
+
+    private
+
+    # Writes a request over the connection.
+    #
+    # @param api_key [Integer] the integer code for the API that is invoked.
+    # @param request [#encode] the request that should be encoded and written.
+    #
+    # @return [nil]
     def write_request(api_key, request)
       @correlation_id += 1
+      @logger.debug "Sending request #{@correlation_id} to #{to_s}"
 
       message = Kafka::Protocol::RequestMessage.new(
         api_key: api_key,
-        api_version: 0,
+        api_version: API_VERSION,
         correlation_id: @correlation_id,
         client_id: @client_id,
         request: request,
       )
 
-      buffer = StringIO.new
-      message_encoder = Kafka::Protocol::Encoder.new(buffer)
-      message.encode(message_encoder)
-
-      @logger.info "Sending request #{@correlation_id} (#{request.class})..."
+      data = Kafka::Protocol::Encoder.encode_with(message)
+      @encoder.write_bytes(data)
 
-      connection_encoder = Kafka::Protocol::Encoder.new(@socket)
-      connection_encoder.write_bytes(buffer.string)
+      nil
     end
 
-    def read_response(response)
-      @logger.info "Reading response #{response.class}"
+    # Reads a response from the connection.
+    #
+    # @param response_class [#decode] an object that can decode the response from
+    #   a given Decoder.
+    #
+    # @return [nil]
+    def read_response(response_class)
+      @logger.debug "Waiting for response #{@correlation_id} from #{to_s}"
 
-      connection_decoder = Kafka::Protocol::Decoder.new(@socket)
-      bytes = connection_decoder.bytes
+      bytes = @decoder.bytes
 
       buffer = StringIO.new(bytes)
       response_decoder = Kafka::Protocol::Decoder.new(buffer)
 
       correlation_id = response_decoder.int32
+      response = response_class.decode(response_decoder)
 
-      @logger.info "Correlation id #{correlation_id}"
+      @logger.debug "Received response #{correlation_id} from #{to_s}"
 
-      response.decode(response_decoder)
+      response
     end
   end
 end

data/lib/kafka/message.rb

@@ -0,0 +1,12 @@
+module Kafka
+  class Message
+    attr_reader :value, :key, :topic, :partition
+
+    def initialize(value, key:, topic:, partition:)
+      @value = value
+      @key = key
+      @topic = topic
+      @partition = partition
+    end
+  end
+end

data/lib/kafka/message_set.rb

@@ -0,0 +1,24 @@
+require "kafka/protocol/message"
+
+module Kafka
+  class MessageSet
+    def initialize(messages)
+      @messages = messages
+    end
+
+    def to_h
+      hsh = {}
+      
+      @messages.each do |message|
+        value, key = message.value, message.key
+        topic, partition = message.topic, message.partition
+
+        hsh[topic] ||= {}
+        hsh[topic][partition] ||= []
+        hsh[topic][partition] << Protocol::Message.new(value: value, key: key)
+      end
+
+      hsh
+    end
+  end
+end

data/lib/kafka/producer.rb

@@ -1,4 +1,5 @@
-require "kafka/protocol/message"
+require "kafka/message"
+require "kafka/message_set"
 
 module Kafka
   class Producer
@@ -6,30 +7,49 @@ module Kafka
     #   acknowledgement from the broker before timing out.
     # @param required_acks [Integer] The number of replicas that must acknowledge
     #   a write.
-    def initialize(cluster:, logger:, timeout: 10_000, required_acks: 1)
-      @cluster = cluster
+    def initialize(broker_pool:, logger:, timeout: 10_000, required_acks: 1)
+      @broker_pool = broker_pool
       @logger = logger
       @required_acks = required_acks
       @timeout = timeout
-      @buffer = {}
+      @buffered_messages = []
     end
 
     def write(value, key:, topic:, partition:)
-      message = Protocol::Message.new(value: value, key: key)
-
-      @buffer[topic] ||= {}
-      @buffer[topic][partition] ||= []
-      @buffer[topic][partition] << message
+      @buffered_messages << Message.new(value, key: key, topic: topic, partition: partition)
     end
 
     def flush
-      @cluster.produce(
-        required_acks: @required_acks,
-        timeout: @timeout,
-        messages_for_topics: @buffer
-      )
+      messages_for_broker = {}
+
+      @buffered_messages.each do |message|
+        broker = @broker_pool.get_leader(message.topic, message.partition)
+
+        messages_for_broker[broker] ||= []
+        messages_for_broker[broker] << message
+      end
+
+      messages_for_broker.each do |broker, messages|
+        @logger.info "Sending #{messages.count} messages to broker #{broker}"
+
+        message_set = MessageSet.new(messages)
+
+        response = broker.produce(
+          messages_for_topics: message_set.to_h,
+          required_acks: @required_acks,
+          timeout: @timeout,
+        )
+
+        if response
+          response.topics.each do |topic_info|
+            topic_info.partitions.each do |partition_info|
+              Protocol.handle_error(partition_info.error_code)
+            end
+          end
+        end
+      end
 
-      @buffer = {}
+      @buffered_messages.clear
     end
   end
 end

data/lib/kafka/protocol.rb

@@ -2,6 +2,21 @@ module Kafka
   module Protocol
     PRODUCE_API_KEY = 0
     TOPIC_METADATA_API_KEY = 3
+
+    def self.handle_error(error_code)
+      case error_code
+      when -1 then raise UnknownError
+      when 0 then nil # no error, yay!
+      when 1 then raise OffsetOutOfRange
+      when 2 then raise CorruptMessage
+      when 3 then raise UnknownTopicOrPartition
+      when 4 then raise InvalidMessageSize
+      when 5 then raise LeaderNotAvailable
+      when 6 then raise NotLeaderForPartition
+      when 7 then raise RequestTimedOut
+      else raise UnknownError, "Unknown error with code #{error_code}"
+      end
+    end
   end
 end
 

data/lib/kafka/protocol/decoder.rb

@@ -1,31 +1,62 @@
 module Kafka
   module Protocol
+
+    # A decoder wraps an IO object, making it easy to read specific data types
+    # from it. The Kafka protocol is not self-describing, so a client must call
+    # these methods in just the right order for things to work.
     class Decoder
+
+      # Initializes a new decoder.
+      #
+      # @param io [IO] an object that acts as an IO.
       def initialize(io)
         @io = io
       end
 
+      # Decodes an 8-bit integer from the IO object.
+      #
+      # @return [Integer]
       def int8
         read(1).unpack("C").first
       end
 
+      # Decodes a 16-bit integer from the IO object.
+      #
+      # @return [Integer]
       def int16
         read(2).unpack("s>").first
       end
 
+      # Decodes a 32-bit integer from the IO object.
+      #
+      # @return [Integer]
       def int32
         read(4).unpack("l>").first
       end
 
+      # Decodes a 64-bit integer from the IO object.
+      #
+      # @return [Integer]
       def int64
         read(8).unpack("q>").first
       end
 
-      def array
+      # Decodes an array from the IO object.
+      #
+      # The provided block will be called once for each item in the array. It is
+      # the responsibility of the block to decode the proper type in the block,
+      # since there's no information that allows the type to be inferred
+      # automatically.
+      #
+      # @return [Array]
+      def array(&block)
         size = int32
-        size.times.map { yield }
+        size.times.map(&block)
       end
 
+      # Decodes a string from the IO object.
+      #
+      # @return [String]
       def string
         size = int16
 
@@ -36,6 +67,9 @@ module Kafka
         end
       end
 
+      # Decodes a list of bytes from the IO object.
+      #
+      # @return [String]
       def bytes
         size = int32
 
@@ -46,6 +80,10 @@ module Kafka
         end
       end
 
+      # Reads the specified number of bytes from the IO object, returning them
+      # as a String.
+      #
+      # @return [String]
       def read(number_of_bytes)
         @io.read(number_of_bytes)
       end

data/lib/kafka/protocol/encoder.rb

@@ -1,36 +1,77 @@
 module Kafka
   module Protocol
+
+    # An encoder wraps an IO object, making it easy to write specific data types
+    # to it.
     class Encoder
+
+      # Initializes a new encoder.
+      #
+      # @param io [IO] an object that acts as an IO.
       def initialize(io)
         @io = io
         @io.set_encoding(Encoding::BINARY)
       end
 
+      # Writes bytes directly to the IO object.
+      #
+      # @param bytes [String]
+      # @return [nil]
       def write(bytes)
         @io.write(bytes)
+
+        nil
       end
 
+      # Writes an 8-bit integer to the IO object.
+      #
+      # @param int [Integer]
+      # @return [nil]
       def write_int8(int)
         write([int].pack("C"))
       end
 
+      # Writes a 16-bit integer to the IO object.
+      #
+      # @param int [Integer]
+      # @return [nil]
       def write_int16(int)
         write([int].pack("s>"))
       end
 
+      # Writes a 32-bit integer to the IO object.
+      #
+      # @param int [Integer]
+      # @return [nil]
       def write_int32(int)
         write([int].pack("l>"))
       end
 
+      # Writes a 64-bit integer to the IO object.
+      #
+      # @param int [Integer]
+      # @return [nil]
       def write_int64(int)
         write([int].pack("q>"))
       end
 
+      # Writes an array to the IO object.
+      #
+      # Each item in the specified array will be yielded to the provided block;
+      # it's the responsibility of the block to write those items using the
+      # encoder.
+      #
+      # @param array [Array]
+      # @return [nil]
       def write_array(array, &block)
         write_int32(array.size)
         array.each(&block)
       end
 
+      # Writes a string to the IO object.
+      #
+      # @param string [String]
+      # @return [nil]
       def write_string(string)
         if string.nil?
           write_int16(-1)
@@ -40,6 +81,10 @@ module Kafka
         end
       end
 
+      # Writes a byte string to the IO object.
+      #
+      # @param bytes [String]
+      # @return [nil]
       def write_bytes(bytes)
         if bytes.nil?
           write_int32(-1)
@@ -48,6 +93,19 @@ module Kafka
           write(bytes)
         end
       end
+
+      # Encodes an object into a new buffer.
+      #
+      # @param object [#encode] the object that will encode itself.
+      # @return [String] the encoded data.
+      def self.encode_with(object)
+        buffer = StringIO.new
+        encoder = new(buffer)
+
+        object.encode(encoder)
+
+        buffer.string
+      end
     end
   end
 end

data/lib/kafka/protocol/metadata_response.rb

@@ -1,7 +1,37 @@
 module Kafka
   module Protocol
+
+    # A response to a {TopicMetadataRequest}.
+    #
+    # The response contains information on the brokers, topics, and partitions in
+    # the cluster.
+    #
+    # * For each broker a node id, host, and port is provided.
+    # * For each topic partition the node id of the broker acting as partition leader,
+    #   as well as a list of node ids for the set of replicas, are given. The `isr` list is
+    #   the subset of replicas that are "in sync", i.e. have fully caught up with the
+    #   leader.
+    #
+    # == API Specification
+    #
+    #     MetadataResponse => [Broker][TopicMetadata]
+    #       Broker => NodeId Host Port  (any number of brokers may be returned)
+    #         NodeId => int32
+    #         Host => string
+    #         Port => int32
+    #
+    #       TopicMetadata => TopicErrorCode TopicName [PartitionMetadata]
+    #         TopicErrorCode => int16
+    #
+    #       PartitionMetadata => PartitionErrorCode PartitionId Leader Replicas Isr
+    #         PartitionErrorCode => int16
+    #         PartitionId => int32
+    #         Leader => int32
+    #         Replicas => [int32]
+    #         Isr => [int32]
+    #
     class MetadataResponse
-      class Broker
+      class BrokerInfo
         attr_reader :node_id, :host, :port
 
         def initialize(node_id:, host:, port:)
@@ -9,9 +39,17 @@ module Kafka
           @host = host
           @port = port
         end
+
+        def inspect
+          "#{host}:#{port} (node_id=#{node_id})"
+        end
       end
 
       class PartitionMetadata
+        attr_reader :partition_id, :leader
+
+        attr_reader :partition_error_code
+
         def initialize(partition_error_code:, partition_id:, leader:, replicas:, isr:)
           @partition_error_code = partition_error_code
           @partition_id = partition_id
@@ -22,6 +60,14 @@ module Kafka
       end
 
       class TopicMetadata
+        # @return [String] the name of the topic
+        attr_reader :topic_name
+
+        # @return [Array<PartitionMetadata>] the partitions in the topic.
+        attr_reader :partitions
+
+        attr_reader :topic_error_code
+
         def initialize(topic_error_code:, topic_name:, partitions:)
           @topic_error_code = topic_error_code
           @topic_name = topic_name
@@ -29,22 +75,65 @@ module Kafka
         end
       end
 
-      attr_reader :brokers, :topics
+      # @return [Array<BrokerInfo>] the list of brokers in the cluster.
+      attr_reader :brokers
+
+      # @return [Array<TopicMetadata>] the list of topics in the cluster.
+      attr_reader :topics
+
+      def initialize(brokers:, topics:)
+        @brokers = brokers
+        @topics = topics
+      end
+
+      # Finds the node id of the broker that is acting as leader for the given topic
+      # and partition per this metadata.
+      #
+      # @param topic [String] the name of the topic.
+      # @param partition [Integer] the partition number.
+      # @return [Integer] the node id of the leader.
+      def find_leader_id(topic, partition)
+        topic_info = @topics.find {|t| t.topic_name == topic }
+
+        if topic_info.nil?
+          raise "no topic #{topic}"
+        end
+
+        partition_info = topic_info.partitions.find {|p| p.partition_id == partition }
+
+        if partition_info.nil?
+          raise "no partition #{partition} in topic #{topic}"
+        end
+
+        partition_info.leader
+      end
+
+      # Finds the broker info for the given node id.
+      #
+      # @param node_id [Integer] the node id of the broker.
+      # @return [BrokerInfo] information about the broker.
+      def find_broker(node_id)
+        @brokers.find {|broker| broker.node_id == node_id }
+      end
 
-      def decode(decoder)
-        @brokers = decoder.array do
+      # Decodes a MetadataResponse from a {Decoder} containing response data.
+      #
+      # @param decoder [Decoder]
+      # @return [MetadataResponse] the metadata response.
+      def self.decode(decoder)
+        brokers = decoder.array do
           node_id = decoder.int32
           host = decoder.string
           port = decoder.int32
 
-          Broker.new(
+          BrokerInfo.new(
             node_id: node_id,
             host: host,
             port: port
           )
         end
 
-        @topics = decoder.array do
+        topics = decoder.array do
           topic_error_code = decoder.int16
           topic_name = decoder.string
 
@@ -64,6 +153,8 @@ module Kafka
             partitions: partitions,
           )
         end
+
+        new(brokers: brokers, topics: topics)
       end
     end
   end

data/lib/kafka/protocol/produce_request.rb

@@ -74,7 +74,7 @@ module Kafka
           # When encoding a message into a message set, the bytesize of the message must
           # precede the actual bytes. Therefore we need to encode the message into a
           # separate buffer first.
-          encoded_message = encode_message(message)
+          encoded_message = Encoder.encode_with(message)
 
           encoder.write_int64(offset)
 
@@ -84,15 +84,6 @@ module Kafka
 
         buffer.string
       end
-
-      def encode_message(message)
-        buffer = StringIO.new
-        encoder = Encoder.new(buffer)
-
-        message.encode(encoder)
-
-        buffer.string
-      end
     end
   end
 end

data/lib/kafka/protocol/produce_response.rb

@@ -4,7 +4,7 @@ module Kafka
       class TopicInfo
         attr_reader :topic, :partitions
 
-        def initialize(topic, partitions)
+        def initialize(topic:, partitions:)
           @topic = topic
           @partitions = partitions
         end
@@ -13,7 +13,7 @@ module Kafka
       class PartitionInfo
         attr_reader :partition, :error_code, :offset
 
-        def initialize(partition, error_code, offset)
+        def initialize(partition:, error_code:, offset:)
           @partition = partition
           @error_code = error_code
           @offset = offset
@@ -22,20 +22,26 @@ module Kafka
 
       attr_reader :topics
 
-      def decode(decoder)
-        @topics = decoder.array do
+      def initialize(topics: [])
+        @topics = topics
+      end
+
+      def self.decode(decoder)
+        topics = decoder.array do
           topic = decoder.string
 
           partitions = decoder.array do
-            partition = decoder.int32
-            error_code = decoder.int16
-            offset = decoder.int64
-
-            PartitionInfo.new(partition, error_code, offset)
+            PartitionInfo.new(
+              partition: decoder.int32,
+              error_code: decoder.int16,
+              offset: decoder.int64,
+            )
           end
 
-          TopicInfo.new(topic, partitions)
+          TopicInfo.new(topic: topic, partitions: partitions)
         end
+
+        new(topics: topics)
       end
     end
   end

data/lib/kafka/version.rb

@@ -1,3 +1,3 @@
 module Kafka
-  VERSION = "0.1.0-alpha"
+  VERSION = "0.1.0-alpha2"
 end

data/lib/ruby-kafka.rb

@@ -0,0 +1,3 @@
+# Needed because the gem is registered as `ruby-kafka`.
+
+require "kafka"

data/test-setup.sh

@@ -0,0 +1,3 @@
+#!/bin/bash
+
+docker-compose run kafka1 /opt/kafka_2.10-0.8.2.0/bin/kafka-topics.sh --create --topic test-messages --replication-factor 3 --partitions 5 --zookeeper zk

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: ruby-kafka
 version: !ruby/object:Gem::Version
-  version: 0.1.0.pre.alpha
+  version: 0.1.0.pre.alpha2
 platform: ruby
 authors:
 - Daniel Schierbeck
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2016-01-19 00:00:00.000000000 Z
+date: 2016-01-21 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -69,10 +69,15 @@ files:
 - Rakefile
 - bin/console
 - bin/setup
+- docker-compose.yml
 - kafka.gemspec
 - lib/kafka.rb
-- lib/kafka/cluster.rb
+- lib/kafka/broker.rb
+- lib/kafka/broker_pool.rb
+- lib/kafka/client.rb
 - lib/kafka/connection.rb
+- lib/kafka/message.rb
+- lib/kafka/message_set.rb
 - lib/kafka/producer.rb
 - lib/kafka/protocol.rb
 - lib/kafka/protocol/decoder.rb
@@ -84,6 +89,8 @@ files:
 - lib/kafka/protocol/request_message.rb
 - lib/kafka/protocol/topic_metadata_request.rb
 - lib/kafka/version.rb
+- lib/ruby-kafka.rb
+- test-setup.sh
 homepage: https://github.com/zendesk/ruby-kafka
 licenses:
 - Apache License Version 2.0

data/lib/kafka/cluster.rb

@@ -1,53 +0,0 @@
-require "logger"
-require "kafka/connection"
-require "kafka/protocol"
-
-module Kafka
-  class Cluster
-    def self.connect(brokers:, client_id:, logger:)
-      host, port = brokers.first.split(":", 2)
-
-      connection = Connection.new(
-        host: host,
-        port: port.to_i,
-        client_id: client_id,
-        logger: logger
-      )
-
-      connection.open
-
-      new(connection: connection, logger: logger)
-    end
-
-    def initialize(connection:, logger: nil)
-      @connection = connection
-      @logger = logger
-    end
-
-    def fetch_metadata(**options)
-      api_key = Protocol::TOPIC_METADATA_API_KEY
-      request = Protocol::TopicMetadataRequest.new(**options)
-      response = Protocol::MetadataResponse.new
-
-      @connection.write_request(api_key, request)
-      @connection.read_response(response)
-
-      response
-    end
-
-    def produce(**options)
-      api_key = Protocol::PRODUCE_API_KEY
-      request = Protocol::ProduceRequest.new(**options)
-
-      @connection.write_request(api_key, request)
-
-      if request.requires_acks?
-        response = Protocol::ProduceResponse.new
-        @connection.read_response(response)
-        response
-      else
-        nil
-      end
-    end
-  end
-end