ruby-kafka 0.1.1 → 0.1.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: a0a1febaafd77046f2840beb708d3ad0d6d4520b
-  data.tar.gz: 5a5824f5799db3d6f7f00193baa3c3a9b4aa6a93
+  metadata.gz: e80cbf966470ab5038d6aab4f88fde89bd5e7f66
+  data.tar.gz: fcb92660786331e9b0f70d9a3e33aa8919590eff
 SHA512:
-  metadata.gz: 8617964bb38c05859f8fed11c77aa6b906ddf2d4bdd86a4e0ab003e4ba797454f886bf2e9611270de30790a4e24cb0bab94d70274fd020fed148fa81d2868d0a
-  data.tar.gz: acc55fc5c0afcf0c9f46e33b04dcd0ec47a1de7fc6f63819d1ac2cdb4b5c46b347a1e570494aec08fc09993ebfa4ae6fbb260514772a1aa49159c6c8ed1a3e46
+  metadata.gz: a62034fba5bf5f64f27d19676ac3429bfc929984f744db234d2fe25db10ae32ae93da6ff1d2635a0285e1d72010ec565951b2fcf00c7e5bc0aba56ba158b03d5
+  data.tar.gz: 236d24d5b6a8fac6fbad9cedc01b0639085ca746bc8bcf12f110aacd2f120fcf0c16f81a58e01c10d92fcc8b2c1c235ce228bc1b731412d4e94ff2fc1dbf43e8

data/README.md

@@ -62,6 +62,8 @@ producer.produce("hello4", topic: "test-messages", partition_key: "yo")
 producer.send_messages
 ```
 
+Read the docs for [Kafka::Producer](http://www.rubydoc.info/gems/ruby-kafka/Kafka/Producer) for more details.
+
 ## Development
 
 After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake spec` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
@@ -84,6 +86,16 @@ The API should now have stabilized and the library should be battle tested enoug
 
 The Consumer API defined by Kafka 0.9 will be implemented.
 
+## Why a new library?
+
+There are a few existing Kafka clients in Ruby:
+
+* [Poseidon](https://github.com/bpot/poseidon) seems to work for Kafka 0.8, but the project has is unmaintained and has known issues.
+* [Hermann](https://github.com/reiseburo/hermann) wraps the C library [librdkafka](https://github.com/edenhill/librdkafka) and seems to be very efficient, but its API and mode of operation is too intrusive for our needs.
+* [jruby-kafka](https://github.com/joekiller/jruby-kafka) is a great option if you're running on JRuby.
+
+We needed a robust client that could be used from our existing Ruby apps, allowed our Ops to monitor operation, and provided flexible error handling. There didn't exist such a client, hence this project. 
+
 ## Contributing
 
 Bug reports and pull requests are welcome on GitHub at https://github.com/zendesk/ruby-kafka.

data/lib/kafka/broker.rb

@@ -24,11 +24,10 @@ module Kafka
     end
 
     def fetch_metadata(**options)
-      api_key = Protocol::TOPIC_METADATA_API_KEY
       request = Protocol::TopicMetadataRequest.new(**options)
       response_class = Protocol::MetadataResponse
 
-      response = @connection.request(api_key, request, response_class)
+      response = @connection.send_request(request, response_class)
 
       response.topics.each do |topic|
         Protocol.handle_error(topic.topic_error_code)
@@ -47,11 +46,10 @@ module Kafka
     end
 
     def produce(**options)
-      api_key = Protocol::PRODUCE_API_KEY
       request = Protocol::ProduceRequest.new(**options)
       response_class = request.requires_acks? ? Protocol::ProduceResponse : nil
 
-      @connection.request(api_key, request, response_class)
+      @connection.send_request(request, response_class)
     end
   end
 end

data/lib/kafka/broker_pool.rb

@@ -15,10 +15,12 @@ module Kafka
     # @param seed_brokers [Array<String>]
     # @param client_id [String]
     # @param logger [Logger]
+    # @param connect_timeout [Integer, nil] see {Connection#initialize}.
     # @param socket_timeout [Integer, nil] see {Connection#initialize}.
-    def initialize(seed_brokers:, client_id:, logger:, socket_timeout: nil)
+    def initialize(seed_brokers:, client_id:, logger:, connect_timeout: nil, socket_timeout: nil)
       @client_id = client_id
       @logger = logger
+      @connect_timeout = connect_timeout
       @socket_timeout = socket_timeout
       @brokers = {}
       @seed_brokers = seed_brokers
@@ -33,13 +35,9 @@ module Kafka
     #
     # @param topic [String]
     # @param partition [Integer]
-    # @return [Integer] the broker id.
-    def get_leader_id(topic, partition)
-      cluster_info.find_leader_id(topic, partition)
-    end
-
-    def get_broker(broker_id)
-      @brokers[broker_id] ||= connect_to_broker(broker_id)
+    # @return [Broker] the broker that's currently leader.
+    def get_leader(topic, partition)
+      get_broker(get_leader_id(topic, partition))
     end
 
     def partitions_for(topic)
@@ -59,6 +57,14 @@ module Kafka
 
     private
 
+    def get_leader_id(topic, partition)
+      cluster_info.find_leader_id(topic, partition)
+    end
+
+    def get_broker(broker_id)
+      @brokers[broker_id] ||= connect_to_broker(broker_id)
+    end
+
     def cluster_info
       @cluster_info ||= fetch_cluster_info
     end
@@ -109,6 +115,7 @@ module Kafka
         port: broker_info.port,
         node_id: broker_info.node_id,
         client_id: @client_id,
+        connect_timeout: @connect_timeout,
         socket_timeout: @socket_timeout,
         logger: @logger,
       )

data/lib/kafka/client.rb

@@ -14,17 +14,21 @@ module Kafka
     #
     # @param logger [Logger]
     #
+    # @param connect_timeout [Integer, nil] the timeout setting for connecting
+    #   to brokers. See {BrokerPool#initialize}.
+    #
     # @param socket_timeout [Integer, nil] the timeout setting for socket
     #   connections. See {BrokerPool#initialize}.
     #
     # @return [Client]
-    def initialize(seed_brokers:, client_id: DEFAULT_CLIENT_ID, logger:, socket_timeout: nil)
+    def initialize(seed_brokers:, client_id: DEFAULT_CLIENT_ID, logger:, connect_timeout: nil, socket_timeout: nil)
       @logger = logger
 
       @broker_pool = BrokerPool.new(
         seed_brokers: seed_brokers,
         client_id: client_id,
         logger: logger,
+        connect_timeout: connect_timeout,
         socket_timeout: socket_timeout,
       )
     end

data/lib/kafka/connection.rb

@@ -1,5 +1,6 @@
 require "stringio"
 require "kafka/socket_with_timeout"
+require "kafka/instrumentation"
 require "kafka/protocol/request_message"
 require "kafka/protocol/encoder"
 require "kafka/protocol/decoder"
@@ -11,8 +12,20 @@ module Kafka
   # Usually you'll need a separate connection to each broker in a cluster, since most
   # requests must be directed specifically to the broker that is currently leader for
   # the set of topic partitions you want to produce to or consumer from.
+  #
+  # ## Instrumentation
+  #
+  # Connections emit a `request.kafka` notification on each request. The following
+  # keys will be found in the payload:
+  #
+  # * `:api` — the name of the API being invoked.
+  # * `:request_size` — the number of bytes in the request.
+  # * `:response_size` — the number of bytes in the response.
+  #
+  # The notification also includes the duration of the request.
+  #
   class Connection
-    SOCKET_TIMEOUT = 5
+    SOCKET_TIMEOUT = 10
     CONNECT_TIMEOUT = 10
 
     # Opens a connection to a Kafka broker.
@@ -26,7 +39,7 @@ module Kafka
     # @param connect_timeout [Integer] the socket timeout for connecting to the broker.
     #   Default is 10 seconds.
     # @param socket_timeout [Integer] the socket timeout for reading and writing to the
-    #   broker. Default is 5 seconds.
+    #   broker. Default is 10 seconds.
     #
     # @return [Connection] a new connection.
     def initialize(host:, port:, client_id:, logger:, connect_timeout: nil, socket_timeout: nil)
@@ -35,33 +48,13 @@ module Kafka
 
       @connect_timeout = connect_timeout || CONNECT_TIMEOUT
       @socket_timeout = socket_timeout || SOCKET_TIMEOUT
-
-      @logger.info "Opening connection to #{@host}:#{@port} with client id #{@client_id}..."
-
-      connect
     end
 
     def to_s
       "#{@host}:#{@port}"
     end
 
-    def connect
-      @socket = SocketWithTimeout.new(@host, @port, connect_timeout: @connect_timeout, timeout: @socket_timeout)
-
-      @encoder = Kafka::Protocol::Encoder.new(@socket)
-      @decoder = Kafka::Protocol::Decoder.new(@socket)
-
-      # Correlation id is initialized to zero and bumped for each request.
-      @correlation_id = 0
-    rescue Errno::ETIMEDOUT => e
-      @logger.error "Timed out while trying to connect to #{self}: #{e}"
-      raise ConnectionError, e
-    rescue SocketError, Errno::ECONNREFUSED => e
-      @logger.error "Failed to connect to #{self}: #{e}"
-      raise ConnectionError, e
-    end
-
-    def connected?
+    def open?
       !@socket.nil?
     end
 
@@ -75,32 +68,24 @@ module Kafka
 
     # Sends a request over the connection.
     #
-    # @param api_key [Integer] the integer code for the API that is invoked.
     # @param request [#encode] the request that should be encoded and written.
     # @param response_class [#decode] an object that can decode the response.
     #
     # @return [Object] the response that was decoded by `response_class`.
-    def request(api_key, request, response_class)
-      connect unless connected?
-
-      write_request(api_key, request)
-
-      unless response_class.nil?
-        loop do
-          correlation_id, response = read_response(response_class)
-
-          # There may have been a previous request that timed out before the client
-          # was able to read the response. In that case, the response will still be
-          # sitting in the socket waiting to be read. If the response we just read
-          # was to a previous request, we can safely skip it.
-          if correlation_id < @correlation_id
-            @logger.error "Received out-of-order response id #{correlation_id}, was expecting #{@correlation_id}"
-          elsif correlation_id > @correlation_id
-            raise Kafka::Error, "Correlation id mismatch: expected #{@correlation_id} but got #{correlation_id}"
-          else
-            break response
-          end
-        end
+    def send_request(request, response_class)
+      Instrumentation.instrument("request.kafka") do |notification|
+        open unless open?
+
+        @correlation_id += 1
+
+        # Look up the API name.
+        notification[:api] = Protocol.api_name(request.api_key)
+
+        # We may not read a response, in which case the size is zero.
+        notification[:response_size] = 0
+
+        write_request(request, notification)
+        wait_for_response(response_class, notification) unless response_class.nil?
       end
     rescue Errno::EPIPE, Errno::ECONNRESET, Errno::ETIMEDOUT, EOFError => e
       @logger.error "Connection error: #{e}"
@@ -112,24 +97,41 @@ module Kafka
 
     private
 
+    def open
+      @logger.info "Opening connection to #{@host}:#{@port} with client id #{@client_id}..."
+
+      @socket = SocketWithTimeout.new(@host, @port, connect_timeout: @connect_timeout, timeout: @socket_timeout)
+
+      @encoder = Kafka::Protocol::Encoder.new(@socket)
+      @decoder = Kafka::Protocol::Decoder.new(@socket)
+
+      # Correlation id is initialized to zero and bumped for each request.
+      @correlation_id = 0
+    rescue Errno::ETIMEDOUT => e
+      @logger.error "Timed out while trying to connect to #{self}: #{e}"
+      raise ConnectionError, e
+    rescue SocketError, Errno::ECONNREFUSED => e
+      @logger.error "Failed to connect to #{self}: #{e}"
+      raise ConnectionError, e
+    end
+
     # Writes a request over the connection.
     #
-    # @param api_key [Integer] the integer code for the API that is invoked.
     # @param request [#encode] the request that should be encoded and written.
     #
     # @return [nil]
-    def write_request(api_key, request)
-      @correlation_id += 1
+    def write_request(request, notification)
       @logger.debug "Sending request #{@correlation_id} to #{to_s}"
 
       message = Kafka::Protocol::RequestMessage.new(
-        api_key: api_key,
+        api_key: request.api_key,
         correlation_id: @correlation_id,
         client_id: @client_id,
         request: request,
       )
 
       data = Kafka::Protocol::Encoder.encode_with(message)
+      notification[:request_size] = data.bytesize
 
       @encoder.write_bytes(data)
 
@@ -145,12 +147,13 @@ module Kafka
     #   a given Decoder.
     #
     # @return [nil]
-    def read_response(response_class)
+    def read_response(response_class, notification)
       @logger.debug "Waiting for response #{@correlation_id} from #{to_s}"
 
-      bytes = @decoder.bytes
+      data = @decoder.bytes
+      notification[:response_size] = data.bytesize
 
-      buffer = StringIO.new(bytes)
+      buffer = StringIO.new(data)
       response_decoder = Kafka::Protocol::Decoder.new(buffer)
 
       correlation_id = response_decoder.int32
@@ -163,5 +166,23 @@ module Kafka
       @logger.error "Timed out while waiting for response #{@correlation_id}"
       raise
     end
+
+    def wait_for_response(response_class, notification)
+      loop do
+        correlation_id, response = read_response(response_class, notification)
+
+        # There may have been a previous request that timed out before the client
+        # was able to read the response. In that case, the response will still be
+        # sitting in the socket waiting to be read. If the response we just read
+        # was to a previous request, we can safely skip it.
+        if correlation_id < @correlation_id
+          @logger.error "Received out-of-order response id #{correlation_id}, was expecting #{@correlation_id}"
+        elsif correlation_id > @correlation_id
+          raise Kafka::Error, "Correlation id mismatch: expected #{@correlation_id} but got #{correlation_id}"
+        else
+          return response
+        end
+      end
+    end
   end
 end

data/lib/kafka/instrumentation.rb

@@ -0,0 +1,13 @@
+module Kafka
+  class NullInstrumentation
+    def self.instrument(name, payload = {})
+      yield payload if block_given?
+    end
+  end
+
+  if defined?(ActiveSupport::Notifications)
+    Instrumentation = ActiveSupport::Notifications 
+  else
+    Instrumentation = NullInstrumentation
+  end
+end

data/lib/kafka/producer.rb

@@ -7,6 +7,20 @@ module Kafka
 
   # Allows sending messages to a Kafka cluster.
   #
+  # Typically you won't instantiate this class yourself, but rather have {Kafka::Client}
+  # do it for you, e.g.
+  #
+  #     # Will instantiate Kafka::Client
+  #     kafka = Kafka.new(...)
+  #
+  #     # Will instantiate Kafka::Producer
+  #     producer = kafka.get_producer
+  #
+  # This is done in order to share a logger as well as a pool of broker connections across
+  # different producers. This also means that you don't need to pass the `broker_pool` and
+  # `logger` options to `#get_producer`. See {#initialize} for the list of other options
+  # you can pass in.
+  #
   # ## Buffering
   #
   # The producer buffers pending messages until {#send_messages} is called. Note that there is
@@ -73,8 +87,10 @@ module Kafka
     # Initializes a new Producer.
     #
     # @param broker_pool [BrokerPool] the broker pool representing the cluster.
+    #   Typically passed in for you.
     #
-    # @param logger [Logger]
+    # @param logger [Logger] the logger that should be used. Typically passed
+    #   in for you.
     #
     # @param ack_timeout [Integer] The number of seconds a broker can wait for
     #   replicas to acknowledge a write before responding with a timeout.
@@ -91,7 +107,7 @@ module Kafka
     # @param max_buffer_size [Integer] the number of messages allowed in the buffer
     #   before new writes will raise BufferOverflow exceptions.
     #
-    def initialize(broker_pool:, logger:, ack_timeout: 10, required_acks: 1, max_retries: 2, retry_backoff: 1, max_buffer_size: 1000)
+    def initialize(broker_pool:, logger:, ack_timeout: 5, required_acks: 1, max_retries: 2, retry_backoff: 1, max_buffer_size: 1000)
       @broker_pool = broker_pool
       @logger = logger
       @required_acks = required_acks

data/lib/kafka/protocol.rb

@@ -1,7 +1,9 @@
 module Kafka
   module Protocol
-    PRODUCE_API_KEY = 0
-    TOPIC_METADATA_API_KEY = 3
+    APIS = {
+      0 => :produce,
+      3 => :topic_metadata,
+    }
 
     ERRORS = {
       -1 => UnknownError,
@@ -23,7 +25,6 @@ module Kafka
       21 => InvalidRequiredAcks,
     }
 
-
     def self.handle_error(error_code)
       if error_code == 0
         # No errors, yay!
@@ -33,6 +34,10 @@ module Kafka
         raise UnknownError, "Unknown error with code #{error_code}"
       end
     end
+
+    def self.api_name(api_key)
+      APIS.fetch(api_key, :unknown)
+    end
   end
 end
 

data/lib/kafka/protocol/produce_request.rb

@@ -33,6 +33,10 @@ module Kafka
         @messages_for_topics = messages_for_topics
       end
 
+      def api_key
+        0
+      end
+
       # Whether this request requires any acknowledgements at all. If no acknowledgements
       # are required, the server will not send back a response at all.
       #

data/lib/kafka/protocol/topic_metadata_request.rb

@@ -5,6 +5,10 @@ module Kafka
         @topics = topics
       end
 
+      def api_key
+        3
+      end
+
       def encode(encoder)
         encoder.write_array(@topics) {|topic| encoder.write_string(topic) }
       end

data/lib/kafka/transmission.rb

@@ -12,18 +12,16 @@ module Kafka
       messages_for_broker = {}
 
       @buffer.each do |topic, partition, messages|
-        broker_id = @broker_pool.get_leader_id(topic, partition)
+        broker = @broker_pool.get_leader(topic, partition)
 
-        @logger.debug "Current leader for #{topic}/#{partition} is node #{broker_id}"
+        @logger.debug "Current leader for #{topic}/#{partition} is node #{broker}"
 
-        messages_for_broker[broker_id] ||= MessageBuffer.new
-        messages_for_broker[broker_id].concat(messages, topic: topic, partition: partition)
+        messages_for_broker[broker] ||= MessageBuffer.new
+        messages_for_broker[broker].concat(messages, topic: topic, partition: partition)
       end
 
-      messages_for_broker.each do |broker_id, message_set|
+      messages_for_broker.each do |broker, message_set|
         begin
-          broker = @broker_pool.get_broker(broker_id)
-
           response = broker.produce(
             messages_for_topics: message_set.to_h,
             required_acks: @required_acks,
@@ -32,7 +30,7 @@ module Kafka
 
           handle_response(response) if response
         rescue ConnectionError => e
-          @logger.error "Could not connect to broker #{broker_id}: #{e}"
+          @logger.error "Could not connect to broker #{broker}: #{e}"
 
           # Mark the broker pool as stale in order to force a cluster metadata refresh.
           @broker_pool.mark_as_stale!

data/lib/kafka/version.rb

@@ -1,3 +1,3 @@
 module Kafka
-  VERSION = "0.1.1"
+  VERSION = "0.1.2"
 end

data/{kafka.gemspec → ruby-kafka.gemspec}

@@ -32,4 +32,5 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency "dotenv"
   spec.add_development_dependency "docker-api"
   spec.add_development_dependency "rspec-benchmark"
+  spec.add_development_dependency "activesupport", ">= 4.2.0", "< 5.1"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: ruby-kafka
 version: !ruby/object:Gem::Version
-  version: 0.1.1
+  version: 0.1.2
 platform: ruby
 authors:
 - Daniel Schierbeck
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2016-02-04 00:00:00.000000000 Z
+date: 2016-02-05 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -108,6 +108,26 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: activesupport
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 4.2.0
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '5.1'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 4.2.0
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '5.1'
 description: |-
   A client library for the Kafka distributed commit log.
 
@@ -129,12 +149,12 @@ files:
 - bin/setup
 - circle.yml
 - examples/simple-producer.rb
-- kafka.gemspec
 - lib/kafka.rb
 - lib/kafka/broker.rb
 - lib/kafka/broker_pool.rb
 - lib/kafka/client.rb
 - lib/kafka/connection.rb
+- lib/kafka/instrumentation.rb
 - lib/kafka/message_buffer.rb
 - lib/kafka/partitioner.rb
 - lib/kafka/producer.rb
@@ -151,6 +171,7 @@ files:
 - lib/kafka/transmission.rb
 - lib/kafka/version.rb
 - lib/ruby-kafka.rb
+- ruby-kafka.gemspec
 homepage: https://github.com/zendesk/ruby-kafka
 licenses:
 - Apache License Version 2.0