ruby-kafka 0.1.0 → 0.1.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 35e88aecf65cde449234f713191dc13758358a68
-  data.tar.gz: cbf1941a1bae86287bf9f10040da64c20d3024bc
+  metadata.gz: a0a1febaafd77046f2840beb708d3ad0d6d4520b
+  data.tar.gz: 5a5824f5799db3d6f7f00193baa3c3a9b4aa6a93
 SHA512:
-  metadata.gz: f3454155cb2e2311044d470a4a33cd8a0fdace0796cabefaa7bf9e08dc5604dc2dd9afe32d50b6022c9a1f9deeb604bcc3b143768e9ce84b438503e5eb71cb91
-  data.tar.gz: e85dea426e4d8f5fb1ec209a658307a5c217fb3c2dc4034828d1fec5181d6411f87e0ceefa37d8ea2377ac6db6ef33f170ada08566709fd9e83bb68afe41391c
+  metadata.gz: 8617964bb38c05859f8fed11c77aa6b906ddf2d4bdd86a4e0ab003e4ba797454f886bf2e9611270de30790a4e24cb0bab94d70274fd020fed148fa81d2868d0a
+  data.tar.gz: acc55fc5c0afcf0c9f46e33b04dcd0ec47a1de7fc6f63819d1ac2cdb4b5c46b347a1e570494aec08fc09993ebfa4ae6fbb260514772a1aa49159c6c8ed1a3e46

data/README.md

@@ -1,8 +1,10 @@
-# Kafka
+# ruby-kafka
+
+[![Circle CI](https://circleci.com/gh/zendesk/ruby-kafka.svg?style=shield)](https://circleci.com/gh/zendesk/ruby-kafka/tree/master)
 
 A Ruby client library for the Kafka distributed log system. The focus of this library will be operational simplicity, with good logging and metrics that can make debugging issues easier.
 
-This library is still in pre-beta stage, but development is ongoing. Current efforts are focused on implementing a solid Producer client. The next step will be implementing a client for the Kafka 0.9 Consumer API.
+Currently, only the Producer API has been implemented, but a fully-fledged Consumer implementation compatible with Kafka 0.9 is on the roadmap.
 
 ## Installation
 
@@ -22,9 +24,9 @@ Or install it yourself as:
 
 ## Usage
 
-Currently, only the Producer API is supported. A Kafka 0.9 compatible Consumer API is on the roadmap.
-
 ```ruby
+require "kafka"
+
 # A client must be initialized with at least one Kafka broker. Each client keeps
 # a separate pool of broker connections. Don't use the same client from more than
 # one thread.
@@ -68,16 +70,14 @@ After checking out the repo, run `bin/setup` to install dependencies. Then, run
 
 ## Roadmap
 
-v0.1 is targeted for release in February. Other milestones do not have firm target dates, but v0.2 will be released as soon as we are confident that it is ready to run in critical production environments and that the API shouldn't be changed.
+The current stable release is v0.1. This release is running in production at Zendesk, but it's still not recommended that you use it when data loss is unacceptable. It will take a little while until all edge cases have been uncovered and handled.
 
-### v0.1: Producer API for non-critical production data
-
-We need to actually run this in production for a while before we can say that it won't lose data, so initially the library should only be deployed for non-critical use cases.
-
-The API may also be changed.
+The API may still be changed in v0.2. 
 
 ### v0.2: Stable Producer API
 
+Target date: end of February.
+
 The API should now have stabilized and the library should be battle tested enough to deploy for critical use cases.
 
 ### v1.0: Consumer API

data/lib/kafka/broker.rb

@@ -38,7 +38,7 @@ module Kafka
             Protocol.handle_error(partition.partition_error_code)
           rescue ReplicaNotAvailable
             # This error can be safely ignored per the protocol specification.
-            @logger.warn "Replica not available for topic #{topic.topic_name}, partition #{partition.partition_id}"
+            @logger.warn "Replica not available for #{topic.topic_name}/#{partition.partition_id}"
           end
         end
       end

data/lib/kafka/client.rb

@@ -3,6 +3,7 @@ require "kafka/producer"
 
 module Kafka
   class Client
+    DEFAULT_CLIENT_ID = "ruby-kafka"
 
     # Initializes a new Kafka client.
     #
@@ -17,7 +18,7 @@ module Kafka
     #   connections. See {BrokerPool#initialize}.
     #
     # @return [Client]
-    def initialize(seed_brokers:, client_id:, logger:, socket_timeout: nil)
+    def initialize(seed_brokers:, client_id: DEFAULT_CLIENT_ID, logger:, socket_timeout: nil)
       @logger = logger
 
       @broker_pool = BrokerPool.new(

data/lib/kafka/connection.rb

@@ -38,7 +38,15 @@ module Kafka
 
       @logger.info "Opening connection to #{@host}:#{@port} with client id #{@client_id}..."
 
-      @socket = SocketWithTimeout.new(@host, @port, timeout: @connect_timeout)
+      connect
+    end
+
+    def to_s
+      "#{@host}:#{@port}"
+    end
+
+    def connect
+      @socket = SocketWithTimeout.new(@host, @port, connect_timeout: @connect_timeout, timeout: @socket_timeout)
 
       @encoder = Kafka::Protocol::Encoder.new(@socket)
       @decoder = Kafka::Protocol::Decoder.new(@socket)
@@ -46,20 +54,23 @@ module Kafka
       # Correlation id is initialized to zero and bumped for each request.
       @correlation_id = 0
     rescue Errno::ETIMEDOUT => e
-      @logger.error "Timed out while trying to connect to #{host}:#{port}: #{e}"
+      @logger.error "Timed out while trying to connect to #{self}: #{e}"
       raise ConnectionError, e
     rescue SocketError, Errno::ECONNREFUSED => e
-      @logger.error "Failed to connect to #{host}:#{port}: #{e}"
+      @logger.error "Failed to connect to #{self}: #{e}"
       raise ConnectionError, e
     end
 
-    def to_s
-      "#{@host}:#{@port}"
+    def connected?
+      !@socket.nil?
     end
 
     def close
       @logger.debug "Closing socket to #{to_s}"
-      @socket.close
+
+      @socket.close if @socket
+
+      @socket = nil
     end
 
     # Sends a request over the connection.
@@ -70,6 +81,8 @@ module Kafka
     #
     # @return [Object] the response that was decoded by `response_class`.
     def request(api_key, request, response_class)
+      connect unless connected?
+
       write_request(api_key, request)
 
       unless response_class.nil?
@@ -89,6 +102,12 @@ module Kafka
           end
         end
       end
+    rescue Errno::EPIPE, Errno::ECONNRESET, Errno::ETIMEDOUT, EOFError => e
+      @logger.error "Connection error: #{e}"
+
+      close
+
+      raise ConnectionError, "Connection error: #{e}"
     end
 
     private
@@ -117,7 +136,7 @@ module Kafka
       nil
     rescue Errno::ETIMEDOUT
       @logger.error "Timed out while writing request #{@correlation_id}"
-      raise ConnectionError
+      raise
     end
 
     # Reads a response from the connection.
@@ -142,7 +161,7 @@ module Kafka
       return correlation_id, response
     rescue Errno::ETIMEDOUT
       @logger.error "Timed out while waiting for response #{@correlation_id}"
-      raise ConnectionError
+      raise
     end
   end
 end

data/lib/kafka/protocol/decoder.rb

@@ -85,7 +85,7 @@ module Kafka
       #
       # @return [String]
       def read(number_of_bytes)
-        @io.read(number_of_bytes)
+        @io.read(number_of_bytes) or raise EOFError
       end
     end
   end

data/lib/kafka/socket_with_timeout.rb

@@ -15,12 +15,15 @@ module Kafka
     #
     # @param host [String]
     # @param port [Integer]
-    # @param timeout [Integer] the connection timeout, in seconds.
+    # @param connect_timeout [Integer] the connection timeout, in seconds.
+    # @param timeout [Integer] the read and write timeout, in seconds.
     # @raise [Errno::ETIMEDOUT] if the timeout is exceeded.
-    def initialize(host, port, timeout: nil)
+    def initialize(host, port, connect_timeout: nil, timeout: nil)
       addr = Socket.getaddrinfo(host, nil)
       sockaddr = Socket.pack_sockaddr_in(port, addr[0][3])
 
+      @timeout = timeout
+
       @socket = Socket.new(Socket.const_get(addr[0][0]), Socket::SOCK_STREAM, 0)
       @socket.setsockopt(Socket::IPPROTO_TCP, Socket::TCP_NODELAY, 1)
 
@@ -32,7 +35,7 @@ module Kafka
       rescue IO::WaitWritable
         # IO.select will block until the socket is writable or the timeout
         # is exceeded, whichever comes first.
-        unless IO.select(nil, [@socket], nil, timeout)
+        unless IO.select(nil, [@socket], nil, connect_timeout)
           # IO.select returns nil when the socket is not ready before timeout 
           # seconds have elapsed
           @socket.close
@@ -51,11 +54,10 @@ module Kafka
     # Reads bytes from the socket, possible with a timeout.
     #
     # @param num_bytes [Integer] the number of bytes to read.
-    # @param timeout [Integer] the number of seconds to wait before timing out.
     # @raise [Errno::ETIMEDOUT] if the timeout is exceeded.
     # @return [String] the data that was read from the socket.
-    def read(num_bytes, timeout: nil)
-      unless IO.select([@socket], nil, nil, timeout)
+    def read(num_bytes)
+      unless IO.select([@socket], nil, nil, @timeout)
         raise Errno::ETIMEDOUT
       end
 
@@ -65,11 +67,10 @@ module Kafka
     # Writes bytes to the socket, possible with a timeout.
     #
     # @param bytes [String] the data that should be written to the socket.
-    # @param timeout [Integer] the number of seconds to wait before timing out.
     # @raise [Errno::ETIMEDOUT] if the timeout is exceeded.
     # @return [Integer] the number of bytes written.
-    def write(bytes, timeout: nil)
-      unless IO.select(nil, [@socket], nil, timeout)
+    def write(bytes)
+      unless IO.select(nil, [@socket], nil, @timeout)
         raise Errno::ETIMEDOUT
       end
 

data/lib/kafka/version.rb

@@ -1,3 +1,3 @@
 module Kafka
-  VERSION = "0.1.0"
+  VERSION = "0.1.1"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: ruby-kafka
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.1.1
 platform: ruby
 authors:
 - Daniel Schierbeck
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2016-02-03 00:00:00.000000000 Z
+date: 2016-02-04 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler