ruby-kafka 0.1.0.pre.beta4 → 0.1.0.pre.beta5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 71fcf73c1adb43253684cb0ef1406a0631c3ce3a
-  data.tar.gz: dde9f9123b31f20475c7d280532f0b135f6a84bf
+  metadata.gz: 2ed5c1319300907fea512494cd9f4f07c3b15b70
+  data.tar.gz: e76e37ec50c63fbd8969478a38e6e804059d5a45
 SHA512:
-  metadata.gz: 8c44cc61c8b50ab3854830596bfd965837e03e1af97c3917c3ecf4386bab38534184688ae5dc1f4c284c8f2878143637e9dffdfe35e95c504a8f1381ca4da6e5
-  data.tar.gz: 06de9ac78b013cf448e03d0ba4ec879059bfbc04c6626c373e30c4c0c95c3b23282b30c3607df8955ab4c617d5d31d67c8ae80033381212baccf107aefda0489
+  metadata.gz: 878fc9de44046cee3895212d766f44d5011de1dd3fd05f0f745efd00d284dc2cdae6e5d2aed3cb1ae794210da5f4b76a993cbff03c1554339cb5f5fb088ab071
+  data.tar.gz: 243bb41e327d1f9d8a9d06eb0c1b6bc585a63d2607ab0b25fe44af0ff7cb936429949620a9a4e8b12b5f0838db59aebfab7f73a4e100e7f8763f566094592405

data/kafka.gemspec

@@ -31,4 +31,5 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency "pry"
   spec.add_development_dependency "dotenv"
   spec.add_development_dependency "docker-api"
+  spec.add_development_dependency "rspec-benchmark"
 end

data/lib/kafka/connection.rb

@@ -1,5 +1,5 @@
-require "socket"
 require "stringio"
+require "kafka/socket_with_timeout"
 require "kafka/protocol/request_message"
 require "kafka/protocol/encoder"
 require "kafka/protocol/decoder"
@@ -38,7 +38,7 @@ module Kafka
 
       @logger.info "Opening connection to #{@host}:#{@port} with client id #{@client_id}..."
 
-      @socket = Socket.tcp(host, port, connect_timeout: @connect_timeout)
+      @socket = SocketWithTimeout.new(@host, @port, timeout: @connect_timeout)
 
       @encoder = Kafka::Protocol::Encoder.new(@socket)
       @decoder = Kafka::Protocol::Decoder.new(@socket)
@@ -112,14 +112,12 @@ module Kafka
 
       data = Kafka::Protocol::Encoder.encode_with(message)
 
-      unless IO.select(nil, [@socket], nil, @socket_timeout)
-        @logger.error "Timed out while writing request #{@correlation_id}"
-        raise ConnectionError
-      end
-
       @encoder.write_bytes(data)
 
       nil
+    rescue Errno::ETIMEDOUT
+      @logger.error "Timed out while writing request #{@correlation_id}"
+      raise ConnectionError
     end
 
     # Reads a response from the connection.
@@ -131,11 +129,6 @@ module Kafka
     def read_response(response_class)
       @logger.debug "Waiting for response #{@correlation_id} from #{to_s}"
 
-      unless IO.select([@socket], nil, nil, @socket_timeout)
-        @logger.error "Timed out while waiting for response #{@correlation_id}"
-        raise ConnectionError
-      end
-
       bytes = @decoder.bytes
 
       buffer = StringIO.new(bytes)
@@ -147,6 +140,9 @@ module Kafka
       @logger.debug "Received response #{correlation_id} from #{to_s}"
 
       return correlation_id, response
+    rescue Errno::ETIMEDOUT
+      @logger.error "Timed out while waiting for response #{@correlation_id}"
+      raise ConnectionError
     end
   end
 end

data/lib/kafka/message_buffer.rb

@@ -4,15 +4,20 @@ module Kafka
   class MessageBuffer
     include Enumerable
 
+    attr_reader :size
+
     def initialize
       @buffer = {}
+      @size = 0
     end
 
     def write(message, topic:, partition:)
+      @size += 1
       buffer_for(topic, partition) << message
     end
 
     def concat(messages, topic:, partition:)
+      @size += messages.count
       buffer_for(topic, partition).concat(messages)
     end
 
@@ -20,10 +25,6 @@ module Kafka
       @buffer
     end
 
-    def size
-      @buffer.values.inject(0) {|sum, messages| messages.values.flatten.size + sum }
-    end
-
     def empty?
       @buffer.empty?
     end
@@ -43,6 +44,8 @@ module Kafka
     #
     # @return [nil]
     def clear_messages(topic:, partition:)
+      @size -= @buffer[topic][partition].count
+
       @buffer[topic].delete(partition)
       @buffer.delete(topic) if @buffer[topic].empty?
     end
@@ -52,6 +55,7 @@ module Kafka
     # @return [nil]
     def clear
       @buffer = {}
+      @size = 0
     end
 
     private

data/lib/kafka/producer.rb

@@ -1,6 +1,7 @@
 require "kafka/partitioner"
 require "kafka/message_buffer"
 require "kafka/protocol/message"
+require "kafka/transmission"
 
 module Kafka
 
@@ -161,11 +162,19 @@ module Kafka
     def send_messages
       attempt = 0
 
+      transmission = Transmission.new(
+        broker_pool: @broker_pool,
+        buffer: @buffer,
+        required_acks: @required_acks,
+        ack_timeout: @ack_timeout,
+        logger: @logger,
+      )
+
       loop do
         @logger.info "Sending #{@buffer.size} messages"
 
         attempt += 1
-        transmit_messages
+        transmission.send_messages
 
         if @buffer.empty?
           @logger.info "Successfully transmitted all messages"
@@ -208,72 +217,5 @@ module Kafka
     def shutdown
       @broker_pool.shutdown
     end
-
-    private
-
-    def transmit_messages
-      messages_for_broker = {}
-
-      @buffer.each do |topic, partition, messages|
-        broker_id = @broker_pool.get_leader_id(topic, partition)
-
-        @logger.debug "Current leader for #{topic}/#{partition} is node #{broker_id}"
-
-        messages_for_broker[broker_id] ||= MessageBuffer.new
-        messages_for_broker[broker_id].concat(messages, topic: topic, partition: partition)
-      end
-
-      messages_for_broker.each do |broker_id, message_set|
-        begin
-          broker = @broker_pool.get_broker(broker_id)
-
-          response = broker.produce(
-            messages_for_topics: message_set.to_h,
-            required_acks: @required_acks,
-            timeout: @ack_timeout * 1000, # Kafka expects the timeout in milliseconds.
-          )
-
-          handle_response(response) if response
-        rescue ConnectionError => e
-          @logger.error "Could not connect to broker #{broker_id}: #{e}"
-
-          # Mark the broker pool as stale in order to force a cluster metadata refresh.
-          @broker_pool.mark_as_stale!
-        end
-      end
-    end
-
-    def handle_response(response)
-      response.each_partition do |topic_info, partition_info|
-        topic = topic_info.topic
-        partition = partition_info.partition
-
-        begin
-          Protocol.handle_error(partition_info.error_code)
-        rescue Kafka::CorruptMessage
-          @logger.error "Corrupt message when writing to #{topic}/#{partition}"
-        rescue Kafka::UnknownTopicOrPartition
-          @logger.error "Unknown topic or partition #{topic}/#{partition}"
-        rescue Kafka::LeaderNotAvailable
-          @logger.error "Leader currently not available for #{topic}/#{partition}"
-          @broker_pool.mark_as_stale!
-        rescue Kafka::NotLeaderForPartition
-          @logger.error "Broker not currently leader for #{topic}/#{partition}"
-          @broker_pool.mark_as_stale!
-        rescue Kafka::RequestTimedOut
-          @logger.error "Timed out while writing to #{topic}/#{partition}"
-        rescue Kafka::NotEnoughReplicas
-          @logger.error "Not enough in-sync replicas for #{topic}/#{partition}"
-        rescue Kafka::NotEnoughReplicasAfterAppend
-          @logger.error "Messages written, but to fewer in-sync replicas than required for #{topic}/#{partition}"
-        else
-          offset = partition_info.offset
-          @logger.info "Successfully sent messages for #{topic}/#{partition}; new offset is #{offset}"
-
-          # The messages were successfully written; clear them from the buffer.
-          @buffer.clear_messages(topic: topic, partition: partition)
-        end
-      end
-    end
   end
 end

data/lib/kafka/socket_with_timeout.rb

@@ -0,0 +1,87 @@
+require "socket"
+
+module Kafka
+
+  # Opens sockets in a non-blocking fashion, ensuring that we're not stalling
+  # for long periods of time.
+  #
+  # It's possible to set timeouts for connecting to the server, for reading data,
+  # and for writing data. Whenever a timeout is exceeded, Errno::ETIMEDOUT is
+  # raised.
+  #
+  class SocketWithTimeout
+
+    # Opens a socket.
+    #
+    # @param host [String]
+    # @param port [Integer]
+    # @param timeout [Integer] the connection timeout, in seconds.
+    # @raise [Errno::ETIMEDOUT] if the timeout is exceeded.
+    def initialize(host, port, timeout: nil)
+      addr = Socket.getaddrinfo(host, nil)
+      sockaddr = Socket.pack_sockaddr_in(port, addr[0][3])
+
+      @socket = Socket.new(Socket.const_get(addr[0][0]), Socket::SOCK_STREAM, 0)
+      @socket.setsockopt(Socket::IPPROTO_TCP, Socket::TCP_NODELAY, 1)
+
+      begin
+        # Initiate the socket connection in the background. If it doesn't fail 
+        # immediately it will raise an IO::WaitWritable (Errno::EINPROGRESS) 
+        # indicating the connection is in progress.
+        @socket.connect_nonblock(sockaddr)
+      rescue IO::WaitWritable
+        # IO.select will block until the socket is writable or the timeout
+        # is exceeded, whichever comes first.
+        unless IO.select(nil, [@socket], nil, timeout)
+          # IO.select returns nil when the socket is not ready before timeout 
+          # seconds have elapsed
+          @socket.close
+          raise Errno::ETIMEDOUT
+        end
+
+        begin
+          # Verify there is now a good connection.
+          @socket.connect_nonblock(sockaddr)
+        rescue Errno::EISCONN
+          # The socket is connected, we're good!
+        end
+      end
+    end
+
+    # Reads bytes from the socket, possible with a timeout.
+    #
+    # @param num_bytes [Integer] the number of bytes to read.
+    # @param timeout [Integer] the number of seconds to wait before timing out.
+    # @raise [Errno::ETIMEDOUT] if the timeout is exceeded.
+    # @return [String] the data that was read from the socket.
+    def read(num_bytes, timeout: nil)
+      unless IO.select([@socket], nil, nil, timeout)
+        raise Errno::ETIMEDOUT
+      end
+
+      @socket.read(num_bytes)
+    end
+
+    # Writes bytes to the socket, possible with a timeout.
+    #
+    # @param bytes [String] the data that should be written to the socket.
+    # @param timeout [Integer] the number of seconds to wait before timing out.
+    # @raise [Errno::ETIMEDOUT] if the timeout is exceeded.
+    # @return [Integer] the number of bytes written.
+    def write(bytes, timeout: nil)
+      unless IO.select(nil, [@socket], nil, timeout)
+        raise Errno::ETIMEDOUT
+      end
+
+      @socket.write(bytes)
+    end
+
+    def close
+      @socket.close
+    end
+
+    def set_encoding(encoding)
+      @socket.set_encoding(encoding)
+    end
+  end
+end

data/lib/kafka/transmission.rb

@@ -0,0 +1,78 @@
+module Kafka
+  class Transmission
+    def initialize(broker_pool:, buffer:, required_acks:, ack_timeout:, logger:)
+      @broker_pool = broker_pool
+      @buffer = buffer
+      @required_acks = required_acks
+      @ack_timeout = ack_timeout
+      @logger = logger
+    end
+
+    def send_messages
+      messages_for_broker = {}
+
+      @buffer.each do |topic, partition, messages|
+        broker_id = @broker_pool.get_leader_id(topic, partition)
+
+        @logger.debug "Current leader for #{topic}/#{partition} is node #{broker_id}"
+
+        messages_for_broker[broker_id] ||= MessageBuffer.new
+        messages_for_broker[broker_id].concat(messages, topic: topic, partition: partition)
+      end
+
+      messages_for_broker.each do |broker_id, message_set|
+        begin
+          broker = @broker_pool.get_broker(broker_id)
+
+          response = broker.produce(
+            messages_for_topics: message_set.to_h,
+            required_acks: @required_acks,
+            timeout: @ack_timeout * 1000, # Kafka expects the timeout in milliseconds.
+          )
+
+          handle_response(response) if response
+        rescue ConnectionError => e
+          @logger.error "Could not connect to broker #{broker_id}: #{e}"
+
+          # Mark the broker pool as stale in order to force a cluster metadata refresh.
+          @broker_pool.mark_as_stale!
+        end
+      end
+    end
+
+    private
+
+    def handle_response(response)
+      response.each_partition do |topic_info, partition_info|
+        topic = topic_info.topic
+        partition = partition_info.partition
+
+        begin
+          Protocol.handle_error(partition_info.error_code)
+        rescue Kafka::CorruptMessage
+          @logger.error "Corrupt message when writing to #{topic}/#{partition}"
+        rescue Kafka::UnknownTopicOrPartition
+          @logger.error "Unknown topic or partition #{topic}/#{partition}"
+        rescue Kafka::LeaderNotAvailable
+          @logger.error "Leader currently not available for #{topic}/#{partition}"
+          @broker_pool.mark_as_stale!
+        rescue Kafka::NotLeaderForPartition
+          @logger.error "Broker not currently leader for #{topic}/#{partition}"
+          @broker_pool.mark_as_stale!
+        rescue Kafka::RequestTimedOut
+          @logger.error "Timed out while writing to #{topic}/#{partition}"
+        rescue Kafka::NotEnoughReplicas
+          @logger.error "Not enough in-sync replicas for #{topic}/#{partition}"
+        rescue Kafka::NotEnoughReplicasAfterAppend
+          @logger.error "Messages written, but to fewer in-sync replicas than required for #{topic}/#{partition}"
+        else
+          offset = partition_info.offset
+          @logger.info "Successfully sent messages for #{topic}/#{partition}; new offset is #{offset}"
+
+          # The messages were successfully written; clear them from the buffer.
+          @buffer.clear_messages(topic: topic, partition: partition)
+        end
+      end
+    end
+  end
+end

data/lib/kafka/version.rb

@@ -1,3 +1,3 @@
 module Kafka
-  VERSION = "0.1.0-beta4"
+  VERSION = "0.1.0-beta5"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: ruby-kafka
 version: !ruby/object:Gem::Version
-  version: 0.1.0.pre.beta4
+  version: 0.1.0.pre.beta5
 platform: ruby
 authors:
 - Daniel Schierbeck
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2016-02-01 00:00:00.000000000 Z
+date: 2016-02-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -94,6 +94,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: rspec-benchmark
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 description: |-
   A client library for the Kafka distributed commit log.
 
@@ -133,6 +147,8 @@ files:
 - lib/kafka/protocol/produce_response.rb
 - lib/kafka/protocol/request_message.rb
 - lib/kafka/protocol/topic_metadata_request.rb
+- lib/kafka/socket_with_timeout.rb
+- lib/kafka/transmission.rb
 - lib/kafka/version.rb
 - lib/ruby-kafka.rb
 homepage: https://github.com/zendesk/ruby-kafka