ruby-kafka 0.1.2 → 0.1.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: e80cbf966470ab5038d6aab4f88fde89bd5e7f66
-  data.tar.gz: fcb92660786331e9b0f70d9a3e33aa8919590eff
+  metadata.gz: a2754b8ff5d028b2883c5657b91d1ab13881fa27
+  data.tar.gz: 71ddafeb0cabe9f03ec326037b6391070372befa
 SHA512:
-  metadata.gz: a62034fba5bf5f64f27d19676ac3429bfc929984f744db234d2fe25db10ae32ae93da6ff1d2635a0285e1d72010ec565951b2fcf00c7e5bc0aba56ba158b03d5
-  data.tar.gz: 236d24d5b6a8fac6fbad9cedc01b0639085ca746bc8bcf12f110aacd2f120fcf0c16f81a58e01c10d92fcc8b2c1c235ce228bc1b731412d4e94ff2fc1dbf43e8
+  metadata.gz: 09421c5b62a1c90e6be2ca25f2b1823d92384e7176d48252c787b8c4de73feac02f7362ac36f98e9da8b1a44968591302e4ee9020f4af5ef02baa8f852f89b7d
+  data.tar.gz: 361c720fd7bc8af0e162c463c0f24a31a5d92b942e23e6b1eeb17bb1843516ffb0b7a3778b43ee411ff492ea69274ba2e7624ab5d99631089f3da333a85fa5ed

data/README.md

@@ -64,6 +64,26 @@ producer.send_messages
 
 Read the docs for [Kafka::Producer](http://www.rubydoc.info/gems/ruby-kafka/Kafka/Producer) for more details.
 
+### Understanding Timeouts
+
+It's important to understand how timeouts work if you have a latency sensitive application. This library allows configuring timeouts on different levels:
+
+**Network timeouts** apply to network connections to individual Kafka brokers. There are two config keys here, each passed to `Kafka.new`:
+
+* `connect_timeout` sets the number of seconds to wait while connecting to a broker for the first time. When ruby-kafka initializes, it needs to connect to at least one host in `seed_brokers` in order to discover the Kafka cluster. Each host is tried until there's one that works. Usually that means the first one, but if your entire cluster is down, or there's a network partition, you could wait up to `n * connect_timeout` seconds, where `n` is the number of seed brokers.
+* `socket_timeout` sets the number of seconds to wait when reading from or writing to a socket connection to a broker. After this timeout expires the connection will be killed. Note that some Kafka operations are by definition long-running, such as waiting for new messages to arrive in a partition, so don't set this value too low. When configuring timeouts relating to specific Kafka operations, make sure to make them shorter than this one.
+
+**Producer timeouts** can be configured when calling `#get_producer` on a client instance:
+
+* `ack_timeout` is a timeout executed by a broker when the client is sending messages to it. It defines the number of seconds the broker should wait for replicas to acknowledge the write before responding to the client with an error. As such, it relates to the `required_acks` setting. It should be set lower than `socket_timeout`.
+* `retry_backoff` configures the number of seconds to wait after a failed attempt to send messages to a Kafka broker before retrying. The `max_retries` setting defines the maximum number of retries to attempt, and so the total duration could be up to `max_retries * retry_backoff` seconds. The timeout can be arbitrarily long, and shouldn't be too short: if a broker goes down its partitions will be handed off to another broker, and that can take tens of seconds.
+
+When sending many messages, it's likely that the client needs to send some messages to each broker in the cluster. Given `n` brokers in the cluster, the total wait time when calling `Kafka::Producer#send_messages` can be up to
+
+    n * (connect_timeout + socket_timeout + retry_backoff) * max_retries
+
+Make sure your application can survive being blocked for so long.
+
 ## Development
 
 After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake spec` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.

data/lib/kafka/broker.rb

@@ -27,22 +27,7 @@ module Kafka
       request = Protocol::TopicMetadataRequest.new(**options)
       response_class = Protocol::MetadataResponse
 
-      response = @connection.send_request(request, response_class)
-
-      response.topics.each do |topic|
-        Protocol.handle_error(topic.topic_error_code)
-
-        topic.partitions.each do |partition|
-          begin
-            Protocol.handle_error(partition.partition_error_code)
-          rescue ReplicaNotAvailable
-            # This error can be safely ignored per the protocol specification.
-            @logger.warn "Replica not available for #{topic.topic_name}/#{partition.partition_id}"
-          end
-        end
-      end
-
-      response
+      @connection.send_request(request, response_class)
     end
 
     def produce(**options)

data/lib/kafka/protocol/metadata_response.rb

@@ -50,7 +50,7 @@ module Kafka
 
         attr_reader :partition_error_code
 
-        def initialize(partition_error_code:, partition_id:, leader:, replicas:, isr:)
+        def initialize(partition_error_code:, partition_id:, leader:, replicas: [], isr: [])
           @partition_error_code = partition_error_code
           @partition_id = partition_id
           @leader = leader
@@ -68,7 +68,7 @@ module Kafka
 
         attr_reader :topic_error_code
 
-        def initialize(topic_error_code:, topic_name:, partitions:)
+        def initialize(topic_error_code: 0, topic_name:, partitions:)
           @topic_error_code = topic_error_code
           @topic_name = topic_name
           @partitions = partitions
@@ -96,13 +96,21 @@ module Kafka
         topic_info = @topics.find {|t| t.topic_name == topic }
 
         if topic_info.nil?
-          raise "no topic #{topic}"
+          raise UnknownTopicOrPartition, "no topic #{topic}"
         end
 
+        Protocol.handle_error(topic_info.topic_error_code)
+
         partition_info = topic_info.partitions.find {|p| p.partition_id == partition }
 
         if partition_info.nil?
-          raise "no partition #{partition} in topic #{topic}"
+          raise UnknownTopicOrPartition, "no partition #{partition} in topic #{topic}"
+        end
+
+        begin
+          Protocol.handle_error(partition_info.partition_error_code)
+        rescue ReplicaNotAvailable
+          # This error can be safely ignored per the protocol specification.
         end
 
         partition_info.leader

data/lib/kafka/version.rb

@@ -1,3 +1,3 @@
 module Kafka
-  VERSION = "0.1.2"
+  VERSION = "0.1.3"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: ruby-kafka
 version: !ruby/object:Gem::Version
-  version: 0.1.2
+  version: 0.1.3
 platform: ruby
 authors:
 - Daniel Schierbeck
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2016-02-05 00:00:00.000000000 Z
+date: 2016-02-08 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler