ruby-kafka 0.3.13.beta1 → 0.3.13.beta2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: e6c48d1b7996a28caddd510f0daf032fdfca84fe
-  data.tar.gz: 302d2211024a2fdb904947c97cb0c258dc8583eb
+  metadata.gz: 3a1680eaa22189f9b23bcbca4789e3fd1ae59f16
+  data.tar.gz: 4bac9d11016756832446f01977bc64eea2ed6c16
 SHA512:
-  metadata.gz: d5524a95270673acb7e7e0ba2d6907eb774c715fb467925a052a633c1ab0e76de0fe5a16806ef3a7b95bbc29d54bd832a2b0a43d496753f4b5f1a162e4f91dde
-  data.tar.gz: 34f6417d7d0fdb987f1c6fdf69595a9de413ad010ca391f1d5d5fa3632fe781fe9514ae3568d20c2ed120887e288bb42848ebb22ab42125215d50ca16c0523da
+  metadata.gz: b3fa4970596e7f62155769a63103cfac3410ab234baf4a539620408f2704bd4e29b2e0977cf507cb3972c0788306aabf339334a90218f130a64dbdac64378eae
+  data.tar.gz: 7fd63b898166156713276a8d792e29af19a1764c9c27e97651ee267a7f11e10e07fcdc8807dea79d664e2c4e6248e9250b2e3cb85b1c0d41b127e73c1e6d65e2

data/lib/kafka/async_producer.rb

@@ -153,11 +153,11 @@ module Kafka
     end
 
     def buffer_overflow(topic)
-      @instrumenter.instrument("buffer_overflow.producer", {
+      @instrumenter.instrument("buffer_overflow.async_producer", {
         topic: topic,
       })
 
-      @logger.error "Buffer overflow: failed to enqueue message for #{topic}"
+      @logger.error "Cannot produce message to #{topic}, max queue size (#{@max_queue_size}) reached"
 
       raise BufferOverflow
     end

data/lib/kafka/connection.rb

@@ -118,16 +118,11 @@ module Kafka
     rescue Errno::ETIMEDOUT => e
       @logger.error "Timed out while trying to connect to #{self}: #{e}"
       raise ConnectionError, e
-    rescue SocketError, Errno::ECONNREFUSED => e
+    rescue SocketError, Errno::ECONNREFUSED, Errno::EHOSTUNREACH => e
       @logger.error "Failed to connect to #{self}: #{e}"
       raise ConnectionError, e
     end
 
-    def reopen
-      close
-      open
-    end
-
     # Writes a request over the connection.
     #
     # @param request [#encode] the request that should be encoded and written.
@@ -145,29 +140,14 @@ module Kafka
       )
 
       data = Kafka::Protocol::Encoder.encode_with(message)
-      retried = false
       notification[:request_size] = data.bytesize
 
-      begin
-        @encoder.write_bytes(data)
-      rescue Errno::ETIMEDOUT
-        @logger.error "Timed out while writing request #{@correlation_id}"
-        raise
-      rescue Errno::EPIPE, Errno::ECONNRESET, EOFError
-        # Kafka brokers automatically close client connections after a period of
-        # inactivity. If this has happened, it's safe to re-open the connection
-        # and retry the request.
-        if retried
-          raise
-        else
-          @logger.warn "Connection has been closed by the server, retrying..."
-          retried = true
-          reopen
-          retry
-        end
-      end
+      @encoder.write_bytes(data)
 
       nil
+    rescue Errno::ETIMEDOUT
+      @logger.error "Timed out while writing request #{@correlation_id}"
+      raise
     end
 
     # Reads a response from the connection.

data/lib/kafka/consumer.rb

@@ -54,6 +54,9 @@ module Kafka
 
       # Whether or not the consumer is currently consuming messages.
       @running = false
+
+      # The maximum number of bytes to fetch from a single partition, by topic.
+      @max_bytes = {}
     end
 
     # Subscribes the consumer to a topic.
@@ -71,12 +74,15 @@ module Kafka
     #   only applies when first consuming a topic partition – once the consumer
     #   has checkpointed its progress, it will always resume from the last
     #   checkpoint.
+    # @param max_bytes_per_partition [Integer] the maximum amount of data fetched
+    #   from a single partition at a time.
     # @return [nil]
-    def subscribe(topic, default_offset: nil, start_from_beginning: true)
+    def subscribe(topic, default_offset: nil, start_from_beginning: true, max_bytes_per_partition: 1048576)
       default_offset ||= start_from_beginning ? :earliest : :latest
 
       @group.subscribe(topic)
       @offset_manager.set_default_offset(topic, default_offset)
+      @max_bytes[topic] = max_bytes_per_partition
 
       nil
     end
@@ -248,10 +254,11 @@ module Kafka
       assigned_partitions.each do |topic, partitions|
         partitions.each do |partition|
           offset = @offset_manager.next_offset_for(topic, partition)
+          max_bytes = @max_bytes.fetch(topic)
 
           @logger.debug "Fetching batch from #{topic}/#{partition} starting at offset #{offset}"
 
-          operation.fetch_from_partition(topic, partition, offset: offset)
+          operation.fetch_from_partition(topic, partition, offset: offset, max_bytes: max_bytes)
         end
       end
 

data/lib/kafka/datadog.rb

@@ -133,6 +133,7 @@ module Kafka
       def produce_message(event)
         client = event.payload.fetch(:client_id)
         topic = event.payload.fetch(:topic)
+        message_size = event.payload.fetch(:message_size)
         buffer_size = event.payload.fetch(:buffer_size)
         max_buffer_size = event.payload.fetch(:max_buffer_size)
         buffer_fill_ratio = buffer_size.to_f / max_buffer_size.to_f
@@ -144,6 +145,8 @@ module Kafka
         # This gets us the write rate.
         increment("producer.produce.messages", tags: tags.merge(topic: topic))
 
+        histogram("producer.produce.message_size", message_size, tags: tags.merge(topic: topic))
+
         # This gets us the avg/max buffer size per producer.
         histogram("producer.buffer.size", buffer_size, tags: tags)
 
@@ -220,10 +223,19 @@ module Kafka
         }
 
         # This gets us the avg/max queue size per producer.
-        histogram("producer.queue.size", queue_size, tags: tags)
+        histogram("async_producer.queue.size", queue_size, tags: tags)
 
         # This gets us the avg/max queue fill ratio per producer.
-        histogram("producer.queue.fill_ratio", queue_fill_ratio, tags: tags)
+        histogram("async_producer.queue.fill_ratio", queue_fill_ratio, tags: tags)
+      end
+
+      def buffer_overflow(event)
+        tags = {
+          client: event.payload.fetch(:client_id),
+          topic: event.payload.fetch(:topic),
+        }
+
+        increment("async_producer.produce.errors", tags: tags)
       end
 
       attach_to "async_producer.kafka"

data/lib/kafka/producer.rb

@@ -195,11 +195,13 @@ module Kafka
       )
 
       if buffer_size >= @max_buffer_size
-        buffer_overflow topic, "Max buffer size (#{@max_buffer_size} messages) exceeded"
+        buffer_overflow topic,
+          "Cannot produce to #{topic}, max buffer size (#{@max_buffer_size} messages) reached"
       end
 
       if buffer_bytesize + message.bytesize >= @max_buffer_bytesize
-        buffer_overflow topic, "Max buffer bytesize (#{@max_buffer_bytesize} bytes) exceeded"
+        buffer_overflow topic,
+          "Cannot produce to #{topic}, max buffer bytesize (#{@max_buffer_bytesize} bytes) reached"
       end
 
       @target_topics.add(topic)
@@ -210,6 +212,7 @@ module Kafka
         key: key,
         topic: topic,
         create_time: create_time,
+        message_size: message.bytesize,
         buffer_size: buffer_size,
         max_buffer_size: @max_buffer_size,
       })

data/lib/kafka/version.rb

@@ -1,3 +1,3 @@
 module Kafka
-  VERSION = "0.3.13.beta1"
+  VERSION = "0.3.13.beta2"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: ruby-kafka
 version: !ruby/object:Gem::Version
-  version: 0.3.13.beta1
+  version: 0.3.13.beta2
 platform: ruby
 authors:
 - Daniel Schierbeck