ruby-kafka 0.1.6 → 0.1.7

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 50a1c28cf71285d37c57e3dbe7a5d156f891576c
-  data.tar.gz: f8a139dc4061ec8a771f86c0a5ca280692ddd5cc
+  metadata.gz: cc49e2526024ee3dd1cb52b5abfef38dbc2e37ea
+  data.tar.gz: a3c0111bccd3e8daf83eec647baec2fa9921fbf8
 SHA512:
-  metadata.gz: 6d3245db50893aba63b50b600903dc0baf345017c940dcaa36b0e2fbe0f50fe231bc15f9af2dc083d4969fd63ba0493781570811a3ba4ba0e7df21b06e0fd993
-  data.tar.gz: d5cddd687cc85f02b0826e95205114e2f64cffd9dad48ef692e717dbb3f7cac41ce2b0afd71ee271e8590e27b9eac42d8f67f247c71fe760dec2915191361273
+  metadata.gz: 94547594275850a3bacd22758fc0ccce8956c83f0e6e7c46dd1c4a5fb4aa5b52d6199383862cb5087a90b89ee2e0ca63b486082b0d44da2e0994f6cc76e794a7
+  data.tar.gz: 69d7522e0ecd82057fd4b5879273026e6669441e0803c8d28bee83213a66bdbdd13f08051d4b3bb0725c3f19ad9df4dedde7c4a6c0236fac56079b1a666c7ed7

data/lib/kafka/async_producer.rb

@@ -18,6 +18,8 @@ module Kafka
   # By default, automatic delivery is disabled and you'll have to call
   # {#deliver_messages} manually.
   #
+  # ## Buffer Overflow and Backpressure
+  #
   # The calling thread communicates with the background thread doing the actual
   # work using a thread safe queue. While the background thread is busy delivering
   # messages, new messages will be buffered in the queue. In order to avoid
@@ -26,6 +28,17 @@ module Kafka
   # number of messages that is allowed to be buffered. You can configure this
   # value by setting `max_queue_size`.
   #
+  # If you produce messages faster than the background producer thread can
+  # deliver them to Kafka you will eventually fill the producer's buffer. Once
+  # this happens, the background thread will stop popping messages off the
+  # queue until it can successfully deliver the buffered messages. The queue
+  # will therefore grow in size, potentially hitting the `max_queue_size` limit.
+  # Once this happens, calls to {#produce} will raise a {BufferOverflow} error.
+  #
+  # Depending on your use case you may want to slow down the rate of messages
+  # being produced or perhaps halt your application completely until the
+  # producer can deliver the buffered messages and clear the message queue.
+  #
   # ## Example
   #
   #     producer = kafka.async_producer(
@@ -91,6 +104,7 @@ module Kafka
     # @return [nil]
     def produce(*args)
       raise BufferOverflow if @queue.size >= @max_queue_size
+
       @queue << [:produce, args]
 
       nil
@@ -146,7 +160,7 @@ module Kafka
 
           case operation
           when :produce
-            @producer.produce(*payload)
+            produce(*payload)
             deliver_messages if threshold_reached?
           when :deliver_messages
             deliver_messages
@@ -166,6 +180,13 @@ module Kafka
 
       private
 
+      def produce(*args)
+        @producer.produce(*args)
+      rescue BufferOverflow
+        deliver_messages
+        retry
+      end
+
       def deliver_messages
         @producer.deliver_messages
       rescue DeliveryFailed

data/lib/kafka/message_buffer.rb

@@ -6,22 +6,28 @@ module Kafka
   class MessageBuffer
     include Enumerable
 
-    attr_reader :size
+    attr_reader :size, :bytesize
 
     def initialize
       @buffer = {}
       @size = 0
+      @bytesize = 0
     end
 
     def write(value:, key:, topic:, partition:)
-      @size += 1
       message = Protocol::Message.new(key: key, value: value)
+
       buffer_for(topic, partition) << message
+
+      @size += 1
+      @bytesize += message.bytesize
     end
 
     def concat(messages, topic:, partition:)
-      @size += messages.count
       buffer_for(topic, partition).concat(messages)
+
+      @size += messages.count
+      @bytesize += messages.map(&:bytesize).reduce(:+)
     end
 
     def to_h
@@ -48,6 +54,7 @@ module Kafka
     # @return [nil]
     def clear_messages(topic:, partition:)
       @size -= @buffer[topic][partition].count
+      @bytesize -= @buffer[topic][partition].map(&:bytesize).reduce(:+)
 
       @buffer[topic].delete(partition)
       @buffer.delete(topic) if @buffer[topic].empty?

data/lib/kafka/pending_message.rb

@@ -2,12 +2,16 @@ module Kafka
   class PendingMessage
     attr_reader :value, :key, :topic, :partition, :partition_key
 
+    attr_reader :bytesize
+
     def initialize(value:, key:, topic:, partition:, partition_key:)
       @key = key
       @value = value
       @topic = topic
       @partition = partition
       @partition_key = partition_key
+
+      @bytesize = key.to_s.bytesize + value.to_s.bytesize
     end
   end
 end

data/lib/kafka/pending_message_queue.rb

@@ -0,0 +1,43 @@
+module Kafka
+
+  # A pending message queue holds messages that have not yet been assigned to
+  # a partition. It's designed to only remove messages once they've been
+  # successfully handled.
+  class PendingMessageQueue
+    attr_reader :size, :bytesize
+
+    def initialize
+      @messages = []
+      @size = 0
+      @bytesize = 0
+    end
+
+    def write(message)
+      @messages << message
+      @size += 1
+      @bytesize += message.bytesize
+    end
+
+    def empty?
+      @messages.empty?
+    end
+
+    # Yields each message in the queue to the provided block, removing the
+    # message after the block has processed it. If the block raises an
+    # exception, the message will be retained in the queue.
+    #
+    # @yieldparam [PendingMessage] message
+    # @return [nil]
+    def dequeue_each(&block)
+      until @messages.empty?
+        message = @messages.first
+
+        yield message
+
+        @size -= 1
+        @bytesize -= message.bytesize
+        @messages.shift
+      end
+    end
+  end
+end

data/lib/kafka/produce_operation.rb

@@ -25,12 +25,13 @@ module Kafka
   # * `sent_message_count` – the number of messages that were successfully sent.
   #
   class ProduceOperation
-    def initialize(cluster:, buffer:, compression_codec:, required_acks:, ack_timeout:, logger:)
+    def initialize(cluster:, buffer:, compression_codec:, compression_threshold:, required_acks:, ack_timeout:, logger:)
       @cluster = cluster
       @buffer = buffer
       @required_acks = required_acks
       @ack_timeout = ack_timeout
       @compression_codec = compression_codec
+      @compression_threshold = compression_threshold
       @logger = logger
     end
 
@@ -77,7 +78,12 @@ module Kafka
           messages_for_topics = {}
 
           message_buffer.each do |topic, partition, messages|
-            message_set = Protocol::MessageSet.new(messages: messages, compression_codec: @compression_codec)
+            message_set = Protocol::MessageSet.new(
+              messages: messages,
+              compression_codec: @compression_codec,
+              compression_threshold: @compression_threshold,
+            )
+
             messages_for_topics[topic] ||= {}
             messages_for_topics[topic][partition] = message_set
           end

data/lib/kafka/producer.rb

@@ -1,6 +1,8 @@
+require "set"
 require "kafka/partitioner"
 require "kafka/message_buffer"
 require "kafka/produce_operation"
+require "kafka/pending_message_queue"
 require "kafka/pending_message"
 require "kafka/compression"
 
@@ -47,10 +49,42 @@ module Kafka
   # not, we do another round of requests, this time with just the remaining messages.
   # We do this for as long as `max_retries` permits.
   #
+  # ## Compression
+  #
+  # Depending on what kind of data you produce, enabling compression may yield improved
+  # bandwidth and space usage. Compression in Kafka is done on entire messages sets
+  # rather than on individual messages. This improves the compression rate and generally
+  # means that compressions works better the larger your buffers get, since the message
+  # sets will be larger by the time they're compressed.
+  #
+  # Since many workloads have variations in throughput and distribution across partitions,
+  # it's possible to configure a threshold for when to enable compression by setting
+  # `compression_threshold`. Only if the defined number of messages are buffered for a
+  # partition will the messages be compressed.
+  #
+  # Compression is enabled by passing the `compression_codec` parameter with the
+  # name of one of the algorithms allowed by Kafka:
+  #
+  # * `:snappy` for [Snappy](http://google.github.io/snappy/) compression.
+  # * `:gzip` for [gzip](https://en.wikipedia.org/wiki/Gzip) compression.
+  #
+  # By default, all message sets will be compressed if you specify a compression
+  # codec. To increase the compression threshold, set `compression_threshold` to
+  # an integer value higher than one.
+  #
   # ## Instrumentation
   #
+  # Whenever {#produce} is called, the notification `produce_message.producer.kafka`
+  # will be emitted with the following payload:
+  #
+  # * `value` – the message value.
+  # * `key` – the message key.
+  # * `topic` – the topic that was produced to.
+  # * `buffer_size` – the buffer size after adding the message.
+  # * `max_buffer_size` – the maximum allowed buffer size for the producer.
+  #
   # After {#deliver_messages} completes, the notification
-  # `deliver_messages.producer.kafka` will be emitted.
+  # `deliver_messages.producer.kafka` will be emitted with the following payload:
   #
   # * `message_count` – the total number of messages that the producer tried to
   #   deliver. Note that not all messages may get delivered.
@@ -116,9 +150,21 @@ module Kafka
     # @param retry_backoff [Integer] the number of seconds to wait between retries.
     #
     # @param max_buffer_size [Integer] the number of messages allowed in the buffer
-    #   before new writes will raise BufferOverflow exceptions.
+    #   before new writes will raise {BufferOverflow} exceptions.
+    #
+    # @param max_buffer_bytesize [Integer] the maximum size of the buffer in bytes.
+    #   attempting to produce messages when the buffer reaches this size will
+    #   result in {BufferOverflow} being raised.
+    #
+    # @param compression_codec [Symbol, nil] the name of the compression codec to
+    #   use, or nil if no compression should be performed. Valid codecs: `:snappy`
+    #   and `:gzip`.
+    #
+    # @param compression_threshold [Integer] the number of messages that needs to
+    #   be in a message set before it should be compressed. Note that message sets
+    #   are per-partition rather than per-topic or per-producer.
     #
-    def initialize(cluster:, logger:, compression_codec: nil, ack_timeout: 5, required_acks: 1, max_retries: 2, retry_backoff: 1, max_buffer_size: 1000)
+    def initialize(cluster:, logger:, compression_codec: nil, compression_threshold: 1, ack_timeout: 5, required_acks: 1, max_retries: 2, retry_backoff: 1, max_buffer_size: 1000, max_buffer_bytesize: 10_000_000)
       @cluster = cluster
       @logger = logger
       @required_acks = required_acks
@@ -126,13 +172,18 @@ module Kafka
       @max_retries = max_retries
       @retry_backoff = retry_backoff
       @max_buffer_size = max_buffer_size
+      @max_buffer_bytesize = max_buffer_bytesize
       @compression_codec = Compression.find_codec(compression_codec)
+      @compression_threshold = compression_threshold
+
+      # The set of topics that are produced to.
+      @target_topics = Set.new
 
       # A buffer organized by topic/partition.
       @buffer = MessageBuffer.new
 
       # Messages added by `#produce` but not yet assigned a partition.
-      @pending_messages = []
+      @pending_message_queue = PendingMessageQueue.new
     end
 
     # Produces a message to the specified topic. Note that messages are buffered in
@@ -165,11 +216,7 @@ module Kafka
     # @raise [BufferOverflow] if the maximum buffer size has been reached.
     # @return [nil]
     def produce(value, key: nil, topic:, partition: nil, partition_key: nil)
-      unless buffer_size < @max_buffer_size
-        raise BufferOverflow, "Max buffer size #{@max_buffer_size} exceeded"
-      end
-
-      @pending_messages << PendingMessage.new(
+      message = PendingMessage.new(
         value: value,
         key: key,
         topic: topic,
@@ -177,6 +224,25 @@ module Kafka
         partition_key: partition_key,
       )
 
+      if buffer_size >= @max_buffer_size
+        raise BufferOverflow, "Max buffer size (#{@max_buffer_size} messages) exceeded"
+      end
+
+      if buffer_bytesize + message.bytesize >= @max_buffer_bytesize
+        raise BufferOverflow, "Max buffer bytesize (#{@max_buffer_bytesize} bytes) exceeded"
+      end
+
+      @target_topics.add(topic)
+      @pending_message_queue.write(message)
+
+      Instrumentation.instrument("produce_message.producer.kafka", {
+        value: value,
+        key: key,
+        topic: topic,
+        buffer_size: buffer_size,
+        max_buffer_size: @max_buffer_size,
+      })
+
       nil
     end
 
@@ -211,7 +277,11 @@ module Kafka
     #
     # @return [Integer] buffer size.
     def buffer_size
-      @pending_messages.size + @buffer.size
+      @pending_message_queue.size + @buffer.size
+    end
+
+    def buffer_bytesize
+      @pending_message_queue.bytesize + @buffer.bytesize
     end
 
     # Closes all connections to the brokers.
@@ -226,9 +296,7 @@ module Kafka
     def deliver_messages_with_retries(notification)
       attempt = 0
 
-      # Make sure we get metadata for this topic.
-      target_topics = @pending_messages.map(&:topic).uniq
-      @cluster.add_target_topics(target_topics)
+      @cluster.add_target_topics(@target_topics)
 
       operation = ProduceOperation.new(
         cluster: @cluster,
@@ -236,6 +304,7 @@ module Kafka
         required_acks: @required_acks,
         ack_timeout: @ack_timeout,
         compression_codec: @compression_codec,
+        compression_threshold: @compression_threshold,
         logger: @logger,
       )
 
@@ -249,7 +318,7 @@ module Kafka
         assign_partitions!
         operation.execute
 
-        if @pending_messages.empty? && @buffer.empty?
+        if buffer_size.zero?
           break
         elsif attempt <= @max_retries
           @logger.warn "Failed to send all messages; attempting retry #{attempt} of #{@max_retries} after #{@retry_backoff}s"
@@ -276,10 +345,7 @@ module Kafka
     end
 
     def assign_partitions!
-      until @pending_messages.empty?
-        # We want to keep the message in the first-stage buffer in case there's an error.
-        message = @pending_messages.first
-
+      @pending_message_queue.dequeue_each do |message|
         partition = message.partition
 
         if partition.nil?
@@ -293,9 +359,6 @@ module Kafka
           topic: message.topic,
           partition: partition,
         )
-
-        # Now it's safe to remove the message from the first-stage buffer.
-        @pending_messages.shift
       end
     rescue Kafka::Error => e
       @logger.error "Failed to assign pending message to a partition: #{e}"

data/lib/kafka/protocol/message.rb

@@ -18,11 +18,15 @@ module Kafka
 
       attr_reader :key, :value, :attributes, :offset
 
+      attr_reader :bytesize
+
       def initialize(value:, key: nil, attributes: 0, offset: -1)
         @key = key
         @value = value
         @attributes = attributes
         @offset = offset
+
+        @bytesize = @key.to_s.bytesize + @value.to_s.bytesize
       end
 
       def encode(encoder)

data/lib/kafka/protocol/message_set.rb

@@ -3,9 +3,14 @@ module Kafka
     class MessageSet
       attr_reader :messages
 
-      def initialize(messages: [], compression_codec: nil)
+      def initialize(messages: [], compression_codec: nil, compression_threshold: 1)
         @messages = messages
         @compression_codec = compression_codec
+        @compression_threshold = compression_threshold
+      end
+
+      def size
+        @messages.size
       end
 
       def ==(other)
@@ -13,10 +18,10 @@ module Kafka
       end
 
       def encode(encoder)
-        if @compression_codec.nil?
-          encode_without_compression(encoder)
-        else
+        if compress?
           encode_with_compression(encoder)
+        else
+          encode_without_compression(encoder)
         end
       end
 
@@ -39,6 +44,10 @@ module Kafka
 
       private
 
+      def compress?
+        !@compression_codec.nil? && size >= @compression_threshold
+      end
+
       def encode_with_compression(encoder)
         codec = @compression_codec
 

data/lib/kafka/version.rb

@@ -1,3 +1,3 @@
 module Kafka
-  VERSION = "0.1.6"
+  VERSION = "0.1.7"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: ruby-kafka
 version: !ruby/object:Gem::Version
-  version: 0.1.6
+  version: 0.1.7
 platform: ruby
 authors:
 - Daniel Schierbeck
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2016-02-22 00:00:00.000000000 Z
+date: 2016-02-24 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -179,6 +179,7 @@ files:
 - lib/kafka/message_buffer.rb
 - lib/kafka/partitioner.rb
 - lib/kafka/pending_message.rb
+- lib/kafka/pending_message_queue.rb
 - lib/kafka/produce_operation.rb
 - lib/kafka/producer.rb
 - lib/kafka/protocol.rb